summaryrefslogtreecommitdiffstats
path: root/src/lua
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
commit133a45c109da5310add55824db21af5239951f93 (patch)
treeba6ac4c0a950a0dda56451944315d66409923918 /src/lua
parentInitial commit. (diff)
downloadrspamd-upstream.tar.xz
rspamd-upstream.zip
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/lua')
-rw-r--r--src/lua/CMakeLists.txt39
-rw-r--r--src/lua/lua_cdb.c391
-rw-r--r--src/lua/lua_cfg_file.c157
-rw-r--r--src/lua/lua_classifier.c230
-rw-r--r--src/lua/lua_common.c2659
-rw-r--r--src/lua/lua_common.h729
-rw-r--r--src/lua/lua_compress.c622
-rw-r--r--src/lua/lua_compress.h37
-rw-r--r--src/lua/lua_config.c4780
-rw-r--r--src/lua/lua_cryptobox.c3065
-rw-r--r--src/lua/lua_dns.c198
-rw-r--r--src/lua/lua_dns_resolver.c754
-rw-r--r--src/lua/lua_dns_resolver.h15
-rw-r--r--src/lua/lua_expression.c512
-rw-r--r--src/lua/lua_html.cxx738
-rw-r--r--src/lua/lua_http.c1270
-rw-r--r--src/lua/lua_ip.c637
-rw-r--r--src/lua/lua_kann.c1361
-rw-r--r--src/lua/lua_logger.c1068
-rw-r--r--src/lua/lua_map.c1421
-rw-r--r--src/lua/lua_map.h38
-rw-r--r--src/lua/lua_mempool.c612
-rw-r--r--src/lua/lua_mimepart.c2304
-rw-r--r--src/lua/lua_parsers.c410
-rw-r--r--src/lua/lua_parsers.h88
-rw-r--r--src/lua/lua_redis.c1662
-rw-r--r--src/lua/lua_regexp.c858
-rw-r--r--src/lua/lua_rsa.c867
-rw-r--r--src/lua/lua_spf.c620
-rw-r--r--src/lua/lua_sqlite3.c379
-rw-r--r--src/lua/lua_task.c7295
-rw-r--r--src/lua/lua_tcp.c2566
-rw-r--r--src/lua/lua_tensor.c817
-rw-r--r--src/lua/lua_tensor.h34
-rw-r--r--src/lua/lua_text.c1789
-rw-r--r--src/lua/lua_thread_pool.cxx369
-rw-r--r--src/lua/lua_thread_pool.h194
-rw-r--r--src/lua/lua_trie.c500
-rw-r--r--src/lua/lua_udp.c594
-rw-r--r--src/lua/lua_upstream.c672
-rw-r--r--src/lua/lua_url.c1481
-rw-r--r--src/lua/lua_url.h87
-rw-r--r--src/lua/lua_util.c3585
-rw-r--r--src/lua/lua_worker.c883
-rw-r--r--src/lua/lua_xmlrpc.c796
-rw-r--r--src/lua/rspamd.luadoc124
46 files changed, 50307 insertions, 0 deletions
diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt
new file mode 100644
index 0000000..a504f99
--- /dev/null
+++ b/src/lua/CMakeLists.txt
@@ -0,0 +1,39 @@
+# Lua support makefile
+SET(LUASRC ${CMAKE_CURRENT_SOURCE_DIR}/lua_common.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_logger.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_task.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_config.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_classifier.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cfg_file.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_regexp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cdb.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_xmlrpc.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_http.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_redis.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_upstream.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_mempool.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_dns_resolver.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_rsa.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_ip.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_expression.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_trie.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_mimepart.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_url.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_tcp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_html.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_sqlite3.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_cryptobox.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_thread_pool.cxx
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_dns.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_udp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_text.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_worker.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_kann.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_spf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_tensor.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_parsers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/lua_compress.c)
+
+SET(RSPAMD_LUA ${LUASRC} PARENT_SCOPE) \ No newline at end of file
diff --git a/src/lua/lua_cdb.c b/src/lua/lua_cdb.c
new file mode 100644
index 0000000..76a5795
--- /dev/null
+++ b/src/lua/lua_cdb.c
@@ -0,0 +1,391 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "cdb.h"
+
+#define CDB_REFRESH_TIME 60
+
+/***
+ * @module rspamd_cdb
+ * Rspamd CDB module is used to read and write key/value pairs to the CDB file
+ *
+ * @example
+local rspamd_cdb = require "rspamd_cdb"
+rspamd_cdb.build('/tmp/test.cdb'):add('test', 'value'):finalize()
+local c = rspamd_cdb.open('/tmp/test.cdb')
+c:find('test')
+-- will return 'value'
+ */
+
+/***
+ * @function rspamd_cdb.open(filename, [ev_base])
+ * Opens an existing CDB for reading. If `ev_base` is specified, then cdb file is added
+ * for monitoring, that will get updates on disk file changes.
+ * @param {string} filename path to file
+ * @param {ev_base} event loop object
+ * @return {rspamd_cdb} cdb object
+ */
+LUA_FUNCTION_DEF(cdb, create);
+/***
+ * @method rspamd_cdb:find(key)
+ * Finds a specific key in cdb and returns a string or nil if a key has not been found
+ * @param {string} key key to find
+ * @return {string/nil} value for the specific key
+ */
+LUA_FUNCTION_DEF(cdb, lookup);
+/***
+ * @method rspamd_cdb:get_name()
+ * Returns filename for the specific cdb
+ * @return {string} filename for cdb
+ */
+LUA_FUNCTION_DEF(cdb, get_name);
+LUA_FUNCTION_DEF(cdb, destroy);
+
+/***
+ * @function rspamd_cdb.build(filename, [mode])
+ * Creates a new cdb in a file (existing one will be overwritten!). The object
+ * returned can be used merely for adding data. Upon finalizing, the data is written to
+ * disk and cdb can no longer be changed.
+ * @param {string} filename path to file
+ * @param {int} mode numeric mode to create a file
+ * @return {rspamd_cdb_builder} cdb builder object (or nil + error message)
+ */
+LUA_FUNCTION_DEF(cdb, build);
+/***
+ * @method rspamd_cdb_builder:add(key, value)
+ * Adds new value to cdb in the builder mode
+ * @param {string} key key to add
+ * @param {string} value value to associate with the key
+ * @return {rspamd_cdb_builder} the same object to allow chaining calls
+ */
+LUA_FUNCTION_DEF(cdb_builder, add);
+/***
+ * @method rspamd_cdb_builder:finalize()
+ * Finalizes the CDB and writes it to disk. This method also closes FD associated with
+ * CDB builder. No further additions are allowed after this point
+ */
+LUA_FUNCTION_DEF(cdb_builder, finalize);
+LUA_FUNCTION_DEF(cdb_builder, dtor);
+
+static const struct luaL_reg cdblib_m[] = {
+ LUA_INTERFACE_DEF(cdb, lookup),
+ {"find", lua_cdb_lookup},
+ LUA_INTERFACE_DEF(cdb, get_name),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cdb_destroy},
+ {NULL, NULL}};
+
+static const struct luaL_reg cdbbuilderlib_m[] = {
+ LUA_INTERFACE_DEF(cdb_builder, add),
+ LUA_INTERFACE_DEF(cdb_builder, finalize),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cdb_builder_dtor},
+ {NULL, NULL}};
+
+static const struct luaL_reg cdblib_f[] = {
+ LUA_INTERFACE_DEF(cdb, create),
+ {"open", lua_cdb_create},
+ {"build", lua_cdb_build},
+ {NULL, NULL}};
+
+static struct cdb *
+lua_check_cdb(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cdb}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'cdb' expected");
+ return ud ? *((struct cdb **) ud) : NULL;
+}
+
+static struct cdb_make *
+lua_check_cdb_builder(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cdb_builder}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'cdb_builder' expected");
+ return ud ? ((struct cdb_make *) ud) : NULL;
+}
+
+static const char *
+lua_cdb_get_input(lua_State *L, int pos, gsize *olen)
+{
+ int t = lua_type(L, pos);
+
+ switch (t) {
+ case LUA_TSTRING:
+ return lua_tolstring(L, pos, olen);
+ case LUA_TNUMBER: {
+ static char numbuf[sizeof(lua_Number)];
+ lua_Number n = lua_tonumber(L, pos);
+ memcpy(numbuf, &n, sizeof(numbuf));
+ *olen = sizeof(n);
+ return numbuf;
+ }
+ case LUA_TUSERDATA: {
+ void *p = rspamd_lua_check_udata_maybe(L, pos, "rspamd{text}");
+ if (p) {
+ struct rspamd_lua_text *t = (struct rspamd_lua_text *) p;
+ *olen = t->len;
+ return t->start;
+ }
+
+ p = rspamd_lua_check_udata_maybe(L, pos, "rspamd{int64}");
+ if (p) {
+ static char numbuf[sizeof(gint64)];
+
+ memcpy(numbuf, p, sizeof(numbuf));
+ *olen = sizeof(numbuf);
+ return numbuf;
+ }
+ }
+ default:
+ break;
+ }
+
+ return NULL;
+}
+
+static gint
+lua_cdb_create(lua_State *L)
+{
+ struct cdb *cdb, **pcdb;
+ const gchar *filename;
+ gint fd;
+
+ struct ev_loop *ev_base = NULL;
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ev_base = lua_check_ev_base(L, 2);
+ }
+
+ filename = luaL_checkstring(L, 1);
+ /* If file begins with cdb://, just skip it */
+ if (g_ascii_strncasecmp(filename, "cdb://", sizeof("cdb://") - 1) == 0) {
+ filename += sizeof("cdb://") - 1;
+ }
+
+ if ((fd = open(filename, O_RDONLY)) == -1) {
+ msg_warn("cannot open cdb: %s, %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ cdb = g_malloc0(sizeof(struct cdb));
+ cdb->filename = g_strdup(filename);
+ if (cdb_init(cdb, fd) == -1) {
+ g_free(cdb->filename);
+ g_free(cdb);
+ msg_warn("cannot open cdb: %s, %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+#ifdef HAVE_READAHEAD
+ struct stat st;
+ /*
+ * Do not readahead more than 100mb,
+ * which is enough for the vast majority of the use cases
+ */
+ static const size_t max_readahead = 100 * 0x100000;
+
+ if (fstat(cdb_fileno(cdb), &st) != 1) {
+ /* Must always be true because cdb_init calls it as well */
+ if (readahead(cdb_fileno(cdb), 0, MIN(max_readahead, st.st_size)) == -1) {
+ msg_warn("cannot readahead cdb: %s, %s", filename, strerror(errno));
+ }
+ }
+#endif
+ if (ev_base) {
+ cdb_add_timer(cdb, ev_base, CDB_REFRESH_TIME);
+ }
+ pcdb = lua_newuserdata(L, sizeof(struct cdb *));
+ rspamd_lua_setclass(L, "rspamd{cdb}", -1);
+ *pcdb = cdb;
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_cdb_get_name(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+
+ if (!cdb) {
+ lua_error(L);
+ return 1;
+ }
+ lua_pushstring(L, cdb->filename);
+ return 1;
+}
+
+static gint
+lua_cdb_lookup(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+ gsize klen;
+ const gchar *what = lua_cdb_get_input(L, 2, &klen);
+
+ if (!cdb || what == NULL) {
+ return lua_error(L);
+ }
+
+ if (cdb_find(cdb, what, klen) > 0) {
+ /* Extract and push value to lua as string */
+ lua_pushlstring(L, cdb_getdata(cdb), cdb_datalen(cdb));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_cdb_destroy(lua_State *L)
+{
+ struct cdb *cdb = lua_check_cdb(L, 1);
+
+ if (cdb) {
+ cdb_free(cdb);
+ if (cdb->cdb_fd != -1) {
+ (void) close(cdb->cdb_fd);
+ }
+ g_free(cdb->filename);
+ g_free(cdb);
+ }
+
+ return 0;
+}
+
+static gint
+lua_cdb_build(lua_State *L)
+{
+ const char *filename = luaL_checkstring(L, 1);
+ int fd, mode = 00755;
+
+ if (filename == NULL) {
+ return luaL_error(L, "invalid arguments, filename expected");
+ }
+
+ /* If file begins with cdb://, just skip it */
+ if (g_ascii_strncasecmp(filename, "cdb://", sizeof("cdb://") - 1) == 0) {
+ filename += sizeof("cdb://") - 1;
+ }
+
+ if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ fd = rspamd_file_xopen(filename, O_RDWR | O_CREAT | O_TRUNC, mode, 0);
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushfstring(L, "cannot open cdb: %s, %s", filename, strerror(errno));
+
+ return 2;
+ }
+
+ struct cdb_make *cdbm = lua_newuserdata(L, sizeof(struct cdb_make));
+
+ g_assert(cdb_make_start(cdbm, fd) == 0);
+ rspamd_lua_setclass(L, "rspamd{cdb_builder}", -1);
+
+ return 1;
+}
+
+static gint
+lua_cdb_builder_add(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+ gsize data_sz, key_sz;
+ const char *key = lua_cdb_get_input(L, 2, &key_sz);
+ const char *data = lua_cdb_get_input(L, 3, &data_sz);
+
+ if (cdbm == NULL || key == NULL || data == NULL || cdbm->cdb_fd == -1) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdb_make_add(cdbm, key, key_sz, data, data_sz) == -1) {
+ lua_pushvalue(L, 1);
+ lua_pushfstring(L, "cannot push value to cdb: %s", strerror(errno));
+
+ return 2;
+ }
+
+ /* Allow chaining */
+ lua_pushvalue(L, 1);
+ return 1;
+}
+
+static gint
+lua_cdb_builder_finalize(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+
+ if (cdbm == NULL || cdbm->cdb_fd == -1) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdb_make_finish(cdbm) == -1) {
+ lua_pushvalue(L, 1);
+ lua_pushfstring(L, "cannot finish value to cdb: %s", strerror(errno));
+
+ return 2;
+ }
+
+ close(cdbm->cdb_fd);
+ cdbm->cdb_fd = -1; /* To distinguish finalized object */
+
+ /* Allow chaining */
+ lua_pushvalue(L, 1);
+ return 1;
+}
+
+static gint
+lua_cdb_builder_dtor(lua_State *L)
+{
+ struct cdb_make *cdbm = lua_check_cdb_builder(L, 1);
+
+ if (cdbm == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cdbm->cdb_fd != -1) {
+ cdb_make_finish(cdbm);
+ close(cdbm->cdb_fd);
+ cdbm->cdb_fd = -1; /* Finalized object */
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_cdb(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cdblib_f);
+
+ return 1;
+}
+
+void luaopen_cdb(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{cdb}", cdblib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{cdb_builder}", cdbbuilderlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cdb", lua_load_cdb);
+}
diff --git a/src/lua/lua_cfg_file.c b/src/lua/lua_cfg_file.c
new file mode 100644
index 0000000..75bc380
--- /dev/null
+++ b/src/lua/lua_cfg_file.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "expression.h"
+#include "src/libserver/composites/composites.h"
+
+/*
+ * This is implementation of lua routines to handle config file params
+ */
+
+/* Process a single item in 'metrics' table */
+static void
+lua_process_metric(lua_State *L, const gchar *name, struct rspamd_config *cfg)
+{
+ gchar *symbol;
+ const gchar *desc = NULL;
+ gdouble *score;
+ struct rspamd_symbol *s;
+
+ /* Now iterate through module table */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* key - -2, value - -1 */
+ symbol = rspamd_mempool_strdup(cfg->cfg_pool, luaL_checkstring(L, -2));
+ if (symbol != NULL) {
+ if (lua_istable(L, -1)) {
+ /* We got a table, so extract individual attributes */
+ lua_pushstring(L, "weight");
+ lua_gettable(L, -2);
+ if (lua_isnumber(L, -1)) {
+ score = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(double));
+ *score = lua_tonumber(L, -1);
+ }
+ else {
+ msg_warn_config("cannot get weight of symbol: %s", symbol);
+ continue;
+ }
+ lua_pop(L, 1);
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+ if (lua_isstring(L, -1)) {
+ desc = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+ }
+ else if (lua_isnumber(L, -1)) {
+ /* Just got weight */
+ score = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(double));
+ *score = lua_tonumber(L, -1);
+ }
+ else {
+ msg_warn_config("cannot get weight of symbol: %s", symbol);
+ continue;
+ }
+ /* Insert symbol */
+ if ((s =
+ g_hash_table_lookup(cfg->symbols, symbol)) != NULL) {
+ msg_info_config("replacing weight for symbol %s: %.2f -> %.2f",
+ symbol,
+ *s->weight_ptr,
+ *score);
+ s->weight_ptr = score;
+ }
+ else {
+ s = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*s));
+ s->name = symbol;
+ s->weight_ptr = score;
+ g_hash_table_insert(cfg->symbols, symbol, s);
+ }
+
+ if (desc) {
+ s->description = rspamd_mempool_strdup(cfg->cfg_pool, desc);
+ }
+ }
+ }
+}
+
+/* Do post load initialization based on lua */
+void rspamd_lua_post_load_config(struct rspamd_config *cfg)
+{
+ lua_State *L = cfg->lua_state;
+ const gchar *name;
+ ucl_object_t *obj;
+ gsize keylen, i;
+
+ /* First check all module options that may be overridden in 'config' global */
+ lua_getglobal(L, "config");
+
+ if (lua_istable(L, -1)) {
+ /* Iterate to get all keys */
+ GPtrArray *names = g_ptr_array_new_full(rspamd_lua_table_size(L, -1),
+ g_free);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 2)) {
+ gchar *tmp;
+ lua_pushvalue(L, -2);
+ name = luaL_checklstring(L, -1, &keylen);
+
+ if (name && lua_istable(L, -2)) {
+ tmp = g_malloc(keylen + 1);
+ rspamd_strlcpy(tmp, name, keylen + 1);
+ g_ptr_array_add(names, tmp);
+ }
+ }
+
+ PTR_ARRAY_FOREACH(names, i, name)
+ {
+ lua_getfield(L, -1, name);
+
+ if (lua_istable(L, -1)) {
+ obj = ucl_object_lua_import(L, lua_gettop(L));
+
+ if (obj != NULL) {
+ ucl_object_sort_keys(obj, UCL_SORT_KEYS_DEFAULT);
+ ucl_object_insert_key_merged(cfg->cfg_ucl_obj,
+ obj,
+ name,
+ strlen(name),
+ true);
+ }
+ }
+ }
+
+ g_ptr_array_free(names, TRUE);
+ }
+
+ /* Check metrics settings */
+ lua_getglobal(L, "metrics");
+
+ if (lua_istable(L, -1)) {
+ /* Iterate */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* 'key' is at index -2 and 'value' is at index -1 */
+ /* Key must be a string and value must be a table */
+ name = luaL_checkstring(L, -2);
+ if (name != NULL && lua_istable(L, -1)) {
+ lua_process_metric(L, name, cfg);
+ }
+ }
+ }
+
+ lua_settop(L, 0);
+
+ rspamd_lua_start_gc(cfg);
+}
diff --git a/src/lua/lua_classifier.c b/src/lua/lua_classifier.c
new file mode 100644
index 0000000..39580a6
--- /dev/null
+++ b/src/lua/lua_classifier.c
@@ -0,0 +1,230 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/* Classifier methods */
+LUA_FUNCTION_DEF(classifier, get_statfiles);
+LUA_FUNCTION_DEF(classifier, get_statfile_by_label);
+LUA_FUNCTION_DEF(classifier, get_param);
+
+static const struct luaL_reg classifierlib_m[] = {
+ LUA_INTERFACE_DEF(classifier, get_statfiles),
+ LUA_INTERFACE_DEF(classifier, get_param),
+ LUA_INTERFACE_DEF(classifier, get_statfile_by_label),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(statfile, get_symbol);
+LUA_FUNCTION_DEF(statfile, get_label);
+LUA_FUNCTION_DEF(statfile, is_spam);
+LUA_FUNCTION_DEF(statfile, get_param);
+
+static const struct luaL_reg statfilelib_m[] = {
+ LUA_INTERFACE_DEF(statfile, get_symbol),
+ LUA_INTERFACE_DEF(statfile, get_label),
+ LUA_INTERFACE_DEF(statfile, is_spam),
+ LUA_INTERFACE_DEF(statfile, get_param),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd_statfile_config *lua_check_statfile(lua_State *L);
+
+/* Classifier implementation */
+
+
+static struct rspamd_classifier_config *
+lua_check_classifier(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{classifier}");
+ luaL_argcheck(L, ud != NULL, 1, "'classifier' expected");
+ return ud ? *((struct rspamd_classifier_config **) ud) : NULL;
+}
+
+/* Return table of statfiles indexed by name */
+static gint
+lua_classifier_get_statfiles(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ GList *cur;
+ struct rspamd_statfile_config *st, **pst;
+ gint i;
+
+ if (ccf) {
+ lua_newtable(L);
+ cur = g_list_first(ccf->statfiles);
+ i = 1;
+ while (cur) {
+ st = cur->data;
+ pst = lua_newuserdata(L, sizeof(struct rspamd_statfile_config *));
+ rspamd_lua_setclass(L, "rspamd{statfile}", -1);
+ *pst = st;
+ lua_rawseti(L, -2, i++);
+
+ cur = g_list_next(cur);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_classifier_get_param(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ const gchar *param;
+ const ucl_object_t *value;
+
+ param = luaL_checkstring(L, 2);
+
+ if (ccf != NULL && param != NULL) {
+ value = ucl_object_lookup(ccf->opts, param);
+
+ if (value != NULL) {
+ ucl_object_push_lua(L, value, true);
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/* Get statfile with specified label */
+static gint
+lua_classifier_get_statfile_by_label(lua_State *L)
+{
+ struct rspamd_classifier_config *ccf = lua_check_classifier(L);
+ struct rspamd_statfile_config *st, **pst;
+ const gchar *label;
+ GList *cur;
+ gint i;
+
+ label = luaL_checkstring(L, 2);
+ if (ccf && label) {
+ cur = g_hash_table_lookup(ccf->labels, label);
+ if (cur) {
+ lua_newtable(L);
+ i = 1;
+ while (cur) {
+ st = cur->data;
+ pst =
+ lua_newuserdata(L,
+ sizeof(struct rspamd_statfile_config *));
+ rspamd_lua_setclass(L, "rspamd{statfile}", -1);
+ *pst = st;
+ lua_rawseti(L, -2, i++);
+ cur = g_list_next(cur);
+ }
+ return 1;
+ }
+ }
+ lua_pushnil(L);
+ return 1;
+}
+
+/* Statfile functions */
+static gint
+lua_statfile_get_symbol(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL) {
+ lua_pushstring(L, st->symbol);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_get_label(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL && st->label != NULL) {
+ lua_pushstring(L, st->label);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_is_spam(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+
+ if (st != NULL) {
+ lua_pushboolean(L, st->is_spam);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_statfile_get_param(lua_State *L)
+{
+ struct rspamd_statfile_config *st = lua_check_statfile(L);
+ const gchar *param;
+ const ucl_object_t *value;
+
+ param = luaL_checkstring(L, 2);
+
+ if (st != NULL && param != NULL) {
+ value = ucl_object_lookup(st->opts, param);
+ if (value != NULL) {
+ lua_pushstring(L, ucl_object_tostring_forced(value));
+ return 1;
+ }
+ }
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static struct rspamd_statfile_config *
+lua_check_statfile(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{statfile}");
+ luaL_argcheck(L, ud != NULL, 1, "'statfile' expected");
+ return ud ? *((struct rspamd_statfile_config **) ud) : NULL;
+}
+
+
+/* Open functions */
+
+void luaopen_classifier(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{classifier}", classifierlib_m);
+ lua_pop(L, 1); /* remove metatable from stack */
+}
+
+void luaopen_statfile(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{statfile}", statfilelib_m);
+ lua_pop(L, 1); /* remove metatable from stack */
+}
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
new file mode 100644
index 0000000..9bf9514
--- /dev/null
+++ b/src/lua/lua_common.c
@@ -0,0 +1,2659 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_compress.h"
+#include "lptree.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "ottery.h"
+#include "lua_thread_pool.h"
+#include "libstat/stat_api.h"
+#include "libserver/rspamd_control.h"
+
+#include <math.h>
+
+
+/* Lua module init function */
+#define MODULE_INIT_FUNC "module_init"
+
+#ifdef WITH_LUA_TRACE
+ucl_object_t *lua_traces;
+#endif
+
+const luaL_reg null_reg[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const char rspamd_modules_state_global[] = "rspamd_plugins_state";
+
+static GQuark
+lua_error_quark(void)
+{
+ return g_quark_from_static_string("lua-routines");
+}
+
+/*
+ * Used to map string to a pointer
+ */
+KHASH_INIT(lua_class_set, const char *, int, 1, rspamd_str_hash, rspamd_str_equal);
+struct rspamd_lua_context {
+ lua_State *L;
+ khash_t(lua_class_set) * classes;
+ struct rspamd_lua_context *prev, *next; /* Expensive but we usually have exactly one lua state */
+};
+struct rspamd_lua_context *rspamd_lua_global_ctx = NULL;
+#define RSPAMD_LUA_NCLASSES 64
+static inline struct rspamd_lua_context *
+rspamd_lua_ctx_by_state(lua_State *L)
+{
+ struct rspamd_lua_context *cur;
+
+ DL_FOREACH(rspamd_lua_global_ctx, cur)
+ {
+ if (cur->L == L) {
+ return cur;
+ }
+ }
+
+ /* When we are using thread pool, this is the case... */
+ return rspamd_lua_global_ctx;
+}
+
+/* Util functions */
+/**
+ * Create new class and store metatable on top of the stack (must be popped if not needed)
+ * @param L
+ * @param classname name of class
+ * @param func table of class methods
+ */
+void rspamd_lua_new_class(lua_State *L,
+ const gchar *classname,
+ const struct luaL_reg *methods)
+{
+ khiter_t k;
+ gint r, nmethods = 0;
+ gboolean seen_index = false;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ if (methods) {
+ for (;;) {
+ if (methods[nmethods].name != NULL) {
+ if (strcmp(methods[nmethods].name, "__index") == 0) {
+ seen_index = true;
+ }
+ nmethods++;
+ }
+ else {
+ break;
+ }
+ }
+ }
+
+ lua_createtable(L, 0, 3 + nmethods);
+
+ if (!seen_index) {
+ lua_pushstring(L, "__index");
+ lua_pushvalue(L, -2); /* pushes the metatable */
+ lua_settable(L, -3); /* metatable.__index = metatable */
+ }
+
+ lua_pushstring(L, "class");
+ lua_pushstring(L, classname);
+ lua_rawset(L, -3);
+
+ if (methods) {
+ luaL_register(L, NULL, methods); /* pushes all methods as MT fields */
+ }
+
+ lua_pushvalue(L, -1); /* Preserves metatable */
+ int offset = luaL_ref(L, LUA_REGISTRYINDEX);
+ k = kh_put(lua_class_set, ctx->classes, classname, &r);
+ kh_value(ctx->classes, k) = offset;
+ /* MT is left on stack ! */
+}
+
+static const gchar *
+rspamd_lua_class_tostring_buf(lua_State *L, gboolean print_pointer, gint pos)
+{
+ static gchar buf[64];
+ const gchar *ret = NULL;
+ gint pop = 0;
+
+ if (!lua_getmetatable(L, pos)) {
+ goto err;
+ }
+
+ pop++;
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ pop++;
+
+ if (!lua_isstring(L, -1)) {
+ goto err;
+ }
+
+ if (print_pointer) {
+ rspamd_snprintf(buf, sizeof(buf), "%s(%p)", lua_tostring(L, -1),
+ lua_touserdata(L, 1));
+ }
+ else {
+ rspamd_snprintf(buf, sizeof(buf), "%s", lua_tostring(L, -1));
+ }
+
+ ret = buf;
+
+err:
+ lua_pop(L, pop);
+
+ return ret;
+}
+
+gint rspamd_lua_class_tostring(lua_State *L)
+{
+ const gchar *p;
+
+ p = rspamd_lua_class_tostring_buf(L, TRUE, 1);
+
+ if (!p) {
+ lua_pushstring(L, "invalid object passed to 'lua_common.c:__tostring'");
+ return lua_error(L);
+ }
+
+ lua_pushstring(L, p);
+
+ return 1;
+}
+
+
+void rspamd_lua_setclass(lua_State *L, const gchar *classname, gint objidx)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (objidx < 0) {
+ objidx--;
+ }
+ lua_setmetatable(L, objidx);
+}
+
+void rspamd_lua_class_metatable(lua_State *L, const gchar *classname)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+}
+
+void rspamd_lua_add_metamethod(lua_State *L, const gchar *classname,
+ luaL_Reg *meth)
+{
+ khiter_t k;
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ g_assert(k != kh_end(ctx->classes));
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ lua_pushcfunction(L, meth->func);
+ lua_setfield(L, -2, meth->name);
+ lua_pop(L, 1); /* remove metatable */
+}
+
+/* assume that table is at the top */
+void rspamd_lua_table_set(lua_State *L, const gchar *index, const gchar *value)
+{
+ lua_pushstring(L, index);
+ if (value) {
+ lua_pushstring(L, value);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ lua_settable(L, -3);
+}
+
+const gchar *
+rspamd_lua_table_get(lua_State *L, const gchar *index)
+{
+ const gchar *result;
+
+ lua_pushstring(L, index);
+ lua_gettable(L, -2);
+ if (!lua_isstring(L, -1)) {
+ return NULL;
+ }
+ result = lua_tostring(L, -1);
+ lua_pop(L, 1);
+ return result;
+}
+
+static void
+lua_add_actions_global(lua_State *L)
+{
+ gint i;
+
+ lua_newtable(L);
+
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ lua_pushstring(L, rspamd_action_to_str(i));
+ lua_pushinteger(L, i);
+ lua_settable(L, -3);
+ }
+ /* Set global table */
+ lua_setglobal(L, "rspamd_actions");
+}
+
+#ifndef __APPLE__
+#define OS_SO_SUFFIX ".so"
+#else
+#define OS_SO_SUFFIX ".dylib"
+#endif
+
+void rspamd_lua_set_path(lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars)
+{
+ const gchar *old_path, *additional_path = NULL;
+ const ucl_object_t *opts = NULL;
+ const gchar *rulesdir = RSPAMD_RULESDIR,
+ *lualibdir = RSPAMD_LUALIBDIR,
+ *libdir = RSPAMD_LIBDIR;
+ const gchar *t;
+
+ gchar path_buf[PATH_MAX];
+
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "path");
+ old_path = luaL_checkstring(L, -1);
+
+ if (strstr(old_path, RSPAMD_LUALIBDIR) != NULL) {
+ /* Path has been already set, do not touch it */
+ lua_pop(L, 2);
+ return;
+ }
+
+ if (cfg_obj) {
+ opts = ucl_object_lookup(cfg_obj, "options");
+ if (opts != NULL) {
+ opts = ucl_object_lookup(opts, "lua_path");
+ if (opts != NULL && ucl_object_type(opts) == UCL_STRING) {
+ additional_path = ucl_object_tostring(opts);
+ }
+ }
+ }
+
+ if (additional_path) {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s;"
+ "%s",
+ additional_path, old_path);
+ }
+ else {
+ /* Try environment */
+ t = getenv("RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = getenv("LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = getenv("LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ t = getenv("RSPAMD_LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ if (vars) {
+ t = g_hash_table_lookup(vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RSPAMD_LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+ }
+
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/lua/?.lua;"
+ "%s/?.lua;"
+ "%s/?.lua;"
+ "%s/?/init.lua;"
+ "%s",
+ RSPAMD_CONFDIR,
+ rulesdir,
+ lualibdir, lualibdir,
+ old_path);
+ }
+
+ lua_pop(L, 1);
+ lua_pushstring(L, path_buf);
+ lua_setfield(L, -2, "path");
+
+ lua_getglobal(L, "package");
+ lua_getfield(L, -1, "cpath");
+ old_path = luaL_checkstring(L, -1);
+
+ additional_path = NULL;
+
+ if (opts != NULL) {
+ opts = ucl_object_lookup(opts, "lua_cpath");
+ if (opts != NULL && ucl_object_type(opts) == UCL_STRING) {
+ additional_path = ucl_object_tostring(opts);
+ }
+ }
+
+ if (additional_path) {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/?%s;"
+ "%s",
+ additional_path,
+ OS_SO_SUFFIX,
+ old_path);
+ }
+ else {
+ rspamd_snprintf(path_buf, sizeof(path_buf),
+ "%s/?%s;"
+ "%s",
+ libdir,
+ OS_SO_SUFFIX,
+ old_path);
+ }
+
+ lua_pop(L, 1);
+ lua_pushstring(L, path_buf);
+ lua_setfield(L, -2, "cpath");
+
+ lua_pop(L, 1);
+}
+
+static gint
+rspamd_lua_cmp_version_components(const gchar *comp1, const gchar *comp2)
+{
+ guint v1, v2;
+
+ v1 = strtoul(comp1, NULL, 10);
+ v2 = strtoul(comp2, NULL, 10);
+
+ return v1 - v2;
+}
+
+static int
+rspamd_lua_rspamd_version_cmp(lua_State *L)
+{
+ const gchar *ver;
+ gchar **components;
+ gint ret = 0;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ ver = lua_tostring(L, 2);
+
+ components = g_strsplit_set(ver, ".-_", -1);
+
+ if (!components) {
+ return luaL_error(L, "invalid arguments to 'cmp': %s", ver);
+ }
+
+ if (components[0]) {
+ ret = rspamd_lua_cmp_version_components(components[0],
+ RSPAMD_VERSION_MAJOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ if (components[1]) {
+ ret = rspamd_lua_cmp_version_components(components[1],
+ RSPAMD_VERSION_MINOR);
+ }
+
+ if (ret) {
+ goto set;
+ }
+
+ /*
+ * XXX: we don't compare git releases assuming that it is meaningless
+ */
+ }
+ else {
+ return luaL_error(L, "invalid arguments to 'cmp'");
+ }
+
+set:
+ g_strfreev(components);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version_numeric(lua_State *L)
+{
+ static gint64 version_num = RSPAMD_VERSION_NUM;
+ const gchar *type;
+
+ if (lua_gettop(L) >= 2 && lua_type(L, 1) == LUA_TSTRING) {
+ type = lua_tostring(L, 1);
+ if (g_ascii_strcasecmp(type, "short") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100 +
+ RSPAMD_VERSION_PATCH_NUM * 10;
+ }
+ else if (g_ascii_strcasecmp(type, "main") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM * 1000 +
+ RSPAMD_VERSION_MINOR_NUM * 100 +
+ RSPAMD_VERSION_PATCH_NUM * 10;
+ }
+ else if (g_ascii_strcasecmp(type, "major") == 0) {
+ version_num = RSPAMD_VERSION_MAJOR_NUM;
+ }
+ else if (g_ascii_strcasecmp(type, "patch") == 0) {
+ version_num = RSPAMD_VERSION_PATCH_NUM;
+ }
+ else if (g_ascii_strcasecmp(type, "minor") == 0) {
+ version_num = RSPAMD_VERSION_MINOR_NUM;
+ }
+ }
+
+ lua_pushinteger(L, version_num);
+
+ return 1;
+}
+
+static int
+rspamd_lua_rspamd_version(lua_State *L)
+{
+ const gchar *result = NULL, *type;
+
+ if (lua_gettop(L) == 0) {
+ result = RVERSION;
+ }
+ else if (lua_gettop(L) >= 1 && lua_type(L, 1) == LUA_TSTRING) {
+ /* We got something like string */
+ type = lua_tostring(L, 1);
+
+ if (g_ascii_strcasecmp(type, "short") == 0) {
+ result = RSPAMD_VERSION_MAJOR
+ "." RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp(type, "main") == 0) {
+ result = RSPAMD_VERSION_MAJOR "." RSPAMD_VERSION_MINOR "." RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp(type, "major") == 0) {
+ result = RSPAMD_VERSION_MAJOR;
+ }
+ else if (g_ascii_strcasecmp(type, "minor") == 0) {
+ result = RSPAMD_VERSION_MINOR;
+ }
+ else if (g_ascii_strcasecmp(type, "patch") == 0) {
+ result = RSPAMD_VERSION_PATCH;
+ }
+ else if (g_ascii_strcasecmp(type, "id") == 0) {
+ result = RID;
+ }
+ else if (g_ascii_strcasecmp(type, "num") == 0) {
+ return rspamd_lua_rspamd_version_numeric(L);
+ }
+ else if (g_ascii_strcasecmp(type, "cmp") == 0) {
+ return rspamd_lua_rspamd_version_cmp(L);
+ }
+ }
+
+ lua_pushstring(L, result);
+
+ return 1;
+}
+
+static gboolean
+rspamd_lua_load_env(lua_State *L, const char *fname, gint tbl_pos, GError **err)
+{
+ gint orig_top = lua_gettop(L), err_idx;
+ gboolean ret = TRUE;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ if (luaL_loadfile(L, fname) != 0) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "cannot load lua file %s: %s",
+ fname,
+ lua_tostring(L, -1));
+ ret = FALSE;
+ }
+
+ if (ret && lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "cannot init lua file %s: %s",
+ fname,
+ lua_tostring(L, -1));
+ ret = FALSE;
+ }
+
+ if (ret && lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ lua_pushvalue(L, -2); /* Store key */
+ lua_pushvalue(L, -2); /* Store value */
+ lua_settable(L, tbl_pos);
+ }
+ }
+ else if (ret) {
+ g_set_error(err, g_quark_from_static_string("lua_env"), errno,
+ "invalid return type when loading env from %s: %s",
+ fname,
+ lua_typename(L, lua_type(L, -1)));
+ ret = FALSE;
+ }
+
+ lua_settop(L, orig_top);
+
+ return ret;
+}
+
+gboolean
+rspamd_lua_set_env(lua_State *L, GHashTable *vars, char **lua_env, GError **err)
+{
+ gint orig_top = lua_gettop(L);
+ gchar **env = g_get_environ();
+
+ /* Set known paths as rspamd_paths global */
+ lua_getglobal(L, "rspamd_paths");
+ if (lua_isnil(L, -1)) {
+ const gchar *confdir = RSPAMD_CONFDIR,
+ *local_confdir = RSPAMD_LOCAL_CONFDIR,
+ *rundir = RSPAMD_RUNDIR,
+ *dbdir = RSPAMD_DBDIR,
+ *logdir = RSPAMD_LOGDIR,
+ *wwwdir = RSPAMD_WWWDIR,
+ *pluginsdir = RSPAMD_PLUGINSDIR,
+ *rulesdir = RSPAMD_RULESDIR,
+ *lualibdir = RSPAMD_LUALIBDIR,
+ *prefix = RSPAMD_PREFIX,
+ *sharedir = RSPAMD_SHAREDIR;
+ const gchar *t;
+
+ /* Try environment */
+ t = g_environ_getenv(env, "SHAREDIR");
+ if (t) {
+ sharedir = t;
+ }
+
+ t = g_environ_getenv(env, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_environ_getenv(env, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_environ_getenv(env, "DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = g_environ_getenv(env, "RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = g_environ_getenv(env, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_environ_getenv(env, "LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+
+ t = g_environ_getenv(env, "WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = g_environ_getenv(env, "CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+ t = g_environ_getenv(env, "LOCAL_CONFDIR");
+ if (t) {
+ local_confdir = t;
+ }
+
+
+ if (vars) {
+ t = g_hash_table_lookup(vars, "SHAREDIR");
+ if (t) {
+ sharedir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "RUNDIR");
+ if (t) {
+ rundir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "WWWDIR");
+ if (t) {
+ wwwdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "CONFDIR");
+ if (t) {
+ confdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LOCAL_CONFDIR");
+ if (t) {
+ local_confdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "DBDIR");
+ if (t) {
+ dbdir = t;
+ }
+
+ t = g_hash_table_lookup(vars, "LOGDIR");
+ if (t) {
+ logdir = t;
+ }
+ }
+
+ lua_createtable(L, 0, 9);
+
+ rspamd_lua_table_set(L, RSPAMD_SHAREDIR_INDEX, sharedir);
+ rspamd_lua_table_set(L, RSPAMD_CONFDIR_INDEX, confdir);
+ rspamd_lua_table_set(L, RSPAMD_LOCAL_CONFDIR_INDEX, local_confdir);
+ rspamd_lua_table_set(L, RSPAMD_RUNDIR_INDEX, rundir);
+ rspamd_lua_table_set(L, RSPAMD_DBDIR_INDEX, dbdir);
+ rspamd_lua_table_set(L, RSPAMD_LOGDIR_INDEX, logdir);
+ rspamd_lua_table_set(L, RSPAMD_WWWDIR_INDEX, wwwdir);
+ rspamd_lua_table_set(L, RSPAMD_PLUGINSDIR_INDEX, pluginsdir);
+ rspamd_lua_table_set(L, RSPAMD_RULESDIR_INDEX, rulesdir);
+ rspamd_lua_table_set(L, RSPAMD_LUALIBDIR_INDEX, lualibdir);
+ rspamd_lua_table_set(L, RSPAMD_PREFIX_INDEX, prefix);
+
+ lua_setglobal(L, "rspamd_paths");
+ }
+
+ lua_getglobal(L, "rspamd_env");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+
+ if (vars != NULL) {
+ GHashTableIter it;
+ gpointer k, v;
+
+ g_hash_table_iter_init(&it, vars);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ rspamd_lua_table_set(L, k, v);
+ }
+ }
+
+ gint hostlen = sysconf(_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen++;
+ }
+
+ gchar *hostbuf = g_alloca(hostlen);
+ memset(hostbuf, 0, hostlen);
+ gethostname(hostbuf, hostlen - 1);
+
+ rspamd_lua_table_set(L, "hostname", hostbuf);
+
+ rspamd_lua_table_set(L, "version", RVERSION);
+ rspamd_lua_table_set(L, "ver_major", RSPAMD_VERSION_MAJOR);
+ rspamd_lua_table_set(L, "ver_minor", RSPAMD_VERSION_MINOR);
+ rspamd_lua_table_set(L, "ver_id", RID);
+ lua_pushstring(L, "ver_num");
+ lua_pushinteger(L, RSPAMD_VERSION_NUM);
+ lua_settable(L, -3);
+
+ if (env) {
+ gint lim = g_strv_length(env);
+
+ for (gint i = 0; i < lim; i++) {
+ if (RSPAMD_LEN_CHECK_STARTS_WITH(env[i], strlen(env[i]), "RSPAMD_")) {
+ const char *var = env[i] + sizeof("RSPAMD_") - 1, *value;
+ gint varlen;
+
+ varlen = strcspn(var, "=");
+ value = var + varlen;
+
+ if (*value == '=') {
+ value++;
+
+ lua_pushlstring(L, var, varlen);
+ lua_pushstring(L, value);
+ lua_settable(L, -3);
+ }
+ }
+ }
+ }
+
+ if (lua_env) {
+ gint lim = g_strv_length(lua_env);
+
+ for (gint i = 0; i < lim; i++) {
+ if (!rspamd_lua_load_env(L, lua_env[i], lua_gettop(L), err)) {
+ return FALSE;
+ }
+ }
+ }
+
+ lua_setglobal(L, "rspamd_env");
+ }
+
+ lua_settop(L, orig_top);
+ g_strfreev(env);
+
+ return TRUE;
+}
+
+void rspamd_lua_set_globals(struct rspamd_config *cfg, lua_State *L)
+{
+ struct rspamd_config **pcfg;
+ gint orig_top = lua_gettop(L);
+
+ /* First check for global variable 'config' */
+ lua_getglobal(L, "config");
+ if (lua_isnil(L, -1)) {
+ /* Assign global table to set up attributes */
+ lua_newtable(L);
+ lua_setglobal(L, "config");
+ }
+
+ lua_getglobal(L, "metrics");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "metrics");
+ }
+
+ lua_getglobal(L, "composites");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "composites");
+ }
+
+ lua_getglobal(L, "rspamd_classifiers");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "rspamd_classifiers");
+ }
+
+ lua_getglobal(L, "classifiers");
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_setglobal(L, "classifiers");
+ }
+
+ lua_getglobal(L, "rspamd_version");
+ if (lua_isnil(L, -1)) {
+ lua_pushcfunction(L, rspamd_lua_rspamd_version);
+ lua_setglobal(L, "rspamd_version");
+ }
+
+ if (cfg != NULL) {
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_setglobal(L, "rspamd_config");
+ }
+
+ lua_settop(L, orig_top);
+}
+
+#ifdef WITH_LUA_TRACE
+static gint
+lua_push_trace_data(lua_State *L)
+{
+ if (lua_traces) {
+ ucl_object_push_lua(L, lua_traces, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+#endif
+
+
+static void *
+rspamd_lua_wipe_realloc(void *ud,
+ void *ptr,
+ size_t osize,
+ size_t nsize) RSPAMD_ATTR_ALLOC_SIZE(4);
+static void *
+rspamd_lua_wipe_realloc(void *ud,
+ void *ptr,
+ size_t osize,
+ size_t nsize)
+{
+ if (nsize == 0) {
+ if (ptr) {
+ rspamd_explicit_memzero(ptr, osize);
+ }
+
+ free(ptr);
+ }
+ else if (ptr == NULL) {
+ return malloc(nsize);
+ }
+ else {
+ if (nsize < osize) {
+ /* Wipe on shrinking (actually never used) */
+ rspamd_explicit_memzero(((unsigned char *) ptr) + nsize, osize - nsize);
+ }
+
+ return realloc(ptr, nsize);
+ }
+
+ return NULL;
+}
+
+#ifndef WITH_LUAJIT
+extern int luaopen_bit(lua_State *L);
+#endif
+
+static unsigned int lua_initialized = 0;
+
+lua_State *
+rspamd_lua_init(bool wipe_mem)
+{
+ lua_State *L;
+
+ if (wipe_mem) {
+#ifdef WITH_LUAJIT
+ /* TODO: broken on luajit without GC64 */
+ L = luaL_newstate();
+#else
+ L = lua_newstate(rspamd_lua_wipe_realloc, NULL);
+#endif
+ }
+ else {
+ L = luaL_newstate();
+ }
+
+ struct rspamd_lua_context *ctx;
+
+ ctx = (struct rspamd_lua_context *) g_malloc0(sizeof(*ctx));
+ ctx->L = L;
+ ctx->classes = kh_init(lua_class_set);
+ kh_resize(lua_class_set, ctx->classes, RSPAMD_LUA_NCLASSES);
+ DL_APPEND(rspamd_lua_global_ctx, ctx);
+
+ lua_gc(L, LUA_GCSTOP, 0);
+ luaL_openlibs(L);
+ luaopen_logger(L);
+ luaopen_mempool(L);
+ luaopen_config(L);
+ luaopen_map(L);
+ luaopen_trie(L);
+ luaopen_task(L);
+ luaopen_textpart(L);
+ luaopen_mimepart(L);
+ luaopen_image(L);
+ luaopen_url(L);
+ luaopen_classifier(L);
+ luaopen_statfile(L);
+ luaopen_regexp(L);
+ luaopen_cdb(L);
+ luaopen_xmlrpc(L);
+ luaopen_http(L);
+ luaopen_redis(L);
+ luaopen_upstream(L);
+ lua_add_actions_global(L);
+ luaopen_dns_resolver(L);
+ luaopen_rsa(L);
+ luaopen_ip(L);
+ luaopen_expression(L);
+ luaopen_text(L);
+ luaopen_util(L);
+ luaopen_tcp(L);
+ luaopen_html(L);
+ luaopen_sqlite3(L);
+ luaopen_cryptobox(L);
+ luaopen_dns(L);
+ luaopen_udp(L);
+ luaopen_worker(L);
+ luaopen_kann(L);
+ luaopen_spf(L);
+ luaopen_tensor(L);
+ luaopen_parsers(L);
+ luaopen_compress(L);
+#ifndef WITH_LUAJIT
+ rspamd_lua_add_preload(L, "bit", luaopen_bit);
+ lua_settop(L, 0);
+#endif
+
+ rspamd_lua_new_class(L, "rspamd{session}", NULL);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "lpeg", luaopen_lpeg);
+ luaopen_ucl(L);
+ rspamd_lua_add_preload(L, "ucl", luaopen_ucl);
+
+ /* Add plugins global */
+ lua_newtable(L);
+ lua_setglobal(L, "rspamd_plugins");
+
+ /* Set PRNG */
+ lua_getglobal(L, "math");
+ lua_pushstring(L, "randomseed"); /* Push math.randomseed function on top of the stack */
+ lua_gettable(L, -2);
+ lua_pushinteger(L, ottery_rand_uint64());
+ g_assert(lua_pcall(L, 1, 0, 0) == 0);
+ lua_pop(L, 1); /* math table */
+
+ /* Modules state */
+ lua_newtable(L);
+ /*
+ * rspamd_plugins_state = {
+ * enabled = {},
+ * disabled_unconfigured = {},
+ * disabled_redis = {},
+ * disabled_explicitly = {},
+ * disabled_failed = {},
+ * disabled_experimental = {},
+ * disabled_unknown = {},
+ * }
+ */
+#define ADD_TABLE(name) \
+ do { \
+ lua_pushstring(L, #name); \
+ lua_newtable(L); \
+ lua_settable(L, -3); \
+ } while (0)
+
+ ADD_TABLE(enabled);
+ ADD_TABLE(disabled_unconfigured);
+ ADD_TABLE(disabled_redis);
+ ADD_TABLE(disabled_explicitly);
+ ADD_TABLE(disabled_failed);
+ ADD_TABLE(disabled_experimental);
+ ADD_TABLE(disabled_unknown);
+
+#undef ADD_TABLE
+ lua_setglobal(L, rspamd_modules_state_global);
+
+#ifdef WITH_LUA_TRACE
+ lua_pushcfunction(L, lua_push_trace_data);
+ lua_setglobal(L, "get_traces");
+#endif
+
+ lua_initialized++;
+
+ return L;
+}
+
+void rspamd_lua_close(lua_State *L)
+{
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ /* TODO: we will leak this memory, but I don't know how to resolve
+ * the chicked-egg problem when lua_close calls GC for many
+ * userdata that requires classes metatables to be represented
+ * For now, it is safe to leave it as is, I'm afraid
+ */
+#if 0
+ int ref;
+ kh_foreach_value(ctx->classes, ref, {
+ luaL_unref(L, LUA_REGISTRYINDEX, ref);
+ });
+#endif
+
+ lua_close(L);
+ DL_DELETE(rspamd_lua_global_ctx, ctx);
+ kh_destroy(lua_class_set, ctx->classes);
+ g_free(ctx);
+
+ lua_initialized--;
+}
+
+bool rspamd_lua_is_initialised(void)
+{
+ return lua_initialized != 0;
+}
+
+void rspamd_lua_start_gc(struct rspamd_config *cfg)
+{
+ lua_State *L = (lua_State *) cfg->lua_state;
+
+ lua_settop(L, 0);
+ /* Set up GC */
+ lua_gc(L, LUA_GCCOLLECT, 0);
+ lua_gc(L, LUA_GCSETSTEPMUL, cfg->lua_gc_step);
+ lua_gc(L, LUA_GCSETPAUSE, cfg->lua_gc_pause);
+ lua_gc(L, LUA_GCRESTART, 0);
+}
+
+
+void rspamd_plugins_table_push_elt(lua_State *L, const gchar *field_name,
+ const gchar *new_elt)
+{
+ lua_getglobal(L, rspamd_modules_state_global);
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, field_name);
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ lua_pushstring(L, new_elt);
+ lua_newtable(L);
+ lua_settable(L, -3);
+ lua_pop(L, 2); /* Global + element */
+ }
+ else {
+ lua_pop(L, 2); /* Global + element */
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ }
+}
+
+gboolean
+rspamd_init_lua_filters(struct rspamd_config *cfg, bool force_load, bool strict)
+{
+ struct rspamd_config **pcfg;
+ struct script_module *module;
+ lua_State *L = cfg->lua_state;
+ gint err_idx, i;
+
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_setglobal(L, "rspamd_config");
+
+ PTR_ARRAY_FOREACH(cfg->script_modules, i, module)
+ {
+ if (module->path) {
+ if (!force_load) {
+ if (!rspamd_config_is_module_enabled(cfg, module->name)) {
+ continue;
+ }
+ }
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ gsize fsize;
+ guint8 *data = rspamd_file_xmap(module->path,
+ PROT_READ, &fsize, TRUE);
+ guchar digest[rspamd_cryptobox_HASHBYTES];
+ gchar *lua_fname;
+
+ if (data == NULL) {
+ msg_err_config("cannot mmap %s failed: %s", module->path,
+ strerror(errno));
+
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ module->digest = rspamd_mempool_alloc(cfg->cfg_pool,
+ rspamd_cryptobox_HASHBYTES * 2 + 1);
+ rspamd_cryptobox_hash(digest, data, fsize, NULL, 0);
+ rspamd_encode_hex_buf(digest, sizeof(digest),
+ module->digest, rspamd_cryptobox_HASHBYTES * 2 + 1);
+ module->digest[rspamd_cryptobox_HASHBYTES * 2] = '\0';
+ lua_fname = g_malloc(strlen(module->path) + 2);
+ rspamd_snprintf(lua_fname, strlen(module->path) + 2, "@%s",
+ module->path);
+
+ if (luaL_loadbuffer(L, data, fsize, lua_fname) != 0) {
+ msg_err_config("load of %s failed: %s", module->path,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+ munmap(data, fsize);
+ g_free(lua_fname);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ munmap(data, fsize);
+ g_free(lua_fname);
+
+ if (lua_pcall(L, 0, 0, err_idx) != 0) {
+ msg_err_config("init of %s failed: %s",
+ module->path,
+ lua_tostring(L, -1));
+
+ lua_settop(L, err_idx - 1);
+ rspamd_plugins_table_push_elt(L, "disabled_failed",
+ module->name);
+
+ if (strict) {
+ return FALSE;
+ }
+
+ continue;
+ }
+
+ if (!force_load) {
+ msg_info_config("init lua module %s from %s; digest: %*s",
+ module->name,
+ module->path,
+ 10, module->digest);
+ }
+
+ lua_pop(L, 1); /* Error function */
+ }
+ }
+
+ return TRUE;
+}
+
+void rspamd_lua_dumpstack(lua_State *L)
+{
+ gint i, t, r = 0;
+ gint top = lua_gettop(L);
+ gchar buf[BUFSIZ];
+
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r, "lua stack: ");
+ for (i = 1; i <= top; i++) { /* repeat for each level */
+ t = lua_type(L, i);
+ switch (t) {
+ case LUA_TSTRING: /* strings */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "str: %s",
+ lua_tostring(L, i));
+ break;
+
+ case LUA_TBOOLEAN: /* booleans */
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r,
+ lua_toboolean(L, i) ? "bool: true" : "bool: false");
+ break;
+
+ case LUA_TNUMBER: /* numbers */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "number: %.2f",
+ lua_tonumber(L, i));
+ break;
+
+ default: /* other values */
+ r += rspamd_snprintf(buf + r,
+ sizeof(buf) - r,
+ "type: %s",
+ lua_typename(L, t));
+ break;
+ }
+ if (i < top) {
+ r += rspamd_snprintf(buf + r, sizeof(buf) - r,
+ " -> "); /* put a separator */
+ }
+ }
+
+ msg_info("%*s", r, buf);
+}
+
+gpointer
+rspamd_lua_check_class(lua_State *L, gint index, const gchar *name)
+{
+ gpointer p;
+ khiter_t k;
+
+ if (lua_type(L, index) == LUA_TUSERDATA) {
+ p = lua_touserdata(L, index);
+ if (p) {
+ if (lua_getmetatable(L, index)) {
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, name);
+
+ if (k == kh_end(ctx->classes)) {
+ lua_pop(L, 1);
+
+ return NULL;
+ }
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */
+ lua_pop(L, 2); /* remove both metatables */
+ return p;
+ }
+ lua_pop(L, 2);
+ }
+ }
+ }
+ return NULL;
+}
+
+int rspamd_lua_typerror(lua_State *L, int narg, const char *tname)
+{
+ const char *msg = lua_pushfstring(L, "%s expected, got %s", tname,
+ luaL_typename(L, narg));
+ return luaL_argerror(L, narg, msg);
+}
+
+
+void rspamd_lua_add_preload(lua_State *L, const gchar *name, lua_CFunction func)
+{
+ lua_getglobal(L, "package");
+ lua_pushstring(L, "preload");
+ lua_gettable(L, -2);
+ lua_pushcfunction(L, func);
+ lua_setfield(L, -2, name);
+ lua_pop(L, 2); /* preload key + global package */
+}
+
+
+gboolean
+rspamd_lua_parse_table_arguments(lua_State *L, gint pos,
+ GError **err,
+ enum rspamd_lua_parse_arguments_flags how,
+ const gchar *extraction_pattern, ...)
+{
+ const gchar *p, *key = NULL, *end, *cls;
+ va_list ap;
+ gboolean required = FALSE, failed = FALSE, is_table;
+ gchar classbuf[128];
+ enum {
+ read_key = 0,
+ read_arg,
+ read_class_start,
+ read_class,
+ read_semicolon
+ } state = read_key;
+ gsize keylen = 0, *valuelen, clslen;
+ gint idx = 0, t, direct_userdata = 0;
+
+ g_assert(extraction_pattern != NULL);
+
+ if (pos < 0) {
+ /* Get absolute pos */
+ pos = lua_gettop(L) + pos + 1;
+ }
+
+ if (lua_type(L, pos) == LUA_TTABLE) {
+ is_table = TRUE;
+ }
+ else {
+ is_table = FALSE;
+ idx = pos;
+ }
+
+ p = extraction_pattern;
+ end = p + strlen(extraction_pattern);
+
+ va_start(ap, extraction_pattern);
+
+ while (p <= end) {
+ switch (state) {
+ case read_key:
+ if (*p == '=') {
+ if (key == NULL) {
+ g_set_error(err, lua_error_quark(), 1, "cannot read key");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_arg;
+ keylen = p - key;
+ }
+ else if (*p == '*' && key == NULL) {
+ required = TRUE;
+ }
+ else if (key == NULL) {
+ key = p;
+ }
+ p++;
+ break;
+ case read_arg:
+ g_assert(keylen != 0);
+
+ if (is_table) {
+ lua_pushlstring(L, key, keylen);
+ lua_gettable(L, pos);
+ idx = -1;
+ }
+
+ t = lua_type(L, idx);
+
+ switch (*p) {
+ case 'S':
+ if (t == LUA_TSTRING) {
+ *(va_arg(ap, const gchar **)) = lua_tostring(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, const gchar **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, gchar **);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)), "string");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'I':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gint64 *)) = lua_tointeger(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint64 *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gint64 *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'i':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gint32 *)) = lua_tointeger(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint32 *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gint32 *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'F':
+ if (t == LUA_TFUNCTION) {
+ if (!is_table) {
+ lua_pushvalue(L, idx);
+ }
+
+ *(va_arg(ap, gint *)) = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gint *)) = -1;
+ }
+ else {
+ (void) va_arg(ap, gint *);
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "function");
+ va_end(ap);
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ return FALSE;
+ }
+
+ /* luaL_ref pops argument from the stack */
+ break;
+
+ case 'B':
+ if (t == LUA_TBOOLEAN) {
+ *(va_arg(ap, gboolean *)) = lua_toboolean(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gboolean *)) = 0;
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "bool");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'N':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gdouble *)) = lua_tonumber(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gdouble *)) = 0;
+ }
+ else {
+ (void) va_arg(ap, gdouble *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "double");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'D':
+ if (t == LUA_TNUMBER) {
+ *(va_arg(ap, gdouble *)) = lua_tonumber(L, idx);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, gdouble *)) = NAN;
+ }
+ else {
+ (void) va_arg(ap, gdouble *);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "double");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+
+ case 'V':
+ valuelen = va_arg(ap, gsize *);
+
+ if (t == LUA_TSTRING) {
+ *(va_arg(ap, const gchar **)) = lua_tolstring(L, idx,
+ valuelen);
+ }
+ else if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, const char **)) = NULL;
+ *valuelen = 0;
+ }
+ else {
+ (void) va_arg(ap, const char **);
+ }
+ }
+ else {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "string");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+ case 'O':
+ if (t != LUA_TNONE) {
+ *(va_arg(ap, ucl_object_t **)) = ucl_object_lua_import(L,
+ idx);
+ }
+ else {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, ucl_object_t **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, ucl_object_t **);
+ }
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ break;
+ case 'U':
+ if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, void **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, void **);
+ }
+ }
+ else if (t != LUA_TUSERDATA) {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_class_start;
+ clslen = 0;
+ direct_userdata = 0;
+ cls = NULL;
+ p++;
+ continue;
+ case 'u':
+ if (t == LUA_TNIL || t == LUA_TNONE) {
+ failed = TRUE;
+
+ if (how != RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING) {
+ *(va_arg(ap, void **)) = NULL;
+ }
+ else {
+ (void) va_arg(ap, void **);
+ }
+ }
+ else if (t != LUA_TUSERDATA) {
+ g_set_error(err,
+ lua_error_quark(),
+ 1,
+ "bad type for key:"
+ " %.*s: '%s', '%s' is expected",
+ (gint) keylen,
+ key,
+ lua_typename(L, lua_type(L, idx)),
+ "int64");
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ state = read_class_start;
+ clslen = 0;
+ direct_userdata = 1;
+ cls = NULL;
+ p++;
+ continue;
+ default:
+ g_assert(0);
+ break;
+ }
+
+ if (failed && required) {
+ g_set_error(err, lua_error_quark(), 2, "required parameter "
+ "%.*s is missing",
+ (gint) keylen, key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ if (!is_table) {
+ idx++;
+ }
+
+ /* Reset read params */
+ state = read_semicolon;
+ failed = FALSE;
+ required = FALSE;
+ keylen = 0;
+ key = NULL;
+ p++;
+ break;
+
+ case read_class_start:
+ if (*p == '{') {
+ cls = p + 1;
+ state = read_class;
+ }
+ else {
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ g_set_error(err, lua_error_quark(), 2, "missing classname for "
+ "%.*s",
+ (gint) keylen, key);
+ va_end(ap);
+
+ return FALSE;
+ }
+ p++;
+ break;
+
+ case read_class:
+ if (*p == '}') {
+ clslen = p - cls;
+ if (clslen == 0) {
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "empty classname for "
+ "%*.s",
+ (gint) keylen,
+ key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ rspamd_snprintf(classbuf, sizeof(classbuf), "rspamd{%*s}",
+ (gint) clslen, cls);
+
+
+ /*
+ * We skip class check here for speed in non-table mode
+ */
+ if (!failed && (!is_table ||
+ rspamd_lua_check_class(L, idx, classbuf))) {
+ if (direct_userdata) {
+ void **arg_p = (va_arg(ap, void **));
+ *arg_p = lua_touserdata(L, idx);
+ }
+ else {
+ *(va_arg(ap,
+ void **)) = *(void **) lua_touserdata(L, idx);
+ }
+ }
+ else {
+ if (!failed) {
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "invalid class for key %.*s, expected %s, got %s",
+ (gint) keylen,
+ key,
+ classbuf,
+ rspamd_lua_class_tostring_buf(L, FALSE, idx));
+ va_end(ap);
+
+ return FALSE;
+ }
+ }
+
+ if (is_table) {
+ lua_pop(L, 1);
+ }
+ else {
+ idx++;
+ }
+
+ if (failed && required) {
+ g_set_error(err,
+ lua_error_quark(),
+ 2,
+ "required parameter "
+ "%.*s is missing",
+ (gint) keylen,
+ key);
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ /* Reset read params */
+ state = read_semicolon;
+ failed = FALSE;
+ required = FALSE;
+ keylen = 0;
+ key = NULL;
+ }
+ p++;
+ break;
+
+ case read_semicolon:
+ if (*p == ';' || p == end) {
+ state = read_key;
+ key = NULL;
+ keylen = 0;
+ failed = FALSE;
+ }
+ else {
+ g_set_error(err, lua_error_quark(), 2, "bad format string: %s,"
+ " at char %c, position %d",
+ extraction_pattern, *p, (int) (p - extraction_pattern));
+ va_end(ap);
+
+ return FALSE;
+ }
+
+ p++;
+ break;
+ }
+ }
+
+ va_end(ap);
+
+ return TRUE;
+}
+
+static void
+rspamd_lua_traceback_string(lua_State *L, luaL_Buffer *buf)
+{
+ gint i = 1, r;
+ lua_Debug d;
+ gchar tmp[256];
+
+ while (lua_getstack(L, i++, &d)) {
+ lua_getinfo(L, "nSl", &d);
+ r = rspamd_snprintf(tmp, sizeof(tmp), " [%d]:{%s:%d - %s [%s]};",
+ i - 1, d.short_src, d.currentline,
+ (d.name ? d.name : "<unknown>"), d.what);
+ luaL_addlstring(buf, tmp, r);
+ }
+}
+
+gint rspamd_lua_traceback(lua_State *L)
+{
+ luaL_Buffer b;
+
+ luaL_buffinit(L, &b);
+ rspamd_lua_get_traceback_string(L, &b);
+ luaL_pushresult(&b);
+
+ return 1;
+}
+
+void rspamd_lua_get_traceback_string(lua_State *L, luaL_Buffer *buf)
+{
+ const gchar *msg = lua_tostring(L, -1);
+
+ if (msg) {
+ luaL_addstring(buf, msg);
+ lua_pop(L, 1); /* Error string */
+ }
+ else {
+ luaL_addstring(buf, "unknown error");
+ }
+
+ luaL_addstring(buf, "; trace:");
+ rspamd_lua_traceback_string(L, buf);
+}
+
+guint rspamd_lua_table_size(lua_State *L, gint tbl_pos)
+{
+ guint tbl_size = 0;
+
+ if (!lua_istable(L, tbl_pos)) {
+ return 0;
+ }
+
+#if LUA_VERSION_NUM >= 502
+ tbl_size = lua_rawlen(L, tbl_pos);
+#else
+ tbl_size = lua_objlen(L, tbl_pos);
+#endif
+
+ return tbl_size;
+}
+
+static void *
+rspamd_lua_check_udata_common(lua_State *L, gint pos, const gchar *classname,
+ gboolean fatal)
+{
+ void *p = lua_touserdata(L, pos);
+ guint i, top = lua_gettop(L);
+ khiter_t k;
+
+ if (p == NULL) {
+ goto err;
+ }
+ else {
+ /* Match class */
+ if (lua_getmetatable(L, pos)) {
+ struct rspamd_lua_context *ctx = rspamd_lua_ctx_by_state(L);
+
+ k = kh_get(lua_class_set, ctx->classes, classname);
+
+ if (k == kh_end(ctx->classes)) {
+ goto err;
+ }
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(ctx->classes, k));
+
+ if (!lua_rawequal(L, -1, -2)) {
+ goto err;
+ }
+ }
+ else {
+ goto err;
+ }
+ }
+
+ lua_settop(L, top);
+
+ return p;
+
+err:
+ if (fatal) {
+ const gchar *actual_classname = NULL;
+
+ if (lua_type(L, pos) == LUA_TUSERDATA && lua_getmetatable(L, pos)) {
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ actual_classname = lua_tostring(L, -1);
+ }
+ else {
+ actual_classname = lua_typename(L, lua_type(L, pos));
+ }
+
+ luaL_Buffer buf;
+ gchar tmp[512];
+ gint r;
+
+ luaL_buffinit(L, &buf);
+ r = rspamd_snprintf(tmp, sizeof(tmp),
+ "expected %s at position %d, but userdata has "
+ "%s metatable; trace: ",
+ classname, pos, actual_classname);
+ luaL_addlstring(&buf, tmp, r);
+ rspamd_lua_traceback_string(L, &buf);
+ r = rspamd_snprintf(tmp, sizeof(tmp), " stack(%d): ", top);
+ luaL_addlstring(&buf, tmp, r);
+
+ for (i = 1; i <= MIN(top, 10); i++) {
+ if (lua_type(L, i) == LUA_TUSERDATA) {
+ const char *clsname;
+
+ if (lua_getmetatable(L, i)) {
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+ clsname = lua_tostring(L, -1);
+ }
+ else {
+ clsname = lua_typename(L, lua_type(L, i));
+ }
+
+ r = rspamd_snprintf(tmp, sizeof(tmp), "[%d: ud=%s] ", i,
+ clsname);
+ luaL_addlstring(&buf, tmp, r);
+ }
+ else {
+ r = rspamd_snprintf(tmp, sizeof(tmp), "[%d: %s] ", i,
+ lua_typename(L, lua_type(L, i)));
+ luaL_addlstring(&buf, tmp, r);
+ }
+ }
+
+ luaL_pushresult(&buf);
+ msg_err("lua type error: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+
+ return NULL;
+}
+
+void *
+rspamd_lua_check_udata(lua_State *L, gint pos, const gchar *classname)
+{
+ return rspamd_lua_check_udata_common(L, pos, classname, TRUE);
+}
+
+void *
+rspamd_lua_check_udata_maybe(lua_State *L, gint pos, const gchar *classname)
+{
+ return rspamd_lua_check_udata_common(L, pos, classname, FALSE);
+}
+
+struct rspamd_async_session *
+lua_check_session(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{session}");
+ luaL_argcheck(L, ud != NULL, pos, "'session' expected");
+ return ud ? *((struct rspamd_async_session **) ud) : NULL;
+}
+
+struct ev_loop *
+lua_check_ev_base(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{ev_base}");
+ luaL_argcheck(L, ud != NULL, pos, "'event_base' expected");
+ return ud ? *((struct ev_loop **) ud) : NULL;
+}
+
+static void rspamd_lua_run_postloads_error(struct thread_entry *thread, int ret, const char *msg);
+
+void rspamd_lua_run_postloads(lua_State *L, struct rspamd_config *cfg,
+ struct ev_loop *ev_base, struct rspamd_worker *w)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+ struct ev_loop **pev_base;
+ struct rspamd_worker **pw;
+
+ /* Execute post load scripts */
+ LL_FOREACH(cfg->on_load_scripts, sc)
+ {
+ struct thread_entry *thread = lua_thread_pool_get_for_config(cfg);
+ thread->error_callback = rspamd_lua_run_postloads_error;
+ L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ pev_base = lua_newuserdata(L, sizeof(*pev_base));
+ *pev_base = ev_base;
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+
+ pw = lua_newuserdata(L, sizeof(*pw));
+ *pw = w;
+ rspamd_lua_setclass(L, "rspamd{worker}", -1);
+
+ lua_thread_call(thread, 3);
+ }
+}
+
+
+void rspamd_lua_run_config_post_init(lua_State *L, struct rspamd_config *cfg)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+
+ LL_FOREACH(cfg->post_init_scripts, sc)
+ {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_config("cannot run config post init script: %s; priority = %d",
+ lua_tostring(L, -1), sc->priority);
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+
+void rspamd_lua_run_config_unload(lua_State *L, struct rspamd_config *cfg)
+{
+ struct rspamd_config_cfg_lua_script *sc;
+ struct rspamd_config **pcfg;
+
+ LL_FOREACH(cfg->config_unload_scripts, sc)
+ {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ *pcfg = cfg;
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_config("cannot run config post init script: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+static void
+rspamd_lua_run_postloads_error(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct rspamd_config *cfg = thread->cfg;
+
+ msg_err_config("error executing post load code: %s", msg);
+}
+
+
+struct rspamd_lua_ref_cbdata {
+ lua_State *L;
+ gint cbref;
+};
+
+static void
+rspamd_lua_ref_dtor(gpointer p)
+{
+ struct rspamd_lua_ref_cbdata *cbdata = p;
+
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->cbref);
+}
+
+void rspamd_lua_add_ref_dtor(lua_State *L, rspamd_mempool_t *pool,
+ gint ref)
+{
+ struct rspamd_lua_ref_cbdata *cbdata;
+
+ if (ref != -1) {
+ cbdata = rspamd_mempool_alloc(pool, sizeof(*cbdata));
+ cbdata->cbref = ref;
+ cbdata->L = L;
+
+ rspamd_mempool_add_destructor(pool, rspamd_lua_ref_dtor, cbdata);
+ }
+}
+
+gboolean
+rspamd_lua_require_function(lua_State *L, const gchar *modname,
+ const gchar *funcname)
+{
+ gint table_pos, err_pos;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_pos = lua_gettop(L);
+ lua_getglobal(L, "require");
+
+ if (lua_isnil(L, -1)) {
+ lua_remove(L, err_pos);
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ lua_pushstring(L, modname);
+
+ /* Now try to call */
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_remove(L, err_pos);
+ msg_warn("require of %s.%s failed: %s", modname,
+ funcname, lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ lua_remove(L, err_pos);
+
+ /* Now we should have a table with results */
+ if (funcname) {
+ if (!lua_istable(L, -1)) {
+ msg_warn("require of %s.%s failed: not a table but %s", modname,
+ funcname, lua_typename(L, lua_type(L, -1)));
+
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+
+ table_pos = lua_gettop(L);
+ lua_pushstring(L, funcname);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ /* Remove table, preserve just a function */
+ lua_remove(L, table_pos);
+
+ return TRUE;
+ }
+ else {
+ msg_warn("require of %s.%s failed: not a function but %s", modname,
+ funcname, lua_typename(L, lua_type(L, -1)));
+ }
+
+ lua_pop(L, 2);
+
+ return FALSE;
+ }
+ else if (lua_isfunction(L, -1)) {
+ return TRUE;
+ }
+ else {
+ msg_warn("require of %s failed: not a function but %s", modname,
+ lua_typename(L, lua_type(L, -1)));
+ lua_pop(L, 1);
+
+ return FALSE;
+ }
+}
+
+gint rspamd_lua_function_ref_from_str(lua_State *L, const gchar *str, gsize slen,
+ const gchar *modname, GError **err)
+{
+ gint err_idx, ref_idx;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Load file */
+ if (luaL_loadbuffer(L, str, slen, modname) != 0) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot load lua script: %s",
+ modname,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1); /* Error function */
+
+ return LUA_NOREF;
+ }
+
+ /* Now call it */
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot init lua script: %s",
+ modname,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ return LUA_NOREF;
+ }
+
+ if (!lua_isfunction(L, -1)) {
+ g_set_error(err,
+ lua_error_quark(),
+ EINVAL,
+ "%s: cannot init lua script: "
+ "must return function not %s",
+ modname,
+ lua_typename(L, lua_type(L, -1)));
+ lua_settop(L, err_idx - 1);
+
+ return LUA_NOREF;
+ }
+
+ ref_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_settop(L, err_idx - 1);
+
+ return ref_idx;
+}
+
+
+gboolean
+rspamd_lua_try_load_redis(lua_State *L, const ucl_object_t *obj,
+ struct rspamd_config *cfg, gint *ref_id)
+{
+ gint err_idx;
+ struct rspamd_config **pcfg;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function(L, "lua_redis", "try_load_redis_servers")) {
+ msg_err_config("cannot require lua_redis");
+ lua_pop(L, 2);
+
+ return FALSE;
+ }
+
+ /* Function arguments */
+ ucl_object_push_lua(L, obj, false);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_pushboolean(L, false); /* no_fallback */
+
+ if (lua_pcall(L, 3, 1, err_idx) != 0) {
+ msg_err_config("cannot call lua try_load_redis_servers script: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return FALSE;
+ }
+
+ if (lua_istable(L, -1)) {
+ if (ref_id) {
+ /* Ref table */
+ lua_pushvalue(L, -1);
+ *ref_id = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_settop(L, 0);
+ }
+ else {
+ /* Leave it on the stack */
+ lua_insert(L, err_idx);
+ lua_settop(L, err_idx);
+ }
+
+ return TRUE;
+ }
+ else {
+ lua_settop(L, 0);
+ }
+
+ return FALSE;
+}
+
+void rspamd_lua_push_full_word(lua_State *L, rspamd_stat_token_t *w)
+{
+ gint fl_cnt;
+
+ lua_createtable(L, 4, 0);
+
+ if (w->stemmed.len > 0) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 1);
+ }
+
+ if (w->normalized.len > 0) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, 2);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 2);
+ }
+
+ if (w->original.len > 0) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, 3);
+ }
+ else {
+ lua_pushstring(L, "");
+ lua_rawseti(L, -2, 3);
+ }
+
+ /* Flags part */
+ fl_cnt = 1;
+ lua_createtable(L, 4, 0);
+
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_NORMALISED) {
+ lua_pushstring(L, "normalised");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE) {
+ lua_pushstring(L, "broken_unicode");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+ lua_pushstring(L, "utf");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ lua_pushstring(L, "text");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) {
+ lua_pushstring(L, "header");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_META | RSPAMD_STAT_TOKEN_FLAG_LUA_META)) {
+ lua_pushstring(L, "meta");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STOP_WORD) {
+ lua_pushstring(L, "stop_word");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES) {
+ lua_pushstring(L, "invisible_spaces");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+ if (w->flags & RSPAMD_STAT_TOKEN_FLAG_STEMMED) {
+ lua_pushstring(L, "stemmed");
+ lua_rawseti(L, -2, fl_cnt++);
+ }
+
+ lua_rawseti(L, -2, 4);
+}
+
+gint rspamd_lua_push_words(lua_State *L, GArray *words,
+ enum rspamd_lua_words_type how)
+{
+ rspamd_stat_token_t *w;
+ guint i, cnt;
+
+ lua_createtable(L, words->len, 0);
+
+ for (i = 0, cnt = 1; i < words->len; i++) {
+ w = &g_array_index(words, rspamd_stat_token_t, i);
+
+ switch (how) {
+ case RSPAMD_LUA_WORDS_STEM:
+ if (w->stemmed.len > 0) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_NORM:
+ if (w->normalized.len > 0) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_RAW:
+ if (w->original.len > 0) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ case RSPAMD_LUA_WORDS_FULL:
+ rspamd_lua_push_full_word(L, w);
+ /* Push to the resulting vector */
+ lua_rawseti(L, -2, cnt++);
+ break;
+ default:
+ break;
+ }
+ }
+
+ return 1;
+}
+
+gchar *
+rspamd_lua_get_module_name(lua_State *L)
+{
+ lua_Debug d;
+ gchar *p;
+ gchar func_buf[128];
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%s:%d", p,
+ d.currentline);
+ }
+
+ return g_strdup(func_buf);
+ }
+
+ return NULL;
+}
+
+bool rspamd_lua_universal_pcall(lua_State *L, gint cbref, const gchar *strloc,
+ gint nret, const gchar *args, GError **err, ...)
+{
+ va_list ap;
+ const gchar *argp = args, *classname;
+ gint err_idx, nargs = 0;
+ gpointer *cls_ptr;
+ gsize sz;
+
+ /* Error function */
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ va_start(ap, err);
+ /* Called function */
+ if (cbref > 0) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+ }
+ else {
+ /* Assume that function was on top of the stack */
+ lua_pushvalue(L, err_idx - 1);
+ }
+ /*
+ * Possible arguments
+ * - i - lua_integer, argument - gint64
+ * - n - lua_number, argument - gdouble
+ * - s - lua_string, argument - const gchar * (zero terminated)
+ * - l - lua_lstring, argument - (size_t + const gchar *) pair
+ * - u - lua_userdata, argument - (const char * + void *) - classname + pointer
+ * - b - lua_boolean, argument - gboolean (not bool due to varargs promotion)
+ * - f - lua_function, argument - int - position of the function on stack (not lua_registry)
+ * - t - lua_text, argument - int - position of the lua_text on stack (not lua_registry)
+ */
+ while (*argp) {
+ switch (*argp) {
+ case 'i':
+ lua_pushinteger(L, va_arg(ap, gint64));
+ nargs++;
+ break;
+ case 'n':
+ lua_pushnumber(L, va_arg(ap, gdouble));
+ nargs++;
+ break;
+ case 's':
+ lua_pushstring(L, va_arg(ap, const gchar *));
+ nargs++;
+ break;
+ case 'l':
+ sz = va_arg(ap, gsize);
+ lua_pushlstring(L, va_arg(ap, const gchar *), sz);
+ nargs++;
+ break;
+ case 'b':
+ lua_pushboolean(L, va_arg(ap, gboolean));
+ nargs++;
+ break;
+ case 'u':
+ classname = va_arg(ap, const gchar *);
+ cls_ptr = (gpointer *) lua_newuserdata(L, sizeof(gpointer));
+ *cls_ptr = va_arg(ap, gpointer);
+ rspamd_lua_setclass(L, classname, -1);
+ nargs++;
+ break;
+ case 'f':
+ case 't':
+ lua_pushvalue(L, va_arg(ap, gint));
+ nargs++;
+ break;
+ default:
+ lua_settop(L, err_idx - 1);
+ g_set_error(err, lua_error_quark(), EINVAL,
+ "invalid argument character: %c at %s",
+ *argp, argp);
+ va_end(ap);
+
+ return false;
+ }
+
+ argp++;
+ }
+
+ if (lua_pcall(L, nargs, nret, err_idx) != 0) {
+ g_set_error(err, lua_error_quark(), EBADF,
+ "error when calling lua function from %s: %s",
+ strloc, lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ va_end(ap);
+
+ return false;
+ }
+
+ lua_remove(L, err_idx);
+ va_end(ap);
+
+ return true;
+}
+
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM <= 502
+gint rspamd_lua_geti(lua_State *L, int pos, int i)
+{
+ pos = lua_absindex(L, pos);
+ lua_pushinteger(L, i);
+ lua_gettable(L, pos);
+
+ return lua_type(L, -1);
+}
+#endif \ No newline at end of file
diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
new file mode 100644
index 0000000..cc2b943
--- /dev/null
+++ b/src/lua/lua_common.h
@@ -0,0 +1,729 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_H
+#define RSPAMD_LUA_H
+
+#include "config.h"
+
+/* Lua headers do not have __cplusplus guards... */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+#ifdef WITH_LUAJIT
+#include <luajit.h>
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#include <stdbool.h>
+
+
+#include "rspamd.h"
+#include "ucl.h"
+#include "lua_ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef lua_open
+#define lua_open() luaL_newstate()
+#endif
+
+#ifndef luaL_reg
+#define luaL_reg luaL_Reg
+#endif
+
+#define LUA_ENUM(L, name, val) \
+ lua_pushlstring(L, #name, sizeof(#name) - 1); \
+ lua_pushinteger(L, val); \
+ lua_settable(L, -3);
+
+#if LUA_VERSION_NUM > 501 && !defined LUA_COMPAT_MODULE
+static inline void
+luaL_register(lua_State *L, const gchar *name, const struct luaL_reg *methods)
+{
+ if (name != NULL) {
+ lua_newtable(L);
+ }
+ luaL_setfuncs(L, methods, 0);
+ if (name != NULL) {
+ lua_pushvalue(L, -1);
+ lua_setglobal(L, name);
+ }
+}
+#endif
+
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM == 501
+
+/* Special hack to work with moonjit of specific version */
+#if !defined(MOONJIT_VERSION) && (!defined(LUAJIT_VERSION_NUM) || LUAJIT_VERSION_NUM != 20200)
+static inline int lua_absindex(lua_State *L, int i)
+{
+ if (i < 0 && i > LUA_REGISTRYINDEX)
+ i += lua_gettop(L) + 1;
+ return i;
+}
+#endif
+
+#endif
+
+/* Interface definitions */
+#define LUA_FUNCTION_DEF(class, name) static int lua_##class##_##name(lua_State *L)
+#define LUA_PUBLIC_FUNCTION_DEF(class, name) int lua_##class##_##name(lua_State *L)
+#define LUA_INTERFACE_DEF(class, name) \
+ { \
+ #name, lua_##class##_##name \
+ }
+
+extern const luaL_reg null_reg[];
+
+#define RSPAMD_LUA_CFG_STATE(cfg) ((lua_State *) ((cfg)->lua_state))
+/**
+* Lua IP address structure
+*/
+struct rspamd_lua_ip {
+ rspamd_inet_addr_t *addr;
+};
+
+#define RSPAMD_TEXT_FLAG_OWN (1u << 0u)
+#define RSPAMD_TEXT_FLAG_MMAPED (1u << 1u)
+#define RSPAMD_TEXT_FLAG_WIPE (1u << 2u)
+#define RSPAMD_TEXT_FLAG_SYSMALLOC (1u << 3u)
+#define RSPAMD_TEXT_FLAG_FAKE (1u << 4u)
+#define RSPAMD_TEXT_FLAG_BINARY (1u << 5u)
+struct rspamd_lua_text {
+ const gchar *start;
+ guint len;
+ guint flags;
+};
+
+struct rspamd_lua_url {
+ struct rspamd_url *url;
+};
+
+struct rspamd_lua_regexp {
+ rspamd_regexp_t *re;
+ gchar *module;
+ gchar *re_pattern;
+ gint re_flags;
+};
+
+struct rspamd_map;
+struct lua_map_callback_data;
+struct radix_tree_compressed;
+struct rspamd_mime_header;
+
+enum rspamd_lua_map_type {
+ RSPAMD_LUA_MAP_RADIX = 0,
+ RSPAMD_LUA_MAP_SET,
+ RSPAMD_LUA_MAP_HASH,
+ RSPAMD_LUA_MAP_REGEXP,
+ RSPAMD_LUA_MAP_REGEXP_MULTIPLE,
+ RSPAMD_LUA_MAP_CALLBACK,
+ RSPAMD_LUA_MAP_CDB,
+ RSPAMD_LUA_MAP_UNKNOWN,
+};
+
+struct rspamd_lua_map {
+ struct rspamd_map *map;
+ enum rspamd_lua_map_type type;
+ guint flags;
+
+ union {
+ struct rspamd_radix_map_helper *radix;
+ struct rspamd_hash_map_helper *hash;
+ struct rspamd_regexp_map_helper *re_map;
+ struct rspamd_cdb_map_helper *cdb_map;
+ struct lua_map_callback_data *cbdata;
+ } data;
+};
+
+struct rspamd_lua_upstream {
+ struct upstream *up;
+ gint upref;
+};
+
+/* Common utility functions */
+
+/**
+* Create and register new class
+*/
+void rspamd_lua_new_class(lua_State *L,
+ const gchar *classname,
+ const struct luaL_reg *methods);
+
+/**
+* Set class name for object at @param objidx position
+*/
+void rspamd_lua_setclass(lua_State *L, const gchar *classname, gint objidx);
+
+/**
+* Pushes the metatable for specific class on top of the stack
+* @param L
+* @param classname
+*/
+void rspamd_lua_class_metatable(lua_State *L, const gchar *classname);
+
+/**
+* Adds a new field to the class (metatable) identified by `classname`
+* @param L
+* @param classname
+* @param meth
+*/
+void rspamd_lua_add_metamethod(lua_State *L, const gchar *classname,
+ luaL_Reg *meth);
+
+/**
+* Set index of table to value (like t['index'] = value)
+*/
+void rspamd_lua_table_set(lua_State *L, const gchar *index, const gchar *value);
+
+/**
+* Get string value of index in a table (return t['index'])
+*/
+const gchar *rspamd_lua_table_get(lua_State *L, const gchar *index);
+
+/**
+* Convert classname to string
+*/
+gint rspamd_lua_class_tostring(lua_State *L);
+
+/**
+* Check whether the argument at specified index is of the specified class
+*/
+gpointer rspamd_lua_check_class(lua_State *L, gint index, const gchar *name);
+
+/**
+* Initialize lua and bindings
+*/
+lua_State *rspamd_lua_init(bool wipe_mem);
+
+/**
+ * Close lua_state and free remainders
+ * @param L
+ */
+void rspamd_lua_close(lua_State *L);
+
+void rspamd_lua_start_gc(struct rspamd_config *cfg);
+
+/**
+* Sets field in a global variable
+* @param L
+* @param global_name
+* @param field_name
+* @param new_elt
+*/
+void rspamd_plugins_table_push_elt(lua_State *L, const gchar *field_name,
+ const gchar *new_elt);
+
+/**
+* Load and initialize lua plugins
+*/
+gboolean
+rspamd_init_lua_filters(struct rspamd_config *cfg, bool force_load, bool strict);
+
+
+/**
+* Push lua ip address
+*/
+void rspamd_lua_ip_push(lua_State *L, rspamd_inet_addr_t *addr);
+
+/**
+* Push rspamd task structure to lua
+*/
+void rspamd_lua_task_push(lua_State *L, struct rspamd_task *task);
+
+/**
+* Return lua ip structure at the specified address
+*/
+struct rspamd_lua_ip *lua_check_ip(lua_State *L, gint pos);
+
+struct rspamd_lua_text *lua_check_text(lua_State *L, gint pos);
+/**
+* Checks for a text or a string. In case of string a pointer to static structure is returned.
+* So it should not be reused or placed to Lua stack anyhow!
+* However, you can use this function up to 4 times and have distinct static structures
+* @param L
+* @param pos
+* @return
+*/
+struct rspamd_lua_text *lua_check_text_or_string(lua_State *L, gint pos);
+/**
+ * Create new text object
+ * @param L
+ * @param start
+ * @param len
+ * @param own
+ * @return
+ */
+struct rspamd_lua_text *lua_new_text(lua_State *L, const gchar *start,
+ gsize len, gboolean own);
+/**
+ * Create new text object from task pool if allocation is needed
+ * @param task
+ * @param L
+ * @param start
+ * @param len
+ * @param own
+ * @return
+ */
+struct rspamd_lua_text *lua_new_text_task(lua_State *L, struct rspamd_task *task,
+ const gchar *start, gsize len, gboolean own);
+/**
+ * Checks if a text has binary characters (non ascii and non-utf8 characters)
+ * @param t
+ * @return
+ */
+bool lua_is_text_binary(struct rspamd_lua_text *t);
+
+struct rspamd_lua_regexp *lua_check_regexp(lua_State *L, gint pos);
+
+struct rspamd_lua_upstream *lua_check_upstream(lua_State *L, int pos);
+
+enum rspamd_lua_task_header_type {
+ RSPAMD_TASK_HEADER_PUSH_SIMPLE = 0,
+ RSPAMD_TASK_HEADER_PUSH_RAW,
+ RSPAMD_TASK_HEADER_PUSH_FULL,
+ RSPAMD_TASK_HEADER_PUSH_COUNT,
+ RSPAMD_TASK_HEADER_PUSH_HAS,
+};
+
+gint rspamd_lua_push_header(lua_State *L,
+ struct rspamd_mime_header *h,
+ enum rspamd_lua_task_header_type how);
+
+/**
+* Push specific header to lua
+*/
+gint rspamd_lua_push_header_array(lua_State *L,
+ const gchar *name,
+ struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how,
+ gboolean strong);
+
+/**
+* Check for task at the specified position
+*/
+struct rspamd_task *lua_check_task(lua_State *L, gint pos);
+
+struct rspamd_task *lua_check_task_maybe(lua_State *L, gint pos);
+
+struct rspamd_lua_map *lua_check_map(lua_State *L, gint pos);
+
+/**
+* Push ip address from a string (nil is pushed if a string cannot be converted)
+*/
+void rspamd_lua_ip_push_fromstring(lua_State *L, const gchar *ip_str);
+
+/**
+* Create type error
+*/
+int rspamd_lua_typerror(lua_State *L, int narg, const char *tname);
+/**
+* Open libraries functions
+*/
+
+/**
+* Add preload function
+*/
+void rspamd_lua_add_preload(lua_State *L, const gchar *name, lua_CFunction func);
+
+void luaopen_task(lua_State *L);
+
+void luaopen_config(lua_State *L);
+
+void luaopen_map(lua_State *L);
+
+void luaopen_trie(lua_State *L);
+
+void luaopen_textpart(lua_State *L);
+
+void luaopen_mimepart(lua_State *L);
+
+void luaopen_image(lua_State *L);
+
+void luaopen_url(lua_State *L);
+
+void luaopen_classifier(lua_State *L);
+
+void luaopen_statfile(lua_State *L);
+
+void luaopen_regexp(lua_State *L);
+
+void luaopen_cdb(lua_State *L);
+
+void luaopen_xmlrpc(lua_State *L);
+
+void luaopen_http(lua_State *L);
+
+void luaopen_redis(lua_State *L);
+
+void luaopen_upstream(lua_State *L);
+
+void luaopen_mempool(lua_State *L);
+
+void luaopen_dns_resolver(lua_State *L);
+
+void luaopen_rsa(lua_State *L);
+
+void luaopen_ip(lua_State *L);
+
+void luaopen_expression(lua_State *L);
+
+void luaopen_logger(lua_State *L);
+
+void luaopen_text(lua_State *L);
+
+void luaopen_util(lua_State *L);
+
+void luaopen_tcp(lua_State *L);
+
+void luaopen_html(lua_State *L);
+
+void luaopen_sqlite3(lua_State *L);
+
+void luaopen_cryptobox(lua_State *L);
+
+void luaopen_dns(lua_State *L);
+
+void luaopen_udp(lua_State *L);
+
+void luaopen_worker(lua_State *L);
+
+void luaopen_kann(lua_State *L);
+
+void luaopen_spf(lua_State *L);
+
+void luaopen_tensor(lua_State *L);
+
+void luaopen_parsers(lua_State *L);
+
+void rspamd_lua_dostring(const gchar *line);
+
+double rspamd_lua_normalize(struct rspamd_config *cfg,
+ long double score,
+ void *params);
+
+/* Config file functions */
+void rspamd_lua_post_load_config(struct rspamd_config *cfg);
+
+void rspamd_lua_dumpstack(lua_State *L);
+
+/* Set lua path according to the configuration */
+void rspamd_lua_set_path(lua_State *L, const ucl_object_t *cfg_obj,
+ GHashTable *vars);
+
+/* Set some lua globals */
+gboolean rspamd_lua_set_env(lua_State *L, GHashTable *vars, char **lua_env,
+ GError **err);
+
+void rspamd_lua_set_globals(struct rspamd_config *cfg, lua_State *L);
+
+struct memory_pool_s *rspamd_lua_check_mempool(lua_State *L, gint pos);
+
+struct rspamd_config *lua_check_config(lua_State *L, gint pos);
+
+struct rspamd_async_session *lua_check_session(lua_State *L, gint pos);
+
+struct ev_loop *lua_check_ev_base(lua_State *L, gint pos);
+
+struct rspamd_dns_resolver *lua_check_dns_resolver(lua_State *L, gint pos);
+
+struct rspamd_lua_url *lua_check_url(lua_State *L, gint pos);
+
+enum rspamd_lua_parse_arguments_flags {
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT = 0,
+ RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING,
+};
+
+/**
+* Extract an arguments from lua table according to format string. Supported arguments are:
+* [*]key=S|I|N|B|V|U{a-z};[key=...]
+* - S - const char *
+* - I - gint64_t
+* - i - int32_t
+* - N - double
+* - B - gboolean
+* - V - size_t + const char *
+* - U{classname} - userdata of the following class (stored in gpointer)
+* - F - function
+* - O - ucl_object_t *
+* - D - same as N but argument is set to NAN not to 0.0
+* - u{classname} - userdata of the following class (stored directly)
+*
+* If any of keys is prefixed with `*` then it is treated as required argument
+* @param L lua state
+* @param pos at which pos start extraction
+* @param err error pointer
+* @param how extraction type (IGNORE_MISSING means that default values will not be set)
+* @param extraction_pattern static pattern
+* @return TRUE if a table has been parsed
+*/
+gboolean rspamd_lua_parse_table_arguments(lua_State *L, gint pos,
+ GError **err,
+ enum rspamd_lua_parse_arguments_flags how,
+ const gchar *extraction_pattern, ...);
+
+
+gint rspamd_lua_traceback(lua_State *L);
+
+/**
+* Returns stack trace as a string. Caller should clear memory.
+* @param L
+* @return
+*/
+void rspamd_lua_get_traceback_string(lua_State *L, luaL_Buffer *buf);
+
+/**
+* Returns size of table at position `tbl_pos`
+*/
+guint rspamd_lua_table_size(lua_State *L, gint tbl_pos);
+
+void lua_push_emails_address_list(lua_State *L, GPtrArray *addrs, int flags);
+
+
+#define TRACE_POINTS 6
+
+struct lua_logger_trace {
+ gint cur_level;
+ gconstpointer traces[TRACE_POINTS];
+};
+
+enum lua_logger_escape_type {
+ LUA_ESCAPE_NONE = (0u),
+ LUA_ESCAPE_UNPRINTABLE = (1u << 0u),
+ LUA_ESCAPE_NEWLINES = (1u << 1u),
+ LUA_ESCAPE_8BIT = (1u << 2u),
+};
+
+#define LUA_ESCAPE_LOG (LUA_ESCAPE_UNPRINTABLE | LUA_ESCAPE_NEWLINES)
+#define LUA_ESCAPE_ALL (LUA_ESCAPE_UNPRINTABLE | LUA_ESCAPE_NEWLINES | LUA_ESCAPE_8BIT)
+
+/**
+* Log lua object to string
+* @param L
+* @param pos
+* @param outbuf
+* @param len
+* @return
+*/
+gsize lua_logger_out_type(lua_State *L, gint pos, gchar *outbuf,
+ gsize len, struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type);
+
+/**
+* Safely checks userdata to match specified class
+* @param L
+* @param pos
+* @param classname
+*/
+void *rspamd_lua_check_udata(lua_State *L, gint pos, const gchar *classname);
+
+#define RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, pos, classname, type, dest) \
+ do { \
+ type **_maybe_ptr = (type **) rspamd_lua_check_udata((L), (pos), (classname)); \
+ if (_maybe_ptr == NULL) { \
+ return luaL_error(L, "%s: invalid arguments; pos = %d; expected = %s", G_STRFUNC, (pos), (classname)); \
+ } \
+ (dest) = *(_maybe_ptr); \
+ } while (0)
+
+/**
+* Safely checks userdata to match specified class
+* @param L
+* @param pos
+* @param classname
+*/
+void *rspamd_lua_check_udata_maybe(lua_State *L, gint pos, const gchar *classname);
+
+/**
+* Call finishing script with the specified task
+* @param sc
+* @param task
+*/
+void lua_call_finish_script(struct rspamd_config_cfg_lua_script *sc,
+ struct rspamd_task *task);
+
+/**
+* Run post-load operations
+* @param L
+* @param cfg
+* @param ev_base
+*/
+void rspamd_lua_run_postloads(lua_State *L, struct rspamd_config *cfg,
+ struct ev_loop *ev_base, struct rspamd_worker *w);
+
+void rspamd_lua_run_config_post_init(lua_State *L, struct rspamd_config *cfg);
+
+void rspamd_lua_run_config_unload(lua_State *L, struct rspamd_config *cfg);
+
+/**
+* Adds new destructor for a local function for specific pool
+* @param L
+* @param pool
+* @param ref
+*/
+void rspamd_lua_add_ref_dtor(lua_State *L, rspamd_mempool_t *pool,
+ gint ref);
+
+/**
+ * Returns a lua reference from a function like string, e.g. `return function(...) end`
+ * @param L
+ * @param str
+ * @return
+ */
+gint rspamd_lua_function_ref_from_str(lua_State *L, const gchar *str, gsize slen,
+ const gchar *modname, GError **err);
+
+/**
+* Tries to load some module using `require` and get some method from it
+* @param L
+* @param modname
+* @param funcname
+* @return TRUE if function exists in that module, the function is pushed in stack, otherwise stack is unchanged and FALSE is returned
+*/
+gboolean rspamd_lua_require_function(lua_State *L, const gchar *modname,
+ const gchar *funcname);
+
+/**
+* Tries to load redis server definition from ucl object specified
+* @param L
+* @param obj
+* @param cfg
+* @return
+*/
+gboolean rspamd_lua_try_load_redis(lua_State *L, const ucl_object_t *obj,
+ struct rspamd_config *cfg, gint *ref_id);
+
+struct rspamd_stat_token_s;
+
+/**
+* Pushes a single word into Lua
+* @param L
+* @param word
+*/
+void rspamd_lua_push_full_word(lua_State *L, struct rspamd_stat_token_s *word);
+
+enum rspamd_lua_words_type {
+ RSPAMD_LUA_WORDS_STEM = 0,
+ RSPAMD_LUA_WORDS_NORM,
+ RSPAMD_LUA_WORDS_RAW,
+ RSPAMD_LUA_WORDS_FULL,
+ RSPAMD_LUA_WORDS_MAX
+};
+
+/**
+* Pushes words (rspamd_stat_token_t) to Lua
+* @param L
+* @param words
+* @param how
+*/
+gint rspamd_lua_push_words(lua_State *L, GArray *words,
+ enum rspamd_lua_words_type how);
+
+/**
+* Returns newly allocated name for caller module name
+* @param L
+* @return
+*/
+gchar *rspamd_lua_get_module_name(lua_State *L);
+
+/**
+* Call Lua function in a universal way. Arguments string:
+* - i - lua_integer, argument - gint64
+* - n - lua_number, argument - gdouble
+* - s - lua_string, argument - const gchar * (zero terminated)
+* - l - lua_lstring, argument - (size_t + const gchar *) pair
+* - u - lua_userdata, argument - (const char * + void *) - classname + pointer
+* - b - lua_boolean, argument - gboolean (not bool due to varargs promotion)
+* - f - lua_function, argument - int - position of the function on stack (not lua_registry)
+* - t - lua_text, argument - int - position of the lua_text on stack (not lua_registry)
+* @param L lua state
+* @param cbref LUA_REGISTRY reference (if it is -1 then a function on top of the stack is called - it must be removed by caller manually)
+* @param strloc where this function is called from
+* @param nret number of results (or LUA_MULTRET)
+* @param args arguments format string
+* @param err error to promote
+* @param ... arguments
+* @return true of pcall returned 0, false + err otherwise
+*/
+bool rspamd_lua_universal_pcall(lua_State *L, gint cbref, const gchar *strloc,
+ gint nret, const gchar *args, GError **err, ...);
+
+/**
+ * Returns true if lua is initialised
+ * @return
+ */
+bool rspamd_lua_is_initialised(void);
+
+/**
+* Wrapper for lua_geti from lua 5.3
+* @param L
+* @param index
+* @param i
+* @return
+*/
+#if defined(LUA_VERSION_NUM) && LUA_VERSION_NUM <= 502
+gint rspamd_lua_geti(lua_State *L, int index, int i);
+#else
+#define rspamd_lua_geti lua_geti
+#endif
+
+/* Paths defs */
+#define RSPAMD_CONFDIR_INDEX "CONFDIR"
+#define RSPAMD_LOCAL_CONFDIR_INDEX "LOCAL_CONFDIR"
+#define RSPAMD_RUNDIR_INDEX "RUNDIR"
+#define RSPAMD_DBDIR_INDEX "DBDIR"
+#define RSPAMD_LOGDIR_INDEX "LOGDIR"
+#define RSPAMD_PLUGINSDIR_INDEX "PLUGINSDIR"
+#define RSPAMD_SHAREDIR_INDEX "SHAREDIR"
+#define RSPAMD_RULESDIR_INDEX "RULESDIR"
+#define RSPAMD_LUALIBDIR_INDEX "LUALIBDIR"
+#define RSPAMD_WWWDIR_INDEX "WWWDIR"
+#define RSPAMD_PREFIX_INDEX "PREFIX"
+#define RSPAMD_VERSION_INDEX "VERSION"
+
+#ifdef WITH_LUA_TRACE
+extern ucl_object_t *lua_traces;
+#define LUA_TRACE_POINT \
+ do { \
+ ucl_object_t *func_obj; \
+ if (lua_traces == NULL) { lua_traces = ucl_object_typed_new(UCL_OBJECT); } \
+ func_obj = (ucl_object_t *) ucl_object_lookup(lua_traces, G_STRFUNC); \
+ if (func_obj == NULL) { \
+ func_obj = ucl_object_typed_new(UCL_INT); \
+ ucl_object_insert_key(lua_traces, func_obj, G_STRFUNC, 0, false); \
+ } \
+ func_obj->value.iv++; \
+ } while (0)
+#else
+#define LUA_TRACE_POINT \
+ do { \
+ } while (0)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RSPAMD_LUA_H */
diff --git a/src/lua/lua_compress.c b/src/lua/lua_compress.c
new file mode 100644
index 0000000..77c82c5
--- /dev/null
+++ b/src/lua/lua_compress.c
@@ -0,0 +1,622 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include <zlib.h>
+
+#ifdef SYS_ZSTD
+#include "zstd.h"
+#include "zstd_errors.h"
+#else
+#include "contrib/zstd/zstd.h"
+#include "contrib/zstd/error_public.h"
+#endif
+
+/***
+ * @module rspamd_compress
+ * This module contains compression/decompression routines (zstd and zlib currently)
+ */
+
+/***
+ * @function zstd.compress_ctx()
+ * Creates new compression ctx
+ * @return {compress_ctx} new compress ctx
+ */
+LUA_FUNCTION_DEF(zstd, compress_ctx);
+
+/***
+ * @function zstd.compress_ctx()
+ * Creates new compression ctx
+ * @return {compress_ctx} new compress ctx
+ */
+LUA_FUNCTION_DEF(zstd, decompress_ctx);
+
+LUA_FUNCTION_DEF(zstd_compress, stream);
+LUA_FUNCTION_DEF(zstd_compress, dtor);
+
+LUA_FUNCTION_DEF(zstd_decompress, stream);
+LUA_FUNCTION_DEF(zstd_decompress, dtor);
+
+static const struct luaL_reg zstd_compress_lib_f[] = {
+ LUA_INTERFACE_DEF(zstd, compress_ctx),
+ LUA_INTERFACE_DEF(zstd, decompress_ctx),
+ {NULL, NULL}};
+
+static const struct luaL_reg zstd_compress_lib_m[] = {
+ LUA_INTERFACE_DEF(zstd_compress, stream),
+ {"__gc", lua_zstd_compress_dtor},
+ {NULL, NULL}};
+
+static const struct luaL_reg zstd_decompress_lib_m[] = {
+ LUA_INTERFACE_DEF(zstd_decompress, stream),
+ {"__gc", lua_zstd_decompress_dtor},
+ {NULL, NULL}};
+
+static ZSTD_CStream *
+lua_check_zstd_compress_ctx(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{zstd_compress}");
+ luaL_argcheck(L, ud != NULL, pos, "'zstd_compress' expected");
+ return ud ? *(ZSTD_CStream **) ud : NULL;
+}
+
+static ZSTD_DStream *
+lua_check_zstd_decompress_ctx(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{zstd_decompress}");
+ luaL_argcheck(L, ud != NULL, pos, "'zstd_decompress' expected");
+ return ud ? *(ZSTD_DStream **) ud : NULL;
+}
+
+int lua_zstd_push_error(lua_State *L, int err)
+{
+ lua_pushnil(L);
+ lua_pushfstring(L, "zstd error %d (%s)", err, ZSTD_getErrorString(err));
+
+ return 2;
+}
+
+gint lua_compress_zstd_compress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz, r;
+ gint comp_level = 1;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ comp_level = lua_tointeger(L, 2);
+ }
+
+ sz = ZSTD_compressBound(t->len);
+
+ if (ZSTD_isError(sz)) {
+ msg_err("cannot compress data: %s", ZSTD_getErrorName(sz));
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ r = ZSTD_compress((void *) res->start, sz, t->start, t->len, comp_level);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot compress data: %s", ZSTD_getErrorName(r));
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ res->len = r;
+
+ return 1;
+}
+
+gint lua_compress_zstd_decompress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize outlen, r;
+ ZSTD_DStream *zstream;
+ ZSTD_inBuffer zin;
+ ZSTD_outBuffer zout;
+ gchar *out;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ zstream = ZSTD_createDStream();
+ ZSTD_initDStream(zstream);
+
+ zin.pos = 0;
+ zin.src = t->start;
+ zin.size = t->len;
+
+ if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) {
+ outlen = ZSTD_DStreamOutSize();
+ }
+
+ out = g_malloc(outlen);
+
+ zout.dst = out;
+ zout.pos = 0;
+ zout.size = outlen;
+
+ while (zin.pos < zin.size) {
+ r = ZSTD_decompressStream(zstream, &zout, &zin);
+
+ if (ZSTD_isError(r)) {
+ msg_err("cannot decompress data: %s", ZSTD_getErrorName(r));
+ ZSTD_freeDStream(zstream);
+ g_free(out);
+ lua_pushstring(L, ZSTD_getErrorName(r));
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ if (zin.pos < zin.size && zout.pos == zout.size) {
+ /* We need to extend output buffer */
+ zout.size = zout.size * 2;
+ out = g_realloc(zout.dst, zout.size);
+ zout.dst = out;
+ }
+ }
+
+ ZSTD_freeDStream(zstream);
+ lua_pushnil(L); /* Error */
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = out;
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ res->len = zout.pos;
+
+ return 2;
+}
+
+gint lua_compress_zlib_decompress(lua_State *L, bool is_gzip)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz;
+ z_stream strm;
+ gint rc;
+ guchar *p;
+ gsize remain;
+ gssize size_limit = -1;
+
+ int windowBits = is_gzip ? (MAX_WBITS + 16) : (MAX_WBITS);
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ size_limit = lua_tointeger(L, 2);
+ if (size_limit <= 0) {
+ return luaL_error(L, "invalid arguments (size_limit)");
+ }
+
+ sz = MIN(t->len * 2, size_limit);
+ }
+ else {
+ sz = t->len * 2;
+ }
+
+ memset(&strm, 0, sizeof(strm));
+ /* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
+
+ /* Here are dragons to distinguish between raw deflate and zlib */
+ if (windowBits == MAX_WBITS && t->len > 0) {
+ if ((int) (unsigned char) ((t->start[0] << 4)) != 0x80) {
+ /* Assume raw deflate */
+ windowBits = -windowBits;
+ }
+ }
+
+ rc = inflateInit2(&strm, windowBits);
+
+ if (rc != Z_OK) {
+ return luaL_error(L, "cannot init zlib");
+ }
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *) t->start;
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ p = (guchar *) res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = inflate(&strm, Z_NO_FLUSH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err("cannot decompress data: %s (last error: %s)",
+ zError(rc), strm.msg);
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ inflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+
+ if (size_limit > 0 || res->len >= G_MAXUINT32 / 2) {
+ if (res->len > size_limit || res->len >= G_MAXUINT32 / 2) {
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ inflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc((gpointer) res->start, res->len * 2);
+ sz = res->len * 2;
+ p = (guchar *) res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ inflateEnd(&strm);
+ res->len = strm.total_out;
+
+ return 1;
+}
+
+gint lua_compress_zlib_compress(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = NULL, *res;
+ gsize sz;
+ z_stream strm;
+ gint rc, comp_level = Z_DEFAULT_COMPRESSION;
+ guchar *p;
+ gsize remain;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_isnumber(L, 2)) {
+ comp_level = lua_tointeger(L, 2);
+
+ if (comp_level > Z_BEST_COMPRESSION || comp_level < Z_BEST_SPEED) {
+ return luaL_error(L, "invalid arguments: compression level must be between %d and %d",
+ Z_BEST_SPEED, Z_BEST_COMPRESSION);
+ }
+ }
+
+
+ memset(&strm, 0, sizeof(strm));
+ rc = deflateInit2(&strm, comp_level, Z_DEFLATED,
+ MAX_WBITS + 16, MAX_MEM_LEVEL - 1, Z_DEFAULT_STRATEGY);
+
+ if (rc != Z_OK) {
+ return luaL_error(L, "cannot init zlib: %s", zError(rc));
+ }
+
+ sz = deflateBound(&strm, t->len);
+
+ strm.avail_in = t->len;
+ strm.next_in = (guchar *) t->start;
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->start = g_malloc(sz);
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ p = (guchar *) res->start;
+ remain = sz;
+
+ while (strm.avail_in != 0) {
+ strm.avail_out = remain;
+ strm.next_out = p;
+
+ rc = deflate(&strm, Z_FINISH);
+
+ if (rc != Z_OK && rc != Z_BUF_ERROR) {
+ if (rc == Z_STREAM_END) {
+ break;
+ }
+ else {
+ msg_err("cannot compress data: %s (last error: %s)",
+ zError(rc), strm.msg);
+ lua_pop(L, 1); /* Text will be freed here */
+ lua_pushnil(L);
+ deflateEnd(&strm);
+
+ return 1;
+ }
+ }
+
+ res->len = strm.total_out;
+
+ if (strm.avail_out == 0 && strm.avail_in != 0) {
+ /* Need to allocate more */
+ remain = res->len;
+ res->start = g_realloc((gpointer) res->start, strm.avail_in + sz);
+ sz = strm.avail_in + sz;
+ p = (guchar *) res->start + remain;
+ remain = sz - remain;
+ }
+ }
+
+ deflateEnd(&strm);
+ res->len = strm.total_out;
+
+ return 1;
+}
+
+/* Stream API interface for Zstd: both compression and decompression */
+
+/* Operations allowed by zstd stream methods */
+static const char *const zstd_stream_op[] = {
+ "continue",
+ "flush",
+ "end",
+ NULL};
+
+static gint
+lua_zstd_compress_ctx(lua_State *L)
+{
+ ZSTD_CCtx *ctx, **pctx;
+
+ pctx = lua_newuserdata(L, sizeof(*pctx));
+ ctx = ZSTD_createCCtx();
+
+ if (!ctx) {
+ return luaL_error(L, "context create failed");
+ }
+
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{zstd_compress}", -1);
+ return 1;
+}
+
+static gint
+lua_zstd_compress_dtor(lua_State *L)
+{
+ ZSTD_CCtx *ctx = lua_check_zstd_compress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_freeCCtx(ctx);
+ }
+
+ return 0;
+}
+
+static gint
+lua_zstd_compress_reset(lua_State *L)
+{
+ ZSTD_CCtx *ctx = lua_check_zstd_compress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_zstd_compress_stream(lua_State *L)
+{
+ ZSTD_CStream *ctx = lua_check_zstd_compress_ctx(L, 1);
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 2);
+ int op = luaL_checkoption(L, 3, zstd_stream_op[0], zstd_stream_op);
+ int err = 0;
+ ZSTD_inBuffer inb;
+ ZSTD_outBuffer onb;
+
+ if (ctx && t) {
+ gsize dlen = 0;
+
+ inb.size = t->len;
+ inb.pos = 0;
+ inb.src = (const void *) t->start;
+
+ onb.pos = 0;
+ onb.size = ZSTD_CStreamInSize(); /* Initial guess */
+ onb.dst = NULL;
+
+ for (;;) {
+ if ((onb.dst = g_realloc(onb.dst, onb.size)) == NULL) {
+ return lua_zstd_push_error(L, ZSTD_error_memory_allocation);
+ }
+
+ dlen = onb.size;
+
+ int res = ZSTD_compressStream2(ctx, &onb, &inb, op);
+
+ if (res == 0) {
+ /* All done */
+ break;
+ }
+
+ if ((err = ZSTD_getErrorCode(res))) {
+ break;
+ }
+
+ onb.size *= 2;
+ res += dlen; /* Hint returned by compression routine */
+
+ /* Either double the buffer, or use the hint provided */
+ if (onb.size < res) {
+ onb.size = res;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (err) {
+ return lua_zstd_push_error(L, err);
+ }
+
+ lua_new_text(L, onb.dst, onb.pos, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_zstd_decompress_dtor(lua_State *L)
+{
+ ZSTD_DStream *ctx = lua_check_zstd_decompress_ctx(L, 1);
+
+ if (ctx) {
+ ZSTD_freeDStream(ctx);
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_zstd_decompress_ctx(lua_State *L)
+{
+ ZSTD_DStream *ctx, **pctx;
+
+ pctx = lua_newuserdata(L, sizeof(*pctx));
+ ctx = ZSTD_createDStream();
+
+ if (!ctx) {
+ return luaL_error(L, "context create failed");
+ }
+
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{zstd_decompress}", -1);
+ return 1;
+}
+
+static gint
+lua_zstd_decompress_stream(lua_State *L)
+{
+ ZSTD_DStream *ctx = lua_check_zstd_decompress_ctx(L, 1);
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 2);
+ int err = 0;
+ ZSTD_inBuffer inb;
+ ZSTD_outBuffer onb;
+
+ if (ctx && t) {
+ gsize dlen = 0;
+
+ if (t->len == 0) {
+ return lua_zstd_push_error(L, ZSTD_error_init_missing);
+ }
+
+ inb.size = t->len;
+ inb.pos = 0;
+ inb.src = (const void *) t->start;
+
+ onb.pos = 0;
+ onb.size = ZSTD_DStreamInSize(); /* Initial guess */
+ onb.dst = NULL;
+
+ for (;;) {
+ if ((onb.dst = g_realloc(onb.dst, onb.size)) == NULL) {
+ return lua_zstd_push_error(L, ZSTD_error_memory_allocation);
+ }
+
+ dlen = onb.size;
+
+ int res = ZSTD_decompressStream(ctx, &onb, &inb);
+
+ if (res == 0) {
+ /* All done */
+ break;
+ }
+
+ if ((err = ZSTD_getErrorCode(res))) {
+ break;
+ }
+
+ onb.size *= 2;
+ res += dlen; /* Hint returned by compression routine */
+
+ /* Either double the buffer, or use the hint provided */
+ if (onb.size < res) {
+ onb.size = res;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (err) {
+ return lua_zstd_push_error(L, err);
+ }
+
+ lua_new_text(L, onb.dst, onb.pos, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_load_zstd(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, zstd_compress_lib_f);
+
+ return 1;
+}
+
+void luaopen_compress(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{zstd_compress}", zstd_compress_lib_m);
+ rspamd_lua_new_class(L, "rspamd{zstd_decompress}", zstd_decompress_lib_m);
+ lua_pop(L, 2);
+
+ rspamd_lua_add_preload(L, "rspamd_zstd", lua_load_zstd);
+}
diff --git a/src/lua/lua_compress.h b/src/lua/lua_compress.h
new file mode 100644
index 0000000..34234de
--- /dev/null
+++ b/src/lua/lua_compress.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_COMPRESS_H
+#define RSPAMD_LUA_COMPRESS_H
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+gint lua_compress_zstd_compress(lua_State *L);
+gint lua_compress_zstd_decompress(lua_State *L);
+gint lua_compress_zlib_compress(lua_State *L);
+gint lua_compress_zlib_decompress(lua_State *L, bool is_gzip);
+
+void luaopen_compress(lua_State *L);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_LUA_COMPRESS_H
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
new file mode 100644
index 0000000..a044827
--- /dev/null
+++ b/src/lua/lua_config.c
@@ -0,0 +1,4780 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libmime/message.h"
+#include "libutil/expression.h"
+#include "src/libserver/composites/composites.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/lang_detection.h"
+#include "lua/lua_map.h"
+#include "lua/lua_thread_pool.h"
+#include "utlist.h"
+#include <math.h>
+
+/***
+ * This module is used to configure rspamd and is normally available as global
+ * variable named `rspamd_config`. Unlike other modules, it is not necessary to
+ * require it before usage.
+ * @module rspamd_config
+ * @example
+-- Register some callback symbol
+local function foo(task)
+ -- do something
+end
+rspamd_config:register_symbol('SYMBOL', 1.0, foo)
+
+-- Get configuration
+local tab = rspamd_config:get_all_opt('module') -- get table for module's options
+local opts = rspamd_config:get_key('options') -- get content of the specified key in rspamd configuration
+ */
+
+/* Config file methods */
+/***
+ * @method rspamd_config:get_module_opt(mname, optname)
+ * Returns value of specified option `optname` for a module `mname`,
+ * @param {string} mname name of module
+ * @param {string} optname option to get
+ * @return {string or table} value of the option or `nil` if option is not found
+ */
+LUA_FUNCTION_DEF(config, get_module_opt);
+/***
+ * @method rspamd_config:get_all_opt(mname)
+ * Returns value of all options for a module `mname`, flattening values into a single table consisting
+ * of all sections with such a name.
+ * @param {string} mname name of module
+ * @return {table} table of all options for `mname` or `nil` if a module's configuration is not found
+ */
+LUA_FUNCTION_DEF(config, get_all_opt);
+
+/***
+ * @method rspamd_config:get_ucl()
+ * Returns full configuration as a native Lua object (ucl to lua conversion).
+ * This method uses caching if possible.
+ * @return {table} table of all options in the configuration
+ */
+LUA_FUNCTION_DEF(config, get_ucl);
+/***
+ * @method rspamd_config:get_mempool()
+ * Returns static configuration memory pool.
+ * @return {mempool} [memory pool](mempool.md) object
+ */
+LUA_FUNCTION_DEF(config, get_mempool);
+/***
+ * @method rspamd_config:get_resolver()
+ * Returns DNS resolver.
+ * @return {dns_resolver} opaque DNS resolver pointer if any
+ */
+LUA_FUNCTION_DEF(config, get_resolver);
+/***
+ * @method rspamd_config:add_radix_map(mapline[, description])
+ * Creates new dynamic map of IP/mask addresses.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} radix tree object
+ * @example
+local ip_map = rspamd_config:add_radix_map ('file:///path/to/file', 'my radix map')
+...
+local function foo(task)
+ local ip = task:get_from_ip()
+ if ip_map:get_key(ip) then
+ return true
+ end
+ return false
+end
+ */
+
+/***
+ * @method rspamd_config:radix_from_config(mname, optname)
+ * Creates new embedded map of IP/mask addresses from config.
+ * @param {string} mname name of module
+ * @param {string} optname option to get
+ * @return {map} radix tree object
+ * @example
+local ip_map = rspamd_config:radix_from_config ('mymodule', 'ips')
+...
+local function foo(task)
+ local ip = task:get_from_ip()
+ if ip_map:get_key(ip) then
+ return true
+ end
+ return false
+end
+ */
+/***
+* @method rspamd_config:radix_from_ucl(obj)
+* Creates new embedded map of IP/mask addresses from object.
+* @param {ucl} obj object
+* @return {map} radix tree object
+*/
+/***
+ * @method rspamd_config:add_hash_map(mapline[, description])
+ * Creates new dynamic map string objects.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} hash set object
+ * @example
+local hash_map = rspamd_config:add_hash_map ('file:///path/to/file', 'my hash map')
+...
+local function foo(task)
+ local from = task:get_from()
+ if hash_map:get_key(from['user']) then
+ return true
+ end
+ return false
+end
+ */
+/***
+ * @method rspamd_config:add_kv_map(mapline[, description])
+ * Creates new dynamic map of key/values associations.
+ * @param {string} mapline URL for a map
+ * @param {string} description optional map description
+ * @return {map} hash table object
+ * @example
+local kv_map = rspamd_config:add_kv_map ('file:///path/to/file', 'my kv map')
+...
+local function foo(task)
+ local from = task:get_from()
+ if from then
+ local value = kv_map:get_key(from['user'])
+ if value then
+ return true,value
+ end
+ end
+ return false
+end
+ */
+/***
+ * @method rspamd_config:add_map({args})
+ * Creates new dynamic map according to the attributes passed.
+ *
+ * - `type`: type of map to be created, can be one of the following set:
+ * + `set`: set of strings
+ * + `radix`: map of IP addresses to strings
+ * + `map`: map of strings to strings
+ * + `regexp`: map of regexps to strings
+ * + `callback`: map processed by lua callback
+ * - `url`: url to load map from
+ * - `description`: map's description
+ * - `callback`: lua callback for the map
+ *
+ * @return {map} `true` if map has been added
+ * @example
+
+local str = ''
+local function process_map(in)
+ str = in
+end
+
+rspamd_config:add_map('http://example.com/map', "settings map", process_map)
+ */
+/***
+* @method rspamd_config:get_maps()
+* Get all maps defined as an array of rspamd{map} objects
+ *
+* @return {table|rspamd{map}}
+*/
+/***
+ * @method rspamd_config:get_classifier(name)
+ * Returns classifier config.
+ * @param {string} name name of classifier (e.g. `bayes`)
+ * @return {classifier} classifier object or `nil`
+ */
+LUA_FUNCTION_DEF(config, get_classifier);
+/***
+ * @method rspamd_config:register_symbol(table)
+ * Register symbol of a specified type in rspamd. This function accepts table of arguments:
+ *
+ * - `name`: name of symbol (can be missing for callback symbols)
+ * - `callback`: function to be called for symbol's check (can be absent for virtual symbols)
+ * - `weight`: weight of symbol (should normally be 1 or missing)
+ * - `priority`: priority of symbol (normally 0 or missing)
+ * - `type`: type of symbol:
+ * + `normal`: executed after prefilters, according to dependency graph or in undefined order
+ * + `callback`: a check that merely inserts virtual symbols
+ * + `connfilter`: executed early; before message body is available
+ * + `idempotent`: cannot change result in any way; executed last
+ * + `postfilter`: executed after most other checks
+ * + `prefilter`: executed before most other checks
+ * + `virtual`: a symbol inserted by its parent check
+ * - `flags`: various flags split by commas or spaces:
+ * + `nice` if symbol can produce negative score;
+ * + `empty` if symbol can be called for empty messages
+ * + `skip` if symbol should be skipped now
+ * + `nostat` if symbol should be excluded from stat tokens
+ * + `trivial` symbol is trivial (e.g. no network requests)
+ * + `explicit_disable` requires explicit disabling (e.g. via settings)
+ * + `ignore_passthrough` executed even if passthrough result has been set
+ * - `parent`: id of parent symbol (useful for virtual symbols)
+ *
+ * @return {number} id of symbol registered
+ */
+LUA_FUNCTION_DEF(config, register_symbol);
+/***
+ * @method rspamd_config:register_symbols(callback, [weight], callback_name, [, symbol, ...])
+ * Register callback function to be called for a set of symbols with initial weight.
+ * @param {function} callback callback function to be called for a specified symbol
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ * @param {string} callback_name symbolic name of callback
+ * @param {list of strings} symbol list of symbols registered by this function
+ */
+LUA_FUNCTION_DEF(config, register_symbols);
+/***
+ * @method rspamd_config:register_virtual_symbol(name, weight,)
+ * Register virtual symbol that is not associated with any callback.
+ *
+ * **This method is deprecated and should not be used in newly written code **
+ * @param {string} virtual name symbol's name
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ */
+LUA_FUNCTION_DEF(config, register_virtual_symbol);
+/***
+ * @method rspamd_config:register_callback_symbol(name, weight, callback)
+ * Register callback function to be called for a specified symbol with initial weight. Symbol itself is
+ * not registered in the metric and is not intended to be visible by a user.
+ *
+ * **This method is deprecated and should not be used in newly written code **
+ * @param {string} name symbol's name (just for unique id purposes)
+ * @param {number} weight initial weight of symbol (can be less than zero to specify non-spam symbols)
+ * @param {function} callback callback function to be called for a specified symbol
+ */
+LUA_FUNCTION_DEF(config, register_callback_symbol);
+LUA_FUNCTION_DEF(config, register_callback_symbol_priority);
+
+/***
+ * @method rspamd_config:register_dependency(id|name, depname)
+ * Create a dependency on symbol identified by name for symbol identified by ID or name.
+ * This affects order of checks only (a symbol is still checked if its dependencies are disabled).
+ * @param {number|string} id id or name of source (numeric id is returned by all register_*_symbol)
+ * @param {string} depname dependency name
+ * @example
+local function cb(task)
+...
+end
+
+local id = rspamd_config:register_symbol('SYM', 1.0, cb)
+rspamd_config:register_dependency(id, 'OTHER_SYM')
+-- Alternative form
+-- Symbol MY_RULE needs result from SPF_CHECK
+rspamd_config:register_dependency('MY_RULE', 'SPF_CHECK')
+ */
+LUA_FUNCTION_DEF(config, register_dependency);
+
+/***
+ * @method rspamd_config:get_symbol_flags(name)
+ * Returns symbol flags
+ * @param {string} name symbols's name
+ * @return {table|string} list of flags for symbol or nil
+ */
+LUA_FUNCTION_DEF(config, get_symbol_flags);
+
+/***
+ * @method rspamd_config:add_symbol_flags(name, flags)
+ * Adds flags to a symbol
+ * @param {string} name symbols's name
+ * @param {table|string} flags flags to add
+ * @return {table|string} new set of flags
+ */
+LUA_FUNCTION_DEF(config, add_symbol_flags);
+
+/**
+ * @method rspamd_config:register_re_selector(name, selector_str, [delimiter, [flatten]])
+ * Registers selector with the specific name to use in regular expressions in form
+ * name=/re/$ or name=/re/{selector}
+ * @param {string} name name of the selector
+ * @param {string} selector_str selector definition
+ * @param {string} delimiter delimiter to use when joining strings if flatten is false
+ * @param {bool} flatten if true then selector will return a table of captures instead of a single string
+ * @return true if selector has been registered
+ */
+LUA_FUNCTION_DEF(config, register_re_selector);
+
+/**
+ * @method rspamd_config:set_symbol({table})
+ * Sets the value of a specified symbol in a metric. This function accepts table with the following elements:
+ *
+ * - `name`: name of symbol (string)
+ * - `score`: score for symbol (number)
+ * - `metric`: name of metric (string, optional)
+ * - `description`: description of symbol (string, optional)
+ * - `group`: name of group for symbol (string, optional)
+ * - `one_shot`: turn off multiple hits for a symbol (boolean, optional)
+ * - `one_param`: turn off multiple options for a symbol (boolean, optional)
+ * - `flags`: comma separated string of flags:
+ * + `ignore`: do not strictly check validity of symbol and corresponding rule
+ * + `one_shot`: turn off multiple hits for a symbol
+ * + `one_param`: allow only one parameter for a symbol
+ * - `priority`: priority of symbol's definition
+ */
+LUA_FUNCTION_DEF(config, set_metric_symbol);
+
+/**
+ * @method rspamd_config:set_action({table})
+ * Sets the score of a specified action in a metric. This function accepts table with the following elements:
+ *
+ * - `action`: name of action (string)
+ * - `score`: score for action (number)
+ * - `metric`: name of metric (string, optional)
+ * - `priority`: priority of action's definition
+ */
+LUA_FUNCTION_DEF(config, set_metric_action);
+
+/**
+ * @method rspamd_config:get_action(name)
+ * Gets data for a specific action in config. This function returns number representing action's score
+ *
+ * @param {string} name name of action
+ * @return {number} action's score or nil in case of undefined score or action
+ */
+LUA_FUNCTION_DEF(config, get_metric_action);
+
+/**
+ * @method rspamd_config:get_all_actions()
+ * Gets data for all action in config
+ * @return {table|str->num} action's score or nil in case of undefined score or action
+ */
+LUA_FUNCTION_DEF(config, get_all_actions);
+
+/**
+ * @method rspamd_config:add_composite(name, expression)
+ * @param {string} name name of composite symbol
+ * @param {string} expression symbolic expression of the composite rule
+ * @return {bool} true if a composite has been added successfully
+ */
+LUA_FUNCTION_DEF(config, add_composite);
+/***
+ * @method rspamd_config:register_pre_filter(callback[, order])
+ * Register function to be called prior to symbols processing.
+ * @param {function} callback callback function
+ * @param {number} order filters are called from lower orders to higher orders, order is equal to 0 by default
+ * @example
+local function check_function(task)
+ -- It is possible to manipulate the task object here: set settings, set pre-action and so on
+ ...
+end
+
+rspamd_config:register_pre_filter(check_function)
+ */
+LUA_FUNCTION_DEF(config, register_pre_filter);
+/***
+ * @method rspamd_config:register_post_filter(callback[, order])
+ * Register function to be called after symbols are processed.
+ *
+ * @param {function} callback callback function
+ * @param {number} order filters are called from lower orders to higher orders, order is equal to 0 by default
+ */
+LUA_FUNCTION_DEF(config, register_post_filter);
+/* XXX: obsoleted */
+LUA_FUNCTION_DEF(config, register_module_option);
+/* XXX: not needed now */
+LUA_FUNCTION_DEF(config, get_api_version);
+/***
+ * @method rspamd_config:get_key(name)
+ * Returns configuration section with the specified `name`.
+ * @param {string} name name of config section
+ * @return {variant} specific value of section
+ * @example
+
+local set_section = rspamd_config:get_key("settings")
+if type(set_section) == "string" then
+ -- Just a map of ucl
+ if rspamd_config:add_map(set_section, "settings map", process_settings_map) then
+ rspamd_config:register_pre_filter(check_settings)
+ end
+elseif type(set_section) == "table" then
+ if process_settings_table(set_section) then
+ rspamd_config:register_pre_filter(check_settings)
+ end
+end
+ */
+LUA_FUNCTION_DEF(config, get_key);
+
+/***
+ * @method rspamd_config:add_condition(symbol, condition)
+ * Adds condition callback for specified symbol
+ * @param {string} symbol symbol's name
+ * @param {function} condition condition callback
+ * @return {boolean} true if condition has been added
+ * @example
+
+local condition_map = rspamd_config:add_map{
+ type = "hash",
+ urls = ['file:///path/to/file'],
+ description = 'SMTP from map that allows FUZZY_DENIED skip for the listed addresses'
+}
+rspamd_config:add_condition('FUZZY_DENIED', function(task)
+ local E = {}
+ -- Check for the smtp from address adding fail safe checks
+ if condition_map:find_key(((task:get_from('smtp') or E)[1] or E).addr) then
+ return false
+ end
+ -- Allow execution otherwise
+ return true
+end)
+ */
+LUA_FUNCTION_DEF(config, add_condition);
+
+/***
+ * @method rspamd_config:enable_symbol(symbol)
+ * Enables execution for the specified symbol
+ * @param {string} symbol symbol's name
+ */
+LUA_FUNCTION_DEF(config, enable_symbol);
+
+/***
+ * @method rspamd_config:disable_symbol(symbol, [disable_parent=true])
+ * Disables execution for the specified symbol
+ * @param {string} symbol symbol's name
+ * @param {boolean} disable_parent if true then disable parent execution in case of a virtual symbol
+ */
+LUA_FUNCTION_DEF(config, disable_symbol);
+
+/***
+ * @method rspamd_config:get_symbol_parent(symbol)
+ * Returns a parent symbol for specific symbol (or symbol itself if top level)
+ * @param {string} symbol symbol's name
+ */
+LUA_FUNCTION_DEF(config, get_symbol_parent);
+
+/***
+ * @method rspamd_config:get_group_symbols(group)
+ * Returns list of symbols for a specific group
+ * @param {string} group group's name
+ * @available 2.0+
+ * @return {list|string} list of all symbols in a specific group
+ */
+LUA_FUNCTION_DEF(config, get_group_symbols);
+
+/***
+ * @method rspamd_config:get_groups([need_private])
+ * Returns list of all groups defined
+ * @param {boolean} need_private optional flag to include private groups
+ * @available 2.3+
+ * @return {list|table} list of all groups
+ */
+LUA_FUNCTION_DEF(config, get_groups);
+
+/***
+ * @method rspamd_config:register_settings_id(name, symbols_enabled, symbols_disabled)
+ * Register new static settings id in config
+ * @param {string} name id name (not numeric!)
+ * @param {map|string->string} symbols_enabled map from symbol's name to boolean (currently)
+ * @param {map|string->string} symbols_disabled map from symbol's name to boolean (currently)
+ * @available 2.0+
+ */
+LUA_FUNCTION_DEF(config, register_settings_id);
+
+/***
+ * @method rspamd_config:__newindex(name, callback)
+ * This metamethod is called if new indices are added to the `rspamd_config` object.
+ * Technically, it is the equivalent of @see rspamd_config:register_symbol where `weight` is 1.0.
+ * There is also table form invocation that allows to control more things:
+ *
+ * - `callback`: has the same meaning and acts as function of task
+ * - `score`: default score for a symbol
+ * - `group`: default group for a symbol
+ * - `description`: default symbol's description
+ * - `priority`: additional priority value
+ * - `one_shot`: default value for one shot attribute
+ * - `condition`: function of task that can enable or disable this specific rule's execution
+ * @param {string} name index name
+ * @param {function/table} callback callback to be called
+ * @return {number} id of the new symbol added
+ * @example
+rspamd_config.R_EMPTY_IMAGE = function (task)
+ parts = task:get_text_parts()
+ if parts then
+ for _,part in ipairs(parts) do
+ if part:is_empty() then
+ images = task:get_images()
+ if images then
+ -- Symbol `R_EMPTY_IMAGE` is inserted
+ return true
+ end
+ return false
+ end
+ end
+ end
+ return false
+end
+
+rspamd_config.SYMBOL = {
+ callback = function(task)
+ ...
+ end,
+ score = 5.1,
+ description = 'sample symbol',
+ group = 'sample symbols',
+ condition = function(task)
+ if task:get_from()[1]['addr'] == 'user@example.com' then
+ return false
+ end
+ return true
+ end
+}
+ */
+LUA_FUNCTION_DEF(config, newindex);
+
+/***
+ * @method rspamd_config:register_regexp(params)
+ * Registers new re for further cached usage
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `pcre_only`: flag regexp as pcre only regexp
+ */
+LUA_FUNCTION_DEF(config, register_regexp);
+
+/***
+ * @method rspamd_config:replace_regexp(params)
+ * Replaces regexp with a new one
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `old_re`* : old regular expression object (must be in the cache)
+ * - `new_re`* : old regular expression object (must not be in the cache)
+ */
+LUA_FUNCTION_DEF(config, replace_regexp);
+
+/***
+ * @method rspamd_config:register_worker_script(worker_type, script)
+ * Registers the following script for workers of a specified type. The exact type
+ * of script function depends on worker type
+ * @param {string} worker_type worker type (e.g. "normal")
+ * @param {function} script script for a worker
+ * @return {boolean} `true` if a script has been registered
+ */
+LUA_FUNCTION_DEF(config, register_worker_script);
+
+/***
+ * @method rspamd_config:add_on_load(function(cfg, ev_base, worker) ... end)
+ * Registers the following script to be executed when configuration is completely loaded
+ * and the worker is already started (forked)
+ * @param {function} script function to be executed
+ * @example
+rspamd_config:add_on_load(function(cfg, ev_base, worker)
+ rspamd_config:add_periodic(ev_base, 1.0, function(cfg, ev_base)
+ local logger = require "rspamd_logger"
+ logger.infox(cfg, "periodic function in worker %s", worker:get_name())
+ return true
+ end)
+end)
+ */
+LUA_FUNCTION_DEF(config, add_on_load);
+
+/***
+ * @method rspamd_config:add_periodic(event_base, timeout, function(cfg, ev_base) ... end, [jitter = false])
+ * Registers function to be periodically executed by Rspamd
+ * @param {ev_base} event_base event base that is needed for async events
+ * @param {number} timeout time in seconds (could be fractional)
+ * @param {function} script function to be executed
+ * @param {boolean} jitter `true` if timeout jittering is needed
+ * @example
+rspamd_config:add_on_load(function(cfg, ev_base)
+ rspamd_config:add_periodic(ev_base, 1.0, function(cfg, ev_base)
+ local logger = require "rspamd_logger"
+ logger.infox(cfg, "periodic function")
+ return true -- if return numeric, a new interval is set. if return false, then the periodic event is removed
+ end)
+end)
+ */
+LUA_FUNCTION_DEF(config, add_periodic);
+
+/***
+ * @method rspamd_config:add_post_init(function(cfg) ... end)
+ * Registers the following script to be executed when configuration is completely loaded
+ * @available 2.0+
+ * @param {function} script function to be executed
+ */
+LUA_FUNCTION_DEF(config, add_post_init);
+
+/***
+ * @method rspamd_config:add_config_unload(function(cfg) ... end)
+ * Registers the following script to be executed when configuration is unloaded
+ * @available 2.0+
+ * @param {function} script function to be executed
+ */
+LUA_FUNCTION_DEF(config, add_config_unload);
+
+/***
+ * @method rspamd_config:get_symbols_count()
+ * Returns number of symbols registered in rspamd configuration
+ * @return {number} number of symbols registered in the configuration
+ */
+LUA_FUNCTION_DEF(config, get_symbols_count);
+
+/***
+ * @method rspamd_config:get_symbols_cksum()
+ * Returns checksum for all symbols in the cache
+ * @return {int64} boxed value of the 64 bit checksum
+ */
+LUA_FUNCTION_DEF(config, get_symbols_cksum);
+
+/***
+ * @method rspamd_config:get_symbols_counters()
+ * Returns table of all counters in the cache (weights, frequencies etc)
+ * @return {table|tables} all symbols indexed by name
+ */
+LUA_FUNCTION_DEF(config, get_symbols_counters);
+
+/***
+ * @method rspamd_config:get_symbols()
+ * Returns table of all scores defined in config. From version 2.0 returns table:
+ * - name
+ * - score
+ * - flags (e.g. `ignore` or `oneparam`)
+ * - nshots (== maxhits)
+ * - group - main group
+ * - groups - array of all groups
+ * @available 2.0+
+ * @return {table|tables} all symbols indexed by name
+ */
+LUA_FUNCTION_DEF(config, get_symbols);
+
+/***
+ * @method rspamd_config:get_symbol(sym_name)
+ * Returns table for a specific symbol getting data from the static config:
+ * - name
+ * - score
+ * - flags (e.g. `ignore` or `oneparam`)
+ * - nshots (== maxhits)
+ * - group - main group
+ * - groups - array of all groups
+ * @available 3.3+
+ * @return {table} symbol data (or nil)
+ */
+LUA_FUNCTION_DEF(config, get_symbol);
+
+/***
+ * @method rspamd_config:get_symbol_callback(name)
+ * Returns callback function for the specified symbol if it is a lua registered callback
+ * @return {function} callback function or nil
+ */
+LUA_FUNCTION_DEF(config, get_symbol_callback);
+
+/***
+ * @method rspamd_config:get_symbol_stat(name)
+ * Returns table with statistics for a specific symbol:
+ * - `frequency`: frequency for symbol's hits
+ * - `stddev`: standard deviation of `frequency`
+ * - `time`: average time in seconds (floating point)
+ * - `count`: total number of hits
+ * @return {table} symbol stats
+ */
+LUA_FUNCTION_DEF(config, get_symbol_stat);
+
+/***
+ * @method rspamd_config:set_symbol_callback(name, callback)
+ * Sets callback for the specified symbol
+ * @return {boolean} true if function has been replaced
+ */
+LUA_FUNCTION_DEF(config, set_symbol_callback);
+
+/***
+ * @method rspamd_config:register_finish_script(callback)
+ * Adds new callback that is called on worker process termination when all
+ * tasks pending are processed
+ *
+ * @param callback {function} a function with one argument (rspamd_task)
+ */
+LUA_FUNCTION_DEF(config, register_finish_script);
+
+/***
+ * @method rspamd_config:register_monitored(url, type, [{params}])
+ * Registers monitored resource to watch its availability. Supported types:
+ *
+ * - `dns`: DNS monitored object
+ *
+ * Params are optional table specific for each type. For DNS it supports the
+ * following options:
+ *
+ * - `prefix`: prefix to add before making request
+ * - `type`: type of request (e.g. 'a' or 'txt')
+ * - `ipnet`: array of ip/networks to expect on reply
+ * - `rcode`: expected return code (e.g. `nxdomain`)
+ *
+ * Returned object has the following methods:
+ *
+ * - `alive`: returns `true` if monitored resource is alive
+ * - `offline`: returns number of seconds of the current offline period (or 0 if alive)
+ * - `total_offline`: returns number of seconds of the overall offline
+ * - `latency`: returns the current average latency in seconds (or 0 if offline)
+ *
+ * @param {string} url resource to monitor
+ * @param {string} type type of monitoring
+ * @param {table} opts optional parameters
+ * @return {rspamd_monitored} rspamd monitored object
+ */
+LUA_FUNCTION_DEF(config, register_monitored);
+
+/***
+ * @method rspamd_config:add_doc(path, option, doc_string, [{params}])
+ * Adds new documentation string for an option `option` at path `path`
+ * Options defines optional params, such as:
+ *
+ * - `default`: default option value
+ * - `type`: type of an option (`string`, `number`, `object`, `array` etc)
+ * - `required`: if an option is required
+ *
+ * @param {string} path documentation path (e.g. module name)
+ * @param {string} option name of the option
+ * @param {string} doc_string documentation string
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(config, add_doc);
+
+/***
+ * @method rspamd_config:add_example(path, option, doc_string, example)
+ * Adds new documentation
+ *
+ * @param {string} path documentation path (e.g. module name or nil for top)
+ * @param {string} option name of the option
+ * @param {string} doc_string documentation string
+ * @param {string} example example in ucl format, comments are also parsed
+ */
+LUA_FUNCTION_DEF(config, add_example);
+
+/***
+ * @method rspamd_config:set_peak_cb(function)
+ * Sets a function that will be called when frequency of some symbol goes out of
+ * stddev * 2 over the last period of refreshment.
+ *
+ * @example
+rspamd_config:set_peak_cb(function(ev_base, sym, mean, stddev, value, error)
+ -- ev_base: event base for async events (e.g. redis)
+ -- sym: symbol's name
+ -- mean: mean frequency value
+ -- stddev: standard deviation of frequency
+ -- value: current frequency value
+ -- error: squared error
+ local logger = require "rspamd_logger"
+ logger.infox(rspamd_config, "symbol %s has changed frequency significantly: %s(%s) over %s(%s)",
+ sym, value, error, mean, stddev)
+end)
+ */
+LUA_FUNCTION_DEF(config, set_peak_cb);
+/***
+ * @method rspamd_config:get_cpu_flags()
+ * Returns architecture dependent flags supported by the CPU
+ * Currently, only x86 flags are supported:
+ * - 'ssse3'
+ * - 'sse42'
+ * - 'avx'
+ * - 'avx2'
+ * @return {table} flag -> true table
+ */
+LUA_FUNCTION_DEF(config, get_cpu_flags);
+
+/***
+ * @method rspamd_config:has_torch()
+ * Returns true if Rspamd is compiled with torch support and the runtime CPU
+ * supports sse4.2 required for torch.
+ * @return {boolean} true if torch is compiled and supported
+ */
+LUA_FUNCTION_DEF(config, has_torch);
+
+/***
+ * @method rspamd_config:experimental_enabled()
+ * Returns true if experimental plugins are enabled
+ * @return {boolean} true if experimental plugins are enabled
+ */
+LUA_FUNCTION_DEF(config, experimental_enabled);
+
+/***
+ * @method rspamd_config:load_ucl(filename[, include_trace])
+ * Loads config from the UCL file (but does not perform parsing using rcl)
+ * @param {string} filename file to load
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF(config, load_ucl);
+
+/***
+ * @method rspamd_config:parse_rcl([skip_sections])
+ * Parses RCL using loaded ucl file
+ * @param {table|string} sections to skip
+ * @return true or false + error message
+ */
+LUA_FUNCTION_DEF(config, parse_rcl);
+
+/***
+ * @method rspamd_config:init_modules()
+ * Initialize lua and internal modules
+ * @return true or false
+ */
+LUA_FUNCTION_DEF(config, init_modules);
+
+/***
+ * @method rspamd_config:init_subsystem(str)
+ * Initialize config subsystem from a comma separated list:
+ * - `modules` - init modules
+ * - `langdet` - language detector
+ * - `dns` - DNS resolver
+ * - TODO: add more
+ */
+LUA_FUNCTION_DEF(config, init_subsystem);
+
+/***
+ * @method rspamd_config:get_tld_path()
+ * Returns path to TLD file
+ * @return {string} path to tld file
+ */
+LUA_FUNCTION_DEF(config, get_tld_path);
+
+/***
+ * @method rspamd_config:get_dns_max_requests()
+ * Returns limit of DNS requests per task
+ * @return {number} number of dns requests allowed
+ */
+LUA_FUNCTION_DEF(config, get_dns_max_requests);
+
+/***
+ * @method rspamd_config:get_dns_timeout()
+ * Returns timeout for a DNS request
+ * @return {number} DNS timeout in second or 0 if not defined
+ */
+LUA_FUNCTION_DEF(config, get_dns_timeout);
+
+static const struct luaL_reg configlib_m[] = {
+ LUA_INTERFACE_DEF(config, get_module_opt),
+ LUA_INTERFACE_DEF(config, get_mempool),
+ LUA_INTERFACE_DEF(config, get_resolver),
+ LUA_INTERFACE_DEF(config, get_all_opt),
+ LUA_INTERFACE_DEF(config, get_ucl),
+ LUA_INTERFACE_DEF(config, add_radix_map),
+ LUA_INTERFACE_DEF(config, radix_from_config),
+ LUA_INTERFACE_DEF(config, radix_from_ucl),
+ LUA_INTERFACE_DEF(config, add_hash_map),
+ LUA_INTERFACE_DEF(config, add_kv_map),
+ LUA_INTERFACE_DEF(config, add_map),
+ LUA_INTERFACE_DEF(config, get_maps),
+ LUA_INTERFACE_DEF(config, get_classifier),
+ LUA_INTERFACE_DEF(config, register_symbol),
+ LUA_INTERFACE_DEF(config, register_symbols),
+ LUA_INTERFACE_DEF(config, register_virtual_symbol),
+ LUA_INTERFACE_DEF(config, register_callback_symbol),
+ LUA_INTERFACE_DEF(config, register_callback_symbol_priority),
+ LUA_INTERFACE_DEF(config, register_dependency),
+ LUA_INTERFACE_DEF(config, register_settings_id),
+ LUA_INTERFACE_DEF(config, get_symbol_flags),
+ LUA_INTERFACE_DEF(config, set_metric_symbol),
+ {"set_symbol", lua_config_set_metric_symbol},
+ LUA_INTERFACE_DEF(config, set_metric_action),
+ {"set_action", lua_config_set_metric_action},
+ {"get_metric_symbol", lua_config_get_symbol},
+ LUA_INTERFACE_DEF(config, get_metric_action),
+ {"get_action", lua_config_get_metric_action},
+ LUA_INTERFACE_DEF(config, get_all_actions),
+ LUA_INTERFACE_DEF(config, add_composite),
+ LUA_INTERFACE_DEF(config, register_module_option),
+ LUA_INTERFACE_DEF(config, register_pre_filter),
+ LUA_INTERFACE_DEF(config, register_post_filter),
+ LUA_INTERFACE_DEF(config, get_api_version),
+ LUA_INTERFACE_DEF(config, get_key),
+ LUA_INTERFACE_DEF(config, add_condition),
+ LUA_INTERFACE_DEF(config, enable_symbol),
+ LUA_INTERFACE_DEF(config, disable_symbol),
+ LUA_INTERFACE_DEF(config, register_regexp),
+ LUA_INTERFACE_DEF(config, replace_regexp),
+ LUA_INTERFACE_DEF(config, register_worker_script),
+ LUA_INTERFACE_DEF(config, register_re_selector),
+ LUA_INTERFACE_DEF(config, add_on_load),
+ LUA_INTERFACE_DEF(config, add_periodic),
+ LUA_INTERFACE_DEF(config, add_post_init),
+ LUA_INTERFACE_DEF(config, add_config_unload),
+ LUA_INTERFACE_DEF(config, get_symbols_count),
+ LUA_INTERFACE_DEF(config, get_symbols_cksum),
+ LUA_INTERFACE_DEF(config, get_symbols_counters),
+ {"get_symbols_scores", lua_config_get_symbols},
+ LUA_INTERFACE_DEF(config, get_symbols),
+ LUA_INTERFACE_DEF(config, get_symbol),
+ LUA_INTERFACE_DEF(config, get_groups),
+ LUA_INTERFACE_DEF(config, get_symbol_callback),
+ LUA_INTERFACE_DEF(config, set_symbol_callback),
+ LUA_INTERFACE_DEF(config, get_symbol_stat),
+ LUA_INTERFACE_DEF(config, get_symbol_parent),
+ LUA_INTERFACE_DEF(config, get_group_symbols),
+ LUA_INTERFACE_DEF(config, register_finish_script),
+ LUA_INTERFACE_DEF(config, register_monitored),
+ LUA_INTERFACE_DEF(config, add_doc),
+ LUA_INTERFACE_DEF(config, add_example),
+ LUA_INTERFACE_DEF(config, set_peak_cb),
+ LUA_INTERFACE_DEF(config, get_cpu_flags),
+ LUA_INTERFACE_DEF(config, has_torch),
+ LUA_INTERFACE_DEF(config, experimental_enabled),
+ LUA_INTERFACE_DEF(config, load_ucl),
+ LUA_INTERFACE_DEF(config, parse_rcl),
+ LUA_INTERFACE_DEF(config, init_modules),
+ LUA_INTERFACE_DEF(config, init_subsystem),
+ LUA_INTERFACE_DEF(config, get_tld_path),
+ LUA_INTERFACE_DEF(config, get_dns_max_requests),
+ LUA_INTERFACE_DEF(config, get_dns_timeout),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__newindex", lua_config_newindex},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(monitored, alive);
+LUA_FUNCTION_DEF(monitored, latency);
+LUA_FUNCTION_DEF(monitored, offline);
+LUA_FUNCTION_DEF(monitored, total_offline);
+
+static const struct luaL_reg monitoredlib_m[] = {
+ LUA_INTERFACE_DEF(monitored, alive),
+ LUA_INTERFACE_DEF(monitored, latency),
+ LUA_INTERFACE_DEF(monitored, offline),
+ LUA_INTERFACE_DEF(monitored, total_offline),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const guint64 rspamd_lua_callback_magic = 0x32c118af1e3263c7ULL;
+
+struct rspamd_config *
+lua_check_config(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{config}");
+ luaL_argcheck(L, ud != NULL, pos, "'config' expected");
+ return ud ? *((struct rspamd_config **) ud) : NULL;
+}
+
+static struct rspamd_monitored *
+lua_check_monitored(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{monitored}");
+ luaL_argcheck(L, ud != NULL, pos, "'monitored' expected");
+ return ud ? *((struct rspamd_monitored **) ud) : NULL;
+}
+
+/*** Config functions ***/
+static gint
+lua_config_get_api_version(lua_State *L)
+{
+ msg_warn("get_api_version is deprecated, do not use it");
+ lua_pushnumber(L, 100);
+
+ return 1;
+}
+
+static gint
+lua_config_get_module_opt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname, *optname;
+ const ucl_object_t *obj;
+
+ if (cfg) {
+ mname = luaL_checkstring(L, 2);
+ optname = luaL_checkstring(L, 3);
+
+ if (mname && optname) {
+ obj = rspamd_config_get_module_opt(cfg, mname, optname);
+ if (obj) {
+ return ucl_object_push_lua(L, obj, TRUE);
+ }
+ }
+ }
+ lua_pushnil(L);
+ return 1;
+}
+
+static int
+lua_config_get_mempool(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t **ppool;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ ppool = lua_newuserdata(L, sizeof(rspamd_mempool_t *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *ppool = cfg->cfg_pool;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static int
+lua_config_get_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_dns_resolver **pres;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL && cfg->dns_resolver) {
+ pres = lua_newuserdata(L, sizeof(*pres));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *pres = cfg->dns_resolver;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_all_opt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname;
+ const ucl_object_t *obj, *cur, *cur_elt;
+ ucl_object_iter_t it = NULL;
+ gint i;
+
+ if (cfg) {
+ mname = luaL_checkstring(L, 2);
+
+ if (mname) {
+ obj = ucl_obj_get_key(cfg->cfg_ucl_obj, mname);
+ /* Flatten object */
+ if (obj != NULL && (ucl_object_type(obj) == UCL_OBJECT ||
+ ucl_object_type(obj) == UCL_ARRAY)) {
+
+ lua_newtable(L);
+ it = ucl_object_iterate_new(obj);
+
+ LL_FOREACH(obj, cur)
+ {
+ it = ucl_object_iterate_reset(it, cur);
+
+ while ((cur_elt = ucl_object_iterate_safe(it, true))) {
+ lua_pushstring(L, ucl_object_key(cur_elt));
+ ucl_object_push_lua(L, cur_elt, true);
+ lua_settable(L, -3);
+ }
+ }
+
+ ucl_object_iterate_free(it);
+
+ return 1;
+ }
+ else if (obj != NULL) {
+ lua_newtable(L);
+ i = 1;
+
+ LL_FOREACH(obj, cur)
+ {
+ lua_pushinteger(L, i++);
+ ucl_object_push_lua(L, cur, true);
+ lua_settable(L, -3);
+ }
+
+ return 1;
+ }
+ }
+ }
+ lua_pushnil(L);
+
+ return 1;
+}
+
+struct rspamd_lua_cached_config {
+ lua_State *L;
+ gint ref;
+};
+
+static void
+lua_config_ucl_dtor(gpointer p)
+{
+ struct rspamd_lua_cached_config *cached = p;
+
+ luaL_unref(cached->L, LUA_REGISTRYINDEX, cached->ref);
+}
+
+static gint
+lua_config_get_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_cached_config *cached;
+
+ if (cfg) {
+ cached = rspamd_mempool_get_variable(cfg->cfg_pool, "ucl_cached");
+
+ if (cached) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cached->ref);
+ }
+ else {
+ if (cfg->cfg_ucl_obj) {
+ ucl_object_push_lua(L, cfg->cfg_ucl_obj, true);
+ lua_pushvalue(L, -1);
+ cached = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*cached));
+ cached->L = L;
+ cached->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_mempool_set_variable(cfg->cfg_pool, "ucl_cached",
+ cached, lua_config_ucl_dtor);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_config_get_classifier(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_classifier_config *clc = NULL, **pclc = NULL;
+ const gchar *name;
+ GList *cur;
+
+ if (cfg) {
+ name = luaL_checkstring(L, 2);
+
+ cur = g_list_first(cfg->classifiers);
+ while (cur) {
+ clc = cur->data;
+ if (g_ascii_strcasecmp(clc->name, name) == 0) {
+ pclc = &clc;
+ break;
+ }
+ cur = g_list_next(cur);
+ }
+ if (pclc) {
+ pclc = lua_newuserdata(L,
+ sizeof(struct rspamd_classifier_config *));
+ rspamd_lua_setclass(L, "rspamd{classifier}", -1);
+ *pclc = clc;
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+struct lua_callback_data {
+ guint64 magic;
+ lua_State *L;
+ gchar *symbol;
+
+ union {
+ gchar *name;
+ gint ref;
+ } callback;
+ gboolean cb_is_ref;
+
+ /* Dynamic data */
+ gint stack_level;
+ gint order;
+ struct rspamd_symcache_dynamic_item *item;
+};
+
+/*
+ * Unref symbol if it is local reference
+ */
+static void
+lua_destroy_cfg_symbol(gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+
+ /* Unref callback */
+ if (cd->cb_is_ref) {
+ luaL_unref(cd->L, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+}
+
+static gint
+lua_config_register_module_option(lua_State *L)
+{
+ return 0;
+}
+
+static gint
+rspamd_compare_order_func(gconstpointer a, gconstpointer b)
+{
+ const struct lua_callback_data *cb1 = a, *cb2 = b;
+
+ /* order of call goes from lower to higher */
+ return cb2->order - cb1->order;
+}
+
+static void
+lua_metric_symbol_callback(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+ struct rspamd_task **ptask;
+ gint level = lua_gettop(cd->L), nresults, err_idx, ret;
+ lua_State *L = cd->L;
+ struct rspamd_symbol_result *s;
+
+ cd->item = item;
+ rspamd_symcache_item_async_inc(task, item, "lua symbol");
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ level++;
+
+ if (cd->cb_is_ref) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+ else {
+ lua_getglobal(L, cd->callback.name);
+ }
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ if ((ret = lua_pcall(L, 1, LUA_MULTRET, err_idx)) != 0) {
+ msg_err_task("call to (%s) failed (%d): %s", cd->symbol, ret,
+ lua_tostring(L, -1));
+ lua_settop(L, err_idx); /* Not -1 here, as err_func is popped below */
+ }
+ else {
+ nresults = lua_gettop(L) - level;
+
+ if (nresults >= 1) {
+ /* Function returned boolean, so maybe we need to insert result? */
+ gint res = 0;
+ gint i;
+ gdouble flag = 1.0;
+ gint type;
+
+ type = lua_type(cd->L, level + 1);
+
+ if (type == LUA_TBOOLEAN) {
+ res = lua_toboolean(L, level + 1);
+ }
+ else if (type == LUA_TNUMBER) {
+ res = lua_tonumber(L, level + 1);
+ }
+ else if (type == LUA_TNIL) {
+ /* Can happen sometimes... */
+ res = FALSE;
+ }
+ else {
+ /* Something bogus has been returned, so we should log it */
+ msg_err_task("invalid return value for %s: %s",
+ cd->symbol, lua_typename(L, type));
+ res = FALSE;
+ }
+
+ if (res) {
+ gint first_opt = 2;
+
+ if (lua_type(L, level + 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, level + 2);
+ /* Shift opt index */
+ first_opt = 3;
+ }
+ else {
+ flag = res;
+ }
+
+ s = rspamd_task_insert_result(task, cd->symbol, flag, NULL);
+
+ if (s) {
+ guint last_pos = lua_gettop(L);
+
+ for (i = level + first_opt; i <= last_pos; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+
+ lua_pop(L, nresults);
+ }
+ }
+
+ lua_pop(L, 1); /* Error function */
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua symbol");
+ g_assert(lua_gettop(L) == level - 1);
+}
+
+static void lua_metric_symbol_callback_return(struct thread_entry *thread_entry,
+ int ret);
+
+static void lua_metric_symbol_callback_error(struct thread_entry *thread_entry,
+ int ret,
+ const char *msg);
+
+static void
+lua_metric_symbol_callback_coro(struct rspamd_task *task,
+ struct rspamd_symcache_dynamic_item *item,
+ gpointer ud)
+{
+ struct lua_callback_data *cd = ud;
+ struct rspamd_task **ptask;
+ struct thread_entry *thread_entry;
+
+ cd->item = item;
+ rspamd_symcache_item_async_inc(task, item, "lua coro symbol");
+ thread_entry = lua_thread_pool_get_for_task(task);
+
+ g_assert(thread_entry->cd == NULL);
+ thread_entry->cd = cd;
+
+ lua_State *thread = thread_entry->lua_state;
+ cd->stack_level = lua_gettop(thread);
+
+ if (cd->cb_is_ref) {
+ lua_rawgeti(thread, LUA_REGISTRYINDEX, cd->callback.ref);
+ }
+ else {
+ lua_getglobal(thread, cd->callback.name);
+ }
+
+ ptask = lua_newuserdata(thread, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(thread, "rspamd{task}", -1);
+ *ptask = task;
+
+ thread_entry->finish_callback = lua_metric_symbol_callback_return;
+ thread_entry->error_callback = lua_metric_symbol_callback_error;
+
+ lua_thread_call(thread_entry, 1);
+}
+
+static void
+lua_metric_symbol_callback_error(struct thread_entry *thread_entry,
+ int ret,
+ const char *msg)
+{
+ struct lua_callback_data *cd = thread_entry->cd;
+ struct rspamd_task *task = thread_entry->task;
+ msg_err_task("call to coroutine (%s) failed (%d): %s", cd->symbol, ret, msg);
+
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua coro symbol");
+}
+
+static void
+lua_metric_symbol_callback_return(struct thread_entry *thread_entry, int ret)
+{
+ struct lua_callback_data *cd = thread_entry->cd;
+ struct rspamd_task *task = thread_entry->task;
+ int nresults;
+ struct rspamd_symbol_result *s;
+
+ (void) ret;
+
+ lua_State *L = thread_entry->lua_state;
+
+ nresults = lua_gettop(L) - cd->stack_level;
+
+ if (nresults >= 1) {
+ /* Function returned boolean, so maybe we need to insert result? */
+ gint res = 0;
+ gint i;
+ gdouble flag = 1.0;
+ gint type;
+
+ type = lua_type(L, cd->stack_level + 1);
+
+ if (type == LUA_TBOOLEAN) {
+ res = lua_toboolean(L, cd->stack_level + 1);
+ }
+ else if (type == LUA_TFUNCTION) {
+ g_assert_not_reached();
+ }
+ else {
+ res = lua_tonumber(L, cd->stack_level + 1);
+ }
+
+ if (res) {
+ gint first_opt = 2;
+
+ if (lua_type(L, cd->stack_level + 2) == LUA_TNUMBER) {
+ flag = lua_tonumber(L, cd->stack_level + 2);
+ /* Shift opt index */
+ first_opt = 3;
+ }
+ else {
+ flag = res;
+ }
+
+ s = rspamd_task_insert_result(task, cd->symbol, flag, NULL);
+
+ if (s) {
+ guint last_pos = lua_gettop(L);
+
+ for (i = cd->stack_level + first_opt; i <= last_pos; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+
+ lua_pop(L, nresults);
+ }
+
+ g_assert(lua_gettop(L) == cd->stack_level); /* we properly cleaned up the stack */
+
+ cd->stack_level = 0;
+ rspamd_symcache_item_async_dec_check(task, cd->item, "lua coro symbol");
+}
+
+static GArray *
+rspamd_process_id_list(const gchar *entries)
+{
+ gchar **sym_elts;
+ GArray *ret;
+
+ sym_elts = g_strsplit_set(entries, ",;", -1);
+ guint nids = g_strv_length(sym_elts);
+ ret = g_array_sized_new(FALSE, FALSE, sizeof(guint32), nids);
+
+ for (guint i = 0; i < nids; i++) {
+ guint32 v = rspamd_config_name_to_id(sym_elts[i], strlen(sym_elts[i]));
+ g_array_append_val(ret, v);
+ }
+
+ g_strfreev(sym_elts);
+
+ return ret;
+}
+
+static gint
+rspamd_register_symbol_fromlua(lua_State *L,
+ struct rspamd_config *cfg,
+ const gchar *name,
+ gint ref,
+ gdouble weight,
+ gint priority,
+ enum rspamd_symbol_type type,
+ gint parent,
+ GArray *allowed_ids,
+ GArray *forbidden_ids,
+ gboolean optional)
+{
+ struct lua_callback_data *cd;
+ gint ret = -1;
+
+ if (priority == 0 && weight < 0) {
+ priority = 1;
+ }
+
+ if ((ret = rspamd_symcache_find_symbol(cfg->cache, name)) != -1) {
+ if (optional) {
+ msg_debug_config("duplicate symbol: %s, skip registering", name);
+
+ return ret;
+ }
+ else {
+ msg_err_config("duplicate symbol: %s, skip registering", name);
+
+ return -1;
+ }
+ }
+
+ if (allowed_ids && !(type & SYMBOL_TYPE_EXPLICIT_DISABLE)) {
+ /* Mark symbol as explicit allow */
+ msg_info_config("mark symbol %s as explicit enable as its execution is"
+ "allowed merely on specific settings ids",
+ name);
+ type |= SYMBOL_TYPE_EXPLICIT_ENABLE;
+ }
+
+ if (ref != -1) {
+ cd = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(struct lua_callback_data));
+ cd->magic = rspamd_lua_callback_magic;
+ cd->cb_is_ref = TRUE;
+ cd->callback.ref = ref;
+ cd->L = L;
+
+ if (name) {
+ cd->symbol = rspamd_mempool_strdup(cfg->cfg_pool, name);
+ }
+
+ if (type & SYMBOL_TYPE_USE_CORO) {
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback_coro,
+ cd,
+ type,
+ parent);
+ }
+ else {
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ lua_metric_symbol_callback,
+ cd,
+ type,
+ parent);
+ }
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) lua_destroy_cfg_symbol,
+ cd);
+ }
+ else {
+ /* No callback */
+ ret = rspamd_symcache_add_symbol(cfg->cache,
+ name,
+ priority,
+ NULL,
+ NULL,
+ type,
+ parent);
+ }
+
+ if (allowed_ids) {
+ rspamd_symcache_set_allowed_settings_ids(cfg->cache, name,
+ &g_array_index(allowed_ids, guint32, 0), allowed_ids->len);
+ }
+
+ if (forbidden_ids) {
+ rspamd_symcache_set_forbidden_settings_ids(cfg->cache, name,
+ &g_array_index(forbidden_ids, guint32, 0), forbidden_ids->len);
+ }
+
+ return ret;
+}
+
+static gint
+lua_config_register_post_filter(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint order = 0, cbref, ret;
+
+ if (cfg) {
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ order = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid type for callback: %s",
+ lua_typename(L, lua_type(L, 2)));
+ }
+
+ msg_warn_config("register_post_filter function is deprecated, "
+ "use register_symbol instead");
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ NULL,
+ cbref,
+ 1.0,
+ order,
+ SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ lua_pushboolean(L, ret);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_pre_filter(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint order = 0, cbref, ret;
+
+ if (cfg) {
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ order = lua_tonumber(L, 3);
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ return luaL_error(L, "invalid type for callback: %s",
+ lua_typename(L, lua_type(L, 2)));
+ }
+
+ msg_warn_config("register_pre_filter function is deprecated, "
+ "use register_symbol instead");
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ NULL,
+ cbref,
+ 1.0,
+ order,
+ SYMBOL_TYPE_PREFILTER | SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ lua_pushboolean(L, ret);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ size_t namelen;
+ const ucl_object_t *val;
+
+ name = luaL_checklstring(L, 2, &namelen);
+ if (name && cfg) {
+ val = ucl_object_lookup_len(cfg->cfg_ucl_obj, name, namelen);
+ if (val != NULL) {
+ ucl_object_push_lua(L, val, val->type != UCL_ARRAY);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static guint
+lua_parse_symbol_flags(const gchar *str)
+{
+ guint ret = 0;
+
+ if (str) {
+ if (strstr(str, "fine") != NULL) {
+ ret |= SYMBOL_TYPE_FINE;
+ }
+ if (strstr(str, "nice") != NULL) {
+ ret |= SYMBOL_TYPE_FINE;
+ }
+ if (strstr(str, "empty") != NULL) {
+ ret |= SYMBOL_TYPE_EMPTY;
+ }
+ if (strstr(str, "skip") != NULL) {
+ ret |= SYMBOL_TYPE_SKIPPED;
+ }
+ if (strstr(str, "nostat") != NULL) {
+ ret |= SYMBOL_TYPE_NOSTAT;
+ }
+ if (strstr(str, "idempotent") != NULL) {
+ ret |= SYMBOL_TYPE_IDEMPOTENT;
+ }
+ if (strstr(str, "trivial") != NULL) {
+ ret |= SYMBOL_TYPE_TRIVIAL;
+ }
+ if (strstr(str, "ghost") != NULL) {
+ ret |= SYMBOL_TYPE_GHOST;
+ }
+ if (strstr(str, "mime") != NULL) {
+ ret |= SYMBOL_TYPE_MIME_ONLY;
+ }
+ if (strstr(str, "ignore_passthrough") != NULL) {
+ ret |= SYMBOL_TYPE_IGNORE_PASSTHROUGH;
+ }
+ if (strstr(str, "explicit_disable") != NULL) {
+ ret |= SYMBOL_TYPE_EXPLICIT_DISABLE;
+ }
+ if (strstr(str, "explicit_enable") != NULL) {
+ ret |= SYMBOL_TYPE_EXPLICIT_ENABLE;
+ }
+ if (strstr(str, "coro") != NULL) {
+ ret |= SYMBOL_TYPE_USE_CORO;
+ }
+ }
+
+ return ret;
+}
+
+static guint
+lua_parse_symbol_type(const gchar *str)
+{
+ guint ret = SYMBOL_TYPE_NORMAL;
+ gchar **vec;
+ guint i, l;
+
+ if (str) {
+ vec = g_strsplit_set(str, ",;", -1);
+
+ if (vec) {
+ l = g_strv_length(vec);
+
+ for (i = 0; i < l; i++) {
+ str = vec[i];
+
+ /* TODO: total shit, rework some day */
+ if (g_ascii_strcasecmp(str, "virtual") == 0) {
+ ret |= SYMBOL_TYPE_VIRTUAL;
+ ret &= ~SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_CALLBACK;
+ }
+ else if (g_ascii_strcasecmp(str, "callback") == 0) {
+ ret |= SYMBOL_TYPE_CALLBACK;
+ ret &= ~SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_VIRTUAL;
+ }
+ else if (g_ascii_strcasecmp(str, "normal") == 0) {
+ ret |= SYMBOL_TYPE_NORMAL;
+ ret &= ~SYMBOL_TYPE_CALLBACK;
+ ret &= ~SYMBOL_TYPE_VIRTUAL;
+ }
+ else if (g_ascii_strcasecmp(str, "prefilter") == 0) {
+ ret |= SYMBOL_TYPE_PREFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "postfilter") == 0) {
+ ret |= SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "connfilter") == 0 ||
+ g_ascii_strcasecmp(str, "conn_filter") == 0) {
+ ret |= SYMBOL_TYPE_CONNFILTER | SYMBOL_TYPE_GHOST;
+ }
+ else if (g_ascii_strcasecmp(str, "idempotent") == 0) {
+ ret |= SYMBOL_TYPE_GHOST |
+ SYMBOL_TYPE_IDEMPOTENT | SYMBOL_TYPE_CALLBACK;
+ }
+ else {
+ gint fl = 0;
+
+ fl = lua_parse_symbol_flags(str);
+
+ if (fl == 0) {
+ msg_warn("bad type: %s", str);
+ }
+ else {
+ ret |= fl;
+ }
+ }
+ }
+
+ g_strfreev(vec);
+ }
+ }
+
+ return ret;
+}
+
+enum lua_push_symbol_flags_opts {
+ LUA_SYMOPT_FLAG_CREATE_ARRAY = 1u << 0u,
+ LUA_SYMOPT_FLAG_CREATE_MAP = 1u << 1u,
+ LUA_SYMOPT_FLAG_USE_MAP = 1u << 2u,
+ LUA_SYMOPT_FLAG_USE_ARRAY = 1u << 3u,
+};
+
+#define LUA_SYMOPT_IS_ARRAY(f) ((f) & (LUA_SYMOPT_FLAG_CREATE_ARRAY | LUA_SYMOPT_FLAG_USE_ARRAY))
+#define LUA_SYMOPT_IS_CREATE(f) ((f) & (LUA_SYMOPT_FLAG_CREATE_ARRAY | LUA_SYMOPT_FLAG_CREATE_MAP))
+#define LUA_OPTION_PUSH(nm) \
+ do { \
+ if (LUA_SYMOPT_IS_ARRAY(fl)) { \
+ lua_pushstring(L, #nm); \
+ lua_rawseti(L, -2, i++); \
+ } \
+ else { \
+ lua_pushboolean(L, true); \
+ lua_setfield(L, -2, #nm); \
+ } \
+ } while (0)
+
+static void
+lua_push_symbol_flags(lua_State *L, guint flags, enum lua_push_symbol_flags_opts fl)
+{
+ guint i = 1;
+
+ if (LUA_SYMOPT_IS_CREATE(fl)) {
+ lua_newtable(L);
+ }
+
+ if (flags & SYMBOL_TYPE_FINE) {
+ LUA_OPTION_PUSH(fine);
+ }
+
+ if (flags & SYMBOL_TYPE_EMPTY) {
+ LUA_OPTION_PUSH(empty);
+ }
+
+ if (flags & SYMBOL_TYPE_EXPLICIT_DISABLE) {
+ LUA_OPTION_PUSH(explicit_disable);
+ }
+
+ if (flags & SYMBOL_TYPE_EXPLICIT_ENABLE) {
+ LUA_OPTION_PUSH(explicit_enable);
+ }
+
+ if (flags & SYMBOL_TYPE_IGNORE_PASSTHROUGH) {
+ LUA_OPTION_PUSH(ignore_passthrough);
+ }
+
+ if (flags & SYMBOL_TYPE_NOSTAT) {
+ LUA_OPTION_PUSH(nostat);
+ }
+
+ if (flags & SYMBOL_TYPE_IDEMPOTENT) {
+ LUA_OPTION_PUSH(idempotent);
+ }
+
+ if (flags & SYMBOL_TYPE_MIME_ONLY) {
+ LUA_OPTION_PUSH(mime);
+ }
+
+ if (flags & SYMBOL_TYPE_TRIVIAL) {
+ LUA_OPTION_PUSH(trivial);
+ }
+
+ if (flags & SYMBOL_TYPE_SKIPPED) {
+ LUA_OPTION_PUSH(skip);
+ }
+
+ if (flags & SYMBOL_TYPE_COMPOSITE) {
+ LUA_OPTION_PUSH(composite);
+ }
+}
+
+static gint
+lua_config_get_symbol_flags(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ guint flags;
+
+ if (cfg && name) {
+ flags = rspamd_symcache_get_symbol_flags(cfg->cache,
+ name);
+
+ if (flags != 0) {
+ lua_push_symbol_flags(L, flags, LUA_SYMOPT_FLAG_CREATE_ARRAY);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL, *type_str = NULL,
+ *description = NULL, *group = NULL;
+ double weight = 0, score = NAN, parent_float = NAN;
+ gboolean one_shot = FALSE;
+ gint ret = -1, cbref = -1;
+ guint type = 0, flags = 0;
+ gint64 parent = 0, priority = 0, nshots = 0;
+ GArray *allowed_ids = NULL, *forbidden_ids = NULL;
+ GError *err = NULL;
+ int prev_top = lua_gettop(L);
+
+ if (cfg) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "name=S;weight=N;callback=F;type=S;priority=I;parent=D;"
+ "score=D;description=S;group=S;one_shot=B;nshots=I",
+ &name, &weight, &cbref, &type_str,
+ &priority, &parent_float,
+ &score, &description, &group, &one_shot, &nshots)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Deal with flags and ids */
+ lua_pushstring(L, "flags");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ flags = lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ flags |= lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ }
+ lua_pop(L, 1); /* Clean flags */
+
+ lua_pushstring(L, "allowed_ids");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ allowed_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(allowed_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "forbidden_ids");
+ lua_gettable(L, 2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(forbidden_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ if (nshots == 0) {
+ nshots = cfg->default_max_shots;
+ }
+
+ type = lua_parse_symbol_type(type_str);
+
+ if (!name && !(type & SYMBOL_TYPE_CALLBACK)) {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "no symbol name but type is not callback");
+ }
+ else if (!(type & SYMBOL_TYPE_VIRTUAL) && cbref == -1) {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "no callback for symbol %s", name);
+ }
+
+ if (isnan(parent_float)) {
+ parent = -1;
+ }
+ else {
+ parent = parent_float;
+ }
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ cbref,
+ weight == 0 ? 1.0 : weight,
+ priority,
+ type | flags,
+ parent,
+ allowed_ids, forbidden_ids,
+ FALSE);
+
+ if (allowed_ids) {
+ g_array_free(allowed_ids, TRUE);
+ }
+
+ if (forbidden_ids) {
+ g_array_free(forbidden_ids, TRUE);
+ }
+
+ if (ret != -1) {
+ if (!isnan(score) || group) {
+ if (one_shot) {
+ nshots = 1;
+ }
+
+ rspamd_config_add_symbol(cfg, name,
+ score, description, group, flags,
+ 0, nshots);
+
+ lua_pushstring(L, "groups");
+ lua_gettable(L, 2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ lua_settop(L, prev_top);
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "augmentations");
+ lua_gettable(L, 2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ int tbl_idx = lua_gettop(L);
+ for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) {
+ size_t len;
+ const char *augmentation = lua_tolstring(L, -1, &len), *eqsign_pos;
+
+ /* Find `=` symbol and use it as a separator */
+ eqsign_pos = memchr(augmentation, '=', len);
+ if (eqsign_pos != NULL && eqsign_pos + 1 < augmentation + len) {
+ rspamd_ftok_t tok;
+
+ tok.begin = augmentation;
+ tok.len = eqsign_pos - augmentation;
+ char *augentation_name = rspamd_ftokdup(&tok);
+
+ tok.begin = eqsign_pos + 1;
+ tok.len = (augmentation + len) - tok.begin;
+
+ char *augmentation_value = rspamd_ftokdup(&tok);
+
+ if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret,
+ augentation_name, augmentation_value)) {
+ lua_settop(L, prev_top);
+ g_free(augmentation_value);
+ g_free(augentation_name);
+
+ return luaL_error(L, "unknown or invalid augmentation %s in symbol %s",
+ augmentation, name);
+ }
+
+ g_free(augmentation_value);
+ g_free(augentation_name);
+ }
+ else {
+ /* Just a value */
+ if (!rspamd_symcache_add_symbol_augmentation(cfg->cache, ret,
+ augmentation, NULL)) {
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "unknown augmentation %s in symbol %s",
+ augmentation, name);
+ }
+ }
+ }
+ }
+ }
+ }
+ else {
+ lua_settop(L, prev_top);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_settop(L, prev_top);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint i, top, idx, ret = -1;
+ const gchar *sym;
+ gdouble weight = 1.0;
+
+ if (lua_gettop(L) < 3) {
+ if (cfg) {
+ msg_err_config("not enough arguments to register a function");
+ }
+
+ lua_error(L);
+
+ return 0;
+ }
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, 2));
+ }
+ else {
+ lua_pushvalue(L, 2);
+ }
+ idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, 3);
+ top = 4;
+ }
+ else {
+ top = 3;
+ }
+ sym = luaL_checkstring(L, top++);
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ sym,
+ idx,
+ weight,
+ 0,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+
+ for (i = top; i <= lua_gettop(L); i++) {
+ if (lua_type(L, i) == LUA_TTABLE) {
+ lua_pushvalue(L, i);
+ lua_pushnil(L);
+ while (lua_next(L, -2)) {
+ lua_pushvalue(L, -2);
+ sym = luaL_checkstring(L, -2);
+ rspamd_symcache_add_symbol(cfg->cache, sym,
+ 0, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, ret);
+ lua_pop(L, 2);
+ }
+ lua_pop(L, 1);
+ }
+ else if (lua_type(L, i) == LUA_TSTRING) {
+ sym = luaL_checkstring(L, i);
+ rspamd_symcache_add_symbol(cfg->cache, sym,
+ 0, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, ret);
+ }
+ }
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_virtual_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ double weight;
+ gint ret = -1, parent = -1;
+
+ if (cfg) {
+ name = luaL_checkstring(L, 2);
+ weight = luaL_checknumber(L, 3);
+
+ if (lua_gettop(L) > 3) {
+ parent = lua_tonumber(L, 4);
+ }
+
+ if (name) {
+ ret = rspamd_symcache_add_symbol(cfg->cache, name,
+ weight > 0 ? 0 : -1, NULL, NULL,
+ SYMBOL_TYPE_VIRTUAL, parent);
+ }
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_callback_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double weight;
+ gint ret = -1, top = 2;
+
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Legacy syntax */
+ name = luaL_checkstring(L, 2);
+ top++;
+ }
+
+ weight = luaL_checknumber(L, top);
+
+ if (lua_type(L, top + 1) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, top + 1));
+ }
+ else {
+ lua_pushvalue(L, top + 1);
+ }
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ weight,
+ 0,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_register_callback_symbol_priority(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double weight;
+ gint priority, ret = -1, top = 2;
+
+ if (cfg) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Legacy syntax */
+ name = luaL_checkstring(L, 2);
+ top++;
+ }
+
+ weight = luaL_checknumber(L, top);
+ priority = luaL_checknumber(L, top + 1);
+
+ if (lua_type(L, top + 2) == LUA_TSTRING) {
+ lua_getglobal(L, luaL_checkstring(L, top + 2));
+ }
+ else {
+ lua_pushvalue(L, top + 2);
+ }
+
+ ret = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ weight,
+ priority,
+ SYMBOL_TYPE_CALLBACK,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+
+static gint
+lua_config_register_dependency(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *parent = NULL, *child = NULL;
+ gint child_id;
+
+ if (cfg == NULL) {
+ lua_error(L);
+ return 0;
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ child_id = luaL_checknumber(L, 2);
+ parent = luaL_checkstring(L, 3);
+
+ return luaL_error(L, "calling for obsolete method to register deps for symbol %d->%s",
+ child_id, parent);
+ }
+ else {
+ child = luaL_checkstring(L, 2);
+ parent = luaL_checkstring(L, 3);
+
+ if (child != NULL && parent != NULL) {
+ rspamd_symcache_add_delayed_dependency(cfg->cache, child,
+ parent);
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_set_metric_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *description = NULL,
+ *group = NULL, *name = NULL, *flags_str = NULL;
+ double score;
+ gboolean one_shot = FALSE, one_param = FALSE;
+ GError *err = NULL;
+ gdouble priority = 0.0;
+ guint flags = 0;
+ gint64 nshots = 0;
+
+ if (cfg) {
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*name=S;score=N;description=S;"
+ "group=S;one_shot=B;one_param=B;priority=N;flags=S;"
+ "nshots=I",
+ &name, &score, &description,
+ &group, &one_shot, &one_param,
+ &priority, &flags_str, &nshots)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+ else {
+ name = luaL_checkstring(L, 2);
+ score = luaL_checknumber(L, 3);
+
+ if (lua_gettop(L) > 3 && lua_type(L, 4) == LUA_TSTRING) {
+ description = luaL_checkstring(L, 4);
+ }
+ if (lua_gettop(L) > 4 && lua_type(L, 5) == LUA_TSTRING) {
+ /* XXX: metrics */
+ }
+ if (lua_gettop(L) > 5 && lua_type(L, 6) == LUA_TSTRING) {
+ group = luaL_checkstring(L, 6);
+ }
+ if (lua_gettop(L) > 6 && lua_type(L, 7) == LUA_TBOOLEAN) {
+ one_shot = lua_toboolean(L, 7);
+ }
+ }
+
+ if (nshots == 0) {
+ nshots = cfg->default_max_shots;
+ }
+
+ if (one_shot) {
+ nshots = 1;
+ }
+ if (one_param) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+
+ if (flags_str) {
+ if (strstr(flags_str, "one_shot") != NULL) {
+ nshots = 1;
+ }
+ if (strstr(flags_str, "ignore") != NULL) {
+ flags |= RSPAMD_SYMBOL_FLAG_IGNORE_METRIC;
+ }
+ if (strstr(flags_str, "one_param") != NULL) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+
+ rspamd_config_add_symbol(cfg, name,
+ score, description, group, flags, (guint) priority, nshots);
+
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushstring(L, "groups");
+ lua_gettable(L, 2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_set_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = NULL;
+ double threshold = NAN;
+ GError *err = NULL;
+ gdouble priority = 0.0;
+ ucl_object_t *obj_tbl = NULL;
+
+ if (cfg) {
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*action=S;score=N;"
+ "priority=N",
+ &name, &threshold,
+ &priority)) {
+ msg_err_config("bad arguments: %e", err);
+ g_error_free(err);
+
+ return 0;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING && lua_type(L, 3) == LUA_TTABLE) {
+ name = lua_tostring(L, 2);
+ obj_tbl = ucl_object_lua_import(L, 3);
+
+ if (obj_tbl) {
+ if (name) {
+ rspamd_config_set_action_score(cfg, name, obj_tbl);
+ ucl_object_unref(obj_tbl);
+ }
+ else {
+ ucl_object_unref(obj_tbl);
+ return luaL_error(L, "invalid first argument, action name expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid second argument, table expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, table expected");
+ }
+
+ if (name != NULL && !isnan(threshold) && threshold != 0) {
+ obj_tbl = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(threshold),
+ "score", 0, false);
+ ucl_object_insert_key(obj_tbl, ucl_object_fromdouble(priority),
+ "priority", 0, false);
+ rspamd_config_set_action_score(cfg, name, obj_tbl);
+ ucl_object_unref(obj_tbl);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_get_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *act_name = luaL_checkstring(L, 2);
+ struct rspamd_action *act;
+
+ if (cfg && act_name) {
+ act = rspamd_config_get_action(cfg, act_name);
+
+ if (act) {
+ if (!isnan(act->threshold)) {
+ lua_pushnumber(L, act->threshold);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 1;
+}
+
+static void
+lua_config_actions_cb(struct rspamd_action *act, void *cbd)
+{
+ lua_State *L = (lua_State *) cbd;
+
+ if (!isnan(act->threshold)) {
+ lua_pushstring(L, act->name);
+ lua_pushnumber(L, act->threshold);
+ lua_settable(L, -3);
+ }
+}
+
+static gint
+lua_config_get_all_actions(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg) {
+ lua_createtable(L, 0, rspamd_config_actions_size(cfg));
+ rspamd_config_actions_foreach(cfg, lua_config_actions_cb, L);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, rspamd_config expected");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_add_composite(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gchar *name;
+ const gchar *expr_str;
+ struct rspamd_composite *composite;
+ gboolean ret = FALSE;
+
+ if (cfg) {
+ name = rspamd_mempool_strdup(cfg->cfg_pool, luaL_checkstring(L, 2));
+ expr_str = luaL_checkstring(L, 3);
+
+ if (name && expr_str) {
+ composite = rspamd_composites_manager_add_from_string(cfg->composites_manager,
+ name, expr_str);
+
+ if (composite) {
+ rspamd_symcache_add_symbol(cfg->cache, name,
+ 0, NULL, composite, SYMBOL_TYPE_COMPOSITE, -1);
+ ret = TRUE;
+ }
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_config_newindex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name;
+ GArray *allowed_ids = NULL, *forbidden_ids = NULL;
+ gint id, nshots;
+ guint flags = 0;
+ gboolean optional = FALSE;
+
+ name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && name != NULL && lua_gettop(L) == 3) {
+
+ if (lua_type(L, 3) == LUA_TFUNCTION) {
+ /* Normal symbol from just a function */
+ lua_pushvalue(L, 3);
+ rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ luaL_ref(L, LUA_REGISTRYINDEX),
+ 1.0,
+ 0,
+ SYMBOL_TYPE_NORMAL,
+ -1,
+ NULL, NULL,
+ FALSE);
+ }
+ else if (lua_type(L, 3) == LUA_TTABLE) {
+ guint type = SYMBOL_TYPE_NORMAL, priority = 0;
+ gint idx;
+ gdouble weight = 1.0, score = NAN;
+ const char *type_str, *group = NULL, *description = NULL;
+
+ /*
+ * Table can have the following attributes:
+ * "callback" - should be a callback function
+ * "weight" - optional weight
+ * "priority" - optional priority
+ * "type" - optional type (normal, virtual, callback)
+ * "flags" - optional flags
+ * -- Metric options
+ * "score" - optional default score (overridden by metric)
+ * "group" - optional default group
+ * "one_shot" - optional one shot mode
+ * "description" - optional description
+ */
+ lua_pushvalue(L, 3);
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 2);
+ msg_info_config("cannot find callback definition for %s",
+ name);
+ return 0;
+ }
+ idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ /* Optional fields */
+ lua_pushstring(L, "weight");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ weight = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "priority");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "optional");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ optional = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "type");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ type_str = lua_tostring(L, -1);
+ type = lua_parse_symbol_type(type_str);
+ }
+ lua_pop(L, 1);
+
+ /* Deal with flags and ids */
+ lua_pushstring(L, "flags");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ flags = lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ flags |= lua_parse_symbol_flags(lua_tostring(L, -1));
+ }
+ }
+ lua_pop(L, 1); /* Clean flags */
+
+ lua_pushstring(L, "allowed_ids");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ allowed_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ allowed_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(allowed_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "forbidden_ids");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ forbidden_ids = rspamd_process_id_list(lua_tostring(L, -1));
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ forbidden_ids = g_array_sized_new(FALSE, FALSE, sizeof(guint32),
+ rspamd_lua_table_size(L, -1));
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ guint32 v = lua_tointeger(L, -1);
+ g_array_append_val(forbidden_ids, v);
+ }
+ }
+ lua_pop(L, 1);
+
+ id = rspamd_register_symbol_fromlua(L,
+ cfg,
+ name,
+ idx,
+ weight,
+ priority,
+ type | flags,
+ -1,
+ allowed_ids, forbidden_ids,
+ optional);
+
+ if (allowed_ids) {
+ g_array_free(allowed_ids, TRUE);
+ }
+
+ if (forbidden_ids) {
+ g_array_free(forbidden_ids, TRUE);
+ }
+
+ if (id != -1) {
+ /* Check for condition */
+ lua_pushstring(L, "condition");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ gint condref;
+
+ /* Here we pop function from the stack, so no lua_pop is required */
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+ g_assert(name != NULL);
+ rspamd_symcache_add_condition_delayed(cfg->cache,
+ name, L, condref);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ /* Check for augmentations */
+ lua_pushstring(L, "augmentations");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+
+ int tbl_idx = lua_gettop(L);
+ for (lua_pushnil(L); lua_next(L, tbl_idx); lua_pop(L, 1)) {
+ rspamd_symcache_add_symbol_augmentation(cfg->cache, id,
+ lua_tostring(L, -1), NULL);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ /*
+ * Now check if a symbol has not been registered in any metric and
+ * insert default value if applicable
+ */
+ struct rspamd_symbol *sym = g_hash_table_lookup(cfg->symbols, name);
+ if (sym == NULL || (sym->flags & RSPAMD_SYMBOL_FLAG_UNSCORED)) {
+ nshots = cfg->default_max_shots;
+
+ lua_pushstring(L, "score");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ score = lua_tonumber(L, -1);
+
+ if (sym) {
+ /* Reset unscored flag */
+ sym->flags &= ~RSPAMD_SYMBOL_FLAG_UNSCORED;
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "group");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ group = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!isnan(score) || group != NULL) {
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ description = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "one_shot");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, -1)) {
+ nshots = 1;
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "one_param");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, -1)) {
+ flags |= RSPAMD_SYMBOL_FLAG_ONEPARAM;
+ }
+ }
+ lua_pop(L, 1);
+
+ /*
+ * Do not override the existing symbols (using zero priority),
+ * since we are defining default values here
+ */
+ if (!isnan(score)) {
+ rspamd_config_add_symbol(cfg, name, score,
+ description, group, flags, 0, nshots);
+ }
+ else if (group) {
+ /* Add with zero score */
+ rspamd_config_add_symbol(cfg, name, NAN,
+ description, group, flags, 0, nshots);
+ }
+
+ lua_pushstring(L, "groups");
+ lua_gettable(L, -2);
+
+ if (lua_istable(L, -1)) {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ rspamd_config_add_symbol_group(cfg, name,
+ lua_tostring(L, -1));
+ }
+ else {
+ return luaL_error(L, "invalid groups element");
+ }
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ /* Fill in missing fields from lua definition if they are not set */
+ if (sym->description == NULL) {
+ lua_pushstring(L, "description");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ description = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (description) {
+ sym->description = rspamd_mempool_strdup(cfg->cfg_pool, description);
+ }
+ }
+
+ /* If ungrouped and there is a group defined in lua, change the primary group
+ * Otherwise, add to the list of groups for this symbol. */
+ lua_pushstring(L, "group");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ group = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+ if (group) {
+ if (sym->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
+ /* Unset the "ungrouped" group */
+ sym->gr = NULL;
+ }
+ /* Add the group. If the symbol was ungrouped, this will
+ * clear RSPAMD_SYMBOL_FLAG_UNGROUPED from the flags. */
+ rspamd_config_add_symbol_group(cfg, name, group);
+ }
+ }
+
+ /* Remove table from stack */
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_add_condition(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+ gint condref;
+
+ if (cfg && sym && lua_type(L, 3) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 3);
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ ret = rspamd_symcache_add_condition_delayed(cfg->cache, sym, L,
+ condref);
+
+ if (!ret) {
+ luaL_unref(L, LUA_REGISTRYINDEX, condref);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static gint
+lua_config_set_peak_cb(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gint condref;
+
+ if (cfg && lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ condref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_symcache_set_peak_callback(cfg->cache,
+ condref);
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_enable_symbol(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *sym = luaL_checkstring(L, 2);
+
+ if (!sym || !cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ rspamd_symcache_enable_symbol_static(cfg->cache, sym);
+
+ return 0;
+}
+
+static gint
+lua_config_disable_symbol(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *sym = luaL_checkstring(L, 2);
+
+ if (!sym || !cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ rspamd_symcache_disable_symbol_static(cfg->cache, sym);
+
+ return 0;
+}
+
+static gint
+lua_config_register_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_regexp *re = NULL;
+ rspamd_regexp_t *cache_re;
+ const gchar *type_str = NULL, *header_str = NULL;
+ gsize header_len = 0;
+ GError *err = NULL;
+ enum rspamd_re_type type = RSPAMD_RE_BODY;
+ gboolean pcre_only = FALSE;
+
+ /*
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `pcre_only`: allow merely pcre for this regexp
+ */
+ if (cfg != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*re=U{regexp};*type=S;header=S;pcre_only=B",
+ &re, &type_str, &header_str, &pcre_only)) {
+ msg_err_config("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+ }
+ else {
+ type = rspamd_re_cache_type_from_string(type_str);
+
+ if ((type == RSPAMD_RE_HEADER ||
+ type == RSPAMD_RE_RAWHEADER ||
+ type == RSPAMD_RE_MIMEHEADER) &&
+ header_str == NULL) {
+ msg_err_config(
+ "header argument is mandatory for header/rawheader regexps");
+ }
+ else {
+ if (pcre_only) {
+ rspamd_regexp_set_flags(re->re,
+ rspamd_regexp_get_flags(re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+
+ if (header_str != NULL) {
+ /* Include the last \0 */
+ header_len = strlen(header_str) + 1;
+ }
+
+ cache_re = rspamd_re_cache_add(cfg->re_cache, re->re, type,
+ (gpointer) header_str, header_len, -1);
+
+ /*
+ * XXX: here are dragons!
+ * Actually, lua regexp contains internal rspamd_regexp_t
+ * and it owns it.
+ * However, after this operation we have some OTHER regexp,
+ * which we really would like to use.
+ * So we do the following:
+ * 1) Remove old re and unref it
+ * 2) Replace the internal re with cached one
+ * 3) Increase its refcount to share ownership between cache and
+ * lua object
+ */
+ if (cache_re != re->re) {
+ rspamd_regexp_unref(re->re);
+ re->re = rspamd_regexp_ref(cache_re);
+
+ if (pcre_only) {
+ rspamd_regexp_set_flags(re->re,
+ rspamd_regexp_get_flags(re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_replace_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_regexp *old_re = NULL, *new_re = NULL;
+ gboolean pcre_only = FALSE;
+ GError *err = NULL;
+
+ if (cfg != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*old_re=U{regexp};*new_re=U{regexp};pcre_only=B",
+ &old_re, &new_re, &pcre_only)) {
+ gint ret = luaL_error(L, "cannot get parameters list: %s",
+ err ? err->message : "invalid arguments");
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+ else {
+
+ if (pcre_only) {
+ rspamd_regexp_set_flags(new_re->re,
+ rspamd_regexp_get_flags(new_re->re) | RSPAMD_REGEXP_FLAG_PCRE_ONLY);
+ }
+
+ rspamd_re_cache_replace(cfg->re_cache, old_re->re, new_re->re);
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_worker_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *worker_type = luaL_checkstring(L, 2), *wtype;
+ struct rspamd_worker_conf *cf;
+ GList *cur;
+ struct rspamd_worker_lua_script *sc;
+ gboolean found = FALSE;
+
+ if (cfg == NULL || worker_type == NULL || lua_type(L, 3) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ for (cur = g_list_first(cfg->workers); cur != NULL; cur = g_list_next(cur)) {
+ cf = cur->data;
+ wtype = g_quark_to_string(cf->type);
+
+ if (g_ascii_strcasecmp(wtype, worker_type) == 0) {
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 3);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cf->scripts, sc);
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+
+ return 1;
+}
+
+static gint
+lua_config_add_on_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cfg->on_load_scripts, sc);
+
+ return 0;
+}
+
+static inline int
+rspamd_post_init_sc_sort(const struct rspamd_config_cfg_lua_script *pra,
+ const struct rspamd_config_cfg_lua_script *prb)
+{
+ /* Inverse sort */
+ return prb->priority - pra->priority;
+}
+
+static gint
+lua_config_add_post_init(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+ guint priority = 0;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, 3);
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 200) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ sc->priority = priority;
+ sc->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ DL_APPEND(cfg->post_init_scripts, sc);
+ DL_SORT(cfg->post_init_scripts, rspamd_post_init_sc_sort);
+
+ return 0;
+}
+
+static gint
+lua_config_add_config_unload(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ sc->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ DL_APPEND(cfg->config_unload_scripts, sc);
+
+ return 0;
+}
+
+
+static void lua_periodic_callback_finish(struct thread_entry *thread, int ret);
+static void lua_periodic_callback_error(struct thread_entry *thread, int ret, const char *msg);
+
+struct rspamd_lua_periodic {
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ gchar *lua_src_pos;
+ lua_State *L;
+ gdouble timeout;
+ ev_timer ev;
+ gint cbref;
+ gboolean need_jitter;
+ ref_entry_t ref;
+};
+
+static void
+lua_periodic_dtor(struct rspamd_lua_periodic *periodic)
+{
+ luaL_unref(periodic->L, LUA_REGISTRYINDEX, periodic->cbref);
+ ev_timer_stop(periodic->event_loop, &periodic->ev);
+}
+
+static void
+lua_periodic_fin(gpointer p)
+{
+ struct rspamd_lua_periodic *periodic = (struct rspamd_lua_periodic *) p;
+
+ REF_RELEASE(periodic);
+}
+
+static void
+lua_periodic_callback(struct ev_loop *loop, ev_timer *w, int revents)
+{
+ struct rspamd_lua_periodic *periodic = (struct rspamd_lua_periodic *) w->data;
+ struct rspamd_config **pcfg, *cfg;
+ struct ev_loop **pev_base;
+ struct thread_entry *thread;
+ lua_State *L;
+
+ REF_RETAIN(periodic);
+ thread = lua_thread_pool_get_for_config(periodic->cfg);
+ thread->cd = periodic;
+ thread->finish_callback = lua_periodic_callback_finish;
+ thread->error_callback = lua_periodic_callback_error;
+
+ L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, periodic->cbref);
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ cfg = periodic->cfg;
+ *pcfg = cfg;
+ pev_base = lua_newuserdata(L, sizeof(*pev_base));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pev_base = periodic->event_loop;
+ lua_pushnumber(L, ev_now(periodic->event_loop));
+
+ lua_thread_call(thread, 3);
+}
+
+static void
+lua_periodic_callback_finish(struct thread_entry *thread, int ret)
+{
+ lua_State *L;
+ struct rspamd_lua_periodic *periodic = thread->cd;
+ gboolean plan_more = FALSE;
+ gdouble timeout = 0.0;
+
+ L = thread->lua_state;
+
+ ev_now_update(periodic->event_loop);
+
+ if (ret == 0) {
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ plan_more = lua_toboolean(L, -1);
+ timeout = periodic->timeout;
+ }
+ else if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ plan_more = timeout > 0 ? TRUE : FALSE;
+ }
+
+ lua_pop(L, 1); /* Return value */
+ }
+
+ if (periodic->cfg->cur_worker) {
+ if (periodic->cfg->cur_worker->state != rspamd_worker_state_running) {
+ /* We are terminating, no more periodics */
+ plan_more = FALSE;
+ }
+ }
+
+ if (plan_more) {
+ if (periodic->need_jitter) {
+ timeout = rspamd_time_jitter(timeout, 0.0);
+ }
+
+ periodic->ev.repeat = timeout;
+ ev_timer_again(periodic->event_loop, &periodic->ev);
+ }
+ else {
+ ev_timer_stop(periodic->event_loop, &periodic->ev);
+ }
+
+ REF_RELEASE(periodic);
+}
+
+static void
+lua_periodic_callback_error(struct thread_entry *thread, int ret, const char *msg)
+{
+ struct rspamd_config *cfg;
+ struct rspamd_lua_periodic *periodic = thread->cd;
+ cfg = periodic->cfg;
+
+ msg_err_config("call to periodic script (registered at %s) failed: %s",
+ periodic->lua_src_pos, msg);
+
+ lua_periodic_callback_finish(thread, ret);
+}
+
+
+static gint
+lua_config_add_periodic(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct ev_loop *ev_base = lua_check_ev_base(L, 2);
+ gdouble timeout = lua_tonumber(L, 3);
+ struct rspamd_lua_periodic *periodic;
+ gboolean need_jitter = FALSE;
+ lua_Debug d;
+ gchar tmp[256], *p;
+
+ if (cfg == NULL || timeout < 0 || lua_type(L, 4) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 5) == LUA_TBOOLEAN) {
+ need_jitter = lua_toboolean(L, 5);
+ }
+
+ if (lua_getstack(L, 1, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 20) {
+ rspamd_snprintf(tmp, sizeof(tmp), "%10s...]:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(tmp, sizeof(tmp), "%s:%d", p,
+ d.currentline);
+ }
+ }
+
+ periodic = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*periodic));
+ periodic->timeout = timeout;
+ periodic->L = L;
+ periodic->cfg = cfg;
+ periodic->event_loop = ev_base;
+ periodic->need_jitter = need_jitter;
+ periodic->lua_src_pos = rspamd_mempool_strdup(cfg->cfg_pool, tmp);
+ lua_pushvalue(L, 4);
+ periodic->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (need_jitter) {
+ timeout = rspamd_time_jitter(timeout, 0.0);
+ }
+
+ ev_timer_init(&periodic->ev, lua_periodic_callback, timeout, 0.0);
+ periodic->ev.data = periodic;
+ ev_timer_start(ev_base, &periodic->ev);
+ REF_INIT_RETAIN(periodic, lua_periodic_dtor);
+
+ rspamd_mempool_add_destructor(cfg->cfg_pool, lua_periodic_fin,
+ periodic);
+
+ return 0;
+}
+
+static gint
+lua_config_get_symbols_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ guint res = 0;
+
+ if (cfg != NULL) {
+ res = rspamd_symcache_stats_symbols_count(cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, res);
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbols_cksum(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ guint64 res = 0, *pres;
+
+ if (cfg != NULL) {
+ res = rspamd_symcache_get_cksum(cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ pres = lua_newuserdata(L, sizeof(res));
+ *pres = res;
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbols_counters(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ ucl_object_t *counters;
+
+ if (cfg != NULL) {
+ counters = rspamd_symcache_counters(cfg->cache);
+ ucl_object_push_lua(L, counters, true);
+ ucl_object_unref(counters);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_metric_symbols_cbdata {
+ lua_State *L;
+ struct rspamd_config *cfg;
+ bool is_table;
+};
+
+static void
+lua_metric_symbol_inserter(gpointer k, gpointer v, gpointer ud)
+{
+ struct lua_metric_symbols_cbdata *cbd = (struct lua_metric_symbols_cbdata *) ud;
+ lua_State *L;
+ const gchar *sym = k;
+ struct rspamd_symbol *s = (struct rspamd_symbol *) v;
+ struct rspamd_symbols_group *gr;
+ gint i;
+
+ L = cbd->L;
+
+ if (cbd->is_table) {
+ lua_pushstring(L, sym); /* Symbol name */
+ }
+
+ lua_createtable(L, 0, 6);
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, s->score);
+ lua_settable(L, -3);
+ lua_pushstring(L, "description");
+ lua_pushstring(L, s->description);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 3);
+
+ if (s->flags & RSPAMD_SYMBOL_FLAG_IGNORE_METRIC) {
+ lua_pushstring(L, "ignore");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM) {
+ lua_pushstring(L, "oneparam");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_UNGROUPED) {
+ lua_pushstring(L, "ungrouped");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (s->flags & RSPAMD_SYMBOL_FLAG_DISABLED) {
+ lua_pushstring(L, "disabled");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+
+ if (s->cache_item) {
+ guint sflags = rspamd_symcache_get_symbol_flags(cbd->cfg->cache, sym);
+
+ lua_push_symbol_flags(L, sflags, LUA_SYMOPT_FLAG_USE_MAP);
+
+ guint nids;
+ const guint *allowed_ids = rspamd_symcache_get_allowed_settings_ids(cbd->cfg->cache,
+ sym, &nids);
+
+ if (allowed_ids && nids > 0) {
+ lua_createtable(L, nids, 0);
+
+ for (i = 0; i < nids; i++) {
+ lua_pushinteger(L, allowed_ids[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_setfield(L, -2, "allowed_ids");
+ }
+
+ const guint *forbidden_ids = rspamd_symcache_get_forbidden_settings_ids(
+ cbd->cfg->cache,
+ sym, &nids);
+
+ if (forbidden_ids && nids > 0) {
+ lua_createtable(L, nids, 0);
+
+ for (i = 0; i < nids; i++) {
+ lua_pushinteger(L, forbidden_ids[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_setfield(L, -2, "forbidden_ids");
+ }
+ }
+
+ lua_settable(L, -3); /* Flags -> flags_table */
+
+ lua_pushstring(L, "nshots");
+ lua_pushinteger(L, s->nshots);
+ lua_settable(L, -3);
+
+ if (s->gr) {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, s->gr->name);
+ lua_settable(L, -3);
+ }
+
+ if (s->groups && s->groups->len > 0) {
+ lua_pushstring(L, "groups");
+ lua_createtable(L, s->groups->len, 0);
+
+ PTR_ARRAY_FOREACH(s->groups, i, gr)
+ {
+ lua_pushstring(L, gr->name);
+ lua_rawseti(L, -2, i + 1); /* Groups[i + 1] = group_name */
+ }
+
+ lua_settable(L, -3); /* Groups -> groups_table */
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_setfield(L, -2, "groups");
+ }
+
+ if (cbd->is_table) {
+ lua_settable(L, -3); /* Symname -> table */
+ }
+}
+
+static gint
+lua_config_get_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ struct lua_metric_symbols_cbdata cbd;
+
+ cbd.L = L;
+ cbd.cfg = cfg;
+ cbd.is_table = true;
+
+ lua_createtable(L, 0, g_hash_table_size(cfg->symbols));
+ g_hash_table_foreach(cfg->symbols,
+ lua_metric_symbol_inserter,
+ &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && sym_name != NULL) {
+ struct lua_metric_symbols_cbdata cbd;
+ struct rspamd_symbol *s = g_hash_table_lookup(cfg->symbols, sym_name);
+
+ if (s) {
+ cbd.L = L;
+ cbd.cfg = cfg;
+ cbd.is_table = false;
+ lua_metric_symbol_inserter((void *) sym_name, s, &cbd);
+ }
+ else {
+ /* No config for a symbol */
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ struct rspamd_abstract_callback_data *abs_cbdata;
+ struct lua_callback_data *cbd;
+
+ if (cfg != NULL && sym != NULL) {
+ abs_cbdata = rspamd_symcache_get_cbdata(cfg->cache, sym);
+
+ if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) {
+ lua_pushnil(L);
+ }
+ else {
+ cbd = (struct lua_callback_data *) abs_cbdata;
+
+ if (cbd->cb_is_ref) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->callback.ref);
+ }
+ else {
+ lua_getglobal(L, cbd->callback.name);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_set_symbol_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ struct rspamd_abstract_callback_data *abs_cbdata;
+ struct lua_callback_data *cbd;
+
+ if (cfg != NULL && sym != NULL && lua_type(L, 3) == LUA_TFUNCTION) {
+ abs_cbdata = rspamd_symcache_get_cbdata(cfg->cache, sym);
+
+ if (abs_cbdata == NULL || abs_cbdata->magic != rspamd_lua_callback_magic) {
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ cbd = (struct lua_callback_data *) abs_cbdata;
+
+ if (cbd->cb_is_ref) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd->callback.ref);
+ }
+ else {
+ cbd->cb_is_ref = TRUE;
+ }
+
+ lua_pushvalue(L, 3);
+ cbd->callback.ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ lua_pushboolean(L, TRUE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2);
+ gdouble freq, stddev, tm;
+ guint hits;
+
+ if (cfg != NULL && sym != NULL) {
+ if (!rspamd_symcache_stat_symbol(cfg->cache, sym, &freq,
+ &stddev, &tm, &hits)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, 0, 4);
+ lua_pushstring(L, "frequency");
+ lua_pushnumber(L, freq);
+ lua_settable(L, -3);
+ lua_pushstring(L, "sttdev");
+ lua_pushnumber(L, stddev);
+ lua_settable(L, -3);
+ lua_pushstring(L, "time");
+ lua_pushnumber(L, tm);
+ lua_settable(L, -3);
+ lua_pushstring(L, "hits");
+ lua_pushinteger(L, hits);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_symbol_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *sym = luaL_checkstring(L, 2), *parent;
+
+ if (cfg != NULL && sym != NULL) {
+ parent = rspamd_symcache_get_parent(cfg->cache, sym);
+
+ if (parent) {
+ lua_pushstring(L, parent);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_group_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *gr_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && gr_name != NULL) {
+ struct rspamd_symbols_group *group;
+
+ group = g_hash_table_lookup(cfg->groups, gr_name);
+
+ if (group == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ guint i = 1;
+ gpointer k, v;
+ GHashTableIter it;
+
+ lua_createtable(L, g_hash_table_size(group->symbols), 0);
+ g_hash_table_iter_init(&it, group->symbols);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ lua_pushstring(L, k);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_groups(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ gboolean need_private;
+ struct rspamd_symbols_group *gr;
+ GHashTableIter it;
+ gpointer k, v;
+
+ if (cfg) {
+ if (lua_isboolean(L, 2)) {
+ need_private = lua_toboolean(L, 2);
+ }
+ else {
+ need_private = !(cfg->public_groups_only);
+ }
+
+ lua_createtable(L, 0, g_hash_table_size(cfg->groups));
+ g_hash_table_iter_init(&it, cfg->groups);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ gr = (struct rspamd_symbols_group *) v;
+
+ if (need_private || (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ lua_createtable(L, 0, 4);
+
+ lua_pushstring(L, gr->description);
+ lua_setfield(L, -2, "description");
+ lua_pushnumber(L, gr->max_score);
+ lua_setfield(L, -2, "max_score");
+ lua_pushboolean(L, (gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC) != 0);
+ lua_setfield(L, -2, "is_public");
+ /* TODO: maybe push symbols as well */
+
+ /* Parent table indexed by group name */
+ lua_setfield(L, -2, gr->name);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_register_finish_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_config_cfg_lua_script *sc;
+
+ if (cfg != NULL && lua_type(L, 2) == LUA_TFUNCTION) {
+ sc = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*sc));
+ lua_pushvalue(L, 2);
+ sc->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ DL_APPEND(cfg->on_term_scripts, sc);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static inline bool
+rspamd_lua_config_check_settings_symbols_object(const ucl_object_t *obj)
+{
+ if (obj == NULL) {
+ /* Semantically valid */
+ return true;
+ }
+
+ if (ucl_object_type(obj) == UCL_OBJECT) {
+ /* Key-value mapping - should be okay */
+ return true;
+ }
+
+ if (ucl_object_type(obj) == UCL_ARRAY) {
+ /* Okay if empty */
+ if (obj->len == 0) {
+ return true;
+ }
+ }
+
+ /* Everything else not okay */
+ return false;
+}
+
+static gint
+lua_config_register_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *settings_name = luaL_checkstring(L, 2);
+
+ if (cfg != NULL && settings_name) {
+ ucl_object_t *sym_enabled, *sym_disabled;
+ enum rspamd_config_settings_policy policy = RSPAMD_SETTINGS_POLICY_DEFAULT;
+
+ sym_enabled = ucl_object_lua_import(L, 3);
+
+ if (!rspamd_lua_config_check_settings_symbols_object(sym_enabled)) {
+ ucl_object_unref(sym_enabled);
+
+ return luaL_error(L, "invalid symbols enabled");
+ }
+
+ sym_disabled = ucl_object_lua_import(L, 4);
+
+ if (!rspamd_lua_config_check_settings_symbols_object(sym_disabled)) {
+ ucl_object_unref(sym_enabled);
+ ucl_object_unref(sym_disabled);
+
+ return luaL_error(L, "invalid symbols enabled");
+ }
+
+ /* Check policy */
+ if (lua_isstring(L, 5)) {
+ const gchar *policy_str = lua_tostring(L, 5);
+
+ if (strcmp(policy_str, "default") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_DEFAULT;
+ }
+ else if (strcmp(policy_str, "implicit_allow") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW;
+ }
+ else if (strcmp(policy_str, "implicit_deny") == 0) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_DENY;
+ }
+ else {
+ return luaL_error(L, "invalid settings policy: %s", policy_str);
+ }
+ }
+ else {
+ /* Apply heuristic */
+ if (!sym_enabled) {
+ policy = RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW;
+ }
+ }
+
+ rspamd_config_register_settings_id(cfg, settings_name, sym_enabled,
+ sym_disabled, policy);
+
+ if (sym_enabled) {
+ ucl_object_unref(sym_enabled);
+ }
+
+ if (sym_disabled) {
+ ucl_object_unref(sym_disabled);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_monitored(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_monitored *m, **pm;
+ const gchar *url, *type;
+ ucl_object_t *params = NULL;
+
+ url = lua_tostring(L, 2);
+ type = lua_tostring(L, 3);
+
+ if (cfg != NULL && url != NULL && type != NULL) {
+ if (g_ascii_strcasecmp(type, "dns") == 0) {
+ lua_Debug ar;
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ params = ucl_object_lua_import(L, 4);
+ }
+
+ /* Get lua line and source */
+ lua_getstack(L, 1, &ar);
+ lua_getinfo(L, "nSl", &ar);
+
+ m = rspamd_monitored_create_(cfg->monitored_ctx, url,
+ RSPAMD_MONITORED_DNS, RSPAMD_MONITORED_DEFAULT,
+ params, ar.short_src);
+
+ if (m) {
+ pm = lua_newuserdata(L, sizeof(*pm));
+ *pm = m;
+ rspamd_lua_setclass(L, "rspamd{monitored}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (params) {
+ ucl_object_unref(params);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid monitored type: %s", type);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_add_doc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg;
+ const gchar *path = NULL, *option, *doc_string;
+ const gchar *type_str = NULL, *default_value = NULL;
+ ucl_type_t type = UCL_NULL;
+ gboolean required = FALSE;
+ GError *err = NULL;
+
+ cfg = lua_check_config(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ path = luaL_checkstring(L, 2);
+ }
+
+ option = luaL_checkstring(L, 3);
+ doc_string = luaL_checkstring(L, 4);
+
+ if (cfg && option && doc_string) {
+ if (lua_type(L, 5) == LUA_TTABLE) {
+ if (!rspamd_lua_parse_table_arguments(L, 5, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "type=S;default=S;required=B",
+ &type_str, &default_value, &required)) {
+ msg_err_config("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ if (type_str) {
+ if (!ucl_object_string_to_type(type_str, &type)) {
+ msg_err_config("invalid type: %s", type_str);
+ }
+ }
+ }
+ }
+
+ rspamd_rcl_add_doc_by_path(cfg, path, doc_string, option,
+ type, NULL, 0, default_value, required);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_add_example(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg;
+ const gchar *path = NULL, *option, *doc_string, *example;
+ gsize example_len;
+
+ cfg = lua_check_config(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ path = luaL_checkstring(L, 2);
+ }
+
+ option = luaL_checkstring(L, 3);
+ doc_string = luaL_checkstring(L, 4);
+ example = luaL_checklstring(L, 5, &example_len);
+
+ if (cfg && option && doc_string && example) {
+
+ rspamd_rcl_add_doc_by_example(cfg, path, doc_string, option,
+ example, example_len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_get_cpu_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_cryptobox_library_ctx *crypto_ctx;
+
+ if (cfg != NULL) {
+ crypto_ctx = cfg->libs_ctx->crypto_ctx;
+ lua_newtable(L);
+
+ if (crypto_ctx->cpu_config & CPUID_SSSE3) {
+ lua_pushstring(L, "ssse3");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE41) {
+ lua_pushstring(L, "sse41");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE42) {
+ lua_pushstring(L, "sse42");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE2) {
+ lua_pushstring(L, "sse2");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_SSE3) {
+ lua_pushstring(L, "sse3");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_AVX) {
+ lua_pushstring(L, "avx");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (crypto_ctx->cpu_config & CPUID_AVX2) {
+ lua_pushstring(L, "avx2");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_has_torch(lua_State *L)
+{
+ msg_warn("use of the obsoleted `has_torch` function");
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_config_experimental_enabled(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushboolean(L, cfg->enable_experimental);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_lua_include_trace_cbdata {
+ lua_State *L;
+ gint cbref;
+};
+
+static void
+lua_include_trace_cb(struct ucl_parser *parser,
+ const ucl_object_t *parent,
+ const ucl_object_t *args,
+ const char *path,
+ size_t pathlen,
+ void *user_data)
+{
+ struct rspamd_lua_include_trace_cbdata *cbdata =
+ (struct rspamd_lua_include_trace_cbdata *) user_data;
+ gint err_idx;
+ lua_State *L;
+
+ L = cbdata->L;
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->cbref);
+ /* Current filename */
+ lua_pushstring(L, ucl_parser_get_cur_file(parser));
+ /* Included filename */
+ lua_pushlstring(L, path, pathlen);
+ /* Params */
+ if (args) {
+ ucl_object_push_lua(L, args, true);
+ }
+ else {
+ lua_newtable(L);
+ }
+ /* Parent */
+ if (parent) {
+ lua_pushstring(L, ucl_object_key(parent));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (lua_pcall(L, 4, 0, err_idx) != 0) {
+ msg_err("lua call to local include trace failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+#define LUA_TABLE_TO_HASH(htb, idx) \
+ do { \
+ lua_pushstring(L, (idx)); \
+ lua_gettable(L, -2); \
+ if (lua_isstring(L, -1)) { \
+ g_hash_table_insert((htb), (idx), g_strdup(lua_tostring(L, -1))); \
+ } \
+ lua_pop(L, 1); \
+ } while (0)
+
+static gint
+lua_config_load_ucl(lua_State *L)
+{
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *filename;
+ GHashTable *paths = g_hash_table_new_full(rspamd_str_hash, rspamd_str_equal,
+ NULL, g_free);
+ GError *err = NULL;
+
+ if (cfg) {
+ if (lua_isstring(L, 2)) {
+ filename = lua_tostring(L, 2);
+ }
+ else {
+ filename = RSPAMD_CONFDIR "/rspamd.conf";
+ }
+
+ /* Convert rspamd_paths */
+ lua_getglobal(L, "rspamd_paths");
+
+ if (lua_istable(L, -1)) {
+ LUA_TABLE_TO_HASH(paths, RSPAMD_CONFDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LOCAL_CONFDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RUNDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_DBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LOGDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_WWWDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PLUGINSDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_RULESDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_LUALIBDIR_INDEX);
+ LUA_TABLE_TO_HASH(paths, RSPAMD_PREFIX_INDEX);
+ }
+
+ lua_pop(L, 1);
+
+ if (lua_isfunction(L, 3)) {
+ struct rspamd_lua_include_trace_cbdata cbd;
+
+ lua_pushvalue(L, 3);
+ cbd.cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cbd.L = L;
+
+ if (!rspamd_config_parse_ucl(cfg, filename, paths,
+ lua_include_trace_cb, &cbd, lua_toboolean(L, 4), &err)) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd.cbref);
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(paths);
+
+ return 2;
+ }
+
+ luaL_unref(L, LUA_REGISTRYINDEX, cbd.cbref);
+ }
+ else {
+ if (!rspamd_config_parse_ucl(cfg, filename, paths, NULL, NULL,
+ lua_toboolean(L, 3), &err)) {
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(paths);
+
+ return 2;
+ }
+ }
+
+ rspamd_rcl_maybe_apply_lua_transform(cfg);
+ rspamd_config_calculate_cksum(cfg);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ g_hash_table_unref(paths);
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+#undef IDX_TO_HASH
+
+static gint
+lua_config_parse_rcl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ GHashTable *excluded = g_hash_table_new_full(rspamd_str_hash, rspamd_str_equal,
+ g_free, NULL);
+ GError *err = NULL;
+ struct rspamd_rcl_sections_map *top;
+
+ if (cfg) {
+ if (lua_istable(L, 2)) {
+ lua_pushvalue(L, 2);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ g_hash_table_insert(excluded, g_strdup(lua_tostring(L, -1)),
+ GINT_TO_POINTER(-1));
+ }
+
+ lua_pop(L, 1);
+ }
+
+ top = rspamd_rcl_config_init(cfg, excluded);
+
+ if (!rspamd_rcl_parse(top, cfg, cfg, cfg->cfg_pool, cfg->cfg_ucl_obj, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushfstring(L, "failed to load config: %s", err->message);
+ g_error_free(err);
+ g_hash_table_unref(excluded);
+ rspamd_rcl_sections_free(top);
+
+ return 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ g_hash_table_unref(excluded);
+ rspamd_rcl_sections_free(top);
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+static gint
+lua_config_init_modules(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ rspamd_lua_post_load_config(cfg);
+ lua_pushboolean(L, rspamd_init_filters(cfg, false, false));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_init_subsystem(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *subsystem = luaL_checkstring(L, 2);
+ gchar **parts;
+ guint nparts, i;
+
+ if (cfg != NULL && subsystem != NULL) {
+ parts = g_strsplit_set(subsystem, ";,", -1);
+ nparts = g_strv_length(parts);
+
+ for (i = 0; i < nparts; i++) {
+ if (strcmp(parts[i], "filters") == 0) {
+ rspamd_lua_post_load_config(cfg);
+ rspamd_init_filters(cfg, false, false);
+ }
+ else if (strcmp(parts[i], "langdet") == 0) {
+ if (!cfg->lang_det) {
+ cfg->lang_det = rspamd_language_detector_init(cfg);
+ rspamd_mempool_add_destructor(cfg->cfg_pool,
+ (rspamd_mempool_destruct_t) rspamd_language_detector_unref,
+ cfg->lang_det);
+ }
+ }
+ else if (strcmp(parts[i], "stat") == 0) {
+ rspamd_stat_init(cfg, NULL);
+ }
+ else if (strcmp(parts[i], "dns") == 0) {
+ struct ev_loop *ev_base = lua_check_ev_base(L, 3);
+
+ if (ev_base) {
+ cfg->dns_resolver = rspamd_dns_resolver_init(rspamd_log_default_logger(),
+ ev_base,
+ cfg);
+ }
+ else {
+ g_strfreev(parts);
+
+ return luaL_error(L, "no event base specified");
+ }
+ }
+ else if (strcmp(parts[i], "symcache") == 0) {
+ rspamd_symcache_init(cfg->cache);
+ }
+ else {
+ int ret = luaL_error(L, "invalid param: %s", parts[i]);
+ g_strfreev(parts);
+
+ return ret;
+ }
+ }
+
+ g_strfreev(parts);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_config_register_re_selector(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ const gchar *selector_str = luaL_checkstring(L, 3);
+ const gchar *delimiter = "";
+ bool flatten = false;
+ gint top = lua_gettop(L);
+ bool res = false;
+
+ if (cfg && name && selector_str) {
+ if (lua_gettop(L) >= 4) {
+ delimiter = luaL_checkstring(L, 4);
+
+ if (lua_isboolean(L, 5)) {
+ flatten = lua_toboolean(L, 5);
+ }
+ }
+
+ if (luaL_dostring(L, "return require \"lua_selectors\"") != 0) {
+ msg_warn_config("cannot require lua_selectors: %s",
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TTABLE) {
+ msg_warn_config("lua selectors must return "
+ "table and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ lua_pushstring(L, "create_selector_closure");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_warn_config("create_selector_closure must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ gint err_idx, ret;
+ struct rspamd_config **pcfg;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ /* Push function */
+ lua_pushvalue(L, -2);
+
+ pcfg = lua_newuserdata(L, sizeof(*pcfg));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ lua_pushstring(L, selector_str);
+ lua_pushstring(L, delimiter);
+ lua_pushboolean(L, flatten);
+
+ if ((ret = lua_pcall(L, 4, 1, err_idx)) != 0) {
+ msg_err_config("call to create_selector_closure lua "
+ "script failed (%d): %s",
+ ret,
+ lua_tostring(L, -1));
+ }
+ else {
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ msg_warn_config("create_selector_closure "
+ "invocation must return "
+ "function and not %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ else {
+ ret = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_re_cache_add_selector(cfg->re_cache,
+ name, ret);
+ res = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_settop(L, top);
+ lua_pushboolean(L, res);
+
+ if (res) {
+ msg_info_config("registered regexp selector %s", name);
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_tld_path(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushstring(L, cfg->tld_file);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_dns_max_requests(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushinteger(L, cfg->dns_max_requests);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_config_get_dns_timeout(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+
+ if (cfg != NULL) {
+ lua_pushnumber(L, cfg->dns_timeout);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_alive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushboolean(L, rspamd_monitored_alive(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_offline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_offline_time(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_total_offline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_total_offline_time(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_monitored_latency(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_monitored *m = lua_check_monitored(L, 1);
+
+ if (m) {
+ lua_pushnumber(L, rspamd_monitored_latency(m));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void luaopen_config(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{config}", configlib_m);
+
+ lua_pop(L, 1);
+
+ rspamd_lua_new_class(L, "rspamd{monitored}", monitoredlib_m);
+
+ lua_pop(L, 1);
+}
+
+void lua_call_finish_script(struct rspamd_config_cfg_lua_script *sc,
+ struct rspamd_task *task)
+{
+
+ struct rspamd_task **ptask;
+ struct thread_entry *thread;
+
+ thread = lua_thread_pool_get_for_task(task);
+ thread->task = task;
+
+ lua_State *L = thread->lua_state;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, sc->cbref);
+
+ ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+
+ lua_thread_call(thread, 1);
+}
diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c
new file mode 100644
index 0000000..70c6f0a
--- /dev/null
+++ b/src/lua/lua_cryptobox.c
@@ -0,0 +1,3065 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/***
+ * @module rspamd_cryptobox
+ * Rspamd cryptobox is a module that operates with digital signatures and
+ * hashes.
+ * @example
+ * local hash = require "rspamd_cryptobox_hash"
+ *
+ * local h = hash.create()
+ * h:update('hello world')
+ * print(h:hex())
+ */
+
+
+#include "lua_common.h"
+#include "libcryptobox/cryptobox.h"
+#include "libcryptobox/keypair.h"
+#include "libcryptobox/keypair_private.h"
+#include "unix-std.h"
+#include "contrib/libottery/ottery.h"
+#include "libutil/ref.h"
+
+#include <stdalign.h>
+#include <openssl/hmac.h>
+
+
+enum lua_cryptobox_hash_type {
+ LUA_CRYPTOBOX_HASH_BLAKE2 = 0,
+ LUA_CRYPTOBOX_HASH_SSL,
+ LUA_CRYPTOBOX_HASH_HMAC,
+ LUA_CRYPTOBOX_HASH_XXHASH64,
+ LUA_CRYPTOBOX_HASH_XXHASH32,
+ LUA_CRYPTOBOX_HASH_XXHASH3,
+ LUA_CRYPTOBOX_HASH_MUM,
+ LUA_CRYPTOBOX_HASH_T1HA,
+};
+
+struct rspamd_lua_cryptobox_hash {
+ union {
+ rspamd_cryptobox_hash_state_t *h;
+ EVP_MD_CTX *c;
+ HMAC_CTX *hmac_c;
+ rspamd_cryptobox_fast_hash_state_t *fh;
+ } content;
+
+ unsigned char out[rspamd_cryptobox_HASHBYTES];
+
+ uint8_t type;
+ uint8_t out_len;
+ uint8_t is_finished;
+
+ ref_entry_t ref;
+};
+
+LUA_FUNCTION_DEF(cryptobox_pubkey, load);
+LUA_FUNCTION_DEF(cryptobox_pubkey, create);
+LUA_FUNCTION_DEF(cryptobox_pubkey, gc);
+LUA_FUNCTION_DEF(cryptobox_keypair, load);
+LUA_FUNCTION_DEF(cryptobox_keypair, create);
+LUA_FUNCTION_DEF(cryptobox_keypair, gc);
+LUA_FUNCTION_DEF(cryptobox_keypair, totable);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_type);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_alg);
+LUA_FUNCTION_DEF(cryptobox_keypair, get_pk);
+LUA_FUNCTION_DEF(cryptobox_signature, create);
+LUA_FUNCTION_DEF(cryptobox_signature, load);
+LUA_FUNCTION_DEF(cryptobox_signature, save);
+LUA_FUNCTION_DEF(cryptobox_signature, gc);
+LUA_FUNCTION_DEF(cryptobox_signature, hex);
+LUA_FUNCTION_DEF(cryptobox_signature, base32);
+LUA_FUNCTION_DEF(cryptobox_signature, base64);
+LUA_FUNCTION_DEF(cryptobox_signature, bin);
+LUA_FUNCTION_DEF(cryptobox_hash, create);
+LUA_FUNCTION_DEF(cryptobox_hash, create_specific);
+LUA_FUNCTION_DEF(cryptobox_hash, create_specific_keyed);
+LUA_FUNCTION_DEF(cryptobox_hash, create_keyed);
+LUA_FUNCTION_DEF(cryptobox_hash, update);
+LUA_FUNCTION_DEF(cryptobox_hash, reset);
+LUA_FUNCTION_DEF(cryptobox_hash, hex);
+LUA_FUNCTION_DEF(cryptobox_hash, base32);
+LUA_FUNCTION_DEF(cryptobox_hash, base64);
+LUA_FUNCTION_DEF(cryptobox_hash, bin);
+LUA_FUNCTION_DEF(cryptobox_hash, gc);
+LUA_FUNCTION_DEF(cryptobox, verify_memory);
+LUA_FUNCTION_DEF(cryptobox, verify_file);
+LUA_FUNCTION_DEF(cryptobox, sign_file);
+LUA_FUNCTION_DEF(cryptobox, sign_memory);
+LUA_FUNCTION_DEF(cryptobox, encrypt_memory);
+LUA_FUNCTION_DEF(cryptobox, encrypt_file);
+LUA_FUNCTION_DEF(cryptobox, decrypt_memory);
+LUA_FUNCTION_DEF(cryptobox, decrypt_file);
+LUA_FUNCTION_DEF(cryptobox, encrypt_cookie);
+LUA_FUNCTION_DEF(cryptobox, decrypt_cookie);
+LUA_FUNCTION_DEF(cryptobox, pbkdf);
+LUA_FUNCTION_DEF(cryptobox, gen_dkim_keypair);
+
+/* Secretbox API: uses libsodium secretbox and blake2b for key derivation */
+LUA_FUNCTION_DEF(cryptobox_secretbox, create);
+LUA_FUNCTION_DEF(cryptobox_secretbox, encrypt);
+LUA_FUNCTION_DEF(cryptobox_secretbox, decrypt);
+LUA_FUNCTION_DEF(cryptobox_secretbox, gc);
+
+static const struct luaL_reg cryptoboxlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox, verify_memory),
+ LUA_INTERFACE_DEF(cryptobox, verify_file),
+ LUA_INTERFACE_DEF(cryptobox, sign_memory),
+ LUA_INTERFACE_DEF(cryptobox, sign_file),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_memory),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_file),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_memory),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_file),
+ LUA_INTERFACE_DEF(cryptobox, encrypt_cookie),
+ LUA_INTERFACE_DEF(cryptobox, decrypt_cookie),
+ LUA_INTERFACE_DEF(cryptobox, pbkdf),
+ LUA_INTERFACE_DEF(cryptobox, gen_dkim_keypair),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxpubkeylib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_pubkey, load),
+ LUA_INTERFACE_DEF(cryptobox_pubkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxpubkeylib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_pubkey_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxkeypairlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_keypair, load),
+ LUA_INTERFACE_DEF(cryptobox_keypair, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxkeypairlib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"totable", lua_cryptobox_keypair_totable},
+ {"get_type", lua_cryptobox_keypair_get_type},
+ {"get_alg", lua_cryptobox_keypair_get_alg},
+ {"type", lua_cryptobox_keypair_get_type},
+ {"alg", lua_cryptobox_keypair_get_alg},
+ {"pk", lua_cryptobox_keypair_get_pk},
+ {"pubkey", lua_cryptobox_keypair_get_pk},
+ {"__gc", lua_cryptobox_keypair_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxsignlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_signature, load),
+ LUA_INTERFACE_DEF(cryptobox_signature, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxsignlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_signature, save),
+ LUA_INTERFACE_DEF(cryptobox_signature, hex),
+ LUA_INTERFACE_DEF(cryptobox_signature, base32),
+ LUA_INTERFACE_DEF(cryptobox_signature, base64),
+ LUA_INTERFACE_DEF(cryptobox_signature, bin),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_signature_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxhashlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_hash, create),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_keyed),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_specific),
+ LUA_INTERFACE_DEF(cryptobox_hash, create_specific_keyed),
+ {NULL, NULL}};
+
+static const struct luaL_reg cryptoboxhashlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_hash, update),
+ LUA_INTERFACE_DEF(cryptobox_hash, reset),
+ LUA_INTERFACE_DEF(cryptobox_hash, hex),
+ LUA_INTERFACE_DEF(cryptobox_hash, base32),
+ LUA_INTERFACE_DEF(cryptobox_hash, base64),
+ LUA_INTERFACE_DEF(cryptobox_hash, bin),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_cryptobox_hash_gc},
+ {NULL, NULL}};
+
+
+static const struct luaL_reg cryptoboxsecretboxlib_f[] = {
+ LUA_INTERFACE_DEF(cryptobox_secretbox, create),
+ {NULL, NULL},
+};
+
+static const struct luaL_reg cryptoboxsecretboxlib_m[] = {
+ LUA_INTERFACE_DEF(cryptobox_secretbox, encrypt),
+ LUA_INTERFACE_DEF(cryptobox_secretbox, decrypt),
+ {"__gc", lua_cryptobox_secretbox_gc},
+ {NULL, NULL},
+};
+
+struct rspamd_lua_cryptobox_secretbox {
+ guchar sk[crypto_secretbox_KEYBYTES];
+};
+
+static struct rspamd_cryptobox_pubkey *
+lua_check_cryptobox_pubkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_pubkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_pubkey' expected");
+ return ud ? *((struct rspamd_cryptobox_pubkey **) ud) : NULL;
+}
+
+static struct rspamd_cryptobox_keypair *
+lua_check_cryptobox_keypair(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_keypair}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_keypair' expected");
+ return ud ? *((struct rspamd_cryptobox_keypair **) ud) : NULL;
+}
+
+static rspamd_fstring_t *
+lua_check_cryptobox_sign(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_signature}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_signature' expected");
+ return ud ? *((rspamd_fstring_t **) ud) : NULL;
+}
+
+struct rspamd_lua_cryptobox_hash *
+lua_check_cryptobox_hash(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_hash}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_hash' expected");
+ return ud ? *((struct rspamd_lua_cryptobox_hash **) ud) : NULL;
+}
+
+static struct rspamd_lua_cryptobox_secretbox *
+lua_check_cryptobox_secretbox(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{cryptobox_secretbox}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'cryptobox_secretbox' expected");
+ return ud ? *((struct rspamd_lua_cryptobox_secretbox **) ud) : NULL;
+}
+
+/***
+ * @function rspamd_cryptobox_pubkey.load(file[, type[, alg]])
+ * Loads public key from base32 encoded file
+ * @param {string} file filename to load
+ * @param {string} type optional 'sign' or 'kex' for signing and encryption
+ * @param {string} alg optional 'default' or 'nist' for curve25519/nistp256 keys
+ * @return {cryptobox_pubkey} new public key
+ */
+static gint
+lua_cryptobox_pubkey_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = NULL, **ppkey;
+ const gchar *filename, *arg;
+ gint type = RSPAMD_KEYPAIR_SIGN;
+ gint alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ guchar *map;
+ gsize len;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ map = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (map == NULL) {
+ msg_err("cannot open pubkey from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* keypair type */
+ arg = lua_tostring(L, 2);
+
+ if (strcmp(arg, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(arg, "kex") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ }
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ /* algorithm */
+ arg = lua_tostring(L, 3);
+
+ if (strcmp(arg, "default") == 0 || strcmp(arg, "curve25519") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else if (strcmp(arg, "nist") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ }
+
+ pkey = rspamd_pubkey_from_base32(map, len, type, alg);
+
+ if (pkey == NULL) {
+ msg_err("cannot open pubkey from file: %s", filename);
+ munmap(map, len);
+ lua_pushnil(L);
+ }
+ else {
+ munmap(map, len);
+ ppkey = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ *ppkey = pkey;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+
+/***
+ * @function rspamd_cryptobox_pubkey.create(data[, type[, alg]])
+ * Loads public key from base32 encoded string
+ * @param {base32 string} base32 string with the key
+ * @param {string} type optional 'sign' or 'kex' for signing and encryption
+ * @param {string} alg optional 'default' or 'nist' for curve25519/nistp256 keys
+ * @return {cryptobox_pubkey} new public key
+ */
+static gint
+lua_cryptobox_pubkey_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = NULL, **ppkey;
+ const gchar *buf, *arg;
+ gsize len;
+ gint type = RSPAMD_KEYPAIR_SIGN;
+ gint alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ buf = luaL_checklstring(L, 1, &len);
+ if (buf != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* keypair type */
+ arg = lua_tostring(L, 2);
+
+ if (strcmp(arg, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(arg, "kex") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ }
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ /* algorithm */
+ arg = lua_tostring(L, 3);
+
+ if (strcmp(arg, "default") == 0 || strcmp(arg, "curve25519") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else if (strcmp(arg, "nist") == 0) {
+ type = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ }
+
+ pkey = rspamd_pubkey_from_base32(buf, len, type, alg);
+
+ if (pkey == NULL) {
+ msg_err("cannot load pubkey from string");
+ lua_pushnil(L);
+ }
+ else {
+ ppkey = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ *ppkey = pkey;
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_pubkey_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pkey = lua_check_cryptobox_pubkey(L, 1);
+
+ if (pkey != NULL) {
+ rspamd_pubkey_unref(pkey);
+ }
+
+ return 0;
+}
+
+/***
+ * @function rspamd_cryptobox_keypair.load(file|table)
+ * Loads public key from UCL file or directly from Lua
+ * @param {string} file filename to load
+ * @return {cryptobox_keypair} new keypair
+ */
+static gint
+lua_cryptobox_keypair_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp, **pkp;
+ const gchar *buf;
+ gsize len;
+ struct ucl_parser *parser;
+ ucl_object_t *obj;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ buf = luaL_checklstring(L, 1, &len);
+ if (buf != NULL) {
+ parser = ucl_parser_new(0);
+
+ if (!ucl_parser_add_chunk(parser, buf, len)) {
+ msg_err("cannot open keypair from data: %s",
+ ucl_parser_get_error(parser));
+ ucl_parser_free(parser);
+ lua_pushnil(L);
+ }
+ else {
+ obj = ucl_parser_get_object(parser);
+ kp = rspamd_keypair_from_ucl(obj);
+ ucl_parser_free(parser);
+
+ if (kp == NULL) {
+ msg_err("cannot load keypair from data");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+ ucl_object_unref(obj);
+ }
+ }
+ }
+ else {
+ luaL_error(L, "bad input arguments");
+ }
+ }
+ else {
+ /* Directly import from lua */
+ obj = ucl_object_lua_import(L, 1);
+ kp = rspamd_keypair_from_ucl(obj);
+
+ if (kp == NULL) {
+ msg_err("cannot load keypair from data");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+ ucl_object_unref(obj);
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_keypair.create([type='encryption'[, alg='curve25519']])
+ * Generates new keypair
+ * @param {string} type type of keypair: 'encryption' (default) or 'sign'
+ * @param {string} alg algorithm of keypair: 'curve25519' (default) or 'nist'
+ * @return {cryptobox_keypair} new keypair
+ */
+static gint
+lua_cryptobox_keypair_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp, **pkp;
+ enum rspamd_cryptobox_keypair_type type = RSPAMD_KEYPAIR_KEX;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ if (lua_isstring(L, 1)) {
+ const gchar *str = lua_tostring(L, 1);
+
+ if (strcmp(str, "sign") == 0) {
+ type = RSPAMD_KEYPAIR_SIGN;
+ }
+ else if (strcmp(str, "encryption") == 0) {
+ type = RSPAMD_KEYPAIR_KEX;
+ }
+ else {
+ return luaL_error(L, "invalid keypair type: %s", str);
+ }
+ }
+
+ if (lua_isstring(L, 2)) {
+ const gchar *str = lua_tostring(L, 2);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid keypair algorithm: %s", str);
+ }
+ }
+
+ kp = rspamd_keypair_new(type, alg);
+
+ pkp = lua_newuserdata(L, sizeof(gpointer));
+ *pkp = kp;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_keypair}", -1);
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_keypair_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp != NULL) {
+ rspamd_keypair_unref(kp);
+ }
+
+ return 0;
+}
+
+/***
+ * @method keypair:totable([hex=false]])
+ * Converts keypair to table (not very safe due to memory leftovers)
+ */
+static gint
+lua_cryptobox_keypair_totable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+ ucl_object_t *obj;
+ gboolean hex = FALSE;
+ gint ret = 1;
+
+ if (kp != NULL) {
+
+ if (lua_isboolean(L, 2)) {
+ hex = lua_toboolean(L, 2);
+ }
+
+ obj = rspamd_keypair_to_ucl(kp, hex ? RSPAMD_KEYPAIR_DUMP_HEX : RSPAMD_KEYPAIR_DUMP_DEFAULT);
+
+ ret = ucl_object_push_lua(L, obj, true);
+ ucl_object_unref(obj);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return ret;
+}
+/***
+ * @method keypair:type()
+ * Returns type of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp) {
+ if (kp->type == RSPAMD_KEYPAIR_KEX) {
+ lua_pushstring(L, "encryption");
+ }
+ else {
+ lua_pushstring(L, "sign");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:alg()
+ * Returns algorithm of keypair as a string: 'encryption' or 'sign'
+ * @return {string} type of keypair as a string
+ */
+static gint
+lua_cryptobox_keypair_get_alg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (kp) {
+ if (kp->alg == RSPAMD_CRYPTOBOX_MODE_25519) {
+ lua_pushstring(L, "curve25519");
+ }
+ else {
+ lua_pushstring(L, "nist");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method keypair:pk()
+ * Returns pubkey for a specific keypair
+ * @return {rspamd_pubkey} pubkey for a keypair
+ */
+static gint
+lua_cryptobox_keypair_get_pk(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = lua_check_cryptobox_keypair(L, 1);
+ struct rspamd_cryptobox_pubkey *pk, **ppk;
+ const guchar *data;
+ guint dlen;
+
+ if (kp) {
+ data = rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_PK, &dlen);
+ pk = rspamd_pubkey_from_bin(data, dlen, kp->type, kp->alg);
+
+ if (pk == NULL) {
+ return luaL_error(L, "invalid keypair");
+ }
+
+ ppk = lua_newuserdata(L, sizeof(*ppk));
+ *ppk = pk;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_pubkey}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_signature.load(file, [alg = 'curve25519'])
+ * Loads signature from raw file
+ * @param {string} file filename to load
+ * @return {cryptobox_signature} new signature
+ */
+static gint
+lua_cryptobox_signature_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig, **psig;
+ const gchar *filename;
+ gpointer data;
+ int fd;
+ struct stat st;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ fd = open(filename, O_RDONLY);
+ if (fd == -1) {
+ msg_err("cannot open signature file: %s, %s", filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (fstat(fd, &st) == -1 ||
+ (data =
+ mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (lua_isstring(L, 2)) {
+ const gchar *str = lua_tostring(L, 2);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ munmap(data, st.st_size);
+ close(fd);
+
+ return luaL_error(L, "invalid keypair algorithm: %s", str);
+ }
+ }
+ if (st.st_size > 0) {
+ sig = rspamd_fstring_new_init(data, st.st_size);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ *psig = sig;
+ }
+ else {
+ msg_err("size of %s mismatches: %d while %d is expected",
+ filename, (int) st.st_size,
+ rspamd_cryptobox_signature_bytes(alg));
+ lua_pushnil(L);
+ }
+
+ munmap(data, st.st_size);
+ }
+ close(fd);
+ }
+ }
+ else {
+ luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_cryptobox_signature:save(file)
+ * Stores signature in raw file
+ * @param {string} file filename to use
+ * @return {boolean} true if signature has been saved
+ */
+static gint
+lua_cryptobox_signature_save(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig;
+ gint fd, flags;
+ const gchar *filename;
+ gboolean forced = FALSE, res = TRUE;
+
+ sig = lua_check_cryptobox_sign(L, 1);
+ filename = luaL_checkstring(L, 2);
+
+ if (!sig || !filename) {
+ luaL_error(L, "bad input arguments");
+ return 1;
+ }
+
+ if (lua_gettop(L) > 2) {
+ forced = lua_toboolean(L, 3);
+ }
+
+ if (sig != NULL && filename != NULL) {
+ flags = O_WRONLY | O_CREAT;
+ if (forced) {
+ flags |= O_TRUNC;
+ }
+ else {
+ flags |= O_EXCL;
+ }
+ fd = open(filename, flags, 00644);
+ if (fd == -1) {
+ msg_err("cannot create a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ while (write(fd, sig->str, sig->len) == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ msg_err("cannot write to a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ res = FALSE;
+ break;
+ }
+ lua_pushboolean(L, res);
+ close(fd);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_signature.create(data)
+ * Creates signature object from raw data
+ * @param {data} raw signature data
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_signature_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig, **psig;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize dlen;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ dlen = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &dlen);
+ }
+
+ if (data != NULL) {
+ if (dlen == rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519)) {
+ sig = rspamd_fstring_new_init(data, dlen);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ *psig = sig;
+ }
+ }
+ else {
+ return luaL_error(L, "bad input arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:hex()
+ * Return hex encoded signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gchar *encoded;
+
+ if (sig) {
+ encoded = rspamd_encode_hex(sig->str, sig->len);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:base32([b32type='default'])
+ * Return base32 encoded signature string
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gchar *encoded;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (sig) {
+ encoded = rspamd_encode_base32(sig->str, sig->len, btype);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:base64()
+ * Return base64 encoded signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+ gsize dlen;
+ gchar *encoded;
+
+ if (sig) {
+ encoded = rspamd_encode_base64(sig->str, sig->len, 0, &dlen);
+ lua_pushlstring(L, encoded, dlen);
+ g_free(encoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_signature:bin()
+ * Return raw signature string
+ * @return {string} raw value of signature
+ */
+static gint
+lua_cryptobox_signature_bin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+
+ if (sig) {
+ lua_pushlstring(L, sig->str, sig->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_signature_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_fstring_t *sig = lua_check_cryptobox_sign(L, 1);
+
+ rspamd_fstring_free(sig);
+
+ return 0;
+}
+
+static void
+rspamd_lua_hash_update(struct rspamd_lua_cryptobox_hash *h,
+ const void *p, gsize len)
+{
+ if (h) {
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ rspamd_cryptobox_hash_update(h->content.h, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestUpdate(h->content.c, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+ HMAC_Update(h->content.hmac_c, p, len);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ case LUA_CRYPTOBOX_HASH_MUM:
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ rspamd_cryptobox_fast_hash_update(h->content.fh, p, len);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ }
+}
+
+static void
+lua_cryptobox_hash_dtor(struct rspamd_lua_cryptobox_hash *h)
+{
+ if (h->type == LUA_CRYPTOBOX_HASH_SSL) {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
+ EVP_MD_CTX_cleanup(h->content.c);
+#else
+ EVP_MD_CTX_reset(h->content.c);
+#endif
+ EVP_MD_CTX_destroy(h->content.c);
+ }
+ else if (h->type == LUA_CRYPTOBOX_HASH_HMAC) {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ HMAC_CTX_cleanup(h->content.hmac_c);
+ g_free(h->content.hmac_c);
+#else
+ HMAC_CTX_free(h->content.hmac_c);
+#endif
+ }
+ else if (h->type == LUA_CRYPTOBOX_HASH_BLAKE2) {
+ rspamd_explicit_memzero(h->content.h, sizeof(*h->content.h));
+ free(h->content.h); /* Allocated by posix_memalign */
+ }
+ else {
+ rspamd_cryptobox_fast_hash_free(h->content.fh);
+ }
+
+ g_free(h);
+}
+
+static inline void
+rspamd_lua_hash_init_default(struct rspamd_lua_cryptobox_hash *h,
+ const gchar *key, gsize keylen)
+{
+ h->type = LUA_CRYPTOBOX_HASH_BLAKE2;
+ if (posix_memalign((void **) &h->content.h,
+ RSPAMD_ALIGNOF(rspamd_cryptobox_hash_state_t),
+ sizeof(*h->content.h)) != 0) {
+ g_assert_not_reached();
+ }
+
+ rspamd_cryptobox_hash_init(h->content.h, key, keylen);
+ h->out_len = rspamd_cryptobox_HASHBYTES;
+}
+
+static void
+rspamd_lua_ssl_hash_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *htype,
+ bool insecure)
+{
+ h->type = LUA_CRYPTOBOX_HASH_SSL;
+ h->content.c = EVP_MD_CTX_create();
+ h->out_len = EVP_MD_size(htype);
+
+ if (insecure) {
+ /* Should never ever be used for crypto/security purposes! */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+ EVP_MD_CTX_set_flags(h->content.c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+ }
+
+ EVP_DigestInit_ex(h->content.c, htype, NULL);
+}
+
+static void
+rspamd_lua_ssl_hmac_create(struct rspamd_lua_cryptobox_hash *h, const EVP_MD *htype,
+ const gchar *key, gsize keylen,
+ bool insecure)
+{
+ h->type = LUA_CRYPTOBOX_HASH_HMAC;
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ h->content.hmac_c = g_malloc0(sizeof(*h->content.hmac_c));
+#else
+ h->content.hmac_c = HMAC_CTX_new();
+#endif
+ h->out_len = EVP_MD_size(htype);
+
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+ if (insecure) {
+ /* Should never ever be used for crypto/security purposes! */
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+ HMAC_CTX_set_flags(h->content.hmac_c, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+ }
+#endif
+
+ HMAC_Init_ex(h->content.hmac_c, key, keylen, htype, NULL);
+}
+
+static struct rspamd_lua_cryptobox_hash *
+rspamd_lua_hash_create(const gchar *type, const gchar *key, gsize keylen)
+{
+ struct rspamd_lua_cryptobox_hash *h;
+
+ h = g_malloc0(sizeof(*h));
+ REF_INIT_RETAIN(h, lua_cryptobox_hash_dtor);
+
+ if (type) {
+ if (g_ascii_strcasecmp(type, "md5") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_md5(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_md5(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha1") == 0 ||
+ g_ascii_strcasecmp(type, "sha") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha1(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha1(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha256") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha256(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha256(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha512") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha512(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha512(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "sha384") == 0) {
+ if (keylen > 0) {
+ rspamd_lua_ssl_hmac_create(h, EVP_sha384(), key, keylen, true);
+ }
+ else {
+ rspamd_lua_ssl_hash_create(h, EVP_sha384(), true);
+ }
+ }
+ else if (g_ascii_strcasecmp(type, "xxh64") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH64;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH64, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "xxh32") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH32;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH32, 0);
+ h->out_len = sizeof(guint32);
+ }
+ else if (g_ascii_strcasecmp(type, "xxh3") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_XXHASH3;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH3, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "mum") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_MUM;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_MUMHASH, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "t1ha") == 0) {
+ h->type = LUA_CRYPTOBOX_HASH_T1HA;
+ h->content.fh = rspamd_cryptobox_fast_hash_new();
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_T1HA, 0);
+ h->out_len = sizeof(guint64);
+ }
+ else if (g_ascii_strcasecmp(type, "blake2") == 0) {
+ rspamd_lua_hash_init_default(h, key, keylen);
+ }
+ else {
+ g_free(h);
+
+ return NULL;
+ }
+ }
+ else {
+ /* Default hash type */
+ rspamd_lua_hash_init_default(h, key, keylen);
+ }
+
+ return h;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create([string])
+ * Creates new hash context
+ * @param {string} data optional string to hash
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *s = NULL;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+
+ h = rspamd_lua_hash_create(NULL, NULL, 0);
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = lua_tolstring(L, 1, &len);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_specific(type, [string])
+ * Creates new hash context
+ * @param {string} type type of hash (blake2, sha256, md5, sha512, mum, xxh64, xxh32, t1ha)
+ * @param {string} string initial data
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *s = NULL, *type = luaL_checkstring(L, 1);
+ gsize len = 0;
+ struct rspamd_lua_text *t;
+
+ if (!type) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ h = rspamd_lua_hash_create(type, NULL, 0);
+
+ if (h == NULL) {
+ return luaL_error(L, "invalid hash type: %s", type);
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ s = lua_tolstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_keyed(key, [string])
+ * Creates new hash context with specified key
+ * @param {string} key key
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_keyed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *key, *s = NULL;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ gsize keylen;
+
+ key = luaL_checklstring(L, 1, &keylen);
+
+ if (key != NULL) {
+ h = rspamd_lua_hash_create(NULL, key, keylen);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ s = lua_tolstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ REF_RELEASE(h);
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox_hash.create_specific_keyed(key, type, [string])
+ * Creates new hash context with specified key
+ * @param {string} key key
+ * @return {cryptobox_hash} hash object
+ */
+static gint
+lua_cryptobox_hash_create_specific_keyed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h, **ph;
+ const gchar *key, *s = NULL, *type = luaL_checkstring(L, 2);
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ gsize keylen;
+
+ key = luaL_checklstring(L, 1, &keylen);
+
+ if (key != NULL && type != NULL) {
+ h = rspamd_lua_hash_create(type, key, keylen);
+
+ if (h == NULL) {
+ return luaL_error(L, "invalid hash type: %s", type);
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ s = lua_tolstring(L, 3, &len);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 3);
+
+ if (!t) {
+ REF_RELEASE(h);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ s = t->start;
+ len = t->len;
+ }
+
+ if (s) {
+ rspamd_lua_hash_update(h, s, len);
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:update(data)
+ * Updates hash with the specified data (hash should not be finalized using `hex` or `bin` methods)
+ * @param {string} data data to hash
+ */
+static gint
+lua_cryptobox_hash_update(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1), **ph;
+ const gchar *data;
+ struct rspamd_lua_text *t;
+ gsize len;
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+ if (lua_isnumber(L, 3)) {
+ gsize nlen = lua_tonumber(L, 3);
+
+ if (nlen > len) {
+ return luaL_error(L, "invalid length: %d while %d is available",
+ (int) nlen, (int) len);
+ }
+
+ len = nlen;
+ }
+
+ if (h && data) {
+ if (!h->is_finished) {
+ rspamd_lua_hash_update(h, data, len);
+ }
+ else {
+ return luaL_error(L, "hash is already finalized");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ REF_RETAIN(h);
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:reset()
+ * Resets hash to the initial state
+ */
+static gint
+lua_cryptobox_hash_reset(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1), **ph;
+
+ if (h) {
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ memset(h->content.h, 0, sizeof(*h->content.h));
+ rspamd_cryptobox_hash_init(h->content.h, NULL, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestInit(h->content.c, EVP_MD_CTX_md(h->content.c));
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
+ (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
+ /* Old openssl is awesome... */
+ HMAC_Init_ex(h->content.hmac_c, NULL, 0, h->content.hmac_c->md, NULL);
+#else
+ HMAC_CTX_reset(h->content.hmac_c);
+#endif
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH64, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH32, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_XXHASH3, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_MUM:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_MUMHASH, 0);
+ break;
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ rspamd_cryptobox_fast_hash_init_specific(h->content.fh,
+ RSPAMD_CRYPTOBOX_T1HA, 0);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ h->is_finished = FALSE;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ph = lua_newuserdata(L, sizeof(void *));
+ *ph = h;
+ REF_RETAIN(h);
+ rspamd_lua_setclass(L, "rspamd{cryptobox_hash}", -1);
+
+ return 1;
+}
+
+static void
+lua_cryptobox_hash_finish(struct rspamd_lua_cryptobox_hash *h)
+{
+ guint64 ll;
+ guchar out[rspamd_cryptobox_HASHBYTES];
+ guint ssl_outlen = sizeof(out);
+
+ switch (h->type) {
+ case LUA_CRYPTOBOX_HASH_BLAKE2:
+ rspamd_cryptobox_hash_final(h->content.h, out);
+ memcpy(h->out, out, sizeof(out));
+ break;
+ case LUA_CRYPTOBOX_HASH_SSL:
+ EVP_DigestFinal_ex(h->content.c, out, &ssl_outlen);
+ h->out_len = ssl_outlen;
+ g_assert(ssl_outlen <= sizeof(h->out));
+ memcpy(h->out, out, ssl_outlen);
+ break;
+ case LUA_CRYPTOBOX_HASH_HMAC:
+ HMAC_Final(h->content.hmac_c, out, &ssl_outlen);
+ h->out_len = ssl_outlen;
+ g_assert(ssl_outlen <= sizeof(h->out));
+ memcpy(h->out, out, ssl_outlen);
+ break;
+ case LUA_CRYPTOBOX_HASH_XXHASH64:
+ case LUA_CRYPTOBOX_HASH_XXHASH32:
+ case LUA_CRYPTOBOX_HASH_XXHASH3:
+ case LUA_CRYPTOBOX_HASH_MUM:
+ case LUA_CRYPTOBOX_HASH_T1HA:
+ ll = rspamd_cryptobox_fast_hash_final(h->content.fh);
+ memcpy(h->out, &ll, sizeof(ll));
+ break;
+ default:
+ g_assert_not_reached();
+ }
+
+ h->is_finished = TRUE;
+}
+
+/***
+ * @method cryptobox_hash:hex()
+ * Finalizes hash and return it as hex string
+ * @return {string} hex value of hash
+ */
+static gint
+lua_cryptobox_hash_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar out_hex[rspamd_cryptobox_HASHBYTES * 2 + 1], *r;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ memset(out_hex, 0, sizeof(out_hex));
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ rspamd_encode_hex_buf(r, dlen, out_hex, sizeof(out_hex));
+ lua_pushstring(L, out_hex);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:base32([b32type])
+ * Finalizes hash and return it as zbase32 (by default) string
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {string} base32 value of hash
+ */
+static gint
+lua_cryptobox_hash_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar out_b32[rspamd_cryptobox_HASHBYTES * 2], *r;
+ guint dlen;
+
+ if (h) {
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ memset(out_b32, 0, sizeof(out_b32));
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ rspamd_encode_base32_buf(r, dlen, out_b32, sizeof(out_b32), btype);
+ lua_pushstring(L, out_b32);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:base64()
+ * Finalizes hash and return it as base64 string
+ * @return {string} base64 value of hash
+ */
+static gint
+lua_cryptobox_hash_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar *b64, *r;
+ gsize len;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ b64 = rspamd_encode_base64(r, dlen, 0, &len);
+ lua_pushlstring(L, b64, len);
+ g_free(b64);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method cryptobox_hash:bin()
+ * Finalizes hash and return it as raw string
+ * @return {string} raw value of hash
+ */
+static gint
+lua_cryptobox_hash_bin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+ guchar *r;
+ guint dlen;
+
+ if (h) {
+ if (!h->is_finished) {
+ lua_cryptobox_hash_finish(h);
+ }
+
+ r = h->out;
+ dlen = h->out_len;
+
+ if (lua_isnumber(L, 2)) {
+ guint lim = lua_tonumber(L, 2);
+
+ if (lim < dlen) {
+ r += dlen - lim;
+ dlen = lim;
+ }
+ }
+
+ lua_pushlstring(L, r, dlen);
+ h->is_finished = TRUE;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_cryptobox_hash_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_cryptobox_hash *h = lua_check_cryptobox_hash(L, 1);
+
+ REF_RELEASE(h);
+
+ return 0;
+}
+
+/***
+ * @function rspamd_cryptobox.verify_memory(pk, sig, data, [alg = 'curve25519'])
+ * Check memory using specified cryptobox key and signature
+ * @param {pubkey} pk public key to verify
+ * @param {sig} signature to check
+ * @param {string} data data to check signature against
+ * @return {boolean} `true` - if string matches cryptobox signature
+ */
+static gint
+lua_cryptobox_verify_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_pubkey *pk;
+ rspamd_fstring_t *signature;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ gsize len;
+ gint ret;
+
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ signature = lua_check_cryptobox_sign(L, 2);
+
+ if (lua_isuserdata(L, 3)) {
+ t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 3, &len);
+ }
+
+ if (lua_isstring(L, 4)) {
+ const gchar *str = lua_tostring(L, 4);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid algorithm: %s", str);
+ }
+ }
+
+ if (pk != NULL && signature != NULL && data != NULL) {
+ ret = rspamd_cryptobox_verify(signature->str, signature->len, data, len,
+ rspamd_pubkey_get_pk(pk, NULL), alg);
+
+ if (ret) {
+ lua_pushboolean(L, 1);
+ }
+ else {
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.verify_file(pk, sig, file, [alg = 'curve25519'])
+ * Check file using specified cryptobox key and signature
+ * @param {pubkey} pk public key to verify
+ * @param {sig} signature to check
+ * @param {string} file to load data from
+ * @return {boolean} `true` - if string matches cryptobox signature
+ */
+static gint
+lua_cryptobox_verify_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fname;
+ struct rspamd_cryptobox_pubkey *pk;
+ rspamd_fstring_t *signature;
+ guchar *map = NULL;
+ enum rspamd_cryptobox_mode alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ gsize len;
+ gint ret;
+
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ signature = lua_check_cryptobox_sign(L, 2);
+ fname = luaL_checkstring(L, 3);
+
+ if (lua_isstring(L, 4)) {
+ const gchar *str = lua_tostring(L, 4);
+
+ if (strcmp(str, "nist") == 0 || strcmp(str, "openssl") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_NIST;
+ }
+ else if (strcmp(str, "curve25519") == 0 || strcmp(str, "default") == 0) {
+ alg = RSPAMD_CRYPTOBOX_MODE_25519;
+ }
+ else {
+ return luaL_error(L, "invalid algorithm: %s", str);
+ }
+ }
+
+ map = rspamd_file_xmap(fname, PROT_READ, &len, TRUE);
+
+ if (map != NULL && pk != NULL && signature != NULL) {
+ ret = rspamd_cryptobox_verify(signature->str, signature->len,
+ map, len,
+ rspamd_pubkey_get_pk(pk, NULL), alg);
+
+ if (ret) {
+ lua_pushboolean(L, 1);
+ }
+ else {
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ if (map != NULL) {
+ munmap(map, len);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (map != NULL) {
+ munmap(map, len);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.sign_memory(kp, data)
+ * Sign data using specified keypair
+ * @param {keypair} kp keypair to sign
+ * @param {string} data
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_sign_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ struct rspamd_lua_text *t;
+ gsize len = 0;
+ rspamd_fstring_t *sig, **psig;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!kp || !data || kp->type == RSPAMD_KEYPAIR_KEX) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sig = rspamd_fstring_sized_new(rspamd_cryptobox_signature_bytes(
+ rspamd_keypair_alg(kp)));
+
+ unsigned long long siglen = sig->len;
+ rspamd_cryptobox_sign(sig->str, &siglen, data,
+ len, rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL), rspamd_keypair_alg(kp));
+
+ sig->len = siglen;
+ psig = lua_newuserdata(L, sizeof(void *));
+ *psig = sig;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.sign_file(kp, file)
+ * Sign file using specified keypair
+ * @param {keypair} kp keypair to sign
+ * @param {string} filename
+ * @return {cryptobox_signature} signature object
+ */
+static gint
+lua_cryptobox_sign_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ gsize len = 0;
+ rspamd_fstring_t *sig, **psig;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+ filename = luaL_checkstring(L, 2);
+
+ if (!kp || !filename) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (data == NULL) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ sig = rspamd_fstring_sized_new(rspamd_cryptobox_signature_bytes(
+ rspamd_keypair_alg(kp)));
+
+ unsigned long long siglen = sig->len;
+
+ rspamd_cryptobox_sign(sig->str, &siglen, data,
+ len, rspamd_keypair_component(kp, RSPAMD_KEYPAIR_COMPONENT_SK, NULL), rspamd_keypair_alg(kp));
+
+ sig->len = siglen;
+ psig = lua_newuserdata(L, sizeof(void *));
+ *psig = sig;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_signature}", -1);
+ munmap(data, len);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.encrypt_memory(kp, data[, nist=false])
+ * Encrypt data using specified keypair/pubkey
+ * @param {keypair|string} kp keypair or pubkey in base32 to use
+ * @param {string|text} data
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = NULL;
+ struct rspamd_cryptobox_pubkey *pk = NULL;
+ const gchar *data;
+ guchar *out = NULL;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen = 0;
+ GError *err = NULL;
+ bool owned_pk = false;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_keypair}")) {
+ kp = lua_check_cryptobox_keypair(L, 1);
+ }
+ else if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_pubkey}")) {
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *b32;
+ gsize blen;
+
+ b32 = lua_tolstring(L, 1, &blen);
+ pk = rspamd_pubkey_from_base32(b32, blen, RSPAMD_KEYPAIR_KEX,
+ lua_toboolean(L, 3) ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ owned_pk = true;
+ }
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ goto err;
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!(kp || pk) || !data) {
+ goto err;
+ }
+
+ if (kp) {
+ if (!rspamd_keypair_encrypt(kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt data: %s", err->message);
+ g_error_free(err);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+ else {
+ if (!rspamd_pubkey_encrypt(pk, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt data: %s", err->message);
+ g_error_free(err);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return 1;
+err:
+
+ if (owned_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return luaL_error(L, "invalid arguments");
+}
+
+/***
+ * @function rspamd_cryptobox.encrypt_file(kp|pk_string, filename[, nist=false])
+ * Encrypt data using specified keypair/pubkey
+ * @param {keypair|string} kp keypair or pubkey in base32 to use
+ * @param {string} filename
+ * @return {rspamd_text} encrypted text
+ */
+static gint
+lua_cryptobox_encrypt_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp = NULL;
+ struct rspamd_cryptobox_pubkey *pk = NULL;
+ const gchar *filename;
+ gchar *data = NULL;
+ guchar *out = NULL;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen = 0;
+ GError *err = NULL;
+ bool own_pk = false;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_keypair}")) {
+ kp = lua_check_cryptobox_keypair(L, 1);
+ }
+ else if (rspamd_lua_check_udata_maybe(L, 1, "rspamd{cryptobox_pubkey}")) {
+ pk = lua_check_cryptobox_pubkey(L, 1);
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *b32;
+ gsize blen;
+
+ b32 = lua_tolstring(L, 1, &blen);
+ pk = rspamd_pubkey_from_base32(b32, blen, RSPAMD_KEYPAIR_KEX,
+ lua_toboolean(L, 3) ? RSPAMD_CRYPTOBOX_MODE_NIST : RSPAMD_CRYPTOBOX_MODE_25519);
+ own_pk = true;
+ }
+
+ filename = luaL_checkstring(L, 2);
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+
+ if (!(kp || pk) || !data) {
+ goto err;
+ }
+
+ if (kp) {
+ if (!rspamd_keypair_encrypt(kp, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt file %s: %s", filename,
+ err->message);
+ g_error_free(err);
+ munmap(data, len);
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+ else if (pk) {
+ if (!rspamd_pubkey_encrypt(pk, data, len, &out, &outlen, &err)) {
+ gint ret = luaL_error(L, "cannot encrypt file %s: %s", filename,
+ err->message);
+ g_error_free(err);
+ munmap(data, len);
+
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return ret;
+ }
+ }
+
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ munmap(data, len);
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+
+ return 1;
+
+err:
+ if (data) {
+ munmap(data, len);
+ }
+ if (own_pk) {
+ rspamd_pubkey_unref(pk);
+ }
+ return luaL_error(L, "invalid arguments");
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_memory(kp, data[, nist = false])
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} data
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_memory(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *t, *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+
+ if (lua_isuserdata(L, 2)) {
+ t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &len);
+ }
+
+
+ if (!kp || !data) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!rspamd_keypair_decrypt(kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ }
+ else {
+ lua_pushboolean(L, true);
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_file(kp, filename)
+ * Encrypt data using specified keypair
+ * @param {keypair} kp keypair to use
+ * @param {string} filename
+ * @return status,{rspamd_text}|error status is boolean variable followed by either unencrypted data or an error message
+ */
+static gint
+lua_cryptobox_decrypt_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_cryptobox_keypair *kp;
+ const gchar *filename;
+ gchar *data;
+ guchar *out;
+ struct rspamd_lua_text *res;
+ gsize len = 0, outlen;
+ GError *err = NULL;
+
+ kp = lua_check_cryptobox_keypair(L, 1);
+ if (!kp) {
+ return luaL_error(L, "invalid arguments; keypair is expected");
+ }
+
+ filename = luaL_checkstring(L, 2);
+ data = rspamd_file_xmap(filename, PROT_READ, &len, TRUE);
+ if (!data) {
+ return luaL_error(L, "invalid arguments; cannot mmap %s: %s",
+ filename, strerror(errno));
+ }
+
+ if (!rspamd_keypair_decrypt(kp, data, len, &out, &outlen, &err)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ }
+ else {
+ lua_pushboolean(L, true);
+ res = lua_newuserdata(L, sizeof(*res));
+ res->flags = RSPAMD_TEXT_FLAG_OWN;
+ res->start = out;
+ res->len = outlen;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+
+ munmap(data, len);
+
+ return 2;
+}
+
+#define RSPAMD_CRYPTOBOX_AES_BLOCKSIZE 16
+#define RSPAMD_CRYPTOBOX_AES_KEYSIZE 16
+
+/***
+ * @function rspamd_cryptobox.encrypt_cookie(secret_key, secret_cookie)
+ * Specialised function that performs AES-CTR encryption of the provided cookie
+ * ```
+ * e := base64(nonce||aesencrypt(nonce, secret_cookie))
+ * nonce := uint32_le(unix_timestamp)||random_64bit
+ * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie)
+ * pad := secret_cookie || 0^(32-len(secret_cookie))
+ * ```
+ * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex)
+ * @param {string} secret_cookie secret cookie as a string for up to 31 character
+ * @return {string} e function value for this sk and cookie
+ */
+static gint
+lua_cryptobox_encrypt_cookie(lua_State *L)
+{
+ guchar aes_block[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE], *blk;
+ guchar padded_cookie[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE];
+ guchar result[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2];
+ guint32 ts;
+
+ const gchar *sk, *cookie;
+ gsize sklen, cookie_len;
+ gint bklen;
+
+ sk = lua_tolstring(L, 1, &sklen);
+ cookie = lua_tolstring(L, 2, &cookie_len);
+
+ if (sk && cookie) {
+ if (sklen == 32) {
+ /* Hex */
+ rspamd_decode_hex_buf(sk, sklen, aes_key, sizeof(aes_key));
+ }
+ else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) {
+ /* Raw */
+ memcpy(aes_key, sk, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid keysize %d", (gint) sklen);
+ }
+
+ if (cookie_len > sizeof(padded_cookie) - 1) {
+ return luaL_error(L, "cookie is too long %d", (gint) cookie_len);
+ }
+
+ /* Fill nonce */
+ ottery_rand_bytes(nonce, sizeof(guint64) + sizeof(guint32));
+ ts = (guint32) rspamd_get_calendar_ticks();
+ ts = GUINT32_TO_LE(ts);
+ memcpy(nonce + sizeof(guint64) + sizeof(guint32), &ts, sizeof(ts));
+
+ /* Prepare padded cookie */
+ memset(padded_cookie, 0, sizeof(padded_cookie));
+ memcpy(padded_cookie, cookie, cookie_len);
+
+ /* Perform AES CTR via AES ECB on nonce */
+ EVP_CIPHER_CTX *ctx;
+ ctx = EVP_CIPHER_CTX_new();
+ EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), NULL, aes_key, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+ bklen = sizeof(aes_block);
+ blk = aes_block;
+ g_assert(EVP_EncryptUpdate(ctx, blk, &bklen, nonce, sizeof(nonce)));
+ blk += bklen;
+ g_assert(EVP_EncryptFinal_ex(ctx, blk, &bklen));
+ EVP_CIPHER_CTX_free(ctx);
+
+ /* Encode result */
+ memcpy(result, nonce, sizeof(nonce));
+ for (guint i = 0; i < sizeof(aes_block); i++) {
+ result[i + sizeof(nonce)] = padded_cookie[i] ^ aes_block[i];
+ }
+
+ gsize rlen;
+ gchar *res = rspamd_encode_base64(result, sizeof(result),
+ 0, &rlen);
+
+ lua_pushlstring(L, res, rlen);
+ g_free(res);
+ rspamd_explicit_memzero(aes_key, sizeof(aes_key));
+ rspamd_explicit_memzero(aes_block, sizeof(aes_block));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.decrypt_cookie(secret_key, encrypted_cookie)
+ * Specialised function that performs AES-CTR decryption of the provided cookie in form
+ * ```
+ * e := base64(nonce||aesencrypt(nonce, secret_cookie))
+ * nonce := int32_le(unix_timestamp)||random_96bit
+ * aesencrypt := aes_ctr(nonce, secret_key) ^ pad(secret_cookie)
+ * pad := secret_cookie || 0^(32-len(secret_cookie))
+ * ```
+ * @param {string} secret_key secret key as a hex string (must be 16 bytes in raw or 32 in hex)
+ * @param {string} encrypted_cookie encrypted cookie as a base64 encoded string
+ * @return {string+number} decrypted value of the cookie and the cookie timestamp
+ */
+static gint
+lua_cryptobox_decrypt_cookie(lua_State *L)
+{
+ guchar *blk;
+ guchar nonce[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE];
+ guchar aes_key[RSPAMD_CRYPTOBOX_AES_KEYSIZE];
+ guchar *src;
+ guint32 ts;
+
+ const gchar *sk, *cookie;
+ gsize sklen, cookie_len;
+ gint bklen;
+
+ sk = lua_tolstring(L, 1, &sklen);
+ cookie = lua_tolstring(L, 2, &cookie_len);
+
+ if (sk && cookie) {
+ if (sklen == 32) {
+ /* Hex */
+ rspamd_decode_hex_buf(sk, sklen, aes_key, sizeof(aes_key));
+ }
+ else if (sklen == RSPAMD_CRYPTOBOX_AES_KEYSIZE) {
+ /* Raw */
+ memcpy(aes_key, sk, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid keysize %d", (gint) sklen);
+ }
+
+ src = g_malloc(cookie_len);
+
+ rspamd_cryptobox_base64_decode(cookie, cookie_len, src, &cookie_len);
+
+ if (cookie_len != RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2) {
+ g_free(src);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ /* Perform AES CTR via AES ECB on nonce */
+ EVP_CIPHER_CTX *ctx;
+ ctx = EVP_CIPHER_CTX_new();
+ /* As per CTR definition, we use encrypt for both encrypt and decrypt */
+ EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), NULL, aes_key, NULL);
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+ /* Copy time */
+ memcpy(&ts, src + sizeof(guint64) + sizeof(guint32), sizeof(ts));
+ ts = GUINT32_FROM_LE(ts);
+ bklen = sizeof(nonce);
+ blk = nonce;
+ g_assert(EVP_EncryptUpdate(ctx, blk, &bklen, src,
+ RSPAMD_CRYPTOBOX_AES_BLOCKSIZE));
+ blk += bklen;
+ g_assert(EVP_EncryptFinal_ex(ctx, blk, &bklen));
+ EVP_CIPHER_CTX_free(ctx);
+
+ /* Decode result */
+ for (guint i = 0; i < RSPAMD_CRYPTOBOX_AES_BLOCKSIZE; i++) {
+ src[i + sizeof(nonce)] ^= nonce[i];
+ }
+
+ if (src[RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2 - 1] != '\0') {
+ /* Bad cookie */
+ lua_pushnil(L);
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, src + sizeof(nonce));
+ lua_pushnumber(L, ts);
+ }
+
+ rspamd_explicit_memzero(src, RSPAMD_CRYPTOBOX_AES_BLOCKSIZE * 2);
+ g_free(src);
+ rspamd_explicit_memzero(aes_key, sizeof(aes_key));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_cryptobox.pbkdf([password, [kdf_alg]])
+ * Function that encrypts password using PBKDF function.
+ * This function either reads password from STDIN or accepts prepared password as
+ * an argument
+ * @param {string} password optional password string
+ * @param {string} kdf_alg algorithm to use (catena or pbkdf2)
+ * @return {string} encrypted password or nil if error occurs
+ */
+static gint
+lua_cryptobox_pbkdf(lua_State *L)
+{
+ const struct rspamd_controller_pbkdf *pbkdf = NULL;
+ const gchar *pbkdf_str = "catena";
+ gchar *password;
+ gsize pwlen;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ pbkdf_str = lua_tostring(L, 2);
+ }
+
+ for (guint i = 0; i < RSPAMD_PBKDF_ID_MAX - 1; i++) {
+ pbkdf = &pbkdf_list[i];
+
+ if (g_ascii_strcasecmp(pbkdf_str, pbkdf->alias) == 0) {
+ break;
+ }
+ if (g_ascii_strcasecmp(pbkdf_str, pbkdf->name) == 0) {
+ break;
+ }
+
+ pbkdf = NULL;
+ }
+
+ if (pbkdf == NULL) {
+ return luaL_error(L, "invalid pbkdf algorithm: %s", pbkdf_str);
+ }
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ password = g_strdup(lua_tolstring(L, 1, &pwlen));
+ }
+ else {
+ pwlen = 8192;
+ password = g_malloc0(pwlen);
+ pwlen = rspamd_read_passphrase(password, pwlen, 0, NULL);
+ }
+
+ if (pwlen == 0) {
+ lua_pushnil(L);
+ g_free(password);
+
+ return 1;
+ }
+
+ guchar *salt, *key;
+ gchar *encoded_salt, *encoded_key;
+ GString *result;
+
+ salt = g_alloca(pbkdf->salt_len);
+ key = g_alloca(pbkdf->key_len);
+ ottery_rand_bytes(salt, pbkdf->salt_len);
+ /* Derive key */
+ rspamd_cryptobox_pbkdf(password, pwlen,
+ salt, pbkdf->salt_len, key, pbkdf->key_len, pbkdf->complexity,
+ pbkdf->type);
+
+ encoded_salt = rspamd_encode_base32(salt, pbkdf->salt_len, RSPAMD_BASE32_DEFAULT);
+ encoded_key = rspamd_encode_base32(key, pbkdf->key_len, RSPAMD_BASE32_DEFAULT);
+
+ result = g_string_new("");
+ rspamd_printf_gstring(result, "$%d$%s$%s", pbkdf->id, encoded_salt,
+ encoded_key);
+
+ g_free(encoded_salt);
+ g_free(encoded_key);
+ rspamd_explicit_memzero(password, pwlen);
+ g_free(password);
+ lua_pushlstring(L, result->str, result->len);
+ g_string_free(result, TRUE);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_cryptobox.gen_dkim_keypair([alg, [nbits]])
+ * Generates DKIM keypair. Returns 2 base64 strings as rspamd_text: privkey and pubkey
+ * @param {string} alg optional algorithm (rsa default, can be ed25519)
+ * @param {number} nbits optional number of bits for rsa (default 1024)
+ * @return {rspamd_text,rspamd_text} private key and public key as base64 encoded strings
+ */
+static gint
+lua_cryptobox_gen_dkim_keypair(lua_State *L)
+{
+ const gchar *alg_str = "rsa";
+ guint nbits = 1024;
+ struct rspamd_lua_text *priv_out, *pub_out;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ alg_str = lua_tostring(L, 1);
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ nbits = lua_tointeger(L, 2);
+ }
+
+ if (strcmp(alg_str, "rsa") == 0) {
+ BIGNUM *e;
+ RSA *r;
+ EVP_PKEY *pk;
+
+ e = BN_new();
+ r = RSA_new();
+ pk = EVP_PKEY_new();
+
+ if (BN_set_word(e, RSA_F4) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "BN_set_word failed");
+ }
+
+ if (RSA_generate_key_ex(r, nbits, e, NULL) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "RSA_generate_key_ex failed");
+ }
+
+ if (EVP_PKEY_set1_RSA(pk, r) != 1) {
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "EVP_PKEY_set1_RSA failed");
+ }
+
+ BIO *mbio;
+ gint rc, len;
+ guchar *data;
+ gchar *b64_data;
+ gsize b64_len;
+
+ mbio = BIO_new(BIO_s_mem());
+
+ /* Process private key */
+ rc = i2d_RSAPrivateKey_bio(mbio, r);
+
+ if (rc == 0) {
+ BIO_free(mbio);
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "i2d_RSAPrivateKey_bio failed");
+ }
+
+ len = BIO_get_mem_data(mbio, &data);
+
+ b64_data = rspamd_encode_base64(data, len, -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ BIO_reset(mbio);
+ rc = i2d_RSA_PUBKEY_bio(mbio, r);
+
+ if (rc == 0) {
+ BIO_free(mbio);
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+
+ return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
+ }
+
+ len = BIO_get_mem_data(mbio, &data);
+
+ b64_data = rspamd_encode_base64(data, len, -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ BN_free(e);
+ RSA_free(r);
+ EVP_PKEY_free(pk);
+ BIO_free(mbio);
+ }
+ else if (strcmp(alg_str, "ed25519") == 0) {
+ rspamd_sig_pk_t pk;
+ rspamd_sig_sk_t sk;
+ gchar *b64_data;
+ gsize b64_len;
+
+ rspamd_cryptobox_keypair_sig(pk, sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ /* Process private key */
+ b64_data = rspamd_encode_base64(sk,
+ rspamd_cryptobox_sk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ b64_data = rspamd_encode_base64(pk,
+ rspamd_cryptobox_pk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ rspamd_explicit_memzero(pk, sizeof(pk));
+ rspamd_explicit_memzero(sk, sizeof(sk));
+ }
+ else if (strcmp(alg_str, "ed25519-seed") == 0) {
+ rspamd_sig_pk_t pk;
+ rspamd_sig_sk_t sk;
+ gchar *b64_data;
+ gsize b64_len;
+
+ rspamd_cryptobox_keypair_sig(pk, sk, RSPAMD_CRYPTOBOX_MODE_25519);
+
+ /* Process private key */
+ b64_data = rspamd_encode_base64(sk,
+ 32,
+ -1, &b64_len);
+
+ priv_out = lua_newuserdata(L, sizeof(*priv_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ priv_out->start = b64_data;
+ priv_out->len = b64_len;
+ priv_out->flags = RSPAMD_TEXT_FLAG_OWN | RSPAMD_TEXT_FLAG_WIPE;
+
+ /* Process public key */
+ b64_data = rspamd_encode_base64(pk,
+ rspamd_cryptobox_pk_sig_bytes(RSPAMD_CRYPTOBOX_MODE_25519),
+ -1, &b64_len);
+
+ pub_out = lua_newuserdata(L, sizeof(*pub_out));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ pub_out->start = b64_data;
+ pub_out->len = b64_len;
+ pub_out->flags = RSPAMD_TEXT_FLAG_OWN;
+
+ rspamd_explicit_memzero(pk, sizeof(pk));
+ rspamd_explicit_memzero(sk, sizeof(sk));
+ }
+ else {
+ return luaL_error(L, "invalid algorithm %s", alg_str);
+ }
+
+ return 2;
+}
+
+/*
+ * Secretbox API
+ */
+/* Ensure that KDF output is suitable for crypto_secretbox_KEYBYTES */
+#ifdef crypto_generichash_BYTES_MIN
+G_STATIC_ASSERT(crypto_secretbox_KEYBYTES >= crypto_generichash_BYTES_MIN);
+#endif
+
+/***
+ * @function rspamd_cryptobox_secretbox.create(secret_string, [params])
+ * Generates a secretbox state by expanding secret string
+ * @param {string/text} secret_string secret string (should have high enough entropy)
+ * @param {table} params optional parameters - NYI
+ * @return {rspamd_cryptobox_secretbox} opaque object with the key expanded
+ */
+static gint
+lua_cryptobox_secretbox_create(lua_State *L)
+{
+ const gchar *in;
+ gsize inlen;
+
+
+ if (lua_isstring(L, 1)) {
+ in = lua_tolstring(L, 1, &inlen);
+ }
+ else if (lua_isuserdata(L, 1)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ if (in == NULL || inlen == 0) {
+ return luaL_error(L, "invalid arguments; non empty secret expected");
+ }
+
+ struct rspamd_lua_cryptobox_secretbox *sbox, **psbox;
+
+ sbox = g_malloc0(sizeof(*sbox));
+ crypto_generichash(sbox->sk, sizeof(sbox->sk), in, inlen, NULL, 0);
+ psbox = lua_newuserdata(L, sizeof(*psbox));
+ *psbox = sbox;
+ rspamd_lua_setclass(L, "rspamd{cryptobox_secretbox}", -1);
+
+ return 1;
+}
+
+
+static gint
+lua_cryptobox_secretbox_gc(lua_State *L)
+{
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+
+ if (sbox != NULL) {
+ sodium_memzero(sbox->sk, sizeof(sbox->sk));
+ g_free(sbox);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+/***
+ * @method rspamd_cryptobox_secretbox:encrypt(input, [nonce])
+ * Encrypts data using secretbox. MAC is prepended to the message
+ * @param {string/text} input input to encrypt
+ * @param {string/text} nonce optional nonce (must be 1 - 192 bits length)
+ * @param {table} params optional parameters - NYI
+ * @return {rspamd_text},{rspamd_text} output with mac + nonce or just output if nonce is there
+ */
+static gint
+lua_cryptobox_secretbox_encrypt(lua_State *L)
+{
+ const gchar *in, *nonce;
+ gsize inlen, nlen;
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+ struct rspamd_lua_text *out;
+
+ if (sbox == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_isstring(L, 2)) {
+ in = lua_tolstring(L, 2, &inlen);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ /* Nonce part */
+ if (!lua_isnoneornil(L, 3)) {
+ if (lua_isstring(L, 3)) {
+ nonce = lua_tolstring(L, 3, &nlen);
+ }
+ else if (lua_isuserdata(L, 3)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ nonce = t->start;
+ nlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ if (nlen < 1 || nlen > crypto_secretbox_NONCEBYTES) {
+ return luaL_error(L, "bad nonce");
+ }
+
+ guchar real_nonce[crypto_secretbox_NONCEBYTES];
+
+ memset(real_nonce, 0, sizeof(real_nonce));
+ memcpy(real_nonce, nonce, nlen);
+
+ out = lua_new_text(L, NULL, inlen + crypto_secretbox_MACBYTES,
+ TRUE);
+ crypto_secretbox_easy((guchar *) out->start, in, inlen,
+ nonce, sbox->sk);
+
+ return 1;
+ }
+ else {
+ /* Random nonce */
+ struct rspamd_lua_text *random_nonce;
+
+ out = lua_new_text(L, NULL, inlen + crypto_secretbox_MACBYTES,
+ TRUE);
+ random_nonce = lua_new_text(L, NULL, crypto_secretbox_NONCEBYTES, TRUE);
+
+ randombytes_buf((guchar *) random_nonce->start, random_nonce->len);
+ crypto_secretbox_easy((guchar *) out->start, in, inlen,
+ random_nonce->start, sbox->sk);
+
+ return 2; /* output + random nonce */
+ }
+}
+
+/***
+ * @method rspamd_cryptobox_secretbox:decrypt(input, nonce)
+ * Decrypts data using secretbox
+ * @param {string/text} nonce nonce used to encrypt
+ * @param {string/text} input input to decrypt
+ * @param {table} params optional parameters - NYI
+ * @return {boolean},{rspamd_text} decryption result + decrypted text
+ */
+static gint
+lua_cryptobox_secretbox_decrypt(lua_State *L)
+{
+ const gchar *in, *nonce;
+ gsize inlen, nlen;
+ struct rspamd_lua_cryptobox_secretbox *sbox =
+ lua_check_cryptobox_secretbox(L, 1);
+ struct rspamd_lua_text *out;
+
+ if (sbox == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Input argument */
+ if (lua_isstring(L, 2)) {
+ in = lua_tolstring(L, 2, &inlen);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ in = t->start;
+ inlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+ /* Nonce argument */
+ if (lua_isstring(L, 3)) {
+ nonce = lua_tolstring(L, 3, &nlen);
+ }
+ else if (lua_isuserdata(L, 3)) {
+ struct rspamd_lua_text *t = lua_check_text(L, 3);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments; userdata is not text");
+ }
+
+ nonce = t->start;
+ nlen = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments; userdata or string are expected");
+ }
+
+
+ if (nlen < 1 || nlen > crypto_secretbox_NONCEBYTES) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "invalid nonce");
+ return 2;
+ }
+
+ if (inlen < crypto_secretbox_MACBYTES) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "too short");
+ return 2;
+ }
+
+ guchar real_nonce[crypto_secretbox_NONCEBYTES];
+
+ memset(real_nonce, 0, sizeof(real_nonce));
+ memcpy(real_nonce, nonce, nlen);
+
+ out = lua_new_text(L, NULL, inlen - crypto_secretbox_MACBYTES,
+ TRUE);
+ gint text_pos = lua_gettop(L);
+
+ if (crypto_secretbox_open_easy((guchar *) out->start, in, inlen,
+ nonce, sbox->sk) == 0) {
+ lua_pushboolean(L, true);
+ lua_pushvalue(L, text_pos); /* Prevent gc by copying in stack */
+ }
+ else {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "authentication error");
+ }
+
+ /* This causes gc method if decryption has failed */
+ lua_remove(L, text_pos);
+
+ return 2;
+}
+
+static gint
+lua_load_pubkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxpubkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_keypair(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxkeypairlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_signature(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxsignlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_hash(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxhashlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_cryptobox_secretbox(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxsecretboxlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_cryptobox(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, cryptoboxlib_f);
+
+ return 1;
+}
+
+void luaopen_cryptobox(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{cryptobox_pubkey}", cryptoboxpubkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_pubkey", lua_load_pubkey);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_keypair}", cryptoboxkeypairlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_keypair", lua_load_keypair);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_signature}", cryptoboxsignlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_signature", lua_load_signature);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_hash}", cryptoboxhashlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_hash", lua_load_hash);
+
+ rspamd_lua_new_class(L, "rspamd{cryptobox_secretbox}",
+ cryptoboxsecretboxlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_cryptobox_secretbox",
+ lua_load_cryptobox_secretbox);
+
+ rspamd_lua_add_preload(L, "rspamd_cryptobox", lua_load_cryptobox);
+
+ lua_settop(L, 0);
+}
diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c
new file mode 100644
index 0000000..cffa312
--- /dev/null
+++ b/src/lua/lua_dns.c
@@ -0,0 +1,198 @@
+/*-
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_dns_resolver.h"
+#include "lua_thread_pool.h"
+
+LUA_FUNCTION_DEF(dns, request);
+
+static const struct luaL_reg dns_f[] = {
+ LUA_INTERFACE_DEF(dns, request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const gchar *M = "rspamd lua dns";
+
+void lua_dns_callback(struct rdns_reply *reply, void *arg);
+
+struct lua_rspamd_dns_cbdata {
+ struct thread_entry *thread;
+ struct rspamd_task *task;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+};
+
+static gint
+lua_dns_request(lua_State *L)
+{
+ GError *err = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct lua_rspamd_dns_cbdata *cbdata = NULL;
+ const gchar *to_resolve = NULL;
+ const gchar *type_str = NULL;
+ struct rspamd_task *task = NULL;
+ rspamd_mempool_t *pool = NULL;
+ gint ret = 0;
+ gboolean forced = FALSE;
+
+ /* Check arguments */
+ if (!rspamd_lua_parse_table_arguments(L, 1, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*name=S;task=U{task};*type=S;forced=B;session=U{session};config=U{config}",
+ &to_resolve,
+ &task,
+ &type_str,
+ &forced,
+ &session,
+ &cfg)) {
+
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task) {
+ session = task->s;
+ pool = task->task_pool;
+ cfg = task->cfg;
+ }
+ else if (session && cfg) {
+ pool = cfg->cfg_pool;
+ }
+ else {
+ return luaL_error(L, "invalid arguments: either task or session/config should be set");
+ }
+
+ enum rdns_request_type type = rdns_type_fromstr(type_str);
+
+ if (type == RDNS_REQUEST_INVALID) {
+ return luaL_error(L, "invalid arguments: this record type is not supported");
+ }
+
+ cbdata = rspamd_mempool_alloc0(pool, sizeof(*cbdata));
+
+ cbdata->task = task;
+
+ if (type == RDNS_REQUEST_PTR) {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ to_resolve = rspamd_mempool_strdup(pool, ptr_str);
+ free(ptr_str);
+ }
+
+
+ if (task == NULL) {
+ ret = (rspamd_dns_resolver_request(cfg->dns_resolver,
+ session,
+ pool,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve) != NULL);
+ }
+ else {
+ if (forced) {
+ ret = rspamd_dns_resolver_request_task_forced(task,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ else {
+ ret = rspamd_dns_resolver_request_task(task,
+ lua_dns_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ }
+
+ if (ret) {
+ cbdata->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ cbdata->s = session;
+
+ if (task) {
+ cbdata->item = rspamd_symcache_get_cur_item(task);
+ rspamd_symcache_item_async_inc(task, cbdata->item, M);
+ }
+
+ return lua_thread_yield(cbdata->thread, 0);
+ }
+ else {
+ lua_pushnil(L);
+ return 1;
+ }
+}
+
+void lua_dns_callback(struct rdns_reply *reply, void *arg)
+{
+ struct lua_rspamd_dns_cbdata *cbdata = arg;
+ lua_State *L = cbdata->thread->lua_state;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, rdns_strerror(reply->code));
+ }
+ else {
+ lua_push_dns_reply(L, reply);
+
+ lua_pushboolean(L, reply->flags & RDNS_AUTH);
+ lua_setfield(L, -3, "authenticated");
+
+ lua_pushboolean(L, reply->flags & RDNS_TRUNCATED);
+ lua_setfield(L, -3, "truncated");
+
+ /* result 1 - not and error */
+ lua_pushboolean(L, true);
+ /* push table into stack, result 2 - results itself */
+ lua_pushvalue(L, -3);
+ }
+
+ lua_thread_resume(cbdata->thread, 2);
+
+ if (cbdata->item) {
+ rspamd_symcache_item_async_dec_check(cbdata->task, cbdata->item, M);
+ }
+}
+
+static gint
+lua_load_dns(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, dns_f);
+
+ return 1;
+}
+
+void luaopen_dns(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_dns", lua_load_dns);
+}
diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c
new file mode 100644
index 0000000..b022e13
--- /dev/null
+++ b/src/lua/lua_dns_resolver.c
@@ -0,0 +1,754 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+
+
+/***
+ * @module rspamd_resolver
+ * This module allows to resolve DNS names from LUA code. All resolving is executed
+ * asynchronously. Here is an example of name resolution:
+ * @example
+local function symbol_callback(task)
+ local host = 'example.com'
+
+ local function dns_cb(resolver, to_resolve, results, err, _, authenticated)
+ if not results then
+ rspamd_logger.infox('DNS resolving of %1 failed: %2', host, err)
+ return
+ end
+ for _,r in ipairs(results) do
+ -- r is of type rspamd{ip} here, but it can be converted to string
+ rspamd_logger.infox('Resolved %1 to %2', host, tostring(r))
+ end
+ end
+
+ task:get_resolver():resolve_a({task = task, name = host, callback = dns_cb})
+end
+ */
+
+static const gchar *M = "rspamd lua dns resolver";
+
+/* Lua bindings */
+LUA_FUNCTION_DEF(dns_resolver, init);
+LUA_FUNCTION_DEF(dns_resolver, resolve_a);
+LUA_FUNCTION_DEF(dns_resolver, resolve_ptr);
+LUA_FUNCTION_DEF(dns_resolver, resolve_txt);
+LUA_FUNCTION_DEF(dns_resolver, resolve_mx);
+LUA_FUNCTION_DEF(dns_resolver, resolve_ns);
+LUA_FUNCTION_DEF(dns_resolver, resolve);
+LUA_FUNCTION_DEF(dns_resolver, idna_convert_utf8);
+
+void lua_push_dns_reply(lua_State *L, const struct rdns_reply *reply);
+
+static const struct luaL_reg dns_resolverlib_f[] = {
+ LUA_INTERFACE_DEF(dns_resolver, init),
+ {NULL, NULL}};
+
+static const struct luaL_reg dns_resolverlib_m[] = {
+ LUA_INTERFACE_DEF(dns_resolver, resolve_a),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_ptr),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_txt),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_mx),
+ LUA_INTERFACE_DEF(dns_resolver, resolve_ns),
+ LUA_INTERFACE_DEF(dns_resolver, resolve),
+ LUA_INTERFACE_DEF(dns_resolver, idna_convert_utf8),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct rspamd_dns_resolver *
+lua_check_dns_resolver(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{resolver}");
+ luaL_argcheck(L, ud != NULL, pos, "'resolver' expected");
+ return ud ? *((struct rspamd_dns_resolver **) ud) : NULL;
+}
+
+struct lua_dns_cbdata {
+ struct rspamd_task *task;
+ rspamd_mempool_t *pool;
+ struct rspamd_dns_resolver *resolver;
+ gint cbref;
+ gchar *to_resolve;
+ gchar *user_str;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+};
+
+static int
+lua_dns_get_type(lua_State *L, int argno)
+{
+ int type = RDNS_REQUEST_A;
+ const gchar *strtype;
+
+ if (lua_type(L, argno) != LUA_TSTRING) {
+ lua_pushvalue(L, argno);
+ lua_gettable(L, lua_upvalueindex(1));
+
+ type = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ if (type == 0) {
+ rspamd_lua_typerror(L, argno, "dns_request_type");
+ }
+ }
+ else {
+ strtype = lua_tostring(L, argno);
+ type = rdns_type_fromstr(strtype);
+ }
+
+ return type;
+}
+
+static void
+lua_dns_resolver_callback(struct rdns_reply *reply, gpointer arg)
+{
+ struct lua_dns_cbdata *cd = arg;
+ struct rspamd_dns_resolver **presolver;
+ lua_State *L;
+ struct lua_callback_state cbs;
+ rspamd_mempool_t *pool;
+ gint err_idx;
+
+ pool = cd->pool;
+ lua_thread_pool_prepare_callback(cd->resolver->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cd->cbref);
+
+ presolver = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+
+ *presolver = cd->resolver;
+ lua_pushstring(L, cd->to_resolve);
+
+ lua_push_dns_reply(L, reply);
+
+ /*
+ * 1 - resolver
+ * 2 - to_resolve
+ * 3 - entries | nil
+ * 4 - error | nil
+ * 5 - user_str
+ * 6 - reply->flags & RDNS_AUTH
+ * 7 - server
+ */
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_pushnil(L);
+ lua_pushstring(L, rdns_strerror(reply->code));
+ }
+ if (cd->user_str != NULL) {
+ lua_pushstring(L, cd->user_str);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ lua_pushboolean(L, reply->flags & RDNS_AUTH);
+
+ const gchar *servname = rdns_request_get_server(reply->request);
+
+ if (servname) {
+ lua_pushstring(L, servname);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (cd->item) {
+ /* We also need to restore the item in case there are some chains */
+ rspamd_symcache_set_cur_item(cd->task, cd->item);
+ }
+
+ if (lua_pcall(L, 7, 0, err_idx) != 0) {
+ msg_err_pool_check("call to dns callback failed: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+
+ /* Unref function */
+ luaL_unref(L, LUA_REGISTRYINDEX, cd->cbref);
+ lua_thread_pool_restore_callback(&cbs);
+
+ if (cd->item) {
+ rspamd_symcache_item_async_dec_check(cd->task, cd->item, M);
+ }
+
+ if (!cd->pool) {
+ g_free(cd->to_resolve);
+ g_free(cd->user_str);
+ g_free(cd);
+ }
+}
+
+void lua_push_dns_reply(lua_State *L, const struct rdns_reply *reply)
+{
+ gint i = 0, naddrs = 0;
+ struct rdns_reply_entry *elt;
+ rspamd_inet_addr_t *addr;
+
+ if (reply->code == RDNS_RC_NOERROR) {
+ LL_FOREACH(reply->entries, elt)
+ {
+ naddrs++;
+ }
+
+ lua_createtable(L, naddrs, 0);
+
+ LL_FOREACH(reply->entries, elt)
+ {
+ if (!rdns_request_has_type(reply->request, elt->type)) {
+ /* Unrequested type has been returned, ignore it */
+ continue;
+ }
+
+ switch (elt->type) {
+ case RDNS_REQUEST_A:
+ addr = rspamd_inet_address_new(AF_INET, &elt->content.a.addr);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_AAAA:
+ addr = rspamd_inet_address_new(AF_INET6, &elt->content.aaa.addr);
+ rspamd_lua_ip_push(L, addr);
+ rspamd_inet_address_free(addr);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_NS:
+ lua_pushstring(L, elt->content.ns.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_PTR:
+ lua_pushstring(L, elt->content.ptr.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_TXT:
+ case RDNS_REQUEST_SPF:
+ lua_pushstring(L, elt->content.txt.data);
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_MX:
+ /* mx['name'], mx['priority'] */
+ lua_createtable(L, 0, 2);
+ rspamd_lua_table_set(L, "name", elt->content.mx.name);
+ lua_pushstring(L, "priority");
+ lua_pushinteger(L, elt->content.mx.priority);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_SOA:
+ lua_createtable(L, 0, 7);
+ rspamd_lua_table_set(L, "ns", elt->content.soa.mname);
+ rspamd_lua_table_set(L, "contact", elt->content.soa.admin);
+ lua_pushstring(L, "serial");
+ lua_pushinteger(L, elt->content.soa.serial);
+ lua_settable(L, -3);
+ lua_pushstring(L, "refresh");
+ lua_pushinteger(L, elt->content.soa.refresh);
+ lua_settable(L, -3);
+ lua_pushstring(L, "retry");
+ lua_pushinteger(L, elt->content.soa.retry);
+ lua_settable(L, -3);
+ lua_pushstring(L, "expiry");
+ lua_pushinteger(L, elt->content.soa.expire);
+ lua_settable(L, -3);
+ /* Negative TTL */
+ lua_pushstring(L, "nx");
+ lua_pushinteger(L, elt->content.soa.minimum);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, ++i);
+ break;
+ case RDNS_REQUEST_CNAME:
+ lua_pushstring(L, elt->content.cname.name);
+ lua_rawseti(L, -2, ++i);
+ break;
+ default:
+ continue;
+ }
+ }
+ lua_pushnil(L);
+ }
+}
+
+/***
+ * @function rspamd_resolver.init(ev_base, config)
+ * @param {event_base} ev_base event base used for asynchronous events
+ * @param {rspamd_config} config rspamd configuration parameters
+ * @return {rspamd_resolver} new resolver object associated with the specified base
+ */
+static int
+lua_dns_resolver_init(lua_State *L)
+{
+ struct rspamd_dns_resolver *resolver, **presolver;
+ struct rspamd_config *cfg, **pcfg;
+ struct ev_loop *base, **pbase;
+
+ /* Check args */
+ pbase = rspamd_lua_check_udata(L, 1, "rspamd{ev_base}");
+ luaL_argcheck(L, pbase != NULL, 1, "'ev_base' expected");
+ base = pbase ? *(pbase) : NULL;
+ pcfg = rspamd_lua_check_udata(L, 2, "rspamd{config}");
+ luaL_argcheck(L, pcfg != NULL, 2, "'config' expected");
+ cfg = pcfg ? *(pcfg) : NULL;
+
+ if (base != NULL && cfg != NULL) {
+ resolver = rspamd_dns_resolver_init(NULL, base, cfg);
+ if (resolver) {
+ presolver = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *presolver = resolver;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_dns_resolver_resolve_common(lua_State *L,
+ struct rspamd_dns_resolver *resolver,
+ enum rdns_request_type type,
+ int first)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session *session = NULL;
+ rspamd_mempool_t *pool = NULL;
+ const gchar *to_resolve = NULL, *user_str = NULL;
+ struct lua_dns_cbdata *cbdata;
+ gint cbref = -1, ret;
+ struct rspamd_task *task = NULL;
+ GError *err = NULL;
+ gboolean forced = FALSE;
+ struct rspamd_symcache_dynamic_item *item = NULL;
+
+ /* Check arguments */
+ if (!rspamd_lua_parse_table_arguments(L, first, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "session=U{session};mempool=U{mempool};*name=S;*callback=F;"
+ "option=S;task=U{task};forced=B",
+ &session, &pool, &to_resolve, &cbref, &user_str, &task, &forced)) {
+
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task) {
+ pool = task->task_pool;
+ session = task->s;
+ item = rspamd_symcache_get_cur_item(task);
+ }
+
+ if (to_resolve != NULL) {
+ if (pool != NULL) {
+ cbdata = rspamd_mempool_alloc0(pool, sizeof(struct lua_dns_cbdata));
+ cbdata->user_str = rspamd_mempool_strdup(pool, user_str);
+
+ if (type != RDNS_REQUEST_PTR) {
+ cbdata->to_resolve = rspamd_mempool_strdup(pool, to_resolve);
+ }
+ else {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ goto err;
+ }
+
+ cbdata->to_resolve = rspamd_mempool_strdup(pool, ptr_str);
+ to_resolve = cbdata->to_resolve;
+ free(ptr_str);
+ }
+ }
+ else {
+ cbdata = g_malloc0(sizeof(struct lua_dns_cbdata));
+ cbdata->user_str = user_str ? g_strdup(user_str) : NULL;
+
+ if (type != RDNS_REQUEST_PTR) {
+ cbdata->to_resolve = g_strdup(to_resolve);
+ }
+ else {
+ char *ptr_str;
+
+ ptr_str = rdns_generate_ptr_from_str(to_resolve);
+
+ if (ptr_str == NULL) {
+ msg_err_task_check("wrong resolve string to PTR request: %s",
+ to_resolve);
+ goto err;
+ }
+
+ cbdata->to_resolve = g_strdup(ptr_str);
+ free(ptr_str);
+ }
+ }
+
+ cbdata->resolver = resolver;
+ cbdata->cbref = cbref;
+ cbdata->task = task;
+ cbdata->pool = pool;
+
+ if (task == NULL) {
+ if (rspamd_dns_resolver_request(resolver,
+ session,
+ pool,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve)) {
+
+ lua_pushboolean(L, TRUE);
+
+ if (session) {
+ cbdata->s = session;
+ }
+ }
+ else {
+ goto err;
+ }
+ }
+ else {
+ /* Fail-safety as this function can, in theory, call
+ * lua_dns_resolver_callback without switching to the event loop
+ */
+ if (item) {
+ rspamd_symcache_item_async_inc(task, item, M);
+ }
+
+ if (forced) {
+ ret = rspamd_dns_resolver_request_task_forced(task,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+ else {
+ ret = rspamd_dns_resolver_request_task(task,
+ lua_dns_resolver_callback,
+ cbdata,
+ type,
+ to_resolve);
+ }
+
+ if (ret) {
+ cbdata->s = session;
+
+ if (item) {
+ cbdata->item = item;
+ rspamd_symcache_item_async_inc(task, item, M);
+ }
+ /* callback was set up */
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+
+ goto err;
+ }
+
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments to lua_resolve");
+ }
+
+ return 1;
+
+err:
+ /* Callback is not called in this case */
+ if (cbdata->cbref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cbref);
+ }
+
+ if (!pool) {
+ /* Free resources */
+ g_free(cbdata->to_resolve);
+ g_free(cbdata->user_str);
+ g_free(cbdata);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_a(table)
+ * Resolve A record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_a(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_A,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_ptr(table)
+ * Resolve PTR record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_ptr(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_PTR,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_txt(table)
+ * Resolve TXT record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_txt(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_TXT,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_mx(table)
+ * Resolve MX record for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_mx(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_MX,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:resolve_ns(table)
+ * Resolve NS records for a specified host.
+ * Table elements:
+ * * `task` - task element (preferred, required to track dependencies) -or-
+ * * `session` - asynchronous session normally associated with rspamd task (`task:get_session()`)
+ * * `mempool` - pool memory pool for storing intermediate data
+ * * `name` - host name to resolve
+ * * `callback` - callback callback function to be called upon name resolution is finished; must be of type `function (resolver, to_resolve, results, err)`
+ * * `forced` - true if needed to override normal limit for DNS requests
+ * @return {boolean} `true` if DNS request has been scheduled
+ */
+static int
+lua_dns_resolver_resolve_ns(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+
+ if (dns_resolver) {
+ return lua_dns_resolver_resolve_common(L,
+ dns_resolver,
+ RDNS_REQUEST_NS,
+ 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/* XXX: broken currently */
+static int
+lua_dns_resolver_resolve(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+ int type;
+
+ type = lua_dns_get_type(L, 2);
+
+ if (dns_resolver && type != 0) {
+ return lua_dns_resolver_resolve_common(L, dns_resolver, type, 3);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method resolver:idna_convert_utf8(hostname[, pool])
+ * Converts domain name from IDN (in utf8 format) to punycode
+ * @return {string} new name converted
+ */
+static int
+lua_dns_resolver_idna_convert_utf8(lua_State *L)
+{
+ struct rspamd_dns_resolver *dns_resolver = lua_check_dns_resolver(L, 1);
+ gsize hlen;
+ guint conv_len = 0;
+ const gchar *hname = luaL_checklstring(L, 2, &hlen);
+ gchar *converted;
+ rspamd_mempool_t *pool = rspamd_lua_check_udata_maybe(L, 3, "rspamd{mempool}");
+
+
+ if (dns_resolver && hname) {
+ if (!rspamd_str_has_8bit(hname, hlen)) {
+ /* No 8 bit, no reasons to call idna */
+ lua_pushlstring(L, hname, hlen);
+ }
+ else {
+ converted = rspamd_dns_resolver_idna_convert_utf8(dns_resolver, pool,
+ hname, hlen, &conv_len);
+
+ if (converted == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, converted, conv_len);
+
+ if (pool == NULL) {
+ g_free(converted);
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_dns_resolver(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, dns_resolverlib_f);
+
+ return 1;
+}
+
+void luaopen_dns_resolver(lua_State *L)
+{
+
+ rspamd_lua_new_class(L, "rspamd{resolver}", dns_resolverlib_m);
+ {
+ LUA_ENUM(L, DNS_A, RDNS_REQUEST_A);
+ LUA_ENUM(L, DNS_PTR, RDNS_REQUEST_PTR);
+ LUA_ENUM(L, DNS_MX, RDNS_REQUEST_MX);
+ LUA_ENUM(L, DNS_TXT, RDNS_REQUEST_TXT);
+ LUA_ENUM(L, DNS_SRV, RDNS_REQUEST_SRV);
+ LUA_ENUM(L, DNS_SPF, RDNS_REQUEST_SPF);
+ LUA_ENUM(L, DNS_AAAA, RDNS_REQUEST_AAAA);
+ LUA_ENUM(L, DNS_SOA, RDNS_REQUEST_SOA);
+ LUA_ENUM(L, DNS_CNAME, RDNS_REQUEST_CNAME);
+ }
+
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_resolver", lua_load_dns_resolver);
+}
diff --git a/src/lua/lua_dns_resolver.h b/src/lua/lua_dns_resolver.h
new file mode 100644
index 0000000..515e9ac
--- /dev/null
+++ b/src/lua/lua_dns_resolver.h
@@ -0,0 +1,15 @@
+#ifndef RSPAMD_LUA_DNS_H
+#define RSPAMD_LUA_DNS_H
+
+struct lua_State;
+struct rdns_reply;
+
+/**
+ * Pushes dns reply onto Lua stack
+ *
+ * @param L
+ * @param reply
+ */
+void lua_push_dns_reply(struct lua_State *L, const struct rdns_reply *reply);
+
+#endif
diff --git a/src/lua/lua_expression.c b/src/lua/lua_expression.c
new file mode 100644
index 0000000..1ac6f86
--- /dev/null
+++ b/src/lua/lua_expression.c
@@ -0,0 +1,512 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "expression.h"
+
+/***
+ * @module rspamd_expression
+ * This module can be used to implement different logic expressions in lua using
+ * rspamd AST optimizer. There are some examples in individual methods definitions to help understanding of this module.
+ */
+
+/***
+ * @function rspamd_expression.create(line, {parse_func, [process_func]}, pool)
+ * Create expression from the line using atom parsing routines and the specified memory pool
+ * @param {string} line expression line
+ * @param {table} atom_functions parse_atom function and optional process_atom function
+ * @param {rspamd_mempool} memory pool to use for this function
+ * @return {expr, err} expression object and error message of `expr` is nil
+ * @example
+require "fun" ()
+local rspamd_expression = require "rspamd_expression"
+local rspamd_mempool = require "rspamd_mempool"
+
+local function parse_func(str)
+ -- extract token till the first space character
+ local token = table.concat(totable(take_while(function(s) return s ~= ' ' end, iter(str))))
+ -- Return token name
+ return token
+end
+
+local function process_func(token)
+ -- Do something using token and task
+end
+
+local pool = rspamd_mempool.create()
+local expr,err = rspamd_expression.create('A & B | !C', {parse_func, process_func}, pool)
+-- Expression is destroyed when the corresponding pool is destroyed
+pool:destroy()
+ */
+LUA_FUNCTION_DEF(expr, create);
+
+/***
+ * @method rspamd_expression:to_string()
+ * Converts rspamd expression to string
+ * @return {string} string representation of rspamd expression
+ */
+LUA_FUNCTION_DEF(expr, to_string);
+
+/***
+ * @method rspamd_expression:process([callback, ]input[, flags])
+ * Executes the expression and pass input to process atom callbacks
+ * @param {function} callback if not specified on process, then callback must be here
+ * @param {any} input input data for processing callbacks
+ * @return {number} result of the expression evaluation
+ */
+LUA_FUNCTION_DEF(expr, process);
+
+/***
+ * @method rspamd_expression:process_traced([callback, ]input[, flags])
+ * Executes the expression and pass input to process atom callbacks. This function also saves the full trace
+ * @param {function} callback if not specified on process, then callback must be here
+ * @param {any} input input data for processing callbacks
+ * @return {number, table of matched atoms} result of the expression evaluation
+ */
+LUA_FUNCTION_DEF(expr, process_traced);
+
+/***
+ * @method rspamd_expression:atoms()
+ * Extract all atoms from the expression as table of strings
+ * @return {table/strings} list of all atoms in the expression
+ */
+LUA_FUNCTION_DEF(expr, atoms);
+
+static const struct luaL_reg exprlib_m[] = {
+ LUA_INTERFACE_DEF(expr, to_string),
+ LUA_INTERFACE_DEF(expr, atoms),
+ LUA_INTERFACE_DEF(expr, process),
+ LUA_INTERFACE_DEF(expr, process_traced),
+ {"__tostring", lua_expr_to_string},
+ {NULL, NULL}};
+
+static const struct luaL_reg exprlib_f[] = {
+ LUA_INTERFACE_DEF(expr, create),
+ {NULL, NULL}};
+
+static rspamd_expression_atom_t *lua_atom_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err);
+static gdouble lua_atom_process(gpointer runtime_ud, rspamd_expression_atom_t *atom);
+
+static const struct rspamd_atom_subr lua_atom_subr = {
+ .parse = lua_atom_parse,
+ .process = lua_atom_process,
+ .priority = NULL,
+ .destroy = NULL};
+
+struct lua_expression {
+ struct rspamd_expression *expr;
+ gint parse_idx;
+ gint process_idx;
+ lua_State *L;
+ rspamd_mempool_t *pool;
+};
+
+static GQuark
+lua_expr_quark(void)
+{
+ return g_quark_from_static_string("lua-expression");
+}
+
+struct lua_expression *
+rspamd_lua_expression(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{expr}");
+ luaL_argcheck(L, ud != NULL, pos, "'expr' expected");
+ return ud ? *((struct lua_expression **) ud) : NULL;
+}
+
+static void
+lua_expr_dtor(gpointer p)
+{
+ struct lua_expression *e = (struct lua_expression *) p;
+
+ if (e->parse_idx != -1) {
+ luaL_unref(e->L, LUA_REGISTRYINDEX, e->parse_idx);
+ }
+
+ if (e->process_idx != -1) {
+ luaL_unref(e->L, LUA_REGISTRYINDEX, e->process_idx);
+ }
+}
+
+static rspamd_expression_atom_t *
+lua_atom_parse(const gchar *line, gsize len,
+ rspamd_mempool_t *pool, gpointer ud, GError **err)
+{
+ struct lua_expression *e = (struct lua_expression *) ud;
+ rspamd_expression_atom_t *atom;
+ gsize rlen;
+ const gchar *tok;
+
+ lua_rawgeti(e->L, LUA_REGISTRYINDEX, e->parse_idx);
+ lua_pushlstring(e->L, line, len);
+
+ if (lua_pcall(e->L, 1, 1, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(e->L, -1));
+ lua_pop(e->L, 1);
+ return NULL;
+ }
+
+ if (lua_type(e->L, -1) != LUA_TSTRING) {
+ g_set_error(err, lua_expr_quark(), 500, "cannot parse lua atom");
+ lua_pop(e->L, 1);
+ return NULL;
+ }
+
+ tok = lua_tolstring(e->L, -1, &rlen);
+ atom = rspamd_mempool_alloc0(e->pool, sizeof(*atom));
+ atom->str = rspamd_mempool_strdup(e->pool, tok);
+ atom->len = rlen;
+ atom->data = ud;
+
+ lua_pop(e->L, 1);
+
+ return atom;
+}
+
+struct lua_atom_process_data {
+ lua_State *L;
+ struct lua_expression *e;
+ gint process_cb_pos;
+ gint stack_item;
+};
+
+static gdouble
+lua_atom_process(gpointer runtime_ud, rspamd_expression_atom_t *atom)
+{
+ struct lua_atom_process_data *pd = (struct lua_atom_process_data *) runtime_ud;
+ gdouble ret = 0;
+ guint nargs;
+ gint err_idx;
+
+ if (pd->stack_item != -1) {
+ nargs = 2;
+ }
+ else {
+ nargs = 1;
+ }
+
+ lua_pushcfunction(pd->L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(pd->L);
+
+ /* Function position */
+ lua_pushvalue(pd->L, pd->process_cb_pos);
+ /* Atom name */
+ lua_pushlstring(pd->L, atom->str, atom->len);
+
+ /* If we have data passed */
+ if (pd->stack_item != -1) {
+ lua_pushvalue(pd->L, pd->stack_item);
+ }
+
+ if (lua_pcall(pd->L, nargs, 1, err_idx) != 0) {
+ msg_info("expression process callback failed: %s", lua_tostring(pd->L, -1));
+ }
+ else {
+ ret = lua_tonumber(pd->L, -1);
+ }
+
+ lua_settop(pd->L, err_idx - 1);
+
+ return ret;
+}
+
+static gint
+lua_expr_process(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_atom_process_data pd;
+ gdouble res;
+ gint flags = 0, old_top;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop(L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction(L, 2)) {
+ return luaL_error(L, "expression process is called with no callback function");
+ }
+
+ pd.process_cb_pos = 2;
+
+ if (lua_type(L, 3) != LUA_TNONE && lua_type(L, 3) != LUA_TNIL) {
+ pd.stack_item = 3;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+
+ if (lua_isnumber(L, 4)) {
+ flags = lua_tointeger(L, 4);
+ }
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop(L);
+
+ if (lua_type(L, 2) != LUA_TNONE && lua_type(L, 2) != LUA_TNIL) {
+ pd.stack_item = 2;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+
+ if (lua_isnumber(L, 3)) {
+ flags = lua_tointeger(L, 3);
+ }
+ }
+
+ res = rspamd_process_expression(e->expr, flags, &pd);
+
+ lua_settop(L, old_top);
+ lua_pushnumber(L, res);
+
+ return 1;
+}
+
+static gint
+lua_expr_process_traced(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_atom_process_data pd;
+ gdouble res;
+ gint flags = 0, old_top;
+ GPtrArray *trace;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop(L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction(L, 2)) {
+ return luaL_error(L, "expression process is called with no callback function");
+ }
+
+ pd.process_cb_pos = 2;
+ pd.stack_item = 3;
+
+ if (lua_isnumber(L, 4)) {
+ flags = lua_tointeger(L, 4);
+ }
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop(L);
+ pd.stack_item = 2;
+
+ if (lua_isnumber(L, 3)) {
+ flags = lua_tointeger(L, 3);
+ }
+ }
+
+ res = rspamd_process_expression_track(e->expr, flags, &pd, &trace);
+
+ lua_settop(L, old_top);
+ lua_pushnumber(L, res);
+
+ lua_createtable(L, trace->len, 0);
+
+ for (guint i = 0; i < trace->len; i++) {
+ struct rspamd_expression_atom_s *atom = g_ptr_array_index(trace, i);
+
+ lua_pushlstring(L, atom->str, atom->len);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_ptr_array_free(trace, TRUE);
+
+ return 2;
+}
+
+static gint
+lua_expr_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e, **pe;
+ const char *line;
+ gsize len;
+ gboolean no_process = FALSE;
+ GError *err = NULL;
+ rspamd_mempool_t *pool;
+
+ /* Check sanity of the arguments */
+ if (lua_type(L, 1) != LUA_TSTRING ||
+ (lua_type(L, 2) != LUA_TTABLE && lua_type(L, 2) != LUA_TFUNCTION) ||
+ rspamd_lua_check_mempool(L, 3) == NULL) {
+ lua_pushnil(L);
+ lua_pushstring(L, "bad arguments");
+ }
+ else {
+ line = lua_tolstring(L, 1, &len);
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ e = rspamd_mempool_alloc(pool, sizeof(*e));
+ e->L = L;
+ e->pool = pool;
+
+ /* Check callbacks */
+ if (lua_istable(L, 2)) {
+ lua_pushvalue(L, 2);
+ lua_pushnumber(L, 1);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ lua_pushstring(L, "bad parse callback");
+
+ return 2;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushnumber(L, 2);
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) != LUA_TFUNCTION) {
+ if (lua_type(L, -1) != LUA_TNIL && lua_type(L, -1) != LUA_TNONE) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ lua_pushstring(L, "bad process callback");
+
+ return 2;
+ }
+ else {
+ no_process = TRUE;
+ }
+ }
+
+ lua_pop(L, 1);
+ /* Table is still on the top of stack */
+
+ lua_pushnumber(L, 1);
+ lua_gettable(L, -2);
+ e->parse_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (!no_process) {
+ lua_pushnumber(L, 2);
+ lua_gettable(L, -2);
+ e->process_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ e->process_idx = -1;
+ }
+
+ lua_pop(L, 1); /* Table */
+ }
+ else {
+ /* Process function is just a function, not a table */
+ lua_pushvalue(L, 2);
+ e->parse_idx = luaL_ref(L, LUA_REGISTRYINDEX);
+ e->process_idx = -1;
+ }
+
+ if (!rspamd_parse_expression(line, len, &lua_atom_subr, e, pool, &err,
+ &e->expr)) {
+ lua_pushnil(L);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+ lua_expr_dtor(e);
+
+ return 2;
+ }
+
+ rspamd_mempool_add_destructor(pool, lua_expr_dtor, e);
+ pe = lua_newuserdata(L, sizeof(struct lua_expression *));
+ rspamd_lua_setclass(L, "rspamd{expr}", -1);
+ *pe = e;
+ lua_pushnil(L);
+ }
+
+ return 2;
+}
+
+static gint
+lua_expr_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ GString *str;
+
+ if (e != NULL && e->expr != NULL) {
+ str = rspamd_expression_tostring(e->expr);
+ if (str) {
+ lua_pushlstring(L, str->str, str->len);
+ g_string_free(str, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+struct lua_expr_atoms_cbdata {
+ lua_State *L;
+ gint idx;
+};
+
+static void
+lua_exr_atom_cb(const rspamd_ftok_t *tok, gpointer ud)
+{
+ struct lua_expr_atoms_cbdata *cbdata = ud;
+
+ lua_pushlstring(cbdata->L, tok->begin, tok->len);
+ lua_rawseti(cbdata->L, -2, cbdata->idx++);
+}
+
+static gint
+lua_expr_atoms(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_expression *e = rspamd_lua_expression(L, 1);
+ struct lua_expr_atoms_cbdata cbdata;
+
+ if (e != NULL && e->expr != NULL) {
+ lua_newtable(L);
+ cbdata.L = L;
+ cbdata.idx = 1;
+ rspamd_expression_atom_foreach(e->expr, lua_exr_atom_cb, &cbdata);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_expression(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, exprlib_f);
+
+ return 1;
+}
+
+void luaopen_expression(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{expr}", exprlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_expression", lua_load_expression);
+}
diff --git a/src/lua/lua_html.cxx b/src/lua/lua_html.cxx
new file mode 100644
index 0000000..6613337
--- /dev/null
+++ b/src/lua/lua_html.cxx
@@ -0,0 +1,738 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "message.h"
+#include "libserver/html/html.h"
+#include "libserver/html/html.hxx"
+#include "libserver/html/html_tag.hxx"
+#include "libserver/html/html_block.hxx"
+#include "images.h"
+
+#include "contrib/ankerl/unordered_dense.h"
+#include <frozen/string.h>
+#include <frozen/unordered_map.h>
+
+/***
+ * @module rspamd_html
+ * This module provides different methods to access HTML tags. To get HTML context
+ * from an HTML part you could use method `part:get_html()`
+ * @example
+rspamd_config.R_EMPTY_IMAGE = function(task)
+ local tp = task:get_text_parts() -- get text parts in a message
+
+ for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs`
+ if p:is_html() then -- if the current part is html part
+ local hc = p:get_html() -- we get HTML context
+ local len = p:get_length() -- and part's length
+
+ if len < 50 then -- if we have a part that has less than 50 bytes of text
+ local images = hc:get_images() -- then we check for HTML images
+
+ if images then -- if there are images
+ for _,i in ipairs(images) do -- then iterate over images in the part
+ if i['height'] + i['width'] >= 400 then -- if we have a large image
+ return true -- add symbol
+ end
+ end
+ end
+ end
+ end
+ end
+end
+ */
+
+/***
+ * @method html:has_tag(name)
+ * Checks if a specified tag `name` is presented in a part
+ * @param {string} name name of tag to check
+ * @return {boolean} `true` if the tag exists in HTML tree
+ */
+LUA_FUNCTION_DEF(html, has_tag);
+
+/***
+ * @method html:check_property(name)
+ * Checks if the HTML has a specific property. Here is the list of available properties:
+ *
+ * - `no_html` - no html tag presented
+ * - `bad_element` - part has some broken elements
+ * - `xml` - part is xhtml
+ * - `unknown_element` - part has some unknown elements
+ * - `duplicate_element` - part has some duplicate elements that should be unique (namely, `title` tag)
+ * - `unbalanced` - part has unbalanced tags
+ * @param {string} name name of property
+ * @return {boolean} true if the part has the specified property
+ */
+LUA_FUNCTION_DEF(html, has_property);
+
+/***
+ * @method html:get_images()
+ * Returns a table of images found in html. Each image is, in turn, a table with the following fields:
+ *
+ * - `src` - link to the source
+ * - `height` - height in pixels
+ * - `width` - width in pixels
+ * - `embedded` - `true` if an image is embedded in a message
+ * @return {table} table of images in html part
+ */
+LUA_FUNCTION_DEF(html, get_images);
+
+/***
+ * @method html:foreach_tag(tagname, callback)
+ * Processes HTML tree calling the specified callback for each tag of the specified
+ * type.
+ *
+ * Callback is called with the following attributes:
+ *
+ * - `tag`: html tag structure
+ * - `content_length`: length of content within a tag
+ *
+ * Callback function should return `true` to **stop** processing and `false` to continue
+ * @return nothing
+ */
+LUA_FUNCTION_DEF(html, foreach_tag);
+
+/***
+ * @method html:get_invisible()
+ * Returns invisible content of the HTML data
+ * @return
+ */
+LUA_FUNCTION_DEF(html, get_invisible);
+
+static const struct luaL_reg htmllib_m[] = {
+ LUA_INTERFACE_DEF(html, has_tag),
+ LUA_INTERFACE_DEF(html, has_property),
+ LUA_INTERFACE_DEF(html, get_images),
+ LUA_INTERFACE_DEF(html, foreach_tag),
+ LUA_INTERFACE_DEF(html, get_invisible),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/***
+ * @method html_tag:get_type()
+ * Returns string representation of HTML type for a tag
+ * @return {string} type of tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_type);
+/***
+ * @method html_tag:get_extra()
+ * Returns extra data associated with the tag
+ * @return {url|image|nil} extra data associated with the tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_extra);
+/***
+ * @method html_tag:get_parent()
+ * Returns parent node for a specified tag
+ * @return {html_tag} parent object for a specified tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_parent);
+
+/***
+ * @method html_tag:get_flags()
+ * Returns flags a specified tag:
+ *
+ * - `closed`: tag is properly closed
+ * - `closing`: tag is a closing tag
+ * - `broken`: tag is somehow broken
+ * - `unbalanced`: tag is unbalanced
+ * - `xml`: tag is xml tag
+ * @return {table} table of flags
+ */
+LUA_FUNCTION_DEF(html_tag, get_flags);
+/***
+ * @method html_tag:get_content()
+ * Returns content of tag (approximate for some cases)
+ * @return {rspamd_text} rspamd text with tag's content
+ */
+LUA_FUNCTION_DEF(html_tag, get_content);
+/***
+ * @method html_tag:get_content_length()
+ * Returns length of a tag's content
+ * @return {number} size of content enclosed within a tag
+ */
+LUA_FUNCTION_DEF(html_tag, get_content_length);
+
+/***
+ * @method html_tag:get_style()
+ * Returns style calculated for the element
+ * @return {table} table associated with the style
+ */
+LUA_FUNCTION_DEF(html_tag, get_style);
+
+/***
+ * @method html_tag:get_attribute(name)
+ * Returns value of attribute for the element
+ * Refer to `html_components_map` in `src/libserver/html/html.cxx` for recognised names
+ * @return {string|nil} value of the attribute
+ */
+LUA_FUNCTION_DEF(html_tag, get_attribute);
+
+static const struct luaL_reg taglib_m[] = {
+ LUA_INTERFACE_DEF(html_tag, get_type),
+ LUA_INTERFACE_DEF(html_tag, get_extra),
+ LUA_INTERFACE_DEF(html_tag, get_parent),
+ LUA_INTERFACE_DEF(html_tag, get_flags),
+ LUA_INTERFACE_DEF(html_tag, get_content),
+ LUA_INTERFACE_DEF(html_tag, get_content_length),
+ LUA_INTERFACE_DEF(html_tag, get_style),
+ LUA_INTERFACE_DEF(html_tag, get_attribute),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd::html::html_content *
+lua_check_html(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{html}");
+ luaL_argcheck(L, ud != NULL, pos, "'html' expected");
+ return ud ? *((struct rspamd::html::html_content **) ud) : NULL;
+}
+
+struct lua_html_tag {
+ rspamd::html::html_content *html;
+ const rspamd::html::html_tag *tag;
+};
+
+static struct lua_html_tag *
+lua_check_html_tag(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{html_tag}");
+ luaL_argcheck(L, ud != NULL, pos, "'html_tag' expected");
+ return ud ? ((struct lua_html_tag *) ud) : NULL;
+}
+
+static gint
+lua_html_has_tag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *tagname = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (hc && tagname) {
+ if (rspamd_html_tag_seen(hc, tagname)) {
+ ret = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+constexpr const auto prop_map = frozen::make_unordered_map<frozen::string, int>({
+ {"no_html", RSPAMD_HTML_FLAG_BAD_START},
+ {"bad_start", RSPAMD_HTML_FLAG_BAD_START},
+ {"bad_element", RSPAMD_HTML_FLAG_BAD_ELEMENTS},
+ {"bad_elements", RSPAMD_HTML_FLAG_BAD_ELEMENTS},
+ {"xml", RSPAMD_HTML_FLAG_XML},
+ {"unknown_element", RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS},
+ {"unknown_elements", RSPAMD_HTML_FLAG_UNKNOWN_ELEMENTS},
+ {"duplicate_element", RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS},
+ {"duplicate_elements", RSPAMD_HTML_FLAG_DUPLICATE_ELEMENTS},
+ {"unbalanced", RSPAMD_HTML_FLAG_UNBALANCED},
+ {"data_urls", RSPAMD_HTML_FLAG_HAS_DATA_URLS},
+});
+
+static gint
+lua_html_has_property(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *propname = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (hc && propname) {
+ auto found_prop = prop_map.find(frozen::string(propname));
+
+ if (found_prop != prop_map.end()) {
+ ret = hc->flags & found_prop->second;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static void
+lua_html_push_image(lua_State *L, const struct html_image *img)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag;
+ struct rspamd_url **purl;
+
+ lua_createtable(L, 0, 7);
+
+ if (img->src) {
+ lua_pushstring(L, "src");
+
+ if (img->flags & RSPAMD_HTML_FLAG_IMAGE_DATA) {
+ struct rspamd_lua_text *t;
+
+ t = static_cast<rspamd_lua_text *>(lua_newuserdata(L, sizeof(*t)));
+ t->start = img->src;
+ t->len = strlen(img->src);
+ t->flags = 0;
+
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+ else {
+ lua_pushstring(L, img->src);
+ }
+
+ lua_settable(L, -3);
+ }
+
+ if (img->url) {
+ lua_pushstring(L, "url");
+ purl = static_cast<rspamd_url **>(lua_newuserdata(L, sizeof(gpointer)));
+ *purl = img->url;
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_settable(L, -3);
+ }
+
+ if (img->tag) {
+ lua_pushstring(L, "tag");
+ ltag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(struct lua_html_tag)));
+ ltag->tag = static_cast<rspamd::html::html_tag *>(img->tag);
+ ltag->html = NULL;
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "height");
+ lua_pushinteger(L, img->height);
+ lua_settable(L, -3);
+ lua_pushstring(L, "width");
+ lua_pushinteger(L, img->width);
+ lua_settable(L, -3);
+ lua_pushstring(L, "embedded");
+ lua_pushboolean(L, img->flags & RSPAMD_HTML_FLAG_IMAGE_EMBEDDED);
+ lua_settable(L, -3);
+ lua_pushstring(L, "data");
+ lua_pushboolean(L, img->flags & RSPAMD_HTML_FLAG_IMAGE_DATA);
+ lua_settable(L, -3);
+}
+
+static gint
+lua_html_get_images(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ guint i = 1;
+
+ if (hc != NULL) {
+ lua_createtable(L, hc->images.size(), 0);
+
+ for (const auto *img: hc->images) {
+ lua_html_push_image(L, img);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+
+ return 1;
+}
+
+static void
+lua_html_push_block(lua_State *L, const struct rspamd::html::html_block *bl)
+{
+ LUA_TRACE_POINT;
+
+ lua_createtable(L, 0, 6);
+
+ if (bl->fg_color_mask) {
+ lua_pushstring(L, "color");
+ lua_createtable(L, 4, 0);
+ lua_pushinteger(L, bl->fg_color.r);
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, bl->fg_color.g);
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, bl->fg_color.b);
+ lua_rawseti(L, -2, 3);
+ lua_pushinteger(L, bl->fg_color.alpha);
+ lua_rawseti(L, -2, 4);
+ lua_settable(L, -3);
+ }
+ if (bl->bg_color_mask) {
+ lua_pushstring(L, "bgcolor");
+ lua_createtable(L, 4, 0);
+ lua_pushinteger(L, bl->bg_color.r);
+ lua_rawseti(L, -2, 1);
+ lua_pushinteger(L, bl->bg_color.g);
+ lua_rawseti(L, -2, 2);
+ lua_pushinteger(L, bl->bg_color.b);
+ lua_rawseti(L, -2, 3);
+ lua_pushinteger(L, bl->bg_color.alpha);
+ lua_rawseti(L, -2, 4);
+ lua_settable(L, -3);
+ }
+
+ if (bl->font_mask) {
+ lua_pushstring(L, "font_size");
+ lua_pushinteger(L, bl->font_size);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "visible");
+ lua_pushboolean(L, bl->is_visible());
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "transparent");
+ lua_pushboolean(L, bl->is_transparent());
+ lua_settable(L, -3);
+}
+
+static gint
+lua_html_foreach_tag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+ const gchar *tagname;
+ gint id;
+ auto any = false;
+ ankerl::unordered_dense::set<int> tags;
+
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ tagname = luaL_checkstring(L, 2);
+ if (strcmp(tagname, "any") == 0) {
+ any = true;
+ }
+ else {
+ id = rspamd_html_tag_by_name(tagname);
+
+ if (id == -1) {
+ return luaL_error(L, "invalid tagname: %s", tagname);
+ }
+
+
+ tags.insert(id);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ tagname = luaL_checkstring(L, -1);
+ if (strcmp(tagname, "any") == 0) {
+ any = TRUE;
+ }
+ else {
+ id = rspamd_html_tag_by_name(tagname);
+
+ if (id == -1) {
+ return luaL_error(L, "invalid tagname: %s", tagname);
+ }
+ tags.insert(id);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (hc && (any || !tags.empty()) && lua_isfunction(L, 3)) {
+ hc->traverse_all_tags([&](const rspamd::html::html_tag *tag) -> bool {
+ if (tag && (any || tags.contains(tag->id))) {
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ auto err_idx = lua_gettop(L);
+ lua_pushvalue(L, 3);
+
+ auto *ltag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(lua_html_tag)));
+ ltag->tag = tag;
+ ltag->html = hc;
+ auto ct = ltag->tag->get_content(hc);
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ lua_pushinteger(L, ct.size());
+
+ /* Leaf flag */
+ if (tag->children.empty()) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ if (lua_pcall(L, 3, 1, err_idx) != 0) {
+ msg_err("error in foreach_tag callback: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+ return false;
+ }
+
+ if (lua_toboolean(L, -1)) {
+ lua_settop(L, err_idx - 1);
+ return false;
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+
+ return true;
+ });
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_html_get_invisible(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ auto *hc = lua_check_html(L, 1);
+
+ if (hc != NULL) {
+ lua_new_text(L, hc->invisible.c_str(), hc->invisible.size(), false);
+ }
+ else {
+ lua_newtable(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ const gchar *tagname;
+
+ if (ltag != NULL) {
+ tagname = rspamd_html_tag_by_id(ltag->tag->id);
+
+ if (tagname) {
+ lua_pushstring(L, tagname);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1), *ptag;
+
+ if (ltag != NULL) {
+ auto *parent = ltag->tag->parent;
+
+ if (parent) {
+ ptag = static_cast<lua_html_tag *>(lua_newuserdata(L, sizeof(*ptag)));
+ ptag->tag = static_cast<rspamd::html::html_tag *>(parent);
+ ptag->html = ltag->html;
+ rspamd_lua_setclass(L, "rspamd{html_tag}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ gint i = 1;
+
+ if (ltag && ltag->tag) {
+ /* Push flags */
+ lua_createtable(L, 4, 0);
+ if (ltag->tag->flags & FL_HREF) {
+ lua_pushstring(L, "href");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_CLOSED) {
+ lua_pushstring(L, "closed");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_BROKEN) {
+ lua_pushstring(L, "broken");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & FL_XML) {
+ lua_pushstring(L, "xml");
+ lua_rawseti(L, -2, i++);
+ }
+ if (ltag->tag->flags & RSPAMD_HTML_FLAG_UNBALANCED) {
+ lua_pushstring(L, "unbalanced");
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (ltag) {
+
+ if (ltag->html) {
+ auto ct = ltag->tag->get_content(ltag->html);
+ if (ct.size() > 0) {
+ t = static_cast<rspamd_lua_text *>(lua_newuserdata(L, sizeof(*t)));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = ct.data();
+ t->len = ct.size();
+ t->flags = 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_content_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+
+ if (ltag) {
+ if (ltag->html) {
+ auto ct = ltag->tag->get_content(ltag->html);
+ lua_pushinteger(L, ct.size());
+ }
+ else {
+ lua_pushinteger(L, ltag->tag->get_content_length());
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_extra(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ struct html_image *img;
+
+ if (ltag) {
+ if (!std::holds_alternative<std::monostate>(ltag->tag->extra)) {
+ if (std::holds_alternative<struct html_image *>(ltag->tag->extra)) {
+ img = std::get<struct html_image *>(ltag->tag->extra);
+ lua_html_push_image(L, img);
+ }
+ else if (std::holds_alternative<struct rspamd_url *>(ltag->tag->extra)) {
+ /* For A that's URL */
+ auto *lua_url = static_cast<rspamd_lua_url *>(lua_newuserdata(L, sizeof(rspamd_lua_url)));
+ lua_url->url = std::get<struct rspamd_url *>(ltag->tag->extra);
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ }
+ else {
+ /* Unknown extra ? */
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_style(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+
+ if (ltag) {
+ if (ltag->tag->block) {
+ lua_html_push_block(L, ltag->tag->block);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_html_tag_get_attribute(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_html_tag *ltag = lua_check_html_tag(L, 1);
+ gsize slen;
+ const gchar *attr_name = luaL_checklstring(L, 2, &slen);
+
+ if (ltag && attr_name) {
+ auto maybe_attr = ltag->tag->find_component(
+ rspamd::html::html_component_from_string({attr_name, slen}));
+
+ if (maybe_attr) {
+ lua_pushlstring(L, maybe_attr->data(), maybe_attr->size());
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void luaopen_html(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{html}", htmllib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{html_tag}", taglib_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
new file mode 100644
index 0000000..713082a
--- /dev/null
+++ b/src/lua/lua_http.c
@@ -0,0 +1,1270 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "libserver/http/http_private.h"
+#include "libutil/upstream.h"
+#include "ref.h"
+#include "unix-std.h"
+#include "zlib.h"
+#include "utlist.h"
+
+/***
+ * @module rspamd_http
+ * Rspamd HTTP module represents HTTP asynchronous client available from LUA code.
+ * This module hides all complexity: DNS resolving, sessions management, zero-copy
+ * text transfers and so on under the hood.
+ * @example
+local rspamd_http = require "rspamd_http"
+
+local function symbol_callback(task)
+ local function http_callback(err_message, code, body, headers)
+ task:insert_result('SYMBOL', 1) -- task is available via closure
+ end
+
+ rspamd_http.request({
+ task=task,
+ url='http://example.com/data',
+ body=task:get_content(),
+ callback=http_callback,
+ headers={Header='Value', OtherHeader='Value'},
+ mime_type='text/plain',
+ })
+ end
+ */
+
+#define MAX_HEADERS_SIZE 8192
+
+static const gchar *M = "rspamd lua http";
+
+LUA_FUNCTION_DEF(http, request);
+
+static const struct luaL_reg httplib_m[] = {
+ LUA_INTERFACE_DEF(http, request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#define RSPAMD_LUA_HTTP_FLAG_TEXT (1 << 0)
+#define RSPAMD_LUA_HTTP_FLAG_NOVERIFY (1 << 1)
+#define RSPAMD_LUA_HTTP_FLAG_RESOLVED (1 << 2)
+#define RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE (1 << 3)
+#define RSPAMD_LUA_HTTP_FLAG_YIELDED (1 << 4)
+
+struct lua_http_cbdata {
+ struct rspamd_http_connection *conn;
+ struct rspamd_async_session *session;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_http_message *msg;
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ struct rspamd_task *task;
+ ev_tstamp timeout;
+ struct rspamd_cryptobox_keypair *local_kp;
+ struct rspamd_cryptobox_pubkey *peer_pk;
+ rspamd_inet_addr_t *addr;
+ gchar *mime_type;
+ gchar *host;
+ gchar *auth;
+ struct upstream *up;
+ const gchar *url;
+ gsize max_size;
+ gint flags;
+ gint fd;
+ gint cbref;
+ struct thread_entry *thread;
+ ref_entry_t ref;
+};
+
+static const gdouble default_http_timeout = 5.0;
+
+static struct rspamd_dns_resolver *
+lua_http_global_resolver(struct ev_loop *ev_base)
+{
+ static struct rspamd_dns_resolver *global_resolver;
+
+ if (global_resolver == NULL) {
+ global_resolver = rspamd_dns_resolver_init(NULL, ev_base, NULL);
+ }
+
+ return global_resolver;
+}
+
+static void
+lua_http_fin(gpointer arg)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) arg;
+
+ if (cbd->cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, cbd->cbref);
+ }
+
+ if (cbd->conn) {
+ /* Here we already have a connection, so we need to unref it */
+ rspamd_http_connection_unref(cbd->conn);
+ }
+ else if (cbd->msg != NULL) {
+ /* We need to free message */
+ rspamd_http_message_unref(cbd->msg);
+ }
+
+ if (cbd->fd != -1) {
+ close(cbd->fd);
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ }
+
+ if (cbd->mime_type) {
+ g_free(cbd->mime_type);
+ }
+
+ if (cbd->auth) {
+ g_free(cbd->auth);
+ }
+
+ if (cbd->host) {
+ g_free(cbd->host);
+ }
+
+ if (cbd->local_kp) {
+ rspamd_keypair_unref(cbd->local_kp);
+ }
+
+ if (cbd->peer_pk) {
+ rspamd_pubkey_unref(cbd->peer_pk);
+ }
+
+ g_free(cbd);
+}
+
+static void
+lua_http_cbd_dtor(struct lua_http_cbdata *cbd)
+{
+ if (cbd->session) {
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_RESOLVED) {
+ /* Event is added merely for resolved events */
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ }
+
+ rspamd_session_remove_event(cbd->session, lua_http_fin, cbd);
+ }
+ }
+ else {
+ lua_http_fin(cbd);
+ }
+}
+
+static void
+lua_http_push_error(struct lua_http_cbdata *cbd, const char *err)
+{
+ struct lua_callback_state lcbd;
+ lua_State *L;
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &lcbd);
+
+ L = lcbd.L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ lua_pushstring(L, err);
+
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 1, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+
+ lua_thread_pool_restore_callback(&lcbd);
+}
+
+static void lua_http_resume_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg, const char *err);
+
+static void
+lua_http_error_handler(struct rspamd_http_connection *conn, GError *err)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+
+ if (cbd->up) {
+ rspamd_upstream_fail(cbd->up, false, err ? err->message : "unknown error");
+ }
+
+ if (cbd->cbref == -1) {
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
+ cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
+ lua_http_resume_handler(conn, NULL, err->message);
+ }
+ else {
+ /* TODO: kill me please */
+ msg_info("lost HTTP error from %s in coroutines mess: %s",
+ rspamd_inet_address_to_string_pretty(cbd->addr),
+ err->message);
+ }
+ }
+ else {
+ lua_http_push_error(cbd, err->message);
+ }
+
+ REF_RELEASE(cbd);
+}
+
+static int
+lua_http_finish_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+ struct rspamd_http_header *h;
+ const gchar *body;
+ gsize body_len;
+
+ struct lua_callback_state lcbd;
+ lua_State *L;
+
+ if (cbd->cbref == -1) {
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_YIELDED) {
+ cbd->flags &= ~RSPAMD_LUA_HTTP_FLAG_YIELDED;
+ lua_http_resume_handler(conn, msg, NULL);
+ }
+ else {
+ /* TODO: kill me please */
+ msg_err("lost HTTP data from %s in coroutines mess",
+ rspamd_inet_address_to_string_pretty(cbd->addr));
+ }
+
+ REF_RELEASE(cbd);
+
+ return 0;
+ }
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &lcbd);
+
+ if (cbd->up) {
+ rspamd_upstream_ok(cbd->up);
+ }
+
+ L = lcbd.L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ /* Error */
+ lua_pushnil(L);
+ /* Reply code */
+ lua_pushinteger(L, msg->code);
+ /* Body */
+ body = rspamd_http_message_get_body(msg, &body_len);
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = body;
+ t->len = body_len;
+ t->flags = 0;
+ }
+ else {
+ if (body_len > 0) {
+ lua_pushlstring(L, body, body_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ /* Headers */
+ lua_newtable(L);
+
+ kh_foreach_value(msg->headers, h, {
+ /*
+ * Lowercase header name, as Lua cannot search in caseless matter
+ */
+ rspamd_str_lc(h->combined->str, h->name.len);
+ lua_pushlstring(L, h->name.begin, h->name.len);
+ lua_pushlstring(L, h->value.begin, h->value.len);
+ lua_settable(L, -3);
+ });
+
+ if (cbd->item) {
+ /* Replace watcher to deal with nested calls */
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 4, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1);
+ }
+
+ REF_RELEASE(cbd);
+
+ lua_thread_pool_restore_callback(&lcbd);
+
+ return 0;
+}
+
+/*
+ * resumes yielded thread
+ */
+static void
+lua_http_resume_handler(struct rspamd_http_connection *conn,
+ struct rspamd_http_message *msg, const char *err)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) conn->ud;
+ lua_State *L = cbd->thread->lua_state;
+ const gchar *body;
+ gsize body_len;
+ struct rspamd_http_header *h;
+
+ if (err) {
+ lua_pushstring(L, err);
+ lua_pushnil(L);
+ }
+ else {
+ /*
+ * 1 - nil (error)
+ * 2 - table:
+ * code (int)
+ * content (string)
+ * headers (table: header -> value)
+ */
+ lua_pushnil(L);// error code
+
+ lua_createtable(L, 0, 3);
+
+ /* code */
+ lua_pushliteral(L, "code");
+ lua_pushinteger(L, msg->code);
+ lua_settable(L, -3);
+
+ /* content */
+ lua_pushliteral(L, "content");
+
+ body = rspamd_http_message_get_body(msg, &body_len);
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_TEXT) {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = body;
+ t->len = body_len;
+ t->flags = 0;
+ }
+ else {
+ if (body_len > 0) {
+ lua_pushlstring(L, body, body_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ lua_settable(L, -3);
+
+ /* headers */
+ lua_pushliteral(L, "headers");
+ lua_newtable(L);
+
+ kh_foreach_value(msg->headers, h, {
+ /*
+ * Lowercase header name, as Lua cannot search in caseless matter
+ */
+ rspamd_str_lc(h->combined->str, h->name.len);
+ lua_pushlstring(L, h->name.begin, h->name.len);
+ lua_pushlstring(L, h->value.begin, h->value.len);
+ lua_settable(L, -3);
+ });
+
+ lua_settable(L, -3);
+ }
+
+ if (cbd->item) {
+ /* Replace watcher to deal with nested calls */
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ lua_thread_resume(cbd->thread, 2);
+}
+
+static gboolean
+lua_http_make_connection(struct lua_http_cbdata *cbd)
+{
+ rspamd_inet_address_set_port(cbd->addr, cbd->msg->port);
+ unsigned http_opts = RSPAMD_HTTP_CLIENT_SIMPLE;
+
+ if (cbd->msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) {
+ http_opts |= RSPAMD_HTTP_CLIENT_SSL;
+ }
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
+ cbd->fd = -1; /* FD is owned by keepalive connection */
+ cbd->conn = rspamd_http_connection_new_client_keepalive(
+ NULL, /* Default context */
+ NULL,
+ lua_http_error_handler,
+ lua_http_finish_handler,
+ http_opts,
+ cbd->addr,
+ cbd->host);
+ }
+ else {
+ cbd->fd = -1;
+ cbd->conn = rspamd_http_connection_new_client(
+ NULL, /* Default context */
+ NULL,
+ lua_http_error_handler,
+ lua_http_finish_handler,
+ http_opts,
+ cbd->addr);
+ }
+
+ if (cbd->conn) {
+ if (cbd->local_kp) {
+ rspamd_http_connection_set_key(cbd->conn, cbd->local_kp);
+ }
+
+ if (cbd->peer_pk) {
+ rspamd_http_message_set_peer_key(cbd->msg, cbd->peer_pk);
+ }
+
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_NOVERIFY) {
+ cbd->msg->flags |= RSPAMD_HTTP_FLAG_SSL_NOVERIFY;
+ }
+
+ if (cbd->max_size) {
+ rspamd_http_connection_set_max_size(cbd->conn, cbd->max_size);
+ }
+
+ if (cbd->auth) {
+ rspamd_http_message_add_header(cbd->msg, "Authorization",
+ cbd->auth);
+ }
+
+ if (cbd->session) {
+ if (cbd->item) {
+ rspamd_session_add_event_full(cbd->session,
+ (event_finalizer_t) lua_http_fin, cbd,
+ M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ rspamd_session_add_event(cbd->session,
+ (event_finalizer_t) lua_http_fin, cbd,
+ M);
+ }
+ cbd->flags |= RSPAMD_LUA_HTTP_FLAG_RESOLVED;
+ }
+
+ if (cbd->task) {
+ cbd->conn->log_tag = cbd->task->task_pool->tag.uid;
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+ }
+ else if (cbd->cfg) {
+ cbd->conn->log_tag = cbd->cfg->cfg_pool->tag.uid;
+ }
+
+ struct rspamd_http_message *msg = cbd->msg;
+
+ /* Message is now owned by a connection object */
+ cbd->msg = NULL;
+
+ return rspamd_http_connection_write_message(cbd->conn, msg,
+ cbd->host, cbd->mime_type, cbd,
+ cbd->timeout);
+ }
+
+ return FALSE;
+}
+
+static void
+lua_http_dns_handler(struct rdns_reply *reply, gpointer ud)
+{
+ struct lua_http_cbdata *cbd = (struct lua_http_cbdata *) ud;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_task *task;
+
+ task = cbd->task;
+ item = cbd->item;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ lua_http_push_error(cbd, "unable to resolve host");
+ REF_RELEASE(cbd);
+ }
+ else {
+ struct rdns_reply_entry *entry;
+
+ DL_FOREACH(reply->entries, entry)
+ {
+ if (entry->type == RDNS_REQUEST_A) {
+ cbd->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ break;
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ cbd->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ break;
+ }
+ }
+
+ if (cbd->addr == NULL) {
+ lua_http_push_error(cbd, "unable to resolve host: no records with such name");
+ REF_RELEASE(cbd);
+ }
+ else {
+ REF_RETAIN(cbd);
+ if (!lua_http_make_connection(cbd)) {
+ lua_http_push_error(cbd, "unable to make connection to the host");
+
+ if (cbd->ref.refcount > 1) {
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+
+ return;
+ }
+ REF_RELEASE(cbd);
+ }
+ }
+
+ if (item) {
+ rspamd_symcache_item_async_dec_check(task, item, M);
+ }
+}
+
+static void
+lua_http_push_headers(lua_State *L, struct rspamd_http_message *msg)
+{
+ const char *name, *value;
+ gint i, sz;
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+
+ lua_pushvalue(L, -2);
+ name = lua_tostring(L, -1);
+ sz = rspamd_lua_table_size(L, -2);
+ if (sz != 0 && name != NULL) {
+ for (i = 1; i <= sz; i++) {
+ lua_rawgeti(L, -2, i);
+ value = lua_tostring(L, -1);
+ if (value != NULL) {
+ rspamd_http_message_add_header(msg, name, value);
+ }
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ value = lua_tostring(L, -2);
+ if (name != NULL && value != NULL) {
+ rspamd_http_message_add_header(msg, name, value);
+ }
+ }
+ lua_pop(L, 2);
+ }
+}
+
+/***
+ * @function rspamd_http.request({params...})
+ * This function creates HTTP request and accepts several parameters as a table using key=value syntax.
+ * Required params are:
+ *
+ * - `url`
+ * - `task`
+ *
+ * In taskless mode, instead of `task` required are:
+ *
+ * - `ev_base`
+ * - `config`
+ *
+ * @param {string} url specifies URL for a request in the standard URI form (e.g. 'http://example.com/path')
+ * @param {function} callback specifies callback function in format `function (err_message, code, body, headers)` that is called on HTTP request completion. if this parameter is missing, the function performs "pseudo-synchronous" call (see [Synchronous and Asynchronous API overview](/doc/lua/sync_async.html#API-example-http-module)
+ * @param {task} task if called from symbol handler it is generally a good idea to use the common task objects: event base, DNS resolver and events session
+ * @param {table} headers optional headers in form `[name='value', name='value']`
+ * @param {string} mime_type MIME type of the HTTP content (for example, `text/html`)
+ * @param {string/text} body full body content, can be opaque `rspamd{text}` to avoid data copying
+ * @param {number} timeout floating point request timeout value in seconds (default is 5.0 seconds)
+ * @param {resolver} resolver to perform DNS-requests. Usually got from either `task` or `config`
+ * @param {boolean} gzip if true, body of the requests will be compressed
+ * @param {boolean} no_ssl_verify disable SSL peer checks
+ * @param {boolean} keepalive enable keep-alive pool
+ * @param {string} user for HTTP authentication
+ * @param {string} password for HTTP authentication, only if "user" present
+ * @return {boolean} `true`, in **async** mode, if a request has been successfully scheduled. If this value is `false` then some error occurred, the callback thus will not be called.
+ * @return In **sync** mode `string|nil, nil|table` In sync mode error message if any and response as table: `int` _code_, `string` _content_ and `table` _headers_ (header -> value)
+ */
+static gint
+lua_http_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop *ev_base;
+ struct rspamd_http_message *msg;
+ struct lua_http_cbdata *cbd;
+ struct rspamd_dns_resolver *resolver;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct rspamd_cryptobox_pubkey *peer_key = NULL;
+ struct rspamd_cryptobox_keypair *local_kp = NULL;
+ struct upstream *up = NULL;
+ const gchar *url, *lua_body;
+ rspamd_fstring_t *body = NULL;
+ gint cbref = -1;
+ gsize bodylen;
+ gdouble timeout = default_http_timeout;
+ gint flags = 0;
+ gchar *mime_type = NULL;
+ gchar *auth = NULL;
+ gsize max_size = 0;
+ gboolean gzip = FALSE;
+
+ if (lua_gettop(L) >= 2) {
+ /* url, callback and event_base format */
+ url = luaL_checkstring(L, 1);
+
+ if (url == NULL || lua_type(L, 2) != LUA_TFUNCTION) {
+ msg_err("http request has bad params");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ if (lua_gettop(L) >= 3 && rspamd_lua_check_udata_maybe(L, 3, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, 3);
+ }
+ else {
+ ev_base = NULL;
+ }
+
+ if (lua_gettop(L) >= 4 && rspamd_lua_check_udata_maybe(L, 4, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, 4);
+ }
+ else {
+ resolver = lua_http_global_resolver(ev_base);
+ }
+
+ if (lua_gettop(L) >= 5 && rspamd_lua_check_udata_maybe(L, 5, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, 5);
+ }
+ else {
+ session = NULL;
+ }
+
+ msg = rspamd_http_message_from_url(url);
+
+ if (msg == NULL) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref);
+ msg_err("cannot create HTTP message from url %s", url);
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "url");
+ lua_gettable(L, 1);
+ url = luaL_checkstring(L, -1);
+ lua_pop(L, 1);
+
+ if (url == NULL) {
+ msg_err("cannot create HTTP message without url");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, 1);
+ if (url == NULL || lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ }
+ else {
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+
+ if (task) {
+ ev_base = task->event_loop;
+ resolver = task->resolver;
+ session = task->s;
+ cfg = task->cfg;
+ }
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ ev_base = NULL;
+ }
+ lua_pop(L, 1);
+
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "config");
+ lua_gettable(L, 1);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{config}")) {
+ cfg = *(struct rspamd_config **) lua_touserdata(L, -1);
+ }
+ else {
+ cfg = NULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "resolver");
+ lua_gettable(L, 1);
+
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, -1);
+ }
+ else {
+ if (cfg && cfg->dns_resolver) {
+ resolver = cfg->dns_resolver;
+ }
+ else {
+ resolver = lua_http_global_resolver(ev_base);
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ msg = rspamd_http_message_from_url(url);
+ if (msg == NULL) {
+ msg_err_task_check("cannot create HTTP message from url %s", url);
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushstring(L, "headers");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_http_push_headers(L, msg);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "mime_type");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ mime_type = g_strdup(lua_tostring(L, -1));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "body");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ lua_body = lua_tolstring(L, -1, &bodylen);
+ body = rspamd_fstring_new_init(lua_body, bodylen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, -1);
+ /* TODO: think about zero-copy possibilities */
+ if (t) {
+ body = rspamd_fstring_new_init(t->start, t->len);
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ g_free(mime_type);
+
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ gsize total_len = 0, nelts = rspamd_lua_table_size(L, -1);
+
+ /* Calculate length and check types */
+ for (gsize i = 0; i < nelts; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+#if LUA_VERSION_NUM >= 502
+ total_len += lua_rawlen(L, -1);
+#else
+ total_len += lua_objlen(L, -1);
+#endif
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ total_len += t->len;
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "invalid body argument: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+
+ lua_pop(L, 1);
+ }
+
+ /* Preallocate body */
+ if (total_len > 0) {
+ body = rspamd_fstring_sized_new(total_len);
+ }
+ else {
+ rspamd_http_message_unref(msg);
+ if (mime_type) {
+ g_free(mime_type);
+ }
+
+ return luaL_error(L, "empty body specified");
+ }
+
+ /* Fill elements */
+ for (gsize i = 0; i < nelts; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ lua_body = lua_tolstring(L, -1, &bodylen);
+ body = rspamd_fstring_append(body, lua_body, bodylen);
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ body = rspamd_fstring_append(body, t->start, t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_type(L, -1) != LUA_TNONE && lua_type(L, -1) != LUA_TNIL) {
+ rspamd_http_message_unref(msg);
+ return luaL_error(L, "invalid body argument type: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "peer_key");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *in;
+ gsize inlen;
+
+ in = lua_tolstring(L, -1, &inlen);
+ peer_key = rspamd_pubkey_from_base32(in, inlen,
+ RSPAMD_KEYPAIR_KEX, RSPAMD_CRYPTOBOX_MODE_25519);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "keypair");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ ucl_object_t *kp_ucl = ucl_object_lua_import(L, -1);
+
+ local_kp = rspamd_keypair_from_ucl(kp_ucl);
+ ucl_object_unref(kp_ucl);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_body");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_TEXT;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "gzip");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ gzip = TRUE;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "no_ssl_verify");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_NOVERIFY;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "keepalive");
+ lua_gettable(L, 1);
+
+ if (!!lua_toboolean(L, -1)) {
+ flags |= RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "max_size");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ max_size = lua_tointeger(L, -1);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "method");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ rspamd_http_message_set_method(msg, lua_tostring(L, -1));
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "upstream");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_upstream *lup = lua_check_upstream(L, -1);
+
+ if (lup) {
+ /* Preserve pointer in case if lup is destructed */
+ up = lup->up;
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "user");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *user = lua_tostring(L, -1);
+
+ lua_pushstring(L, "password");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *password = lua_tostring(L, -1);
+ gchar *tmpbuf;
+ gsize tlen;
+
+ tlen = strlen(user) + strlen(password) + 1;
+ tmpbuf = g_malloc(tlen + 1);
+ rspamd_snprintf(tmpbuf, tlen + 1, "%s:%s", user, password);
+ tlen *= 2;
+ tlen += sizeof("Basic ") - 1;
+ auth = g_malloc(tlen + 1);
+ rspamd_snprintf(auth, tlen + 1, "Basic %Bs", tmpbuf);
+ g_free(tmpbuf);
+ }
+ else {
+ msg_warn("HTTP user must have password, disabling auth");
+ }
+
+ lua_pop(L, 1); /* password */
+ }
+
+ lua_pop(L, 1); /* username */
+ }
+ else {
+ msg_err("http request has bad params");
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ if (session && rspamd_session_blocked(session)) {
+ lua_pushboolean(L, FALSE);
+
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return 1;
+ }
+ if (task == NULL && cfg == NULL) {
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return luaL_error(L,
+ "Bad params to rspamd_http:request(): either task or config should be set");
+ }
+
+ if (ev_base == NULL) {
+ g_free(auth);
+ rspamd_http_message_unref(msg);
+ if (body) {
+ rspamd_fstring_free(body);
+ }
+ if (local_kp) {
+ rspamd_keypair_unref(local_kp);
+ }
+
+ return luaL_error(L,
+ "Bad params to rspamd_http:request(): ev_base isn't passed");
+ }
+
+ cbd = g_malloc0(sizeof(*cbd));
+ cbd->cbref = cbref;
+ cbd->msg = msg;
+ cbd->event_loop = ev_base;
+ cbd->mime_type = mime_type;
+ cbd->timeout = timeout;
+ cbd->fd = -1;
+ cbd->cfg = cfg;
+ cbd->peer_pk = peer_key;
+ cbd->local_kp = local_kp;
+ cbd->flags = flags;
+ cbd->max_size = max_size;
+ cbd->url = url;
+ cbd->auth = auth;
+ cbd->task = task;
+
+ if (up) {
+ cbd->up = rspamd_upstream_ref(up);
+ }
+
+ if (cbd->cbref == -1) {
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ }
+
+ REF_INIT_RETAIN(cbd, lua_http_cbd_dtor);
+
+ if (task) {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+ }
+
+
+ if (body) {
+ if (gzip) {
+ if (rspamd_fstring_gzip(&body)) {
+ rspamd_http_message_add_header(msg, "Content-Encoding", "gzip");
+ }
+ }
+
+ rspamd_http_message_set_body_from_fstring_steal(msg, body);
+ }
+
+ if (session) {
+ cbd->session = session;
+ }
+
+ bool numeric_ip = false;
+
+ /* Check if we can skip resolving */
+
+ gsize hostlen = 0;
+ const gchar *host = rspamd_http_message_get_http_host(msg, &hostlen);
+
+ if (host) {
+ cbd->host = g_malloc(hostlen + 1);
+ rspamd_strlcpy(cbd->host, host, hostlen + 1);
+
+ /* Keep-alive entry is available */
+ if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) {
+ const rspamd_inet_addr_t *ka_addr = rspamd_http_context_has_keepalive(NULL,
+ cbd->host,
+ msg->port,
+ msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL);
+
+ if (ka_addr) {
+ cbd->addr = rspamd_inet_address_copy(ka_addr, NULL);
+ numeric_ip = true;
+ }
+ }
+
+ /*
+ * No keep-alive stuff, check if we have upstream or if we can parse host as
+ * a numeric address
+ */
+ if (!cbd->addr) {
+ if (cbd->up) {
+ numeric_ip = true;
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+ }
+ else {
+ /* We use msg->host here, not cbd->host ! */
+ if (rspamd_parse_inet_address(&cbd->addr,
+ msg->host->str, msg->host->len,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ numeric_ip = true;
+ }
+ }
+ }
+ }
+ else {
+ if (cbd->up) {
+ numeric_ip = true;
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+ }
+ cbd->host = NULL;
+ }
+
+ if (numeric_ip) {
+ /* Host is numeric IP, no need to resolve */
+ gboolean ret;
+
+ REF_RETAIN(cbd);
+ ret = lua_http_make_connection(cbd);
+
+ if (!ret) {
+ if (cbd->up) {
+ rspamd_upstream_fail(cbd->up, true, "HTTP connection failed");
+ }
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ REF_RELEASE(cbd);
+ }
+ else {
+ if (!cbd->host) {
+ REF_RELEASE(cbd);
+
+ return luaL_error(L, "no host has been specified");
+ }
+ if (task == NULL) {
+
+ REF_RETAIN(cbd);
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_http_dns_handler, cbd,
+ RDNS_REQUEST_A,
+ cbd->host)) {
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ REF_RELEASE(cbd);
+ }
+ else {
+ REF_RETAIN(cbd);
+
+ if (!rspamd_dns_resolver_request_task_forced(task, lua_http_dns_handler, cbd,
+ RDNS_REQUEST_A, cbd->host)) {
+ if (cbd->ref.refcount > 1) {
+ /* Not released by make_connection */
+ REF_RELEASE(cbd);
+ }
+
+ REF_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+ else if (cbd->item) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+
+ REF_RELEASE(cbd);
+ }
+ }
+
+ if (cbd->cbref == -1) {
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+ cbd->flags |= RSPAMD_LUA_HTTP_FLAG_YIELDED;
+
+ return lua_thread_yield(cbd->thread, 0);
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_http(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, httplib_m);
+
+ return 1;
+}
+
+void luaopen_http(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_http", lua_load_http);
+}
diff --git a/src/lua/lua_ip.c b/src/lua/lua_ip.c
new file mode 100644
index 0000000..ac24dc5
--- /dev/null
+++ b/src/lua/lua_ip.c
@@ -0,0 +1,637 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libserver/maps/map_helpers.h"
+
+/***
+ * @module rspamd_ip
+ * `rspamd_ip` is a helper module to simplify IP addresses manipulations.
+ * @example
+local print_octets = function(ip)
+ print('Normal order octets:')
+ for _,o in ipairs(ip:str_octets()) do
+ print(o)
+ end
+ print('Reversed order octets:')
+ for _,o in ipairs(ip:inversed_str_octets()) do
+ print(o)
+ end
+ print('Numeric octets:')
+ for _,o in ipairs(ip:to_table()) do
+ print(o)
+ end
+end
+
+local rspamd_ip = require "rspamd_ip"
+-- Create ipv4
+local ip4 = rspamd_ip.from_string('127.0.0.1')
+-- Implicit conversion to string
+print(ip4)
+-- Numeric version
+print(ip4:get_version())
+print_octets(ip4)
+
+-- Create a sample ipv6 address
+local ip6 = rspamd_ip.from_string('2001:41d0:8:dd9a::100')
+print(ip6)
+print(ip6:get_version())
+print_octets(ip6)
+ */
+
+/***
+ * @method ip:to_string([pretty=false])
+ * Converts valid IP address to string
+ * @param {bool} pretty print IP address with port and braces (for IPv6)
+ * @return {string or nil} string representation of IP or `nil` if IP is invalid
+ */
+LUA_FUNCTION_DEF(ip, to_string);
+/***
+ * @method ip:to_number()
+ * Converts valid IP address to number or list of numbers in case of IPv6
+ * @return {integer(s) or nil} numeric representation of IP in *host* byte order or `nil` if IP is invalid
+ */
+LUA_FUNCTION_DEF(ip, to_number);
+
+/***
+ * @method ip:to_table()
+ * Converts valid IP address to the table of numeric octets
+ * @return {table or nil} numeric octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('127.0.0.1')
+for _,o in ipairs(ip:to_table()) do
+ print(o)
+end
+-- Output:
+-- 127
+-- 0
+-- 0
+-- 1
+ */
+LUA_FUNCTION_DEF(ip, to_table);
+/***
+ * @method ip:str_octets()
+ * Converts valid IP address to the table of string octets. The difference from
+ * @see ip:to_table() is that this method returns just hex strings for ipv6
+ * addresses.
+ * @return {table or nil} string octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('fe80::11')
+print(table.concat(ip:str_octets(), "."))
+-- Output:
+-- f.e.8.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.1.1
+ */
+LUA_FUNCTION_DEF(ip, str_octets);
+/***
+ * @method ip:inversed_str_octets()
+ * Converts valid IP address to the table of string octets in reversed order. The difference from
+ * @see ip:to_table() is that this method returns just hex strings for ipv6
+ * addresses in reversed order.
+ * @return {table or nil} string octets of IP address or `nil` if IP is invalid
+ * @example
+local ip = rspamd_ip.from_string('fe80::11')
+print(table.concat(ip:inversed_str_octets(), "."))
+-- Output:
+-- 1.1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f
+ */
+LUA_FUNCTION_DEF(ip, inversed_str_octets);
+/***
+ * @function rspamd_ip.from_string(line)
+ * Create IP address from its string representation.
+ * @param {string} line valid IP address string (either ipv4 or ipv6)
+ * @return {ip} new ip object or `nil` if input is invalid
+ */
+LUA_FUNCTION_DEF(ip, from_string);
+/***
+ * @method ip:__gc()
+ * Automatically destroys IP object.
+ */
+LUA_FUNCTION_DEF(ip, destroy);
+/***
+ * @method ip:get_version()
+ * Gets numeric version of ip address
+ * @return {number} `4` for IPv4 and `6` for IPv6
+ */
+LUA_FUNCTION_DEF(ip, get_version);
+/***
+ * @method ip:is_valid()
+ * Checks if an IP object is a valid IP address.
+ * @return {boolean} `true` if IP is valid and `false` otherwise
+ */
+LUA_FUNCTION_DEF(ip, is_valid);
+/***
+ * @method ip:apply_mask(mask)
+ * Applies mask to IP address, resetting up to `mask` least significant bits to zero.
+ * @param {integer} mask how many bits to reset
+ * @return {ip} new IP object with `mask` bits reset
+ */
+LUA_FUNCTION_DEF(ip, apply_mask);
+/***
+ * @method ip:__eq(other)
+ * Compares two IP addresses
+ * @param {ip} other IP to compare
+ * @return {boolean} `true` if two objects are the same
+ */
+LUA_FUNCTION_DEF(ip, equal);
+/***
+ * @method ip:copy()
+ * Performs deep copy of IP address.
+ * @return {ip} a fresh copy of IP address
+ */
+LUA_FUNCTION_DEF(ip, copy);
+
+/**
+ * @method ip:get_port()
+ * Returns associated port for this IP address
+ * @return {number} port number or nil
+ */
+LUA_FUNCTION_DEF(ip, get_port);
+/***
+ * @method ip:is_local()
+ * Returns true if address is local one
+ * @return {boolean} `true` if address is local
+ */
+LUA_FUNCTION_DEF(ip, is_local);
+
+/***
+ * @method ip:less_than(other)
+ * Returns true if address is less than other
+ * @return {boolean}
+ */
+LUA_FUNCTION_DEF(ip, less_than);
+
+static const struct luaL_reg iplib_m[] = {
+ LUA_INTERFACE_DEF(ip, to_string),
+ LUA_INTERFACE_DEF(ip, to_table),
+ LUA_INTERFACE_DEF(ip, to_number),
+ LUA_INTERFACE_DEF(ip, str_octets),
+ LUA_INTERFACE_DEF(ip, inversed_str_octets),
+ LUA_INTERFACE_DEF(ip, get_version),
+ LUA_INTERFACE_DEF(ip, get_port),
+ LUA_INTERFACE_DEF(ip, is_valid),
+ LUA_INTERFACE_DEF(ip, apply_mask),
+ LUA_INTERFACE_DEF(ip, copy),
+ LUA_INTERFACE_DEF(ip, is_local),
+ {"tostring", lua_ip_to_string},
+ {"totable", lua_ip_to_table},
+ {"tonumber", lua_ip_to_number},
+ {"__tostring", lua_ip_to_string},
+ {"__eq", lua_ip_equal},
+ {"__gc", lua_ip_destroy},
+ {"__lt", lua_ip_less_than},
+ {NULL, NULL}};
+
+static const struct luaL_reg iplib_f[] = {
+ LUA_INTERFACE_DEF(ip, from_string),
+ {"fromstring", lua_ip_from_string},
+ {"fromip", lua_ip_copy},
+ {"from_ip", lua_ip_copy},
+ {NULL, NULL}};
+
+static struct rspamd_lua_ip *
+lua_ip_new(lua_State *L, struct rspamd_lua_ip *old)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ ip = g_malloc0(sizeof(*ip));
+
+ if (old != NULL && old->addr != NULL) {
+ ip->addr = rspamd_inet_address_copy(old->addr, NULL);
+ }
+
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+
+
+ return ip;
+}
+
+struct rspamd_lua_ip *
+lua_check_ip(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{ip}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'ip' expected");
+ return ud ? *((struct rspamd_lua_ip **) ud) : NULL;
+}
+
+static gint
+lua_ip_to_table(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ lua_createtable(L, max, 0);
+
+ for (i = 1; i <= max; i++, ptr++) {
+ lua_pushinteger(L, *ptr);
+ lua_rawseti(L, -2, i);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_str_octets(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+ gint af;
+ char numbuf[8];
+
+ if (ip != NULL && ip->addr) {
+ af = rspamd_inet_address_get_af(ip->addr);
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ lua_createtable(L, max * 2, 0);
+
+ for (i = 1; i <= max; i++, ptr++) {
+ if (af == AF_INET) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%d", *ptr);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i);
+ }
+ else {
+ rspamd_snprintf(numbuf,
+ sizeof(numbuf),
+ "%xd",
+ (*ptr & 0xf0) >> 4);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2 - 1);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%xd", *ptr & 0x0f);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_inversed_str_octets(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint max, i;
+ guint8 *ptr;
+ char numbuf[4];
+ gint af;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+ af = rspamd_inet_address_get_af(ip->addr);
+ lua_createtable(L, max * 2, 0);
+
+ ptr += max - 1;
+ for (i = 1; i <= max; i++, ptr--) {
+ if (af == AF_INET) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%d", *ptr);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i);
+ }
+ else {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%xd", *ptr & 0x0f);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2 - 1);
+ rspamd_snprintf(numbuf,
+ sizeof(numbuf),
+ "%xd",
+ (*ptr & 0xf0) >> 4);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, i * 2);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip != NULL && ip->addr) {
+ if (lua_isboolean(L, 2) && lua_toboolean(L, 2) == true) {
+ lua_pushstring(L, rspamd_inet_address_to_string_pretty(ip->addr));
+ }
+ else {
+ lua_pushstring(L, rspamd_inet_address_to_string(ip->addr));
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip != NULL && ip->addr) {
+ lua_pushinteger(L, rspamd_inet_address_get_port(ip->addr));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_from_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip;
+ const gchar *ip_str;
+ gsize len;
+
+ ip_str = luaL_checklstring(L, 1, &len);
+ if (ip_str) {
+ ip = lua_ip_new(L, NULL);
+
+ if (!rspamd_parse_inet_address(&ip->addr,
+ ip_str, len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ msg_warn("cannot parse ip: %*s", (gint) len, ip_str);
+ ip->addr = NULL;
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_to_number(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ guint32 c;
+ guint max, i;
+ guchar *ptr;
+
+ if (ip != NULL && ip->addr) {
+ ptr = rspamd_inet_address_get_hash_key(ip->addr, &max);
+
+ for (i = 0; i < max / sizeof(c); i++) {
+ memcpy(&c, ptr + i * sizeof(c), sizeof(c));
+ lua_pushinteger(L, ntohl(c));
+ }
+
+ return max / sizeof(c);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_ip_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ if (ip->addr) {
+ rspamd_inet_address_free(ip->addr);
+ }
+ g_free(ip);
+ }
+
+ return 0;
+}
+
+static gint
+lua_ip_get_version(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip && ip->addr) {
+ lua_pushinteger(L, rspamd_inet_address_get_af(ip->addr) == AF_INET6 ? 6 : 4);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_is_valid(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ lua_pushboolean(L, ip->addr != NULL);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_apply_mask(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1), *nip;
+ gint mask;
+
+ mask = lua_tonumber(L, 2);
+ if (mask > 0 && ip != NULL && ip->addr) {
+ nip = lua_ip_new(L, ip);
+ rspamd_inet_address_apply_mask(nip->addr, mask);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_equal(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip1 = lua_check_ip(L, 1),
+ *ip2 = lua_check_ip(L, 2);
+ gboolean res = FALSE;
+
+ if (ip1 && ip2 && ip1->addr && ip2->addr) {
+ res = rspamd_inet_address_compare(ip1->addr, ip2->addr, TRUE) == 0;
+ }
+
+ lua_pushboolean(L, res);
+
+ return 1;
+}
+
+static gint
+lua_ip_copy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+
+ if (ip) {
+ lua_ip_new(L, ip);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_is_local(lua_State *L)
+{
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1);
+ gboolean check_laddrs = TRUE;
+
+ if (ip && ip->addr) {
+
+ if (lua_type(L, 2) == LUA_TBOOLEAN) {
+ check_laddrs = lua_toboolean(L, 2);
+ }
+
+ if (rspamd_inet_address_is_local(ip->addr)) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ else if (check_laddrs) {
+ struct rspamd_radix_map_helper *local_addrs =
+ rspamd_inet_library_get_lib_ctx();
+ if (local_addrs) {
+ if (rspamd_match_radix_map_addr(local_addrs, ip->addr) != NULL) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ }
+ }
+
+ lua_pushboolean(L, false);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_ip_less_than(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 1),
+ *other = lua_check_ip(L, 2);
+
+ if (ip && other) {
+ lua_pushboolean(L,
+ rspamd_inet_address_compare(ip->addr, other->addr, true) < 0);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+void rspamd_lua_ip_push(lua_State *L, rspamd_inet_addr_t *addr)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ if (addr) {
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+ ip->addr = rspamd_inet_address_copy(addr, NULL);
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+ }
+ else {
+ lua_pushnil(L);
+ }
+}
+
+void rspamd_lua_ip_push_fromstring(lua_State *L, const gchar *ip_str)
+{
+ struct rspamd_lua_ip *ip, **pip;
+
+ if (ip_str == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+
+ if (rspamd_parse_inet_address(&ip->addr,
+ ip_str, strlen(ip_str), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+
+ pip = lua_newuserdata(L, sizeof(struct rspamd_lua_ip *));
+ rspamd_lua_setclass(L, "rspamd{ip}", -1);
+ *pip = ip;
+ }
+ else {
+ g_free(ip);
+ lua_pushnil(L);
+ }
+ }
+}
+
+static gint
+lua_load_ip(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, iplib_f);
+
+ return 1;
+}
+
+void luaopen_ip(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{ip}", iplib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_ip", lua_load_ip);
+}
diff --git a/src/lua/lua_kann.c b/src/lua/lua_kann.c
new file mode 100644
index 0000000..e42fbfb
--- /dev/null
+++ b/src/lua/lua_kann.c
@@ -0,0 +1,1361 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_tensor.h"
+#include "contrib/kann/kann.h"
+
+/***
+ * @module rspamd_kann
+ * `rspamd_kann` is a Lua interface to kann library
+ */
+
+#define KANN_NODE_CLASS "rspamd{kann_node}"
+#define KANN_NETWORK_CLASS "rspamd{kann}"
+
+/* Simple macros to define behaviour */
+#define KANN_LAYER_DEF(name) static int lua_kann_layer_##name(lua_State *L)
+#define KANN_LAYER_INTERFACE(name) \
+ { \
+ #name, lua_kann_layer_##name \
+ }
+
+#define KANN_TRANSFORM_DEF(name) static int lua_kann_transform_##name(lua_State *L)
+#define KANN_TRANSFORM_INTERFACE(name) \
+ { \
+ #name, lua_kann_transform_##name \
+ }
+
+#define KANN_LOSS_DEF(name) static int lua_kann_loss_##name(lua_State *L)
+#define KANN_LOSS_INTERFACE(name) \
+ { \
+ #name, lua_kann_loss_##name \
+ }
+
+#define KANN_NEW_DEF(name) static int lua_kann_new_##name(lua_State *L)
+#define KANN_NEW_INTERFACE(name) \
+ { \
+ #name, lua_kann_new_##name \
+ }
+
+
+/*
+ * Forwarded declarations
+ */
+static kad_node_t *lua_check_kann_node(lua_State *L, int pos);
+
+/* Layers */
+KANN_LAYER_DEF(input);
+KANN_LAYER_DEF(dense);
+KANN_LAYER_DEF(layernorm);
+KANN_LAYER_DEF(rnn);
+KANN_LAYER_DEF(lstm);
+KANN_LAYER_DEF(gru);
+KANN_LAYER_DEF(conv2d);
+KANN_LAYER_DEF(conv1d);
+KANN_LAYER_DEF(cost);
+
+static luaL_reg rspamd_kann_layers_f[] = {
+ KANN_LAYER_INTERFACE(input),
+ KANN_LAYER_INTERFACE(dense),
+ KANN_LAYER_INTERFACE(layernorm),
+ KANN_LAYER_INTERFACE(rnn),
+ KANN_LAYER_INTERFACE(lstm),
+ KANN_LAYER_INTERFACE(gru),
+ KANN_LAYER_INTERFACE(conv2d),
+ KANN_LAYER_INTERFACE(conv1d),
+ KANN_LAYER_INTERFACE(cost),
+ {NULL, NULL},
+};
+
+/* Transition and composition functions */
+
+/* General transform */
+KANN_TRANSFORM_DEF(add);
+KANN_TRANSFORM_DEF(sub);
+KANN_TRANSFORM_DEF(mul);
+KANN_TRANSFORM_DEF(cmul);
+KANN_TRANSFORM_DEF(matmul);
+
+KANN_TRANSFORM_DEF(square);
+KANN_TRANSFORM_DEF(sigm);
+KANN_TRANSFORM_DEF(tanh);
+KANN_TRANSFORM_DEF(relu);
+KANN_TRANSFORM_DEF(softmax);
+KANN_TRANSFORM_DEF(1minus);
+KANN_TRANSFORM_DEF(exp);
+KANN_TRANSFORM_DEF(log);
+KANN_TRANSFORM_DEF(sin);
+static luaL_reg rspamd_kann_transform_f[] = {
+ KANN_TRANSFORM_INTERFACE(add),
+ KANN_TRANSFORM_INTERFACE(sub),
+ KANN_TRANSFORM_INTERFACE(mul),
+ KANN_TRANSFORM_INTERFACE(cmul),
+ KANN_TRANSFORM_INTERFACE(matmul),
+
+ KANN_TRANSFORM_INTERFACE(square),
+ KANN_TRANSFORM_INTERFACE(sigm),
+ KANN_TRANSFORM_INTERFACE(tanh),
+ KANN_TRANSFORM_INTERFACE(relu),
+ KANN_TRANSFORM_INTERFACE(softmax),
+ KANN_TRANSFORM_INTERFACE(1minus),
+ KANN_TRANSFORM_INTERFACE(exp),
+ KANN_TRANSFORM_INTERFACE(log),
+ KANN_TRANSFORM_INTERFACE(sin),
+ {NULL, NULL},
+};
+
+/* Loss functions */
+KANN_LOSS_DEF(mse);
+KANN_LOSS_DEF(ce_multi);
+KANN_LOSS_DEF(ce_bin);
+KANN_LOSS_DEF(ce_bin_neg);
+KANN_LOSS_DEF(ce_multi_weighted);
+static luaL_reg rspamd_kann_loss_f[] = {
+ KANN_LOSS_INTERFACE(mse),
+ KANN_LOSS_INTERFACE(ce_multi),
+ KANN_LOSS_INTERFACE(ce_bin),
+ KANN_LOSS_INTERFACE(ce_bin_neg),
+ KANN_LOSS_INTERFACE(ce_multi_weighted),
+ {NULL, NULL},
+};
+
+/* Creation functions */
+KANN_NEW_DEF(leaf);
+KANN_NEW_DEF(scalar);
+KANN_NEW_DEF(weight);
+KANN_NEW_DEF(bias);
+KANN_NEW_DEF(weight_conv2d);
+KANN_NEW_DEF(weight_conv1d);
+KANN_NEW_DEF(kann);
+
+static luaL_reg rspamd_kann_new_f[] = {
+ KANN_NEW_INTERFACE(leaf),
+ KANN_NEW_INTERFACE(scalar),
+ KANN_NEW_INTERFACE(weight),
+ KANN_NEW_INTERFACE(bias),
+ KANN_NEW_INTERFACE(weight_conv2d),
+ KANN_NEW_INTERFACE(weight_conv1d),
+ KANN_NEW_INTERFACE(kann),
+ {NULL, NULL},
+};
+
+LUA_FUNCTION_DEF(kann, load);
+LUA_FUNCTION_DEF(kann, destroy);
+LUA_FUNCTION_DEF(kann, save);
+LUA_FUNCTION_DEF(kann, train1);
+LUA_FUNCTION_DEF(kann, apply1);
+
+static luaL_reg rspamd_kann_m[] = {
+ LUA_INTERFACE_DEF(kann, save),
+ LUA_INTERFACE_DEF(kann, train1),
+ LUA_INTERFACE_DEF(kann, apply1),
+ {"__gc", lua_kann_destroy},
+ {NULL, NULL},
+};
+
+static int
+rspamd_kann_table_to_flags(lua_State *L, int table_pos)
+{
+ int result = 0;
+
+ lua_pushvalue(L, table_pos);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ int fl = lua_tointeger(L, -1);
+
+ result |= fl;
+ }
+
+ lua_pop(L, 1);
+
+ return result;
+}
+
+static gint
+lua_load_kann(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Flags */
+ lua_pushstring(L, "flag");
+ lua_newtable(L);
+ lua_pushinteger(L, KANN_F_IN);
+ lua_setfield(L, -2, "in");
+ lua_pushinteger(L, KANN_F_COST);
+ lua_setfield(L, -2, "cost");
+ lua_pushinteger(L, KANN_F_OUT);
+ lua_setfield(L, -2, "out");
+ lua_pushinteger(L, KANN_F_TRUTH);
+ lua_setfield(L, -2, "truth");
+ lua_settable(L, -3);
+
+ /* Cost type */
+ lua_pushstring(L, "cost");
+ lua_newtable(L);
+ /* binary cross-entropy cost, used with sigmoid */
+ lua_pushinteger(L, KANN_C_CEB);
+ lua_setfield(L, -2, "ceb");
+ /* multi-class cross-entropy cost, used with softmax */
+ lua_pushinteger(L, KANN_C_CEM);
+ lua_setfield(L, -2, "cem");
+ /* binary cross-entropy-like cost, used with tanh */
+ lua_pushinteger(L, KANN_C_CEB_NEG);
+ lua_setfield(L, -2, "ceb_neg");
+ lua_pushinteger(L, KANN_C_MSE);
+ lua_setfield(L, -2, "mse");
+ lua_settable(L, -3);
+
+ /* RNN flag */
+ lua_pushstring(L, "rnn");
+ lua_newtable(L);
+ /* apply layer normalization */
+ lua_pushinteger(L, KANN_RNN_NORM);
+ lua_setfield(L, -2, "norm");
+ /* take the initial hidden values as variables */
+ lua_pushinteger(L, KANN_RNN_VAR_H0);
+ lua_setfield(L, -2, "var_h0");
+ lua_settable(L, -3);
+
+ /* Layers */
+ lua_pushstring(L, "layer");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_layers_f);
+ lua_settable(L, -3);
+
+ /* Transforms */
+ lua_pushstring(L, "transform");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_transform_f);
+ lua_settable(L, -3);
+
+ /* Cost */
+ lua_pushstring(L, "loss");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_loss_f);
+ lua_settable(L, -3);
+
+ /* Create functions */
+ lua_pushstring(L, "new");
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_kann_new_f);
+ lua_settable(L, -3);
+
+ /* Load ann from memory or file */
+ lua_pushstring(L, "load");
+ lua_pushcfunction(L, lua_kann_load);
+ lua_settable(L, -3);
+
+ return 1;
+}
+
+static kad_node_t *
+lua_check_kann_node(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, KANN_NODE_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'kann_node' expected");
+ return ud ? *((kad_node_t **) ud) : NULL;
+}
+
+static kann_t *
+lua_check_kann(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, KANN_NETWORK_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'kann' expected");
+ return ud ? *((kann_t **) ud) : NULL;
+}
+
+void luaopen_kann(lua_State *L)
+{
+ /* Metatables */
+ rspamd_lua_new_class(L, KANN_NODE_CLASS, NULL); /* TODO: add methods */
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_new_class(L, KANN_NETWORK_CLASS, rspamd_kann_m);
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_add_preload(L, "rspamd_kann", lua_load_kann);
+ lua_settop(L, 0);
+}
+
+/* Layers implementation */
+#define PUSH_KAD_NODE(n) \
+ do { \
+ kad_node_t **pt; \
+ pt = lua_newuserdata(L, sizeof(kad_node_t *)); \
+ *pt = (n); \
+ rspamd_lua_setclass(L, KANN_NODE_CLASS, -1); \
+ } while (0)
+
+#define PUSH_KAN_NETWORK(n) \
+ do { \
+ kann_t **pn; \
+ pn = lua_newuserdata(L, sizeof(kann_t *)); \
+ *pn = (n); \
+ rspamd_lua_setclass(L, KANN_NETWORK_CLASS, -1); \
+ } while (0)
+
+#define PROCESS_KAD_FLAGS(n, pos) \
+ do { \
+ int fl = 0; \
+ if (lua_type(L, (pos)) == LUA_TTABLE) { fl = rspamd_kann_table_to_flags(L, (pos)); } \
+ else if (lua_type(L, (pos)) == LUA_TNUMBER) { \
+ fl = lua_tointeger(L, (pos)); \
+ } \
+ (n)->ext_flag |= fl; \
+ } while (0)
+
+/***
+ * @function kann.layer.input(ninputs[, flags])
+ * Creates an input layer for ANN
+ * @param {int} ninputs number of inputs
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_input(lua_State *L)
+{
+ gint nnodes = luaL_checkinteger(L, 1);
+
+ if (nnodes > 0) {
+ kad_node_t *t;
+
+ t = kann_layer_input(nnodes);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dense(in, ninputs[, flags])
+ * Creates a dense layer (e.g. for hidden layer)
+ * @param {kann_node} in kann node
+ * @param {int} ninputs number of dense nodes
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_dense(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ t = kann_layer_dense(in, nnodes);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dropout(in, ratio[, flags])
+ * Creates a dropout layer
+ * @param {kann_node} in kann node
+ * @param {float} ratio drop ratio
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_layerdropout(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ double r = luaL_checknumber(L, 2);
+
+ if (in != NULL) {
+ kad_node_t *t;
+
+ t = kann_layer_dropout(in, r);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + rate required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.dropout(in [, flags])
+ * Creates a normalisation layer
+ * @param {kann_node} in kann node
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_layernorm(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+
+ if (in != NULL) {
+ kad_node_t *t;
+
+ t = kann_layer_layernorm(in);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.rnn(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_rnn(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_rnn(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.lstm(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer using LSTM cells
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_lstm(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_lstm(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.rnn(in, nnodes[, rnn_flags, [, flags]])
+ * Creates a recursive NN layer using GRU cells
+ * @param {kann_node} in kann node
+ * @param {int} nnodes number of cells
+ * @param {int} rnnflags rnn flags
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_gru(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ gint nnodes = luaL_checkinteger(L, 2);
+ gint rnnflags = 0;
+
+ if (in != NULL && nnodes > 0) {
+ kad_node_t *t;
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ rnnflags = lua_tointeger(L, 3);
+ }
+
+ t = kann_layer_gru(in, nnodes, rnnflags);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input + nnodes required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.conv2d(in, n_flt, k_rows, k_cols, stride_rows, stride_cols, pad_rows, pad_columns[, flags])
+ * Creates a 2D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} n_flt number of filters
+ * @param {int} k_rows kernel rows
+ * @param {int} k_cols kernel columns
+ * @param {int} stride_rows stride rows
+ * @param {int} stride_cols stride columns
+ * @param {int} pad_rows padding rows
+ * @param {int} pad_columns padding columns
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_conv2d(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int n_flt = luaL_checkinteger(L, 2);
+ int k_rows = luaL_checkinteger(L, 3);
+ int k_cols = luaL_checkinteger(L, 4);
+ int stride_r = luaL_checkinteger(L, 5);
+ int stride_c = luaL_checkinteger(L, 6);
+ int pad_r = luaL_checkinteger(L, 7);
+ int pad_c = luaL_checkinteger(L, 8);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = kann_layer_conv2d(in, n_flt, k_rows, k_cols, stride_r, stride_c,
+ pad_r, pad_c);
+
+ PROCESS_KAD_FLAGS(t, 9);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nflt, kx, ky, stridex, stridey, padx, pady are required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.conv1d(in, n_flt, kern_size, stride_size, pad_size[, flags])
+ * Creates 1D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} n_flt number of filters
+ * @param {int} kern_size kernel rows
+ * @param {int} stride_size stride rows
+ * @param {int} pad_size padding rows
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_conv1d(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int n_flt = luaL_checkinteger(L, 2);
+ int k_size = luaL_checkinteger(L, 3);
+ int stride = luaL_checkinteger(L, 4);
+ int pad = luaL_checkinteger(L, 5);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = kann_layer_conv1d(in, n_flt, k_size, stride, pad);
+
+ PROCESS_KAD_FLAGS(t, 6);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nflt, k, stride, pad required");
+ }
+
+ return 1;
+}
+
+/***
+ * @function kann.layer.cost(in, nout, cost_type[, flags])
+ * Creates 1D convolution layer
+ * @param {kann_node} in kann node
+ * @param {int} nout number of outputs
+ * @param {int} cost_type see kann.cost table
+ * @param {table|int} flags optional flags
+ * @return {kann_node} kann node object (should be used to combine ANN)
+*/
+static int
+lua_kann_layer_cost(lua_State *L)
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+ int nout = luaL_checkinteger(L, 2);
+ int cost_type = luaL_checkinteger(L, 3);
+
+ if (in != NULL && nout > 0) {
+ kad_node_t *t;
+ t = kann_layer_cost(in, nout, cost_type);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, input, nout and cost_type are required");
+ }
+
+ return 1;
+}
+
+/* Generic helpers */
+static int
+lua_kann_call_unary_function(lua_State *L, const char *name,
+ kad_node_t *(*func)(kad_node_t *) )
+{
+ kad_node_t *in = lua_check_kann_node(L, 1);
+
+ if (in != NULL) {
+ kad_node_t *t;
+ t = func(in);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for %s, input required", name);
+ }
+
+ return 1;
+}
+static int
+lua_kann_call_binary_function(lua_State *L, const char *name,
+ kad_node_t *(*func)(kad_node_t *, kad_node_t *) )
+{
+ kad_node_t *x = lua_check_kann_node(L, 1);
+ kad_node_t *y = lua_check_kann_node(L, 2);
+
+ if (x != NULL && y != NULL) {
+ kad_node_t *t;
+ t = func(x, y);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for %s, 2 inputs required", name);
+ }
+
+ return 1;
+}
+
+#define LUA_UNARY_TRANSFORM_FUNC_IMPL(name) \
+ static int lua_kann_transform_##name(lua_State *L) \
+ { \
+ return lua_kann_call_unary_function(L, #name, kad_##name); \
+ }
+
+#define LUA_BINARY_TRANSFORM_FUNC_IMPL(name) \
+ static int lua_kann_transform_##name(lua_State *L) \
+ { \
+ return lua_kann_call_binary_function(L, #name, kad_##name); \
+ }
+
+#define LUA_LOSS_FUNC_IMPL(name) \
+ static int lua_kann_loss_##name(lua_State *L) \
+ { \
+ return lua_kann_call_binary_function(L, #name, kad_##name); \
+ }
+
+/* Transform functions registered via macro helpers */
+LUA_BINARY_TRANSFORM_FUNC_IMPL(add)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(sub)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(mul)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(cmul)
+LUA_BINARY_TRANSFORM_FUNC_IMPL(matmul)
+
+LUA_UNARY_TRANSFORM_FUNC_IMPL(square)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(sigm)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(tanh)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(relu)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(softmax)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(1minus)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(exp)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(log)
+LUA_UNARY_TRANSFORM_FUNC_IMPL(sin)
+
+/* Generic cost functions */
+LUA_LOSS_FUNC_IMPL(mse)
+LUA_LOSS_FUNC_IMPL(ce_multi)
+LUA_LOSS_FUNC_IMPL(ce_bin)
+LUA_LOSS_FUNC_IMPL(ce_bin_neg)
+
+/* The only case of ternary weight function */
+static int
+lua_kann_loss_ce_multi_weighted(lua_State *L)
+{
+ kad_node_t *pred = lua_check_kann_node(L, 1);
+ kad_node_t *truth = lua_check_kann_node(L, 2);
+ kad_node_t *weight = lua_check_kann_node(L, 3);
+
+ if (pred != NULL && truth != NULL && weight != NULL) {
+ kad_node_t *t;
+ t = kad_ce_multi_weighted(pred, truth, weight);
+
+ PUSH_KAD_NODE(t);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for ce_multi_weighted, 3 inputs required");
+ }
+
+ return 1;
+}
+
+/* Creation functions */
+static int
+lua_kann_new_scalar(lua_State *L)
+{
+ gint flag = luaL_checkinteger(L, 1);
+ double x = luaL_checknumber(L, 2);
+ kad_node_t *t;
+
+ t = kann_new_scalar(flag, x);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight(lua_State *L)
+{
+ gint nrow = luaL_checkinteger(L, 1);
+ gint ncol = luaL_checkinteger(L, 2);
+ kad_node_t *t;
+
+ t = kann_new_weight(nrow, ncol);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_bias(lua_State *L)
+{
+ gint n = luaL_checkinteger(L, 1);
+ kad_node_t *t;
+
+ t = kann_new_bias(n);
+
+ PROCESS_KAD_FLAGS(t, 2);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight_conv2d(lua_State *L)
+{
+ gint nout = luaL_checkinteger(L, 1);
+ gint nin = luaL_checkinteger(L, 2);
+ gint krow = luaL_checkinteger(L, 3);
+ gint kcol = luaL_checkinteger(L, 4);
+ kad_node_t *t;
+
+ t = kann_new_weight_conv2d(nout, nin, krow, kcol);
+
+ PROCESS_KAD_FLAGS(t, 5);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_weight_conv1d(lua_State *L)
+{
+ gint nout = luaL_checkinteger(L, 1);
+ gint nin = luaL_checkinteger(L, 2);
+ gint klen = luaL_checkinteger(L, 3);
+ kad_node_t *t;
+
+ t = kann_new_weight_conv1d(nout, nin, klen);
+
+ PROCESS_KAD_FLAGS(t, 4);
+ PUSH_KAD_NODE(t);
+
+ return 1;
+}
+
+static int
+lua_kann_new_leaf(lua_State *L)
+{
+ int dim = luaL_checkinteger(L, 1), i, *ar;
+ kad_node_t *t;
+
+ if (dim >= 1 && dim < KAD_MAX_DIM && lua_istable(L, 2)) {
+ ar = g_new0(int, KAD_MAX_DIM);
+
+ for (i = 0; i < dim; i++) {
+ lua_rawgeti(L, 2, i + 1);
+ ar[i] = lua_tointeger(L, -1);
+ lua_pop(L, 1);
+ }
+
+ t = kann_new_leaf_array(NULL, NULL, 0, 0.0, dim, ar);
+
+ PROCESS_KAD_FLAGS(t, 3);
+ PUSH_KAD_NODE(t);
+
+ g_free(ar);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for new.leaf, "
+ "dim and vector of elements are required");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_new_kann(lua_State *L)
+{
+ kad_node_t *cost = lua_check_kann_node(L, 1);
+ kann_t *k;
+
+ if (cost) {
+ k = kann_new(cost, 0);
+
+ PUSH_KAN_NETWORK(k);
+ }
+ else {
+ return luaL_error(L, "invalid arguments for new.kann, "
+ "cost node is required");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_destroy(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+
+ kann_delete(k);
+
+ return 0;
+}
+
+static int
+lua_kann_save(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+
+ if (k) {
+ if (lua_istable(L, 2)) {
+ lua_getfield(L, 2, "filename");
+
+ if (lua_isstring(L, -1)) {
+ const gchar *fname = lua_tostring(L, -1);
+ FILE *f;
+
+ f = fopen(fname, "w");
+
+ if (!f) {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "cannot open %s for writing: %s",
+ fname, strerror(errno));
+ }
+
+ kann_save_fp(f, k);
+ fclose(f);
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments: missing filename");
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ /* Save to Rspamd text */
+#ifndef HAVE_OPENMEMSTREAM
+ return luaL_error(L, "no support of saving to memory on your system");
+#endif
+ FILE *f;
+ char *buf = NULL;
+ size_t buflen;
+ struct rspamd_lua_text *t;
+
+ f = open_memstream(&buf, &buflen);
+ g_assert(f != NULL);
+
+ kann_save_fp(f, k);
+ fclose(f);
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ t->start = (const gchar *) buf;
+ t->len = buflen;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_load(lua_State *L)
+{
+ kann_t *k;
+ FILE *f = NULL;
+
+ if (lua_istable(L, 1)) {
+ lua_getfield(L, 2, "filename");
+
+ if (lua_isstring(L, -1)) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ f = fopen(fname, "rb");
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments: missing filename");
+ }
+
+ lua_pop(L, 1);
+ }
+ else if (lua_isstring(L, 1)) {
+ gsize dlen;
+ const gchar *data;
+
+ data = lua_tolstring(L, 1, &dlen);
+
+#ifndef HAVE_FMEMOPEN
+ return luaL_error(L, "no support of loading from memory on your system");
+#endif
+ f = fmemopen((void *) data, dlen, "rb");
+ }
+ else if (lua_isuserdata(L, 1)) {
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+#ifndef HAVE_FMEMOPEN
+ return luaL_error(L, "no support of loading from memory on your system");
+#endif
+ f = fmemopen((void *) t->start, t->len, "rb");
+ }
+
+ if (f == NULL) {
+ return luaL_error(L, "invalid arguments or cannot open file");
+ }
+
+ k = kann_load_fp(f);
+ fclose(f);
+
+ if (k == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ PUSH_KAN_NETWORK(k);
+ }
+
+ return 1;
+}
+
+struct rspamd_kann_train_cbdata {
+ lua_State *L;
+ kann_t *k;
+ gint cbref;
+};
+
+static void
+lua_kann_train_cb(int iter, float train_cost, float val_cost, void *ud)
+{
+ struct rspamd_kann_train_cbdata *cbd = (struct rspamd_kann_train_cbdata *) ud;
+
+ if (cbd->cbref != -1) {
+ gint err_idx;
+ lua_State *L = cbd->L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ lua_pushinteger(L, iter);
+ lua_pushnumber(L, train_cost);
+ lua_pushnumber(L, val_cost);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err("cannot run lua train callback: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+}
+
+#define FREE_VEC(a, n) \
+ do { \
+ for (int i = 0; i < (n); i++) g_free((a)[i]); \
+ g_free(a); \
+ } while (0)
+
+static int
+lua_kann_train1(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+ struct rspamd_lua_tensor *pca = NULL;
+
+ /* Default train params */
+ double lr = 0.001;
+ gint64 mini_size = 64;
+ gint64 max_epoch = 25;
+ gint64 max_drop_streak = 10;
+ double frac_val = 0.1;
+ gint cbref = -1;
+
+ if (k && lua_istable(L, 2) && lua_istable(L, 3)) {
+ int n = rspamd_lua_table_size(L, 2);
+ int n_in = kann_dim_in(k);
+ int n_out = kann_dim_out(k);
+
+ if (n_in <= 0) {
+ return luaL_error(L, "invalid inputs count: %d", n_in);
+ }
+
+ if (n_out <= 0) {
+ return luaL_error(L, "invalid outputs count: %d", n_out);
+ }
+
+ if (n != rspamd_lua_table_size(L, 3) || n == 0) {
+ return luaL_error(L, "invalid dimensions: outputs size must be "
+ "equal to inputs and non zero");
+ }
+
+ if (lua_istable(L, 4)) {
+ GError *err = NULL;
+
+ if (!rspamd_lua_parse_table_arguments(L, 4, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_IGNORE_MISSING,
+ "lr=N;mini_size=I;max_epoch=I;max_drop_streak=I;frac_val=N;cb=F;pca=u{tensor}",
+ &lr, &mini_size, &max_epoch, &max_drop_streak, &frac_val, &cbref, &pca)) {
+ n = luaL_error(L, "invalid params: %s",
+ err ? err->message : "unknown error");
+ g_error_free(err);
+
+ return n;
+ }
+ }
+
+ if (pca) {
+ /* Check pca matrix validity */
+ if (pca->ndims != 2) {
+ return luaL_error(L, "invalid pca tensor: matrix expected, got a row");
+ }
+
+ if (pca->dim[0] != n_in) {
+ return luaL_error(L, "invalid pca tensor: "
+ "matrix must have %d rows and it has %d rows instead",
+ n_in, pca->dim[0]);
+ }
+ }
+
+ float **x, **y, *tmp_row = NULL;
+
+ /* Fill vectors row by row */
+ x = (float **) g_malloc0(sizeof(float *) * n);
+ y = (float **) g_malloc0(sizeof(float *) * n);
+
+ if (pca) {
+ tmp_row = g_malloc(sizeof(float) * pca->dim[1]);
+ }
+
+ for (int s = 0; s < n; s++) {
+ /* Inputs */
+ lua_rawgeti(L, 2, s + 1);
+ x[s] = (float *) g_malloc(sizeof(float) * n_in);
+
+ if (pca == NULL) {
+ if (rspamd_lua_table_size(L, -1) != n_in) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+
+ n = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ n_in);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+ for (int i = 0; i < n_in; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ x[s][i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ if (rspamd_lua_table_size(L, -1) != pca->dim[1]) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+
+ n = luaL_error(L, "(pca on) invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ pca->dim[1]);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+
+ for (int i = 0; i < pca->dim[1]; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ tmp_row[i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ kad_sgemm_simple(0, 1, 1, n_in,
+ pca->dim[1], tmp_row, pca->data,
+ x[s]);
+ }
+
+ lua_pop(L, 1);
+
+ /* Outputs */
+ y[s] = (float *) g_malloc(sizeof(float) * n_out);
+ lua_rawgeti(L, 3, s + 1);
+
+ if (rspamd_lua_table_size(L, -1) != n_out) {
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+
+ n = luaL_error(L, "invalid params at pos %d: "
+ "bad output dimension %d; "
+ "%d expected",
+ s + 1,
+ (int) rspamd_lua_table_size(L, -1),
+ n_out);
+ lua_pop(L, 1);
+
+ return n;
+ }
+
+ for (int i = 0; i < n_out; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ y[s][i] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ struct rspamd_kann_train_cbdata cbd;
+
+ cbd.cbref = cbref;
+ cbd.k = k;
+ cbd.L = L;
+
+ int niters = kann_train_fnn1(k, lr,
+ mini_size, max_epoch, max_drop_streak,
+ frac_val, n, x, y, lua_kann_train_cb, &cbd);
+
+ lua_pushinteger(L, niters);
+
+ FREE_VEC(x, n);
+ FREE_VEC(y, n);
+ g_free(tmp_row);
+ }
+ else {
+ return luaL_error(L, "invalid arguments: kann, inputs, outputs and"
+ " optional params are expected");
+ }
+
+ return 1;
+}
+
+static int
+lua_kann_apply1(lua_State *L)
+{
+ kann_t *k = lua_check_kann(L, 1);
+ struct rspamd_lua_tensor *pca = NULL;
+
+ if (k) {
+ if (lua_istable(L, 2)) {
+ gsize vec_len = rspamd_lua_table_size(L, 2);
+ float *vec = (float *) g_malloc(sizeof(float) * vec_len),
+ *pca_out = NULL;
+ int i_out;
+ int n_in = kann_dim_in(k);
+
+ if (n_in <= 0) {
+ g_free(vec);
+ return luaL_error(L, "invalid inputs count: %d", n_in);
+ }
+
+ if (lua_isuserdata(L, 3)) {
+ pca = lua_check_tensor(L, 3);
+
+ if (pca) {
+ if (pca->ndims != 2) {
+ g_free(vec);
+ return luaL_error(L, "invalid pca tensor: matrix expected, got a row");
+ }
+
+ if (pca->dim[0] != n_in) {
+ g_free(vec);
+ return luaL_error(L, "invalid pca tensor: "
+ "matrix must have %d rows and it has %d rows instead",
+ n_in, pca->dim[0]);
+ }
+ }
+ else {
+ g_free(vec);
+ return luaL_error(L, "invalid params: pca matrix expected");
+ }
+ }
+ else {
+ if (n_in != vec_len) {
+ g_free(vec);
+ return luaL_error(L, "invalid params: bad input dimension %d; %d expected",
+ (int) vec_len, n_in);
+ }
+ }
+
+ for (gsize i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+ vec[i] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+
+ i_out = kann_find(k, KANN_F_OUT, 0);
+
+ if (i_out <= 0) {
+ g_free(vec);
+ return luaL_error(L, "invalid ANN: output layer is missing or is "
+ "at the input pos");
+ }
+
+ kann_set_batch_size(k, 1);
+ if (pca) {
+ pca_out = g_malloc(sizeof(float) * n_in);
+
+ kad_sgemm_simple(0, 1, 1, n_in,
+ vec_len, vec, pca->data,
+ pca_out);
+
+ kann_feed_bind(k, KANN_F_IN, 0, &pca_out);
+ }
+ else {
+ kann_feed_bind(k, KANN_F_IN, 0, &vec);
+ }
+
+ kad_eval_at(k->n, k->v, i_out);
+
+ gsize outlen = kad_len(k->v[i_out]);
+ lua_createtable(L, outlen, 0);
+
+ for (gsize i = 0; i < outlen; i++) {
+ lua_pushnumber(L, k->v[i_out]->x[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_free(vec);
+ g_free(pca_out);
+ }
+ else if (lua_isuserdata(L, 2)) {
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 2);
+
+ if (t && t->ndims == 1) {
+ int i_out;
+ int n_in = kann_dim_in(k);
+
+ if (n_in != t->dim[0]) {
+ return luaL_error(L, "invalid params: bad input dimension %d; %d expected",
+ (int) t->dim[0], n_in);
+ }
+
+ i_out = kann_find(k, KANN_F_OUT, 0);
+
+ if (i_out <= 0) {
+ return luaL_error(L, "invalid ANN: output layer is missing or is "
+ "at the input pos");
+ }
+
+ kann_set_batch_size(k, 1);
+ kann_feed_bind(k, KANN_F_IN, 0, &t->data);
+ kad_eval_at(k->n, k->v, i_out);
+
+ gint outlen = kad_len(k->v[i_out]);
+ struct rspamd_lua_tensor *out;
+ out = lua_newtensor(L, 1, &outlen, false, false);
+ /* Ensure that kann and tensor have the same understanding of floats */
+ G_STATIC_ASSERT(sizeof(float) == sizeof(rspamd_tensor_num_t));
+ memcpy(out->data, k->v[i_out]->x, outlen * sizeof(float));
+ }
+ else {
+ return luaL_error(L, "invalid arguments: 1D rspamd{tensor} expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments: 1D rspamd{tensor} expected");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments: rspamd{kann} expected");
+ }
+
+ return 1;
+} \ No newline at end of file
diff --git a/src/lua/lua_logger.c b/src/lua/lua_logger.c
new file mode 100644
index 0000000..f4f8f3d
--- /dev/null
+++ b/src/lua/lua_logger.c
@@ -0,0 +1,1068 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_private.h"
+
+/***
+ * @module rspamd_logger
+ * Rspamd logger module is used to log messages from LUA API to the main rspamd logger.
+ * It supports legacy and modern interfaces allowing highly customized an convenient log functions.
+ * Here is an example of logger usage:
+ * @example
+local rspamd_logger = require "rspamd_logger"
+
+local a = 'string'
+local b = 1.5
+local c = 1
+local d = {
+ 'aa',
+ 1,
+ 'bb'
+}
+local e = {
+ key = 'value',
+ key2 = 1.0
+}
+
+-- New extended interface
+-- %<number> means numeric arguments and %s means the next argument
+-- for example %1, %2, %s: %s would mean the third argument
+
+rspamd_logger.info('a=%1, b=%2, c=%3, d=%4, e=%s', a, b, c, d, e)
+-- Output: a=string, b=1.50000, c=1, d={[1] = aa, [2] = 1, [3] = bb} e={[key]=value, [key2]=1.0}
+
+-- Create string using logger API
+local str = rspamd_logger.slog('a=%1, b=%2, c=%3, d=%4, e=%5', a, b, c, d, e)
+
+print(str)
+-- Output: a=string, b=1.50000, c=1, d={[1] = aa, [2] = 1, [3] = bb} e={[key]=value, [key2]=1.0}
+ */
+
+/* Logger methods */
+/***
+ * @function logger.err(msg)
+ * Log message as an error
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, err);
+/***
+ * @function logger.warn(msg)
+ * Log message as a warning
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, warn);
+/***
+ * @function logger.info(msg)
+ * Log message as an informational message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, info);
+/***
+ * @function logger.message(msg)
+ * Log message as an notice message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, message);
+/***
+ * @function logger.debug(msg)
+ * Log message as a debug message
+ * @param {string} msg string to be logged
+ */
+LUA_FUNCTION_DEF(logger, debug);
+/***
+ * @function logger.errx(fmt[, args)
+ * Extended interface to make an error log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, errx);
+/***
+ * @function logger.warn(fmt[, args)
+ * Extended interface to make a warning log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, warnx);
+/***
+ * @function logger.infox(fmt[, args)
+ * Extended interface to make an informational log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, infox);
+/***
+ * @function logger.infox(fmt[, args)
+ * Extended interface to make an informational log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, messagex);
+/***
+ * @function logger.debugx(fmt[, args)
+ * Extended interface to make a debug log message
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, debugx);
+
+/***
+ * @function logger.debugm(module, id, fmt[, args)
+ * Extended interface to make a debug log message
+ * @param {string} module debug module
+ * @param {task|cfg|pool|string} id id to log
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, debugm);
+/***
+ * @function logger.slog(fmt[, args)
+ * Create string replacing percent params with corresponding arguments
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ * @return {string} string with percent parameters substituted
+ */
+LUA_FUNCTION_DEF(logger, slog);
+
+/***
+ * @function logger.logx(level, module, id, fmt[, args)
+ * Extended interface to make a generic log message on any level
+ * @param {number} log level as a number (see GLogLevelFlags enum for values)
+ * @param {task|cfg|pool|string} id id to log
+ * @param {string} fmt format string, arguments are encoded as %<number>
+ * @param {any} args list of arguments to be replaced in %<number> positions
+ */
+LUA_FUNCTION_DEF(logger, logx);
+
+/***
+ * @function logger.log_level()
+ * Returns log level for a logger
+ * @return {string} current log level
+ */
+LUA_FUNCTION_DEF(logger, log_level);
+
+static const struct luaL_reg loggerlib_f[] = {
+ LUA_INTERFACE_DEF(logger, err),
+ LUA_INTERFACE_DEF(logger, warn),
+ LUA_INTERFACE_DEF(logger, message),
+ {"msg", lua_logger_message},
+ LUA_INTERFACE_DEF(logger, info),
+ LUA_INTERFACE_DEF(logger, debug),
+ LUA_INTERFACE_DEF(logger, errx),
+ LUA_INTERFACE_DEF(logger, warnx),
+ LUA_INTERFACE_DEF(logger, infox),
+ LUA_INTERFACE_DEF(logger, messagex),
+ {"msgx", lua_logger_messagex},
+ LUA_INTERFACE_DEF(logger, debugx),
+ LUA_INTERFACE_DEF(logger, debugm),
+ LUA_INTERFACE_DEF(logger, slog),
+ LUA_INTERFACE_DEF(logger, logx),
+ LUA_INTERFACE_DEF(logger, log_level),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static void
+lua_common_log_line(GLogLevelFlags level,
+ lua_State *L,
+ const gchar *msg,
+ const gchar *uid,
+ const gchar *module,
+ gint stack_level)
+{
+ lua_Debug d;
+ gchar func_buf[128], *p;
+
+ if (lua_getstack(L, stack_level, &d) == 1) {
+ (void) lua_getinfo(L, "Sl", &d);
+ if ((p = strrchr(d.short_src, '/')) == NULL) {
+ p = d.short_src;
+ }
+ else {
+ p++;
+ }
+
+ if (strlen(p) > 30) {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%27s...:%d", p,
+ d.currentline);
+ }
+ else {
+ rspamd_snprintf(func_buf, sizeof(func_buf), "%s:%d", p,
+ d.currentline);
+ }
+
+ rspamd_common_log_function(NULL,
+ level,
+ module,
+ uid,
+ func_buf,
+ "%s",
+ msg);
+ }
+ else {
+ rspamd_common_log_function(NULL,
+ level,
+ module,
+ uid,
+ G_STRFUNC,
+ "%s",
+ msg);
+ }
+}
+
+/*** Logger interface ***/
+static gint
+lua_logger_err(lua_State *L)
+{
+ return lua_logger_errx(L);
+}
+
+static gint
+lua_logger_warn(lua_State *L)
+{
+ return lua_logger_warnx(L);
+}
+
+static gint
+lua_logger_info(lua_State *L)
+{
+ return lua_logger_infox(L);
+}
+
+static gint
+lua_logger_message(lua_State *L)
+{
+ return lua_logger_messagex(L);
+}
+
+static gint
+lua_logger_debug(lua_State *L)
+{
+ return lua_logger_debugx(L);
+}
+
+static inline bool
+lua_logger_char_safe(int t, unsigned int esc_type)
+{
+ if (t & 0x80) {
+ if (esc_type & LUA_ESCAPE_8BIT) {
+ return false;
+ }
+
+ return true;
+ }
+
+ if (esc_type & LUA_ESCAPE_UNPRINTABLE) {
+ if (!g_ascii_isprint(t) && !g_ascii_isspace(t)) {
+ return false;
+ }
+ }
+
+ if (esc_type & LUA_ESCAPE_NEWLINES) {
+ if (t == '\r' || t == '\n') {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static gsize
+lua_logger_out_str(lua_State *L, gint pos,
+ gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gsize slen, flen;
+ const gchar *str = lua_tolstring(L, pos, &slen);
+ static const gchar hexdigests[16] = "0123456789abcdef";
+ gsize r = 0, s;
+
+ if (str) {
+ gboolean normal = TRUE;
+ flen = MIN(slen, len - 1);
+
+ for (r = 0; r < flen; r++) {
+ if (!lua_logger_char_safe(str[r], esc_type)) {
+ normal = FALSE;
+ break;
+ }
+ }
+
+ if (normal) {
+ r = rspamd_strlcpy(outbuf, str, flen + 1);
+ }
+ else {
+ /* Need to escape non-printed characters */
+ r = 0;
+ s = 0;
+
+ while (slen > 0 && len > 1) {
+ if (!lua_logger_char_safe(str[s], esc_type)) {
+ if (len >= 3) {
+ outbuf[r++] = '\\';
+ outbuf[r++] = hexdigests[((str[s] >> 4) & 0xF)];
+ outbuf[r++] = hexdigests[((str[s]) & 0xF)];
+
+ len -= 2;
+ }
+ else {
+ outbuf[r++] = '?';
+ }
+ }
+ else {
+ outbuf[r++] = str[s];
+ }
+
+ s++;
+ slen--;
+ len--;
+ }
+
+ outbuf[r] = '\0';
+ }
+ }
+
+ return r;
+}
+
+static gsize
+lua_logger_out_num(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gdouble num = lua_tonumber(L, pos);
+ glong inum;
+ gsize r = 0;
+
+ if ((gdouble) (glong) num == num) {
+ inum = num;
+ r = rspamd_snprintf(outbuf, len + 1, "%l", inum);
+ }
+ else {
+ r = rspamd_snprintf(outbuf, len + 1, "%f", num);
+ }
+
+ return r;
+}
+
+static gsize
+lua_logger_out_boolean(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gboolean val = lua_toboolean(L, pos);
+ gsize r = 0;
+
+ r = rspamd_strlcpy(outbuf, val ? "true" : "false", len + 1);
+
+ return r;
+}
+
+static gsize
+lua_logger_out_userdata(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace)
+{
+ gint r = 0, top;
+ const gchar *str = NULL;
+ gboolean converted_to_str = FALSE;
+
+ top = lua_gettop(L);
+
+ if (!lua_getmetatable(L, pos)) {
+ return 0;
+ }
+
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+
+ if (!lua_istable(L, -1)) {
+
+ if (lua_isfunction(L, -1)) {
+ /* Functional metatable, try to get __tostring directly */
+ lua_pushstring(L, "__tostring");
+ lua_gettable(L, -3);
+
+ if (lua_isfunction(L, -1)) {
+ lua_pushvalue(L, pos);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ r = rspamd_snprintf(outbuf, len, "%s", str);
+ }
+
+ lua_settop(L, top);
+
+ return r;
+ }
+ }
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ lua_pushstring(L, "__tostring");
+ lua_gettable(L, -2);
+
+ if (lua_isfunction(L, -1)) {
+ lua_pushvalue(L, pos);
+
+ if (lua_pcall(L, 1, 1, 0) != 0) {
+ lua_settop(L, top);
+
+ return 0;
+ }
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ converted_to_str = TRUE;
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+
+ if (lua_isstring(L, -1)) {
+ str = lua_tostring(L, -1);
+ converted_to_str = TRUE;
+ }
+ }
+
+ if (converted_to_str) {
+ r = rspamd_snprintf(outbuf, len, "%s", str);
+ }
+ else {
+ /* Print raw pointer */
+ r = rspamd_snprintf(outbuf, len, "%s(%p)", str, lua_touserdata(L, pos));
+ }
+
+ lua_settop(L, top);
+
+ return r;
+}
+
+#define MOVE_BUF(d, remain, r) \
+ (d) += (r); \
+ (remain) -= (r); \
+ if ((remain) == 0) { \
+ lua_settop(L, old_top); \
+ break; \
+ }
+
+static gsize
+lua_logger_out_table(lua_State *L, gint pos, gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gchar *d = outbuf;
+ gsize remain = len, r;
+ gboolean first = TRUE;
+ gconstpointer self = NULL;
+ gint i, tpos, last_seq = -1, old_top;
+
+ if (!lua_istable(L, pos) || remain == 0) {
+ return 0;
+ }
+
+ old_top = lua_gettop(L);
+ self = lua_topointer(L, pos);
+
+ /* Check if we have seen this pointer */
+ for (i = 0; i < TRACE_POINTS; i++) {
+ if (trace->traces[i] == self) {
+ r = rspamd_snprintf(d, remain + 1, "ref(%p)", self);
+
+ d += r;
+
+ return (d - outbuf);
+ }
+ }
+
+ trace->traces[trace->cur_level % TRACE_POINTS] = self;
+
+ lua_pushvalue(L, pos);
+ r = rspamd_snprintf(d, remain + 1, "{");
+ remain -= r;
+ d += r;
+
+ /* Get numeric keys (ipairs) */
+ for (i = 1;; i++) {
+ lua_rawgeti(L, -1, i);
+
+ if (lua_isnil(L, -1)) {
+ lua_pop(L, 1);
+ break;
+ }
+
+ last_seq = i;
+
+ if (!first) {
+ r = rspamd_snprintf(d, remain + 1, ", ");
+ MOVE_BUF(d, remain, r);
+ }
+
+ r = rspamd_snprintf(d, remain + 1, "[%d] = ", i);
+ MOVE_BUF(d, remain, r);
+ tpos = lua_gettop(L);
+
+ if (lua_topointer(L, tpos) == self) {
+ r = rspamd_snprintf(d, remain + 1, "__self");
+ }
+ else {
+ r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type);
+ }
+ MOVE_BUF(d, remain, r);
+
+ first = FALSE;
+ lua_pop(L, 1);
+ }
+
+ /* Get string keys (pairs) */
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ /* 'key' is at index -2 and 'value' is at index -1 */
+ if (lua_type(L, -2) == LUA_TNUMBER) {
+ if (last_seq > 0) {
+ lua_pushvalue(L, -2);
+
+ if (lua_tonumber(L, -1) <= last_seq + 1) {
+ lua_pop(L, 1);
+ /* Already seen */
+ continue;
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+
+ if (!first) {
+ r = rspamd_snprintf(d, remain + 1, ", ");
+ MOVE_BUF(d, remain, r);
+ }
+
+ /* Preserve key */
+ lua_pushvalue(L, -2);
+ r = rspamd_snprintf(d, remain + 1, "[%s] = ",
+ lua_tostring(L, -1));
+ lua_pop(L, 1); /* Remove key */
+ MOVE_BUF(d, remain, r);
+ tpos = lua_gettop(L);
+
+ if (lua_topointer(L, tpos) == self) {
+ r = rspamd_snprintf(d, remain + 1, "__self");
+ }
+ else {
+ r = lua_logger_out_type(L, tpos, d, remain, trace, esc_type);
+ }
+ MOVE_BUF(d, remain, r);
+
+ first = FALSE;
+ }
+
+ lua_settop(L, old_top);
+
+ r = rspamd_snprintf(d, remain + 1, "}");
+ d += r;
+
+ return (d - outbuf);
+}
+
+#undef MOVE_BUF
+
+gsize lua_logger_out_type(lua_State *L, gint pos,
+ gchar *outbuf, gsize len,
+ struct lua_logger_trace *trace,
+ enum lua_logger_escape_type esc_type)
+{
+ gint type;
+ gsize r = 0;
+
+ if (len == 0) {
+ return 0;
+ }
+
+ type = lua_type(L, pos);
+ trace->cur_level++;
+
+ switch (type) {
+ case LUA_TNUMBER:
+ r = lua_logger_out_num(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TBOOLEAN:
+ r = lua_logger_out_boolean(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TTABLE:
+ r = lua_logger_out_table(L, pos, outbuf, len, trace, esc_type);
+ break;
+ case LUA_TUSERDATA:
+ r = lua_logger_out_userdata(L, pos, outbuf, len, trace);
+ break;
+ case LUA_TFUNCTION:
+ r = rspamd_snprintf(outbuf, len + 1, "function");
+ break;
+ case LUA_TLIGHTUSERDATA:
+ r = rspamd_snprintf(outbuf, len + 1, "0x%p", lua_topointer(L, pos));
+ break;
+ case LUA_TNIL:
+ r = rspamd_snprintf(outbuf, len + 1, "nil");
+ break;
+ case LUA_TNONE:
+ r = rspamd_snprintf(outbuf, len + 1, "no value");
+ break;
+ default:
+ /* Try to push everything as string using tostring magic */
+ r = lua_logger_out_str(L, pos, outbuf, len, trace, esc_type);
+ break;
+ }
+
+ trace->cur_level--;
+
+ return r;
+}
+
+static const gchar *
+lua_logger_get_id(lua_State *L, gint pos, GError **err)
+{
+ const gchar *uid = NULL, *clsname;
+
+ if (lua_getmetatable(L, pos) != 0) {
+ uid = "";
+ lua_pushstring(L, "__index");
+ lua_gettable(L, -2);
+
+ lua_pushstring(L, "class");
+ lua_gettable(L, -2);
+
+ clsname = lua_tostring(L, -1);
+
+ if (strcmp(clsname, "rspamd{task}") == 0) {
+ struct rspamd_task *task = lua_check_task(L, pos);
+
+ if (task) {
+ uid = task->task_pool->tag.uid;
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{task}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{mempool}") == 0) {
+ rspamd_mempool_t *pool;
+
+ pool = rspamd_lua_check_mempool(L, pos);
+
+ if (pool) {
+ uid = pool->tag.uid;
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{mempool}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{config}") == 0) {
+ struct rspamd_config *cfg;
+
+ cfg = lua_check_config(L, pos);
+
+ if (cfg) {
+ if (cfg->checksum) {
+ uid = cfg->checksum;
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{config}");
+ }
+ }
+ else if (strcmp(clsname, "rspamd{map}") == 0) {
+ struct rspamd_lua_map *map;
+
+ map = lua_check_map(L, pos);
+
+ if (map) {
+ if (map->map) {
+ uid = map->map->tag;
+ }
+ else {
+ uid = "embedded";
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "invalid rspamd{map}");
+ }
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "unknown class: %s", clsname);
+ }
+
+
+ /* Metatable, __index, classname */
+ lua_pop(L, 3);
+ }
+ else {
+ g_set_error(err, g_quark_from_static_string("lua_logger"),
+ EINVAL, "no metatable found for userdata");
+ }
+
+ return uid;
+}
+
+static gboolean
+lua_logger_log_format(lua_State *L, gint fmt_pos, gboolean is_string,
+ gchar *logbuf, gsize remain)
+{
+ gchar *d;
+ const gchar *s, *c;
+ gsize r, cpylen = 0;
+ guint arg_num = 0, cur_arg;
+ bool num_arg = false;
+ struct lua_logger_trace tr;
+ enum {
+ copy_char = 0,
+ got_percent,
+ parse_arg_num
+ } state = copy_char;
+
+ d = logbuf;
+ s = lua_tostring(L, fmt_pos);
+ c = s;
+ cur_arg = fmt_pos;
+
+ if (s == NULL) {
+ return FALSE;
+ }
+
+ while (remain > 0 && *s != '\0') {
+ switch (state) {
+ case copy_char:
+ if (*s == '%') {
+ state = got_percent;
+ s++;
+ if (cpylen > 0) {
+ memcpy(d, c, cpylen);
+ d += cpylen;
+ }
+ cpylen = 0;
+ }
+ else {
+ s++;
+ cpylen++;
+ remain--;
+ }
+ break;
+ case got_percent:
+ if (g_ascii_isdigit(*s) || *s == 's') {
+ state = parse_arg_num;
+ c = s;
+ }
+ else {
+ *d++ = *s++;
+ c = s;
+ state = copy_char;
+ }
+ break;
+ case parse_arg_num:
+ if (g_ascii_isdigit(*s)) {
+ s++;
+ num_arg = true;
+ }
+ else {
+ if (num_arg) {
+ arg_num = strtoul(c, NULL, 10);
+ arg_num += fmt_pos - 1;
+ /* Update the current argument */
+ cur_arg = arg_num;
+ }
+ else {
+ /* We have non numeric argument, e.g. %s */
+ arg_num = cur_arg++;
+ s++;
+ }
+
+ if (arg_num < 1 || arg_num > (guint) lua_gettop(L) + 1) {
+ msg_err("wrong argument number: %ud", arg_num);
+
+ return FALSE;
+ }
+
+ memset(&tr, 0, sizeof(tr));
+ r = lua_logger_out_type(L, arg_num + 1, d, remain, &tr,
+ is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG);
+ g_assert(r <= remain);
+ remain -= r;
+ d += r;
+ state = copy_char;
+ c = s;
+ }
+ break;
+ }
+ }
+
+ if (state == parse_arg_num) {
+ if (num_arg) {
+ arg_num = strtoul(c, NULL, 10);
+ arg_num += fmt_pos - 1;
+ }
+ else {
+ /* We have non numeric argument, e.g. %s */
+ arg_num = cur_arg;
+ }
+
+ if (arg_num < 1 || arg_num > (guint) lua_gettop(L) + 1) {
+ msg_err("wrong argument number: %ud", arg_num);
+
+ return FALSE;
+ }
+
+ memset(&tr, 0, sizeof(tr));
+ r = lua_logger_out_type(L, arg_num + 1, d, remain, &tr,
+ is_string ? LUA_ESCAPE_UNPRINTABLE : LUA_ESCAPE_LOG);
+ g_assert(r <= remain);
+ remain -= r;
+ d += r;
+ }
+ else if (state == copy_char) {
+ if (cpylen > 0 && remain > 0) {
+ memcpy(d, c, cpylen);
+ d += cpylen;
+ }
+ }
+
+ *d = '\0';
+
+
+ return TRUE;
+}
+
+static gint
+lua_logger_do_log(lua_State *L,
+ GLogLevelFlags level,
+ gboolean is_string,
+ gint start_pos)
+{
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ const gchar *uid = NULL;
+ gint fmt_pos = start_pos;
+ gint ret;
+ GError *err = NULL;
+
+ if (lua_type(L, start_pos) == LUA_TSTRING) {
+ fmt_pos = start_pos;
+ }
+ else if (lua_type(L, start_pos) == LUA_TUSERDATA) {
+ fmt_pos = start_pos + 1;
+
+ uid = lua_logger_get_id(L, start_pos, &err);
+
+ if (uid == NULL) {
+ ret = luaL_error(L, "bad userdata for logging: %s",
+ err ? err->message : "unknown error");
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+ }
+ else {
+ /* Bad argument type */
+ return luaL_error(L, "bad format string type: %s",
+ lua_typename(L, lua_type(L, start_pos)));
+ }
+
+ ret = lua_logger_log_format(L, fmt_pos, is_string,
+ logbuf, sizeof(logbuf) - 1);
+
+ if (ret) {
+ if (is_string) {
+ lua_pushstring(L, logbuf);
+ return 1;
+ }
+ else {
+ lua_common_log_line(level, L, logbuf, uid, "lua", 1);
+ }
+ }
+ else {
+ if (is_string) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_logger_errx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_CRITICAL, FALSE, 1);
+}
+
+static gint
+lua_logger_warnx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_WARNING, FALSE, 1);
+}
+
+static gint
+lua_logger_infox(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_INFO, FALSE, 1);
+}
+
+static gint
+lua_logger_messagex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_MESSAGE, FALSE, 1);
+}
+
+static gint
+lua_logger_debugx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_logger_do_log(L, G_LOG_LEVEL_DEBUG, FALSE, 1);
+}
+
+static gint
+lua_logger_logx(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GLogLevelFlags flags = lua_tonumber(L, 1);
+ const gchar *modname = lua_tostring(L, 2), *uid = NULL;
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ gboolean ret;
+ gint stack_pos = 1;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ uid = luaL_checkstring(L, 3);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ uid = lua_logger_get_id(L, 3, NULL);
+ }
+ else {
+ uid = "???";
+ }
+
+ if (uid && modname) {
+ if (lua_type(L, 4) == LUA_TSTRING) {
+ ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else if (lua_type(L, 4) == LUA_TNUMBER) {
+ stack_pos = lua_tonumber(L, 4);
+ ret = lua_logger_log_format(L, 5, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else {
+ return luaL_error(L, "invalid argument on pos 4");
+ }
+
+ if (ret) {
+ lua_common_log_line(flags, L, logbuf, uid, modname, stack_pos);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_logger_debugm(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar logbuf[RSPAMD_LOGBUF_SIZE - 128];
+ const gchar *uid = NULL, *module = NULL;
+ gint stack_pos = 1;
+ gboolean ret;
+
+ module = luaL_checkstring(L, 1);
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ uid = luaL_checkstring(L, 2);
+ }
+ else {
+ uid = lua_logger_get_id(L, 2, NULL);
+ }
+
+ if (uid && module) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ ret = lua_logger_log_format(L, 3, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else if (lua_type(L, 3) == LUA_TNUMBER) {
+ stack_pos = lua_tonumber(L, 3);
+ ret = lua_logger_log_format(L, 4, FALSE, logbuf, sizeof(logbuf) - 1);
+ }
+ else {
+ return luaL_error(L, "invalid argument on pos 3");
+ }
+
+ if (ret) {
+ lua_common_log_line(G_LOG_LEVEL_DEBUG, L, logbuf, uid, module, stack_pos);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_logger_slog(lua_State *L)
+{
+ return lua_logger_do_log(L, 0, TRUE, 1);
+}
+
+static gint
+lua_logger_log_level(lua_State *L)
+{
+ gint log_level = rspamd_log_get_log_level(NULL);
+
+ lua_pushstring(L, rspamd_get_log_severity_string(log_level));
+
+ return 1;
+}
+
+/*** Init functions ***/
+
+static gint
+lua_load_logger(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, loggerlib_f);
+
+ return 1;
+}
+
+void luaopen_logger(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_logger", lua_load_logger);
+}
diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c
new file mode 100644
index 0000000..54cfb4b
--- /dev/null
+++ b/src/lua/lua_map.c
@@ -0,0 +1,1421 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "libserver/maps/map.h"
+#include "libserver/maps/map_helpers.h"
+#include "libserver/maps/map_private.h"
+#include "contrib/libucl/lua_ucl.h"
+
+/***
+ * This module is used to manage rspamd maps and map like objects
+ *
+ * @module rspamd_map
+ *
+ * All maps could be obtained by function `rspamd_config:get_maps()`
+ * Also see [`lua_maps` module description](lua_maps.html).
+ *
+ * **Important notice** maps cannot be queried outside of the worker context.
+ * For example, you cannot add even a file map and query some keys from it during
+ * some module initialisation, you need to add the appropriate event loop context
+ * for a worker (e.g. you cannot use `get_key` outside of the symbols callbacks or
+ * a worker `on_load` scripts).
+ *
+@example
+
+local hash_map = rspamd_config:add_map{
+ type = "hash",
+ urls = ['file:///path/to/file'],
+ description = 'sample map'
+}
+
+local function sample_symbol_cb(task)
+ -- Check whether hash map contains from address of message
+ if hash_map:get_key((task:get_from() or {})[1]) then
+ -- key found
+ end
+end
+
+rspamd_config:register_symbol{
+ name = 'SAMPLE_SYMBOL',
+ type = 'normal',
+ score = 1.0,
+ description = "A sample symbol",
+ callback = sample_symbol_cb,
+}
+ */
+
+/***
+ * @method map:get_key(in)
+ * Variable method for different types of maps:
+ *
+ * - For hash maps it returns boolean and accepts string
+ * - For kv maps it returns string (or nil) and accepts string
+ * - For radix maps it returns boolean and accepts IP address (as object, string or number)
+ *
+ * @param {vary} in input to check
+ * @return {bool|string} if a value is found then this function returns string or `True` if not - then it returns `nil` or `False`
+ */
+LUA_FUNCTION_DEF(map, get_key);
+
+
+/***
+ * @method map:is_signed()
+ * Returns `True` if a map is signed
+ * @return {bool} signed value
+ */
+LUA_FUNCTION_DEF(map, is_signed);
+
+/***
+ * @method map:get_proto()
+ * Returns protocol of map as string:
+ *
+ * - `http`: for HTTP map
+ * - `file`: for file map
+ * @return {string} string representation of the map protocol
+ */
+LUA_FUNCTION_DEF(map, get_proto);
+
+/***
+ * @method map:get_sign_key()
+ * Returns pubkey used for signing as base32 string or nil
+ * @return {string} base32 encoded string or nil
+ */
+LUA_FUNCTION_DEF(map, get_sign_key);
+
+/***
+ * @method map:set_sign_key(key)
+ * Set trusted key for signatures for this map
+ * @param {string} key base32 encoded string or nil
+ */
+LUA_FUNCTION_DEF(map, set_sign_key);
+
+/***
+ * @method map:set_callback(cb)
+ * Set callback for a specified callback map.
+ * @param {function} cb map callback function
+ */
+LUA_FUNCTION_DEF(map, set_callback);
+
+/***
+ * @method map:get_uri()
+ * Get uri for a specified map
+ * @return {string} map's URI
+ */
+LUA_FUNCTION_DEF(map, get_uri);
+
+/***
+ * @method map:get_stats(reset)
+ * Get statistics for specific map. It returns table in form:
+ * [key] => [nhits]
+ * @param {boolean} reset reset stats if true
+ * @return {table} map's stat
+ */
+LUA_FUNCTION_DEF(map, get_stats);
+
+/***
+ * @method map:foreach(callback, is_text)
+ * Iterate over map elements and call callback for each element.
+ * @param {function} callback callback function, that accepts two arguments: key and value, if it returns true then iteration is stopped
+ * @param {boolean} is_text if true then callback accepts rspamd_text instead of Lua strings
+ * @return {number} number of elements iterated
+ */
+LUA_FUNCTION_DEF(map, foreach);
+
+/***
+ * @method map:on_load(callback)
+ * Sets a callback for a map that is called when map is loaded
+ * @param {function} callback callback function, that accepts no arguments (pass maps in a closure if needed)
+ */
+LUA_FUNCTION_DEF(map, on_load);
+
+/***
+ * @method map:get_data_digest()
+ * Get data digest for specific map
+ * @return {string} 64 bit number represented as string (due to Lua limitations)
+ */
+LUA_FUNCTION_DEF(map, get_data_digest);
+
+/***
+ * @method map:get_nelts()
+ * Get number of elements for specific map
+ * @return {number} number of elements in the map
+ */
+LUA_FUNCTION_DEF(map, get_nelts);
+
+static const struct luaL_reg maplib_m[] = {
+ LUA_INTERFACE_DEF(map, get_key),
+ LUA_INTERFACE_DEF(map, is_signed),
+ LUA_INTERFACE_DEF(map, get_proto),
+ LUA_INTERFACE_DEF(map, get_sign_key),
+ LUA_INTERFACE_DEF(map, set_sign_key),
+ LUA_INTERFACE_DEF(map, set_callback),
+ LUA_INTERFACE_DEF(map, get_uri),
+ LUA_INTERFACE_DEF(map, get_stats),
+ LUA_INTERFACE_DEF(map, foreach),
+ LUA_INTERFACE_DEF(map, on_load),
+ LUA_INTERFACE_DEF(map, get_data_digest),
+ LUA_INTERFACE_DEF(map, get_nelts),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct lua_map_callback_data {
+ lua_State *L;
+ gint ref;
+ gboolean opaque;
+ rspamd_fstring_t *data;
+ struct rspamd_lua_map *lua_map;
+};
+
+struct rspamd_lua_map *
+lua_check_map(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{map}");
+ luaL_argcheck(L, ud != NULL, pos, "'map' expected");
+ return ud ? *((struct rspamd_lua_map **) ud) : NULL;
+}
+
+gint lua_config_add_radix_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid radix map %s", map_line);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_radix_from_config(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *mname, *optname;
+ const ucl_object_t *obj;
+ struct rspamd_lua_map *map, **pmap;
+ ucl_object_t *fake_obj;
+ struct rspamd_map *m;
+
+ if (!cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ mname = luaL_checkstring(L, 2);
+ optname = luaL_checkstring(L, 3);
+
+ if (mname && optname) {
+ obj = rspamd_config_get_module_opt(cfg, mname, optname);
+
+ if (obj) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ fake_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(fake_obj, ucl_object_ref(obj),
+ "data", 0, false);
+ ucl_object_insert_key(fake_obj, ucl_object_fromstring("static"),
+ "url", 0, false);
+
+ if ((m = rspamd_map_add_from_ucl(cfg, fake_obj, "static radix map",
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_err_config("invalid radix map static");
+ lua_pushnil(L);
+ ucl_object_unref(fake_obj);
+
+ return 1;
+ }
+
+ ucl_object_unref(fake_obj);
+ pmap = lua_newuserdata(L, sizeof(void *));
+ map->map = m;
+ m->lua_map = map;
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ msg_warn_config("Couldnt find config option [%s][%s]", mname,
+ optname);
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+gint lua_config_radix_from_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ ucl_object_t *obj;
+ struct rspamd_lua_map *map, **pmap;
+ ucl_object_t *fake_obj;
+ struct rspamd_map *m;
+
+ if (!cfg) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ obj = ucl_object_lua_import(L, 2);
+
+ if (obj) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ fake_obj = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(fake_obj, ucl_object_ref(obj),
+ "data", 0, false);
+ ucl_object_insert_key(fake_obj, ucl_object_fromstring("static"),
+ "url", 0, false);
+
+ if ((m = rspamd_map_add_from_ucl(cfg, fake_obj, "static radix map",
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_err_config("invalid radix map static");
+ lua_pushnil(L);
+ ucl_object_unref(fake_obj);
+ ucl_object_unref(obj);
+
+ return 1;
+ }
+
+ ucl_object_unref(fake_obj);
+ ucl_object_unref(obj);
+ pmap = lua_newuserdata(L, sizeof(void *));
+ map->map = m;
+ m->lua_map = map;
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_add_hash_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_SET;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid set map %s", map_line);
+ lua_pushnil(L);
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+gint lua_config_add_kv_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *map_line, *description;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+
+ if (cfg) {
+ map_line = luaL_checkstring(L, 2);
+ description = lua_tostring(L, 3);
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_HASH;
+
+ if ((m = rspamd_map_add(cfg, map_line, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ msg_warn_config("invalid hash map %s", map_line);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gchar *
+lua_map_read(gchar *chunk, gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct lua_map_callback_data *cbdata, *old;
+
+ if (data->cur_data == NULL) {
+ old = (struct lua_map_callback_data *) data->prev_data;
+ cbdata = old;
+ cbdata->L = old->L;
+ cbdata->ref = old->ref;
+ cbdata->lua_map = old->lua_map;
+ data->cur_data = cbdata;
+ data->prev_data = NULL;
+ }
+ else {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ }
+
+ if (cbdata->data == NULL) {
+ cbdata->data = rspamd_fstring_new_init(chunk, len);
+ }
+ else {
+ cbdata->data = rspamd_fstring_append(cbdata->data, chunk, len);
+ }
+
+ return NULL;
+}
+
+static void
+lua_map_fin(struct map_cb_data *data, void **target)
+{
+ struct lua_map_callback_data *cbdata;
+ struct rspamd_lua_map **pmap;
+ struct rspamd_map *map;
+
+ map = data->map;
+
+ if (data->errored) {
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ if (cbdata->ref != -1) {
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+ }
+
+ if (cbdata->data) {
+ rspamd_fstring_free(cbdata->data);
+ }
+
+ data->cur_data = NULL;
+ }
+ }
+ else {
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ }
+ else {
+ msg_err_map("no data read for map");
+ return;
+ }
+
+ if (cbdata->ref == -1) {
+ msg_err_map("map has no callback set");
+ }
+ else if (cbdata->data != NULL && cbdata->data->len != 0) {
+
+ lua_pushcfunction(cbdata->L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(cbdata->L);
+
+ lua_rawgeti(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+
+ if (!cbdata->opaque) {
+ lua_pushlstring(cbdata->L, cbdata->data->str, cbdata->data->len);
+ }
+ else {
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(cbdata->L, sizeof(*t));
+ rspamd_lua_setclass(cbdata->L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->len = cbdata->data->len;
+ t->start = cbdata->data->str;
+ }
+
+ pmap = lua_newuserdata(cbdata->L, sizeof(void *));
+ *pmap = cbdata->lua_map;
+ rspamd_lua_setclass(cbdata->L, "rspamd{map}", -1);
+
+ gint ret = lua_pcall(cbdata->L, 2, 0, err_idx);
+
+ if (ret != 0) {
+ msg_info_map("call to %s failed (%d): %s", "map fin function",
+ ret,
+ lua_tostring(cbdata->L, -1));
+ }
+
+ lua_settop(cbdata->L, err_idx - 1);
+ }
+
+ cbdata->data = rspamd_fstring_assign(cbdata->data, "", 0);
+
+ if (target) {
+ *target = data->cur_data;
+ }
+
+ if (data->prev_data) {
+ data->prev_data = NULL;
+ }
+ }
+}
+
+static void
+lua_map_dtor(struct map_cb_data *data)
+{
+ struct lua_map_callback_data *cbdata;
+
+ if (data->cur_data) {
+ cbdata = (struct lua_map_callback_data *) data->cur_data;
+ if (cbdata->ref != -1) {
+ luaL_unref(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref);
+ }
+
+ if (cbdata->data) {
+ rspamd_fstring_free(cbdata->data);
+ }
+ }
+}
+
+gint lua_config_add_map(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const char *description = NULL;
+ const gchar *type = NULL;
+ ucl_object_t *map_obj = NULL;
+ struct lua_map_callback_data *cbdata;
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+ gboolean opaque_data = FALSE;
+ int cbidx = -1, ret;
+ GError *err = NULL;
+
+ if (cfg) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*url=O;description=S;callback=F;type=S;opaque_data=B",
+ &map_obj, &description, &cbidx, &type, &opaque_data)) {
+ ret = luaL_error(L, "invalid table arguments: %s", err->message);
+ g_error_free(err);
+ if (map_obj) {
+ ucl_object_unref(map_obj);
+ }
+
+ return ret;
+ }
+
+ g_assert(map_obj != NULL);
+
+ if (type == NULL && cbidx != -1) {
+ type = "callback";
+ }
+ else if (type == NULL) {
+ return luaL_error(L, "invalid map type");
+ }
+
+ if (strcmp(type, "callback") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->type = RSPAMD_LUA_MAP_CALLBACK;
+ map->data.cbdata = rspamd_mempool_alloc0(cfg->cfg_pool,
+ sizeof(*map->data.cbdata));
+ cbdata = map->data.cbdata;
+ cbdata->L = L;
+ cbdata->data = NULL;
+ cbdata->lua_map = map;
+ cbdata->ref = cbidx;
+ cbdata->opaque = opaque_data;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ lua_map_read,
+ lua_map_fin,
+ lua_map_dtor,
+ (void **) &map->data.cbdata,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+
+ if (cbidx != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbidx);
+ }
+
+ if (map_obj) {
+ ucl_object_unref(map_obj);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "set") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_SET;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "map") == 0 || strcmp(type, "hash") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.hash = NULL;
+ map->type = RSPAMD_LUA_MAP_HASH;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_kv_list_read,
+ rspamd_kv_list_fin,
+ rspamd_kv_list_dtor,
+ (void **) &map->data.hash,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "radix") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.radix = NULL;
+ map->type = RSPAMD_LUA_MAP_RADIX;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_radix_read,
+ rspamd_radix_fin,
+ rspamd_radix_dtor,
+ (void **) &map->data.radix,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "regexp") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_regexp_list_read_single,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "regexp_multi") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP_MULTIPLE;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_regexp_list_read_multiple,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "glob") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_glob_list_read_single,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "glob_multi") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.re_map = NULL;
+ map->type = RSPAMD_LUA_MAP_REGEXP_MULTIPLE;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_glob_list_read_multiple,
+ rspamd_regexp_list_fin,
+ rspamd_regexp_list_dtor,
+ (void **) &map->data.re_map,
+ NULL, RSPAMD_MAP_DEFAULT)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else if (strcmp(type, "cdb") == 0) {
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+ map->data.cdb_map = NULL;
+ map->type = RSPAMD_LUA_MAP_CDB;
+
+ if ((m = rspamd_map_add_from_ucl(cfg, map_obj, description,
+ rspamd_cdb_list_read,
+ rspamd_cdb_list_fin,
+ rspamd_cdb_list_dtor,
+ (void **) &map->data.cdb_map,
+ NULL, RSPAMD_MAP_FILE_ONLY | RSPAMD_MAP_FILE_NO_READ)) == NULL) {
+ lua_pushnil(L);
+ ucl_object_unref(map_obj);
+
+ return 1;
+ }
+ m->lua_map = map;
+ }
+ else {
+ ret = luaL_error(L, "invalid arguments: unknown type '%s'", type);
+ ucl_object_unref(map_obj);
+
+ return ret;
+ }
+
+ map->map = m;
+ pmap = lua_newuserdata(L, sizeof(void *));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ucl_object_unref(map_obj);
+
+ return 1;
+}
+
+gint lua_config_get_maps(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ struct rspamd_lua_map *map, **pmap;
+ struct rspamd_map *m;
+ gint i = 1;
+ GList *cur;
+
+ if (cfg) {
+ lua_newtable(L);
+ cur = g_list_first(cfg->maps);
+
+ while (cur) {
+ m = cur->data;
+
+ if (m->lua_map) {
+ map = m->lua_map;
+ }
+ else {
+ /* Implement heuristic */
+ map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*map));
+
+ if (m->read_callback == rspamd_radix_read) {
+ map->type = RSPAMD_LUA_MAP_RADIX;
+ map->data.radix = *m->user_data;
+ }
+ else if (m->read_callback == rspamd_kv_list_read) {
+ map->type = RSPAMD_LUA_MAP_HASH;
+ map->data.hash = *m->user_data;
+ }
+ else {
+ map->type = RSPAMD_LUA_MAP_UNKNOWN;
+ }
+
+ map->map = m;
+ m->lua_map = map;
+ }
+
+ pmap = lua_newuserdata(L, sizeof(*pmap));
+ *pmap = map;
+ rspamd_lua_setclass(L, "rspamd{map}", -1);
+ lua_rawseti(L, -2, i);
+
+ cur = g_list_next(cur);
+ i++;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static const gchar *
+lua_map_process_string_key(lua_State *L, gint pos, gsize *len)
+{
+ struct rspamd_lua_text *t;
+
+ if (lua_type(L, pos) == LUA_TSTRING) {
+ return lua_tolstring(L, pos, len);
+ }
+ else if (lua_type(L, pos) == LUA_TUSERDATA) {
+ t = lua_check_text(L, pos);
+
+ if (t) {
+ *len = t->len;
+ return t->start;
+ }
+ }
+
+ return NULL;
+}
+
+/* Radix and hash table functions */
+static gint
+lua_map_get_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_radix_map_helper *radix;
+ struct rspamd_lua_ip *addr = NULL;
+ const gchar *key, *value = NULL;
+ gpointer ud;
+ gsize len;
+ guint32 key_num = 0;
+ gboolean ret = FALSE;
+
+ if (map) {
+ if (map->type == RSPAMD_LUA_MAP_RADIX) {
+ radix = map->data.radix;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *addr_str;
+
+ addr_str = luaL_checklstring(L, 2, &len);
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = g_alloca(rspamd_inet_address_storage_size());
+
+ if (!rspamd_parse_inet_address_ip(addr_str, len, addr->addr)) {
+ addr = NULL;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ud = rspamd_lua_check_udata(L, 2, "rspamd{ip}");
+ if (ud != NULL) {
+ addr = *((struct rspamd_lua_ip **) ud);
+
+ if (addr->addr == NULL) {
+ addr = NULL;
+ }
+ }
+ else {
+ msg_err("invalid userdata type provided, rspamd{ip} expected");
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TNUMBER) {
+ key_num = luaL_checkinteger(L, 2);
+ key_num = htonl(key_num);
+ }
+
+ if (radix) {
+ gconstpointer p = NULL;
+
+ if (addr != NULL) {
+ if ((p = rspamd_match_radix_map_addr(radix, addr->addr)) != NULL) {
+ ret = TRUE;
+ }
+ else {
+ p = 0;
+ }
+ }
+ else if (key_num != 0) {
+ if ((p = rspamd_match_radix_map(radix,
+ (guint8 *) &key_num, sizeof(key_num))) != NULL) {
+ ret = TRUE;
+ }
+ else {
+ p = 0;
+ }
+ }
+
+ value = (const char *) p;
+ }
+
+ if (ret) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_SET) {
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.hash) {
+ ret = rspamd_match_hash_map(map->data.hash, key, len) != NULL;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_REGEXP) {
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.re_map) {
+ value = rspamd_match_regexp_map_single(map->data.re_map, key,
+ len);
+
+ if (value) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_REGEXP_MULTIPLE) {
+ GPtrArray *ar;
+ guint i;
+ const gchar *val;
+
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.re_map) {
+ ar = rspamd_match_regexp_map_all(map->data.re_map, key,
+ len);
+
+ if (ar) {
+ lua_createtable(L, ar->len, 0);
+
+ PTR_ARRAY_FOREACH(ar, i, val)
+ {
+ lua_pushstring(L, val);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ g_ptr_array_free(ar, TRUE);
+
+ return 1;
+ }
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_HASH) {
+ /* key-value map */
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.hash) {
+ value = rspamd_match_hash_map(map->data.hash, key, len);
+ }
+
+ if (value) {
+ lua_pushstring(L, value);
+ return 1;
+ }
+ }
+ else if (map->type == RSPAMD_LUA_MAP_CDB) {
+ /* cdb map */
+ const rspamd_ftok_t *tok = NULL;
+
+ key = lua_map_process_string_key(L, 2, &len);
+
+ if (key && map->data.cdb_map) {
+ tok = rspamd_match_cdb_map(map->data.cdb_map, key, len);
+ }
+
+ if (tok) {
+ lua_pushlstring(L, tok->begin, tok->len);
+ return 1;
+ }
+ }
+ else {
+ /* callback map or unknown type map */
+ lua_pushnil(L);
+ return 1;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static gboolean
+lua_map_traverse_cb(gconstpointer key,
+ gconstpointer value, gsize hits, gpointer ud)
+{
+ lua_State *L = (lua_State *) ud;
+
+ lua_pushstring(L, key);
+ lua_pushinteger(L, hits);
+ lua_settable(L, -3);
+
+ return TRUE;
+}
+
+static gint
+lua_map_get_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean do_reset = FALSE;
+
+ if (map != NULL) {
+ if (lua_isboolean(L, 2)) {
+ do_reset = lua_toboolean(L, 2);
+ }
+
+ lua_createtable(L, 0, map->map->nelts);
+
+ if (map->map->traverse_function) {
+ rspamd_map_traverse(map->map, lua_map_traverse_cb, L, do_reset);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_map_traverse_cbdata {
+ lua_State *L;
+ gint cbref;
+ gboolean use_text;
+};
+
+static gboolean
+lua_map_foreach_cb(gconstpointer key, gconstpointer value, gsize _hits, gpointer ud)
+{
+ struct lua_map_traverse_cbdata *cbdata = ud;
+ lua_State *L = cbdata->L;
+
+ lua_pushvalue(L, cbdata->cbref);
+
+ if (cbdata->use_text) {
+ lua_new_text(L, key, strlen(key), 0);
+ lua_new_text(L, value, strlen(value), 0);
+ }
+ else {
+ lua_pushstring(L, key);
+ lua_pushstring(L, value);
+ }
+
+ if (lua_pcall(L, 2, 1, 0) != 0) {
+ msg_err("call to map foreach callback failed: %s", lua_tostring(L, -1));
+ lua_pop(L, 1); /* error */
+
+ return FALSE;
+ }
+ else {
+ if (lua_isboolean(L, -1)) {
+ lua_pop(L, 2);
+
+ return lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1); /* result */
+ }
+
+ return TRUE;
+}
+
+static gint
+lua_map_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean use_text = FALSE;
+
+ if (map != NULL && lua_isfunction(L, 2)) {
+ if (lua_isboolean(L, 3)) {
+ use_text = lua_toboolean(L, 3);
+ }
+
+ struct lua_map_traverse_cbdata cbdata;
+ cbdata.L = L;
+ lua_pushvalue(L, 2); /* func */
+ cbdata.cbref = lua_gettop(L);
+ cbdata.use_text = use_text;
+
+ if (map->map->traverse_function) {
+ rspamd_map_traverse(map->map, lua_map_foreach_cb, &cbdata, FALSE);
+ }
+
+ /* Remove callback */
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_map_get_data_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gchar numbuf[64];
+
+ if (map != NULL) {
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", map->map->digest);
+ lua_pushstring(L, numbuf);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_map_get_nelts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (map != NULL) {
+ lua_pushinteger(L, map->map->nelts);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_map_is_signed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ gboolean ret = FALSE;
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ if (map->map) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ if (bk->is_signed && bk->protocol == MAP_PROTO_FILE) {
+ ret = TRUE;
+ break;
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ return 1;
+}
+
+static int
+lua_map_get_proto(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ const gchar *ret = "undefined";
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ switch (bk->protocol) {
+ case MAP_PROTO_FILE:
+ ret = "file";
+ break;
+ case MAP_PROTO_HTTP:
+ ret = "http";
+ break;
+ case MAP_PROTO_HTTPS:
+ ret = "https";
+ break;
+ case MAP_PROTO_STATIC:
+ ret = "static";
+ break;
+ }
+ lua_pushstring(L, ret);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return map->map->backends->len;
+}
+
+static int
+lua_map_get_sign_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ guint i;
+ GString *ret = NULL;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+
+ if (bk->trusted_pubkey) {
+ ret = rspamd_pubkey_print(bk->trusted_pubkey,
+ RSPAMD_KEYPAIR_PUBKEY | RSPAMD_KEYPAIR_BASE32);
+ }
+ else {
+ ret = NULL;
+ }
+
+ if (ret) {
+ lua_pushlstring(L, ret->str, ret->len);
+ g_string_free(ret, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return map->map->backends->len;
+}
+
+static int
+lua_map_set_sign_key(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ const gchar *pk_str;
+ struct rspamd_cryptobox_pubkey *pk;
+ gsize len;
+ guint i;
+
+ pk_str = lua_tolstring(L, 2, &len);
+
+ if (map && pk_str) {
+ pk = rspamd_pubkey_from_base32(pk_str, len, RSPAMD_KEYPAIR_SIGN,
+ RSPAMD_CRYPTOBOX_MODE_25519);
+
+ if (!pk) {
+ return luaL_error(L, "invalid pubkey string");
+ }
+
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ if (bk->trusted_pubkey) {
+ /* Unref old pk */
+ rspamd_pubkey_unref(bk->trusted_pubkey);
+ }
+
+ bk->trusted_pubkey = rspamd_pubkey_ref(pk);
+ }
+
+ rspamd_pubkey_unref(pk);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_map_set_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (!map || map->type != RSPAMD_LUA_MAP_CALLBACK || map->data.cbdata == NULL) {
+ return luaL_error(L, "invalid map");
+ }
+
+ if (lua_type(L, 2) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid callback");
+ }
+
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ map->data.cbdata->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return 0;
+}
+
+static int
+lua_map_get_uri(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+ struct rspamd_map_backend *bk;
+ guint i;
+
+ if (map != NULL) {
+ for (i = 0; i < map->map->backends->len; i++) {
+ bk = g_ptr_array_index(map->map->backends, i);
+ lua_pushstring(L, bk->uri);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return map->map->backends->len;
+}
+
+struct lua_map_on_load_cbdata {
+ lua_State *L;
+ gint ref;
+};
+
+static void
+lua_map_on_load_dtor(gpointer p)
+{
+ struct lua_map_on_load_cbdata *cbd = p;
+
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->ref);
+ g_free(cbd);
+}
+
+static void
+lua_map_on_load_handler(struct rspamd_map *map, gpointer ud)
+{
+ struct lua_map_on_load_cbdata *cbd = ud;
+ lua_State *L = cbd->L;
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->ref);
+
+ if (lua_pcall(L, 0, 0, 0) != 0) {
+ msg_err_map("call to on_load function failed: %s", lua_tostring(L, -1));
+ }
+}
+
+static gint
+lua_map_on_load(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_map *map = lua_check_map(L, 1);
+
+ if (map == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ struct lua_map_on_load_cbdata *cbd = g_malloc(sizeof(struct lua_map_on_load_cbdata));
+ cbd->L = L;
+ cbd->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_map_set_on_load_function(map->map, lua_map_on_load_handler, cbd, lua_map_on_load_dtor);
+ }
+ else {
+ return luaL_error(L, "invalid callback");
+ }
+
+ return 0;
+}
+
+void luaopen_map(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{map}", maplib_m);
+
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_map.h b/src/lua/lua_map.h
new file mode 100644
index 0000000..70677de
--- /dev/null
+++ b/src/lua/lua_map.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LUA_LUA_MAP_H_
+#define SRC_LUA_LUA_MAP_H_
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+LUA_PUBLIC_FUNCTION_DEF(config, add_radix_map);
+LUA_PUBLIC_FUNCTION_DEF(config, radix_from_config);
+LUA_PUBLIC_FUNCTION_DEF(config, radix_from_ucl);
+LUA_PUBLIC_FUNCTION_DEF(config, add_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_hash_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_kv_map);
+LUA_PUBLIC_FUNCTION_DEF(config, add_map);
+LUA_PUBLIC_FUNCTION_DEF(config, get_maps);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LUA_LUA_MAP_H_ */
diff --git a/src/lua/lua_mempool.c b/src/lua/lua_mempool.c
new file mode 100644
index 0000000..4897d15
--- /dev/null
+++ b/src/lua/lua_mempool.c
@@ -0,0 +1,612 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/***
+ * @module rspamd_mempool
+ * Rspamd memory pool is used to allocate memory attached to specific objects,
+ * namely it was initially used for memory allocation for rspamd_task.
+ *
+ * All memory allocated by the pool is destroyed when the associated object is
+ * destroyed. This allows a sort of controlled garbage collection for memory
+ * allocated from the pool. Memory pools are extensively used by rspamd internal
+ * components and provide some powerful features, such as destructors or
+ * persistent variables.
+ * @example
+local mempool = require "rspamd_mempool"
+local pool = mempool.create()
+
+pool:set_variable('a', 'bcd', 1, 1.01, false)
+local v1, v2, v3, v4 = pool:get_variable('a', 'string,double,double,bool')
+pool:destroy()
+ */
+
+/* Lua bindings */
+/***
+ * @function mempool.create([size])
+ * Creates a memory pool of a specified `size` or platform dependent optimal size (normally, a page size)
+ * @param {number} size size of a page inside pool
+ * @return {rspamd_mempool} new pool object (that should be removed by explicit call to `pool:destroy()`)
+ */
+LUA_FUNCTION_DEF(mempool, create);
+/***
+ * @method mempool:add_destructor(func)
+ * Adds new destructor function to the pool
+ * @param {function} func function to be called when the pool is destroyed
+ */
+LUA_FUNCTION_DEF(mempool, add_destructor);
+/***
+ * @method mempool:destroy()
+ * Destroys memory pool cleaning all variables and calling all destructors registered (both C and Lua ones)
+ */
+LUA_FUNCTION_DEF(mempool, delete);
+LUA_FUNCTION_DEF(mempool, stat);
+LUA_FUNCTION_DEF(mempool, suggest_size);
+/***
+ * @method mempool:set_variable(name, [value1[, value2 ...]])
+ * Sets a variable that's valid during memory pool lifetime. This function allows
+ * to pack multiple values inside a single variable. Currently supported types are:
+ *
+ * - `string`: packed as null terminated C string (so no `\0` are allowed)
+ * - `number`: packed as C double
+ * - `boolean`: packed as bool
+ * @param {string} name variable's name to set
+ */
+LUA_FUNCTION_DEF(mempool, set_variable);
+/***
+ * @method mempool:set_bucket(name, num_values, [value1...valuen]|[table])
+ * Stores a variable bucket of numbers where the first number is number of elements to pack
+ * and then there should be either n numeric values or a plain table of numeric values
+ * @param {string} name variable's name to set
+ * @param {number} num_values number of variables in the bucket
+ * @param {table|list} values values
+ */
+LUA_FUNCTION_DEF(mempool, set_bucket);
+/***
+ * @method mempool:get_variable(name[, type])
+ * Unpacks mempool variable to lua If `type` is not specified, then a variable is
+ * assumed to be zero-terminated C string. Otherwise, `type` is a comma separated (spaces are ignored)
+ * list of types that should be unpacked from a variable's content. The following types
+ * are supported:
+ *
+ * - `string`: null terminated C string (so no `\0` are allowed)
+ * - `double`: returned as lua number
+ * - `int`: unpack a single integer
+ * - `int64`: unpack 64-bits integer
+ * - `boolean`: unpack boolean
+ * - `bucket`: bucket of numbers represented as a Lua table
+ * - `fstrings`: list of rspamd_fstring_t (GList) represented as a Lua table
+ * @param {string} name variable's name to get
+ * @param {string} type list of types to be extracted
+ * @return {variable list} list of variables extracted (but **not** a table)
+ */
+LUA_FUNCTION_DEF(mempool, get_variable);
+/***
+ * @method mempool:has_variable(name)
+ * Checks if the specified variable `name` exists in the memory pool
+ * @param {string} name variable's name to get
+ * @return {boolean} `true` if variable exists and `false` otherwise
+ */
+LUA_FUNCTION_DEF(mempool, has_variable);
+
+/***
+ * @method mempool:delete_variable(name)
+ * Removes the specified variable `name` from the memory pool
+ * @param {string} name variable's name to remove
+ * @return {boolean} `true` if variable exists and has been removed
+ */
+LUA_FUNCTION_DEF(mempool, delete_variable);
+/**
+ * @method mempool:topointer()
+ *
+ * Returns raw C pointer (lightuserdata) associated with mempool. This might be
+ * broken with luajit and GC64, use with caution.
+ */
+LUA_FUNCTION_DEF(mempool, topointer);
+
+static const struct luaL_reg mempoollib_m[] = {
+ LUA_INTERFACE_DEF(mempool, add_destructor),
+ LUA_INTERFACE_DEF(mempool, stat),
+ LUA_INTERFACE_DEF(mempool, suggest_size),
+ LUA_INTERFACE_DEF(mempool, set_variable),
+ LUA_INTERFACE_DEF(mempool, set_bucket),
+ LUA_INTERFACE_DEF(mempool, get_variable),
+ LUA_INTERFACE_DEF(mempool, has_variable),
+ LUA_INTERFACE_DEF(mempool, delete_variable),
+ LUA_INTERFACE_DEF(mempool, topointer),
+ LUA_INTERFACE_DEF(mempool, delete),
+ {"destroy", lua_mempool_delete},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static const struct luaL_reg mempoollib_f[] = {
+ LUA_INTERFACE_DEF(mempool, create),
+ {NULL, NULL}};
+
+/*
+ * Struct for lua destructor
+ */
+
+struct lua_mempool_udata {
+ lua_State *L;
+ gint cbref;
+ rspamd_mempool_t *mempool;
+};
+
+struct memory_pool_s *
+rspamd_lua_check_mempool(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{mempool}");
+ luaL_argcheck(L, ud != NULL, pos, "'mempool' expected");
+ return ud ? *((struct memory_pool_s **) ud) : NULL;
+}
+
+
+static int
+lua_mempool_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_mempool_new(
+ rspamd_mempool_suggest_size(), "lua", 0),
+ **pmempool;
+
+ if (mempool) {
+ pmempool = lua_newuserdata(L, sizeof(struct memory_pool_s *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *pmempool = mempool;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static void
+lua_mempool_destructor_func(gpointer p)
+{
+ struct lua_mempool_udata *ud = p;
+
+ lua_rawgeti(ud->L, LUA_REGISTRYINDEX, ud->cbref);
+ if (lua_pcall(ud->L, 0, 0, 0) != 0) {
+ msg_info("call to destructor failed: %s", lua_tostring(ud->L, -1));
+ lua_pop(ud->L, 1);
+ }
+ luaL_unref(ud->L, LUA_REGISTRYINDEX, ud->cbref);
+}
+
+static int
+lua_mempool_add_destructor(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ struct lua_mempool_udata *ud;
+
+ if (mempool) {
+ if (lua_isfunction(L, 2)) {
+ ud = rspamd_mempool_alloc(mempool,
+ sizeof(struct lua_mempool_udata));
+ lua_pushvalue(L, 2);
+ /* Get a reference */
+ ud->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ ud->L = L;
+ ud->mempool = mempool;
+ rspamd_mempool_add_destructor(mempool,
+ lua_mempool_destructor_func,
+ ud);
+ }
+ else {
+ msg_err("trying to add destructor without function");
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_delete(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ rspamd_mempool_delete(mempool);
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_suggest_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+
+ if (mempool) {
+ lua_pushinteger(L, rspamd_mempool_suggest_size());
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+struct lua_numbers_bucket {
+ guint nelts;
+ gdouble elts[0];
+};
+
+static int
+lua_mempool_set_bucket(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ struct lua_numbers_bucket *bucket;
+ gint nelts = luaL_checknumber(L, 3), i;
+
+ if (var && nelts > 0) {
+ bucket = rspamd_mempool_alloc(mempool,
+ sizeof(*bucket) + sizeof(gdouble) * nelts);
+ bucket->nelts = nelts;
+
+ if (lua_type(L, 4) == LUA_TTABLE) {
+ /* Table version */
+ for (i = 1; i <= nelts; i++) {
+ lua_rawgeti(L, 4, i);
+ bucket->elts[i - 1] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ for (i = 0; i <= nelts; i++) {
+ bucket->elts[i] = lua_tonumber(L, 4 + i);
+ }
+ }
+
+ rspamd_mempool_set_variable(mempool, var, bucket, NULL);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_mempool_set_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gpointer value;
+ struct lua_numbers_bucket *bucket;
+ gchar *vp;
+ union {
+ gdouble d;
+ const gchar *s;
+ gboolean b;
+ } val;
+ gsize slen;
+ gint i, j, len = 0, type;
+
+ if (mempool && var) {
+
+ for (i = 3; i <= lua_gettop(L); i++) {
+ type = lua_type(L, i);
+
+ if (type == LUA_TNUMBER) {
+ /* We have some ambiguity here between integer and double */
+ len += sizeof(gdouble);
+ }
+ else if (type == LUA_TBOOLEAN) {
+ len += sizeof(gboolean);
+ }
+ else if (type == LUA_TSTRING) {
+ (void) lua_tolstring(L, i, &slen);
+ len += slen + 1;
+ }
+ else if (type == LUA_TTABLE) {
+ /* We assume it as a bucket of numbers so far */
+ slen = rspamd_lua_table_size(L, i);
+ len += sizeof(gdouble) * slen + sizeof(*bucket);
+ }
+ else {
+ msg_err("cannot handle lua type %s", lua_typename(L, type));
+ }
+ }
+
+ if (len == 0) {
+ msg_err("no values specified");
+ }
+ else {
+ value = rspamd_mempool_alloc(mempool, len);
+ vp = value;
+
+ for (i = 3; i <= lua_gettop(L); i++) {
+ type = lua_type(L, i);
+
+ if (type == LUA_TNUMBER) {
+ val.d = lua_tonumber(L, i);
+ memcpy(vp, &val, sizeof(gdouble));
+ vp += sizeof(gdouble);
+ }
+ else if (type == LUA_TBOOLEAN) {
+ val.b = lua_toboolean(L, i);
+ memcpy(vp, &val, sizeof(gboolean));
+ vp += sizeof(gboolean);
+ }
+ else if (type == LUA_TSTRING) {
+ val.s = lua_tolstring(L, i, &slen);
+ memcpy(vp, val.s, slen + 1);
+ vp += slen + 1;
+ }
+ else if (type == LUA_TTABLE) {
+ slen = rspamd_lua_table_size(L, i);
+ /* XXX: Ret, ret, ret: alignment issues */
+ bucket = (struct lua_numbers_bucket *) vp;
+ bucket->nelts = slen;
+
+ for (j = 0; j < slen; j++) {
+ lua_rawgeti(L, i, j + 1);
+ bucket->elts[j] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+
+ vp += sizeof(gdouble) * slen + sizeof(*bucket);
+ }
+ else {
+ msg_err("cannot handle lua type %s", lua_typename(L, type));
+ }
+ }
+
+ rspamd_mempool_set_variable(mempool, var, value, NULL);
+ }
+
+ return 0;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static int
+lua_mempool_get_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ const gchar *type = NULL, *pt;
+ struct lua_numbers_bucket bucket;
+ const gchar *value, *pv;
+ guint len, nvar, slen, i;
+
+ if (mempool && var) {
+ value = rspamd_mempool_get_variable(mempool, var);
+
+ if (lua_gettop(L) >= 3) {
+ type = luaL_checkstring(L, 3);
+ }
+
+ if (value) {
+
+ if (type) {
+ pt = type;
+ pv = value;
+ nvar = 0;
+
+ while ((len = strcspn(pt, ", ")) > 0) {
+ if (len == sizeof("double") - 1 &&
+ g_ascii_strncasecmp(pt, "double", len) == 0) {
+ gdouble num;
+ memcpy(&num, pv, sizeof(gdouble));
+ lua_pushnumber(L, num);
+ pv += sizeof(gdouble);
+ }
+ else if (len == sizeof("int") - 1 &&
+ g_ascii_strncasecmp(pt, "int", len) == 0) {
+ gint num;
+ memcpy(&num, pv, sizeof(gint));
+ lua_pushinteger(L, num);
+ pv += sizeof(gint);
+ }
+ else if (len == sizeof("int64") - 1 &&
+ g_ascii_strncasecmp(pt, "int64", len) == 0) {
+ gint64 num;
+ memcpy(&num, pv, sizeof(gint64));
+ lua_pushinteger(L, num);
+ pv += sizeof(gint64);
+ }
+ else if (len == sizeof("bool") - 1 &&
+ g_ascii_strncasecmp(pt, "bool", len) == 0) {
+ gboolean num;
+ memcpy(&num, pv, sizeof(gboolean));
+ lua_pushboolean(L, num);
+ pv += sizeof(gboolean);
+ }
+ else if (len == sizeof("string") - 1 &&
+ g_ascii_strncasecmp(pt, "string", len) == 0) {
+ slen = strlen((const gchar *) pv);
+ lua_pushlstring(L, (const gchar *) pv, slen);
+ pv += slen + 1;
+ }
+ else if (len == sizeof("gstring") - 1 &&
+ g_ascii_strncasecmp(pt, "gstring", len) == 0) {
+ GString *st = (GString *) pv;
+ lua_pushlstring(L, st->str, st->len);
+ pv += sizeof(GString *);
+ }
+ else if (len == sizeof("bucket") - 1 &&
+ g_ascii_strncasecmp(pt, "bucket", len) == 0) {
+ memcpy(&bucket, pv, sizeof(bucket));
+ lua_createtable(L, bucket.nelts, 0);
+ pv += sizeof(struct lua_numbers_bucket);
+
+ for (i = 0; i < bucket.nelts; i++) {
+ gdouble num;
+ memcpy(&num, pv, sizeof(num));
+ lua_pushnumber(L, num);
+ lua_rawseti(L, -2, i + 1);
+ pv += sizeof(num);
+ }
+ }
+ else if (len == sizeof("fstrings") - 1 &&
+ g_ascii_strncasecmp(pt, "fstrings", len) == 0) {
+ GList *cur;
+ rspamd_fstring_t *fstr;
+
+ cur = (GList *) pv;
+ lua_newtable(L);
+
+ i = 1;
+ while (cur != NULL) {
+ fstr = cur->data;
+ lua_pushlstring(L, fstr->str, fstr->len);
+ lua_rawseti(L, -2, i);
+ i++;
+ cur = g_list_next(cur);
+ }
+
+ pv += sizeof(GList *);
+ }
+ else {
+ msg_err("unknown type for get_variable: %s", pt);
+ lua_pushnil(L);
+ }
+
+ pt += len;
+ pt += strspn(pt, ", ");
+
+ nvar++;
+ }
+
+ return nvar;
+ }
+ else {
+ /* No type specified, return string */
+ lua_pushstring(L, value);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static int
+lua_mempool_has_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (mempool && var) {
+ if (rspamd_mempool_get_variable(mempool, var) != NULL) {
+ ret = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static int
+lua_mempool_delete_variable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct memory_pool_s *mempool = rspamd_lua_check_mempool(L, 1);
+ const gchar *var = luaL_checkstring(L, 2);
+ gboolean ret = FALSE;
+
+ if (mempool && var) {
+ if (rspamd_mempool_get_variable(mempool, var) != NULL) {
+ ret = TRUE;
+
+ rspamd_mempool_remove_variable(mempool, var);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_mempool_topointer(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 1);
+
+ if (pool) {
+ /* XXX: this might cause issues on arm64 and LuaJIT */
+ lua_pushlightuserdata(L, pool);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_mempool(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, mempoollib_f);
+
+ return 1;
+}
+
+void luaopen_mempool(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{mempool}", mempoollib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_mempool", lua_load_mempool);
+}
diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c
new file mode 100644
index 0000000..5d4b8b7
--- /dev/null
+++ b/src/lua/lua_mimepart.c
@@ -0,0 +1,2304 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_url.h"
+#include "libmime/message.h"
+#include "libmime/lang_detection.h"
+#include "libstat/stat_api.h"
+#include "libcryptobox/cryptobox.h"
+#include "libutil/shingles.h"
+
+#include "contrib/uthash/utlist.h"
+
+/* Textpart methods */
+/***
+ * @module rspamd_textpart
+ * This module provides different methods to manipulate text parts data. Text parts
+ * could be obtained from the `rspamd_task` by using of method `task:get_text_parts()`
+@example
+rspamd_config.R_EMPTY_IMAGE = function (task)
+ parts = task:get_text_parts()
+ if parts then
+ for _,part in ipairs(parts) do
+ if part:is_empty() then
+ texts = task:get_texts()
+ if texts then
+ return true
+ end
+ return false
+ end
+ end
+ end
+ return false
+end
+ */
+
+/***
+ * @method text_part:is_utf()
+ * Return TRUE if part is a valid utf text
+ * @return {boolean} true if part is valid `UTF8` part
+ */
+LUA_FUNCTION_DEF(textpart, is_utf);
+
+/***
+ * @method text_part:has_8bit_raw()
+ * Return TRUE if a part has raw 8bit characters
+ * @return {boolean} true if a part has raw 8bit characters
+ */
+LUA_FUNCTION_DEF(textpart, has_8bit_raw);
+
+/***
+ * @method text_part:has_8bit()
+ * Return TRUE if a part has raw 8bit characters
+ * @return {boolean} true if a part has encoded 8bit characters
+ */
+LUA_FUNCTION_DEF(textpart, has_8bit);
+
+/***
+ * @method text_part:get_content([type])
+ * Get the text of the part (html tags stripped). Optional `type` defines type of content to get:
+ * - `content` (default): utf8 content with HTML tags stripped and newlines preserved
+ * - `content_oneline`: utf8 content with HTML tags and newlines stripped
+ * - `raw`: raw content, not mime decoded nor utf8 converted
+ * - `raw_parsed`: raw content, mime decoded, not utf8 converted
+ * - `raw_utf`: raw content, mime decoded, utf8 converted (but with HTML tags and newlines)
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_content);
+/***
+ * @method text_part:get_raw_content()
+ * Get the original text of the part
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_raw_content);
+/***
+ * @method text_part:get_content_oneline()
+ *Get the text of the part (html tags and newlines stripped)
+ * @return {text} `UTF8` encoded content of the part (zero-copy if not converted to a lua string)
+ */
+LUA_FUNCTION_DEF(textpart, get_content_oneline);
+/***
+ * @method text_part:get_length()
+ * Get length of the text of the part
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_length);
+/***
+ * @method mime_part:get_raw_length()
+ * Get length of the **raw** content of the part (e.g. HTML with tags unstripped)
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_raw_length);
+/***
+ * @method mime_part:get_urls_length()
+ * Get length of the urls within the part
+ * @return {integer} length of urls in **bytes**
+ */
+LUA_FUNCTION_DEF(textpart, get_urls_length);
+/***
+ * @method mime_part:get_lines_count()
+ * Get lines number in the part
+ * @return {integer} number of lines in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_lines_count);
+/***
+ * @method mime_part:get_stats()
+ * Returns a table with the following data:
+ * - `lines`: number of lines
+ * - `spaces`: number of spaces
+ * - `double_spaces`: double spaces
+ * - `empty_lines`: number of empty lines
+ * - `non_ascii_characters`: number of non ascii characters
+ * - `ascii_characters`: number of ascii characters
+ * @return {table} table of stats
+ */
+LUA_FUNCTION_DEF(textpart, get_stats);
+/***
+ * @method mime_part:get_words_count()
+ * Get words number in the part
+ * @return {integer} number of words in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_words_count);
+
+/***
+ * @method mime_part:get_words([how])
+ * Get words in the part. Optional `how` argument defines type of words returned:
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ * @return {table/strings} words in the part
+ */
+LUA_FUNCTION_DEF(textpart, get_words);
+
+/***
+ * @method mime_part:filter_words(regexp, [how][, max]])
+ * Filter words using some regexp:
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ * @param {rspamd_regexp} regexp regexp to match
+ * @param {string} how what words to extract
+ * @param {number} max maximum number of hits returned (all hits if <= 0 or nil)
+ * @return {table/strings} words matching regexp
+ */
+LUA_FUNCTION_DEF(textpart, filter_words);
+
+/***
+ * @method text_part:is_empty()
+ * Returns `true` if the specified part is empty
+ * @return {bool} whether a part is empty
+ */
+LUA_FUNCTION_DEF(textpart, is_empty);
+/***
+ * @method text_part:is_html()
+ * Returns `true` if the specified part has HTML content
+ * @return {bool} whether a part is HTML part
+ */
+LUA_FUNCTION_DEF(textpart, is_html);
+/***
+ * @method text_part:get_html()
+ * Returns html content of the specified part
+ * @return {html} html content
+ */
+LUA_FUNCTION_DEF(textpart, get_html);
+/***
+ * @method text_part:get_language()
+ * Returns the code of the most used unicode script in the text part. Does not work with raw parts
+ * @return {string} short abbreviation (such as `ru`) for the script's language
+ */
+LUA_FUNCTION_DEF(textpart, get_language);
+
+/***
+ * @method text_part:get_charset()
+ * Returns part real charset
+ * @return {string} charset of the part
+ */
+LUA_FUNCTION_DEF(textpart, get_charset);
+/***
+ * @method text_part:get_languages()
+ * Returns array of tables of all languages detected for a part:
+ * - 'code': language code (short string)
+ * - 'prob': logarithm of probability
+ * @return {array|tables} all languages detected for the part
+ */
+LUA_FUNCTION_DEF(textpart, get_languages);
+/***
+ * @method text_part:get_fuzzy_hashes(mempool)
+ * @param {rspamd_mempool} mempool - memory pool (usually task pool)
+ * Returns direct hash of textpart as a string and array [1..32] of shingles each represented as a following table:
+ * - [1] - 64 bit fuzzy hash represented as a string
+ * - [2..4] - strings used to generate this hash
+ * @return {string,array|tables} fuzzy hashes calculated
+ */
+LUA_FUNCTION_DEF(textpart, get_fuzzy_hashes);
+/***
+ * @method text_part:get_mimepart()
+ * Returns the mime part object corresponding to this text part
+ * @return {mimepart} mimepart object
+ */
+LUA_FUNCTION_DEF(textpart, get_mimepart);
+
+static const struct luaL_reg textpartlib_m[] = {
+ LUA_INTERFACE_DEF(textpart, is_utf),
+ LUA_INTERFACE_DEF(textpart, has_8bit_raw),
+ LUA_INTERFACE_DEF(textpart, has_8bit),
+ LUA_INTERFACE_DEF(textpart, get_content),
+ LUA_INTERFACE_DEF(textpart, get_raw_content),
+ LUA_INTERFACE_DEF(textpart, get_content_oneline),
+ LUA_INTERFACE_DEF(textpart, get_length),
+ LUA_INTERFACE_DEF(textpart, get_raw_length),
+ LUA_INTERFACE_DEF(textpart, get_urls_length),
+ LUA_INTERFACE_DEF(textpart, get_lines_count),
+ LUA_INTERFACE_DEF(textpart, get_words_count),
+ LUA_INTERFACE_DEF(textpart, get_words),
+ LUA_INTERFACE_DEF(textpart, filter_words),
+ LUA_INTERFACE_DEF(textpart, is_empty),
+ LUA_INTERFACE_DEF(textpart, is_html),
+ LUA_INTERFACE_DEF(textpart, get_html),
+ LUA_INTERFACE_DEF(textpart, get_language),
+ LUA_INTERFACE_DEF(textpart, get_charset),
+ LUA_INTERFACE_DEF(textpart, get_languages),
+ LUA_INTERFACE_DEF(textpart, get_mimepart),
+ LUA_INTERFACE_DEF(textpart, get_stats),
+ LUA_INTERFACE_DEF(textpart, get_fuzzy_hashes),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Mimepart methods */
+
+/***
+ * @module rspamd_mimepart
+ * This module provides access to mime parts found in a message
+@example
+rspamd_config.MISSING_CONTENT_TYPE = function(task)
+ local parts = task:get_parts()
+ if parts and #parts > 1 then
+ -- We have more than one part
+ for _,p in ipairs(parts) do
+ local ct = p:get_header('Content-Type')
+ -- And some parts have no Content-Type header
+ if not ct then
+ return true
+ end
+ end
+ end
+ return false
+end
+ */
+
+/***
+ * @method mime_part:get_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} decoded value of a header
+ */
+LUA_FUNCTION_DEF(mimepart, get_header);
+/***
+ * @method mime_part:get_header_raw(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} raw value of a header
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_raw);
+/***
+ * @method mime_part:get_header_full(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter. This method returns more
+ * information about the header as a list of tables with the following structure:
+ *
+ * - `name` - name of a header
+ * - `value` - raw value of a header
+ * - `decoded` - decoded value of a header
+ * - `tab_separated` - `true` if a header and a value are separated by `tab` character
+ * - `empty_separator` - `true` if there are no separator between a header and a value
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {list of tables} all values of a header as specified above
+@example
+function check_header_delimiter_tab(task, header_name)
+ for _,rh in ipairs(task:get_header_full(header_name)) do
+ if rh['tab_separated'] then return true end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_full);
+/***
+ * @method mimepart:get_header_count(name[, case_sensitive])
+ * Lightweight version if you need just a header's count
+ * * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {number} number of header's occurrences or 0 if not found
+ */
+LUA_FUNCTION_DEF(mimepart, get_header_count);
+
+/***
+ * @method mimepart:get_raw_headers()
+ * Get all undecoded headers of a mime part as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(mimepart, get_raw_headers);
+
+/***
+ * @method mimepart:get_headers()
+ * Get all undecoded headers of a mime part as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(mimepart, get_headers);
+
+/***
+ * @method mime_part:get_content()
+ * Get the parsed content of part
+ * @return {text} opaque text object (zero-copy if not casted to lua string)
+ */
+LUA_FUNCTION_DEF(mimepart, get_content);
+/***
+ * @method mime_part:get_raw_content()
+ * Get the raw content of part
+ * @return {text} opaque text object (zero-copy if not casted to lua string)
+ */
+LUA_FUNCTION_DEF(mimepart, get_raw_content);
+/***
+ * @method mime_part:get_length()
+ * Get length of the content of the part
+ * @return {integer} length of part in **bytes**
+ */
+LUA_FUNCTION_DEF(mimepart, get_length);
+/***
+ * @method mime_part:get_type()
+ * Extract content-type string of the mime part
+ * @return {string,string} content type in form 'type','subtype'
+ */
+LUA_FUNCTION_DEF(mimepart, get_type);
+
+/***
+ * @method mime_part:get_type_full()
+ * Extract content-type string of the mime part with all attributes
+ * @return {string,string,table} content type in form 'type','subtype', {attrs}
+ */
+LUA_FUNCTION_DEF(mimepart, get_type_full);
+
+/***
+ * @method mime_part:get_detected_type()
+ * Extract content-type string of the mime part. Use lua_magic detection
+ * @return {string,string} content type in form 'type','subtype'
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_type);
+
+/***
+ * @method mime_part:get_detected_type_full()
+ * Extract content-type string of the mime part with all attributes. Use lua_magic detection
+ * @return {string,string,table} content type in form 'type','subtype', {attrs}
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_type_full);
+
+/***
+ * @method mime_part:get_detected_ext()
+ * Returns a msdos extension name according to lua_magic detection
+ * @return {string} detected extension (see lua_magic.types)
+ */
+LUA_FUNCTION_DEF(mimepart, get_detected_ext);
+
+/***
+ * @method mime_part:get_cte()
+ * Extract content-transfer-encoding for a part
+ * @return {string} content transfer encoding (e.g. `base64` or `7bit`)
+ */
+LUA_FUNCTION_DEF(mimepart, get_cte);
+
+/***
+ * @method mime_part:get_filename()
+ * Extract filename associated with mime part if it is an attachment
+ * @return {string} filename or `nil` if no file is associated with this part
+ */
+LUA_FUNCTION_DEF(mimepart, get_filename);
+/***
+ * @method mime_part:is_image()
+ * Returns true if mime part is an image
+ * @return {bool} true if a part is an image
+ */
+LUA_FUNCTION_DEF(mimepart, is_image);
+/***
+ * @method mime_part:get_image()
+ * Returns rspamd_image structure associated with this part. This structure has
+ * the following methods:
+ *
+ * * `get_width` - return width of an image in pixels
+ * * `get_height` - return height of an image in pixels
+ * * `get_type` - return string representation of image's type (e.g. 'jpeg')
+ * * `get_filename` - return string with image's file name
+ * * `get_size` - return size in bytes
+ * @return {rspamd_image} image structure or nil if a part is not an image
+ */
+LUA_FUNCTION_DEF(mimepart, get_image);
+/***
+ * @method mime_part:is_archive()
+ * Returns true if mime part is an archive
+ * @return {bool} true if a part is an archive
+ */
+LUA_FUNCTION_DEF(mimepart, is_archive);
+/***
+ * @method mime_part:is_attachment()
+ * Returns true if mime part looks like an attachment
+ * @return {bool} true if a part looks like an attachment
+ */
+LUA_FUNCTION_DEF(mimepart, is_attachment);
+
+/***
+ * @method mime_part:get_archive()
+ * Returns rspamd_archive structure associated with this part. This structure has
+ * the following methods:
+ *
+ * * `get_files` - return list of strings with filenames inside archive
+ * * `get_files_full` - return list of tables with all information about files
+ * * `is_encrypted` - return true if an archive is encrypted
+ * * `get_type` - return string representation of image's type (e.g. 'zip')
+ * * `get_filename` - return string with archive's file name
+ * * `get_size` - return size in bytes
+ * @return {rspamd_archive} archive structure or nil if a part is not an archive
+ */
+LUA_FUNCTION_DEF(mimepart, get_archive);
+/***
+ * @method mime_part:is_multipart()
+ * Returns true if mime part is a multipart part
+ * @return {bool} true if a part is is a multipart part
+ */
+LUA_FUNCTION_DEF(mimepart, is_multipart);
+/***
+ * @method mime_part:is_message()
+ * Returns true if mime part is a message part (message/rfc822)
+ * @return {bool} true if a part is is a message part
+ */
+LUA_FUNCTION_DEF(mimepart, is_message);
+/***
+ * @method mime_part:get_boundary()
+ * Returns boundary for a part (extracted from parent multipart for normal parts and
+ * from the part itself for multipart)
+ * @return {string} boundary value or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_boundary);
+
+/***
+ * @method mime_part:get_enclosing_boundary()
+ * Returns an enclosing boundary for a part even for multiparts. For normal parts
+ * this method is identical to `get_boundary`
+ * @return {string} boundary value or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_enclosing_boundary);
+
+/***
+ * @method mime_part:get_children()
+ * Returns rspamd_mimepart table of part's childer. Returns nil if mime part is not multipart
+ * or a message part.
+ * @return {rspamd_mimepart} table of children
+ */
+LUA_FUNCTION_DEF(mimepart, get_children);
+/***
+ * @method mime_part:is_text()
+ * Returns true if mime part is a text part
+ * @return {bool} true if a part is a text part
+ */
+LUA_FUNCTION_DEF(mimepart, is_text);
+/***
+ * @method mime_part:get_text()
+ * Returns rspamd_textpart structure associated with this part.
+ * @return {rspamd_textpart} textpart structure or nil if a part is not an text
+ */
+LUA_FUNCTION_DEF(mimepart, get_text);
+
+/***
+ * @method mime_part:get_digest()
+ * Returns the unique digest for this mime part
+ * @return {string} 128 characters hex string with digest of the part
+ */
+LUA_FUNCTION_DEF(mimepart, get_digest);
+
+/***
+ * @method mime_part:get_id()
+ * Returns the order of the part in parts list
+ * @return {number} index of the part (starting from 1 as it is Lua API)
+ */
+LUA_FUNCTION_DEF(mimepart, get_id);
+/***
+ * @method mime_part:is_broken()
+ * Returns true if mime part has incorrectly specified content type
+ * @return {bool} true if a part has bad content type
+ */
+LUA_FUNCTION_DEF(mimepart, is_broken);
+/***
+ * @method mime_part:headers_foreach(callback, [params])
+ * This method calls `callback` for each header that satisfies some condition.
+ * By default, all headers are iterated unless `callback` returns `true`. Nil or
+ * false means continue of iterations.
+ * Params could be as following:
+ *
+ * - `full`: header value is full table of all attributes @see task:get_header_full for details
+ * - `regexp`: return headers that satisfies the specified regexp
+ * @param {function} callback function from header name and header value
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(mimepart, headers_foreach);
+/***
+ * @method mime_part:get_parent()
+ * Returns parent part for this part
+ * @return {rspamd_mimepart} parent part or nil
+ */
+LUA_FUNCTION_DEF(mimepart, get_parent);
+
+/***
+ * @method mime_part:get_specific()
+ * Returns specific lua content for this part
+ * @return {any} specific lua content
+ */
+LUA_FUNCTION_DEF(mimepart, get_specific);
+
+/***
+ * @method mime_part:set_specific(<any>)
+ * Sets a specific content for this part
+ * @return {any} previous specific lua content (or nil)
+ */
+LUA_FUNCTION_DEF(mimepart, set_specific);
+
+/***
+ * @method mime_part:is_specific(<any>)
+ * Returns true if part has specific lua content
+ * @return {boolean} flag
+ */
+LUA_FUNCTION_DEF(mimepart, is_specific);
+
+/***
+ * @method mime_part:get_urls([need_emails|list_protos][, need_images])
+ * Get all URLs found in a mime part. Telephone urls and emails are not included unless explicitly asked in `list_protos`
+ * @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
+ * @param {boolean} need_images return urls from images (<img src=...>) as well
+ * @return {table rspamd_url} list of all urls found
+ */
+LUA_FUNCTION_DEF(mimepart, get_urls);
+
+static const struct luaL_reg mimepartlib_m[] = {
+ LUA_INTERFACE_DEF(mimepart, get_content),
+ LUA_INTERFACE_DEF(mimepart, get_raw_content),
+ LUA_INTERFACE_DEF(mimepart, get_length),
+ LUA_INTERFACE_DEF(mimepart, get_type),
+ LUA_INTERFACE_DEF(mimepart, get_type_full),
+ LUA_INTERFACE_DEF(mimepart, get_detected_type),
+ LUA_INTERFACE_DEF(mimepart, get_detected_ext),
+ LUA_INTERFACE_DEF(mimepart, get_detected_type_full),
+ LUA_INTERFACE_DEF(mimepart, get_cte),
+ LUA_INTERFACE_DEF(mimepart, get_filename),
+ LUA_INTERFACE_DEF(mimepart, get_boundary),
+ LUA_INTERFACE_DEF(mimepart, get_enclosing_boundary),
+ LUA_INTERFACE_DEF(mimepart, get_header),
+ LUA_INTERFACE_DEF(mimepart, get_header_raw),
+ LUA_INTERFACE_DEF(mimepart, get_header_full),
+ LUA_INTERFACE_DEF(mimepart, get_header_count),
+ LUA_INTERFACE_DEF(mimepart, get_raw_headers),
+ LUA_INTERFACE_DEF(mimepart, get_headers),
+ LUA_INTERFACE_DEF(mimepart, is_image),
+ LUA_INTERFACE_DEF(mimepart, get_image),
+ LUA_INTERFACE_DEF(mimepart, is_archive),
+ LUA_INTERFACE_DEF(mimepart, get_archive),
+ LUA_INTERFACE_DEF(mimepart, is_multipart),
+ LUA_INTERFACE_DEF(mimepart, is_message),
+ LUA_INTERFACE_DEF(mimepart, get_children),
+ LUA_INTERFACE_DEF(mimepart, get_parent),
+ LUA_INTERFACE_DEF(mimepart, get_urls),
+ LUA_INTERFACE_DEF(mimepart, is_text),
+ LUA_INTERFACE_DEF(mimepart, is_broken),
+ LUA_INTERFACE_DEF(mimepart, is_attachment),
+ LUA_INTERFACE_DEF(mimepart, get_text),
+ LUA_INTERFACE_DEF(mimepart, get_digest),
+ LUA_INTERFACE_DEF(mimepart, get_id),
+ LUA_INTERFACE_DEF(mimepart, headers_foreach),
+ LUA_INTERFACE_DEF(mimepart, get_specific),
+ LUA_INTERFACE_DEF(mimepart, set_specific),
+ LUA_INTERFACE_DEF(mimepart, is_specific),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+
+static struct rspamd_mime_text_part *
+lua_check_textpart(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{textpart}");
+ luaL_argcheck(L, ud != NULL, 1, "'textpart' expected");
+ return ud ? *((struct rspamd_mime_text_part **) ud) : NULL;
+}
+
+static struct rspamd_mime_part *
+lua_check_mimepart(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{mimepart}");
+ luaL_argcheck(L, ud != NULL, 1, "'mimepart' expected");
+ return ud ? *((struct rspamd_mime_part **) ud) : NULL;
+}
+
+
+static gint
+lua_textpart_is_utf(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_UTF(part));
+
+ return 1;
+}
+
+
+static gint
+lua_textpart_has_8bit_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part) {
+ if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_has_8bit(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part) {
+ if (part->flags & RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_textpart_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_text *t;
+ gsize len;
+ const gchar *start, *type = NULL;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ type = lua_tostring(L, 2);
+ }
+
+ if (!type) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+ start = part->utf_content.begin;
+ len = part->utf_content.len;
+ }
+ else if (strcmp(type, "content") == 0) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_content.begin;
+ len = part->utf_content.len;
+ }
+ else if (strcmp(type, "content_oneline") == 0) {
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_stripped_content->data;
+ len = part->utf_stripped_content->len;
+ }
+ else if (strcmp(type, "raw_parsed") == 0) {
+ if (part->parsed.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->parsed.begin;
+ len = part->parsed.len;
+ }
+ else if (strcmp(type, "raw_utf") == 0) {
+ if (part->utf_raw_content == NULL || part->utf_raw_content->len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->utf_raw_content->data;
+ len = part->utf_raw_content->len;
+ }
+ else if (strcmp(type, "raw") == 0) {
+ if (part->raw.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ start = part->raw.begin;
+ len = part->raw.len;
+ }
+ else {
+ return luaL_error(L, "invalid content type: %s", type);
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ t->start = start;
+ t->len = len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_raw_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw.begin;
+ t->len = part->raw.len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_content_oneline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL || IS_TEXT_PART_EMPTY(part)) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_new_text(L, part->utf_stripped_content->data, part->utf_stripped_content->len, FALSE);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_content.len == 0) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->utf_content.len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_raw_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushinteger(L, part->raw.len);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_urls_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ GList *cur;
+ guint total = 0;
+ struct rspamd_process_exception *ex;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ for (cur = part->exceptions; cur != NULL; cur = g_list_next(cur)) {
+ ex = cur->data;
+
+ if (ex->type == RSPAMD_EXCEPTION_URL) {
+ total += ex->len;
+ }
+ }
+
+ lua_pushinteger(L, total);
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_lines_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part)) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->nlines);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_words_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_pushinteger(L, 0);
+ }
+ else {
+ lua_pushinteger(L, part->nwords);
+ }
+
+ return 1;
+}
+
+static inline enum rspamd_lua_words_type
+word_extract_type_from_string(const gchar *how_str)
+{
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_MAX;
+
+ if (strcmp(how_str, "stem") == 0) {
+ how = RSPAMD_LUA_WORDS_STEM;
+ }
+ else if (strcmp(how_str, "norm") == 0) {
+ how = RSPAMD_LUA_WORDS_NORM;
+ }
+ else if (strcmp(how_str, "raw") == 0) {
+ how = RSPAMD_LUA_WORDS_RAW;
+ }
+ else if (strcmp(how_str, "full") == 0) {
+ how = RSPAMD_LUA_WORDS_FULL;
+ }
+
+ return how;
+}
+
+static gint
+lua_textpart_get_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 2);
+
+ how = word_extract_type_from_string(how_str);
+
+ if (how == RSPAMD_LUA_WORDS_MAX) {
+ return luaL_error(L, "invalid extraction type: %s", how_str);
+ }
+ }
+
+ return rspamd_lua_push_words(L, part->utf_words, how);
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_filter_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 2);
+ gint lim = -1;
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (part == NULL || re == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ how = word_extract_type_from_string(how_str);
+
+ if (how == RSPAMD_LUA_WORDS_MAX) {
+ return luaL_error(L, "invalid extraction type: %s", how_str);
+ }
+ }
+
+ if (lua_type(L, 4) == LUA_TNUMBER) {
+ lim = lua_tointeger(L, 4);
+ }
+
+ guint cnt, i;
+
+ lua_createtable(L, 8, 0);
+
+ for (i = 0, cnt = 1; i < part->utf_words->len; i++) {
+ rspamd_stat_token_t *w = &g_array_index(part->utf_words,
+ rspamd_stat_token_t, i);
+
+ switch (how) {
+ case RSPAMD_LUA_WORDS_STEM:
+ if (w->stemmed.len > 0) {
+ if (rspamd_regexp_match(re->re, w->stemmed.begin,
+ w->stemmed.len, FALSE)) {
+ lua_pushlstring(L, w->stemmed.begin, w->stemmed.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_NORM:
+ if (w->normalized.len > 0) {
+ if (rspamd_regexp_match(re->re, w->normalized.begin,
+ w->normalized.len, FALSE)) {
+ lua_pushlstring(L, w->normalized.begin, w->normalized.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_RAW:
+ if (w->original.len > 0) {
+ if (rspamd_regexp_match(re->re, w->original.begin,
+ w->original.len, TRUE)) {
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, cnt++);
+ }
+ }
+ break;
+ case RSPAMD_LUA_WORDS_FULL:
+ if (rspamd_regexp_match(re->re, w->normalized.begin,
+ w->normalized.len, FALSE)) {
+ rspamd_lua_push_full_word(L, w);
+ /* Push to the resulting vector */
+ lua_rawseti(L, -2, cnt++);
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (lim > 0 && cnt >= lim) {
+ break;
+ }
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_is_empty(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_EMPTY(part));
+
+ return 1;
+}
+
+static gint
+lua_textpart_is_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushboolean(L, IS_TEXT_PART_HTML(part));
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct html_content **phc;
+
+ if (part == NULL || part->html == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ phc = lua_newuserdata(L, sizeof(*phc));
+ rspamd_lua_setclass(L, "rspamd{html}", -1);
+ *phc = part->html;
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_language(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ if (part->language != NULL && part->language[0] != '\0') {
+ lua_pushstring(L, part->language);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_charset(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ if (part->real_charset != NULL) {
+ lua_pushstring(L, part->real_charset);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_textpart_get_languages(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ guint i;
+ struct rspamd_lang_detector_res *cur;
+
+ if (part != NULL) {
+ if (part->languages != NULL) {
+ lua_createtable(L, part->languages->len, 0);
+
+ PTR_ARRAY_FOREACH(part->languages, i, cur)
+ {
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "code");
+ lua_pushstring(L, cur->lang);
+ lua_settable(L, -3);
+ lua_pushstring(L, "prob");
+ lua_pushnumber(L, cur->prob);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct lua_shingle_data {
+ guint64 hash;
+ rspamd_ftok_t t1;
+ rspamd_ftok_t t2;
+ rspamd_ftok_t t3;
+};
+
+struct lua_shingle_filter_cbdata {
+ struct rspamd_mime_text_part *part;
+ rspamd_mempool_t *pool;
+};
+
+#define STORE_TOKEN(i, t) \
+ do { \
+ if ((i) < part->utf_words->len) { \
+ word = &g_array_index(part->utf_words, rspamd_stat_token_t, (i)); \
+ sd->t.begin = word->stemmed.begin; \
+ sd->t.len = word->stemmed.len; \
+ } \
+ } while (0)
+
+static guint64
+lua_shingles_filter(guint64 *input, gsize count,
+ gint shno, const guchar *key, gpointer ud)
+{
+ guint64 minimal = G_MAXUINT64;
+ gsize i, min_idx = 0;
+ struct lua_shingle_data *sd;
+ rspamd_stat_token_t *word;
+ struct lua_shingle_filter_cbdata *cbd = (struct lua_shingle_filter_cbdata *) ud;
+ struct rspamd_mime_text_part *part;
+
+ part = cbd->part;
+
+ for (i = 0; i < count; i++) {
+ if (minimal > input[i]) {
+ minimal = input[i];
+ min_idx = i;
+ }
+ }
+
+ sd = rspamd_mempool_alloc0(cbd->pool, sizeof(*sd));
+ sd->hash = minimal;
+
+
+ STORE_TOKEN(min_idx, t1);
+ STORE_TOKEN(min_idx + 1, t2);
+ STORE_TOKEN(min_idx + 2, t3);
+
+ return GPOINTER_TO_SIZE(sd);
+}
+
+#undef STORE_TOKEN
+
+static gint
+lua_textpart_get_fuzzy_hashes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 2);
+ guchar key[rspamd_cryptobox_HASHBYTES], digest[rspamd_cryptobox_HASHBYTES],
+ hexdigest[rspamd_cryptobox_HASHBYTES * 2 + 1], numbuf[64];
+ struct rspamd_shingle *sgl;
+ guint i;
+ struct lua_shingle_data *sd;
+ rspamd_cryptobox_hash_state_t st;
+ rspamd_stat_token_t *word;
+ struct lua_shingle_filter_cbdata cbd;
+
+
+ if (part == NULL || pool == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_TEXT_PART_EMPTY(part) || part->utf_words == NULL) {
+ lua_pushnil(L);
+ lua_pushnil(L);
+ }
+ else {
+ /* TODO: add keys and algorithms support */
+ rspamd_cryptobox_hash(key, "rspamd", strlen("rspamd"), NULL, 0);
+
+ /* TODO: add short text support */
+
+ /* Calculate direct hash */
+ rspamd_cryptobox_hash_init(&st, key, rspamd_cryptobox_HASHKEYBYTES);
+
+ for (i = 0; i < part->utf_words->len; i++) {
+ word = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+ rspamd_cryptobox_hash_update(&st,
+ word->stemmed.begin, word->stemmed.len);
+ }
+
+ rspamd_cryptobox_hash_final(&st, digest);
+
+ rspamd_encode_hex_buf(digest, sizeof(digest), hexdigest,
+ sizeof(hexdigest));
+ lua_pushlstring(L, hexdigest, sizeof(hexdigest) - 1);
+
+ cbd.pool = pool;
+ cbd.part = part;
+ sgl = rspamd_shingles_from_text(part->utf_words, key,
+ pool, lua_shingles_filter, &cbd, RSPAMD_SHINGLES_MUMHASH);
+
+ if (sgl == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, G_N_ELEMENTS(sgl->hashes), 0);
+
+ for (i = 0; i < G_N_ELEMENTS(sgl->hashes); i++) {
+ sd = GSIZE_TO_POINTER(sgl->hashes[i]);
+
+ lua_createtable(L, 4, 0);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", sd->hash);
+ lua_pushstring(L, numbuf);
+ lua_rawseti(L, -2, 1);
+
+ /* Tokens */
+ lua_pushlstring(L, sd->t1.begin, sd->t1.len);
+ lua_rawseti(L, -2, 2);
+
+ lua_pushlstring(L, sd->t2.begin, sd->t2.len);
+ lua_rawseti(L, -2, 3);
+
+ lua_pushlstring(L, sd->t3.begin, sd->t3.len);
+ lua_rawseti(L, -2, 4);
+
+ lua_rawseti(L, -2, i + 1); /* Store table */
+ }
+ }
+ }
+
+ return 2;
+}
+
+static gint
+lua_textpart_get_mimepart(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+ struct rspamd_mime_part **pmime;
+
+ if (part != NULL) {
+ if (part->mime_part != NULL) {
+ pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ *pmime = part->mime_part;
+
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method mime_part:get_stats()
+ * Returns a table with the following data:
+ * -
+ * - `lines`: number of lines
+ * - `spaces`: number of spaces
+ * - `double_spaces`: double spaces
+ * - `empty_lines`: number of empty lines
+ * - `non_ascii_characters`: number of non ascii characters
+ * - `ascii_characters`: number of ascii characters
+ * @return {table} table of stats
+ */
+static gint
+lua_textpart_get_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_text_part *part = lua_check_textpart(L);
+
+ if (part != NULL) {
+ lua_createtable(L, 0, 9);
+
+ lua_pushstring(L, "lines");
+ lua_pushinteger(L, part->nlines);
+ lua_settable(L, -3);
+ lua_pushstring(L, "empty_lines");
+ lua_pushinteger(L, part->empty_lines);
+ lua_settable(L, -3);
+ lua_pushstring(L, "spaces");
+ lua_pushinteger(L, part->spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "non_spaces");
+ lua_pushinteger(L, part->non_spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "double_spaces");
+ lua_pushinteger(L, part->double_spaces);
+ lua_settable(L, -3);
+ lua_pushstring(L, "ascii_characters");
+ lua_pushinteger(L, part->ascii_chars);
+ lua_settable(L, -3);
+ lua_pushstring(L, "non_ascii_characters");
+ lua_pushinteger(L, part->non_ascii_chars);
+ lua_settable(L, -3);
+ lua_pushstring(L, "capital_letters");
+ lua_pushinteger(L, part->capital_letters);
+ lua_settable(L, -3);
+ lua_pushstring(L, "numeric_characters");
+ lua_pushinteger(L, part->numeric_characters);
+ lua_settable(L, -3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Mimepart implementation */
+
+static gint
+lua_mimepart_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->parsed_data.begin;
+ t->len = part->parsed_data.len;
+ t->flags = 0;
+
+ if (lua_is_text_binary(t)) {
+ t->flags |= RSPAMD_TEXT_FLAG_BINARY;
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_raw_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw_data.begin;
+ t->len = part->raw_data.len;
+ t->flags = 0;
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushinteger(L, part->parsed_data.len);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_type_common(lua_State *L, struct rspamd_content_type *ct,
+ gboolean full)
+{
+
+ GHashTableIter it;
+ gpointer k, v;
+ struct rspamd_content_type_param *param;
+
+ if (ct == NULL) {
+ lua_pushnil(L);
+ lua_pushnil(L);
+ return 2;
+ }
+
+ lua_pushlstring(L, ct->type.begin, ct->type.len);
+ lua_pushlstring(L, ct->subtype.begin, ct->subtype.len);
+
+ if (!full) {
+ return 2;
+ }
+
+ lua_createtable(L, 0, 2 + (ct->attrs ? g_hash_table_size(ct->attrs) : 0));
+
+ if (ct->charset.len > 0) {
+ lua_pushstring(L, "charset");
+ lua_pushlstring(L, ct->charset.begin, ct->charset.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->boundary.len > 0) {
+ lua_pushstring(L, "boundary");
+ lua_pushlstring(L, ct->boundary.begin, ct->boundary.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->attrs) {
+ g_hash_table_iter_init(&it, ct->attrs);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ param = v;
+
+ if (param->name.len > 0 && param->value.len > 0) {
+ /* TODO: think about multiple values here */
+ lua_pushlstring(L, param->name.begin, param->name.len);
+ lua_pushlstring(L, param->value.begin, param->value.len);
+ lua_settable(L, -3);
+ }
+ }
+ }
+
+ return 3;
+}
+
+static gint
+lua_mimepart_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->ct, FALSE);
+}
+
+static gint
+lua_mimepart_get_type_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->ct, TRUE);
+}
+
+static gint
+lua_mimepart_get_detected_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->detected_ct, FALSE);
+}
+
+static gint
+lua_mimepart_get_detected_type_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return lua_mimepart_get_type_common(L, part->detected_ct, TRUE);
+}
+
+static gint
+lua_mimepart_get_detected_ext(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->detected_ext) {
+ lua_pushstring(L, part->detected_ext);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_cte(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushstring(L, rspamd_cte_to_string(part->cte));
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL || part->cd == NULL || part->cd->filename.len == 0) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ lua_pushlstring(L, part->cd->filename.begin, part->cd->filename.len);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_boundary(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L), *parent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (IS_PART_MULTIPART(part)) {
+ lua_pushlstring(L, part->specific.mp->boundary.begin,
+ part->specific.mp->boundary.len);
+ }
+ else {
+ parent = part->parent_part;
+
+ if (!parent || !IS_PART_MULTIPART(parent)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, parent->specific.mp->boundary.begin,
+ parent->specific.mp->boundary.len);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_enclosing_boundary(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L), *parent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ parent = part->parent_part;
+
+ if (!parent || !IS_PART_MULTIPART(parent)) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushlstring(L, parent->specific.mp->boundary.begin,
+ parent->specific.mp->boundary.len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_header_common(lua_State *L, enum rspamd_lua_task_header_type how)
+{
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ const gchar *name;
+ gboolean strong = FALSE;
+
+ name = luaL_checkstring(L, 2);
+
+ if (name && part) {
+
+ if (lua_isboolean(L, 3)) {
+ strong = lua_toboolean(L, 3);
+ }
+
+ return rspamd_lua_push_header_array(L,
+ name,
+ rspamd_message_get_header_from_hash(part->raw_headers, name, FALSE),
+ how,
+ strong);
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_header_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_FULL);
+}
+
+static gint
+lua_mimepart_get_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_SIMPLE);
+}
+
+static gint
+lua_mimepart_get_header_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_RAW);
+}
+
+static gint
+lua_mimepart_get_header_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ return lua_mimepart_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_COUNT);
+}
+
+static gint
+lua_mimepart_get_raw_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_lua_text *t;
+
+ if (part) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = part->raw_headers_str;
+ t->len = part->raw_headers_len;
+ t->flags = 0;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ bool need_modified = lua_isnoneornil(L, 2) ? false : lua_toboolean(L, 2);
+
+ if (part) {
+ struct rspamd_mime_header *cur;
+ int i = 1;
+
+ lua_createtable(L, rspamd_mime_headers_count(part->raw_headers), 0);
+ LL_FOREACH2(part->headers_order, cur, ord_next)
+ {
+ if (need_modified && cur->modified_chain) {
+ struct rspamd_mime_header *cur_modified;
+
+ LL_FOREACH(cur->modified_chain, cur_modified)
+ {
+ rspamd_lua_push_header(L, cur_modified, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ rspamd_lua_push_header(L, cur, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+
+static gint
+lua_mimepart_is_image(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_IMAGE);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_archive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_ARCHIVE);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_multipart(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, IS_PART_MULTIPART(part) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, IS_PART_MESSAGE(part) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_attachment(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->cd && part->cd->type == RSPAMD_CT_ATTACHMENT) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ /* if has_name and not (image and Content-ID_header_present) */
+ if (part->cd && part->cd->filename.len > 0) {
+ if (part->part_type != RSPAMD_MIME_PART_IMAGE &&
+ rspamd_message_get_header_from_hash(part->raw_headers,
+ "Content-Id", FALSE) == NULL) {
+ /* Filename is presented but no content id and not image */
+ lua_pushboolean(L, true);
+ }
+ else {
+ /* Image or an embedded object */
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ /* No filename */
+ lua_pushboolean(L, false);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_TEXT);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_broken(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->ct) {
+ lua_pushboolean(L, (part->ct->flags & RSPAMD_CONTENT_TYPE_BROKEN) ? true : false);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_image(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_image **pimg;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_IMAGE || part->specific.img == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ pimg = lua_newuserdata(L, sizeof(*pimg));
+ *pimg = part->specific.img;
+ rspamd_lua_setclass(L, "rspamd{image}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_archive(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_archive **parch;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_ARCHIVE || part->specific.arch == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ parch = lua_newuserdata(L, sizeof(*parch));
+ *parch = part->specific.arch;
+ rspamd_lua_setclass(L, "rspamd{archive}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_children(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_part **pcur, *cur;
+ guint i;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!IS_PART_MULTIPART(part) || part->specific.mp->children == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, part->specific.mp->children->len, 0);
+
+ PTR_ARRAY_FOREACH(part->specific.mp->children, i, cur)
+ {
+ pcur = lua_newuserdata(L, sizeof(*pcur));
+ *pcur = cur;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_parent(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_part **pparent;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->parent_part) {
+ pparent = lua_newuserdata(L, sizeof(*pparent));
+ *pparent = part->parent_part;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_mimepart_get_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ struct rspamd_mime_text_part **ppart;
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_TEXT || part->specific.txt == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ ppart = lua_newuserdata(L, sizeof(*ppart));
+ *ppart = part->specific.txt;
+ rspamd_lua_setclass(L, "rspamd{textpart}", -1);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ gchar digestbuf[rspamd_cryptobox_HASHBYTES * 2 + 1];
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ memset(digestbuf, 0, sizeof(digestbuf));
+ rspamd_encode_hex_buf(part->digest, sizeof(part->digest),
+ digestbuf, sizeof(digestbuf));
+ lua_pushstring(L, digestbuf);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, part->part_number);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_headers_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+ enum rspamd_lua_task_header_type how = RSPAMD_TASK_HEADER_PUSH_SIMPLE;
+ struct rspamd_lua_regexp *re = NULL;
+ struct rspamd_mime_header *hdr, *cur;
+ gint old_top;
+
+ if (part && lua_isfunction(L, 2)) {
+ if (lua_istable(L, 3)) {
+ lua_pushstring(L, "full");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_FULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_RAW;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "regexp");
+ lua_gettable(L, 3);
+
+ if (lua_isuserdata(L, -1)) {
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, -1, "rspamd{regexp}",
+ struct rspamd_lua_regexp, re);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (part->headers_order) {
+ hdr = part->headers_order;
+
+ LL_FOREACH2(hdr, cur, ord_next)
+ {
+ if (re && re->re) {
+ if (!rspamd_regexp_match(re->re, cur->name,
+ strlen(cur->name), FALSE)) {
+ continue;
+ }
+ }
+
+ old_top = lua_gettop(L);
+ lua_pushvalue(L, 2);
+ lua_pushstring(L, cur->name);
+ rspamd_lua_push_header(L, cur, how);
+
+ if (lua_pcall(L, 2, LUA_MULTRET, 0) != 0) {
+ msg_err("call to header_foreach failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, old_top);
+ break;
+ }
+ else {
+ if (lua_gettop(L) > old_top) {
+ if (lua_isboolean(L, old_top + 1)) {
+ if (lua_toboolean(L, old_top + 1)) {
+ lua_settop(L, old_top);
+ break;
+ }
+ }
+ }
+ }
+
+ lua_settop(L, old_top);
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_mimepart_get_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ }
+
+ return 1;
+}
+
+static gint
+lua_mimepart_get_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0, i;
+
+ if (part->urls == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ if (!lua_url_cbdata_fill(L, 2, &cb, default_protocols_mask,
+ ~(0), max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = part->urls->len;
+
+ lua_createtable(L, sz, 0);
+
+ PTR_ARRAY_FOREACH(part->urls, i, u)
+ {
+ lua_tree_url_callback(u, u, &cb);
+ }
+
+ lua_url_cbdata_dtor(&cb);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_is_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA);
+
+ return 1;
+}
+
+static gint
+lua_mimepart_set_specific(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_part *part = lua_check_mimepart(L);
+
+ if (part == NULL || lua_isnil(L, 2)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (part->part_type != RSPAMD_MIME_PART_UNDEFINED &&
+ part->part_type != RSPAMD_MIME_PART_CUSTOM_LUA) {
+ return luaL_error(L,
+ "internal error: trying to set specific lua content on part of type %d",
+ part->part_type);
+ }
+
+ if (part->part_type == RSPAMD_MIME_PART_CUSTOM_LUA) {
+ /* Push old specific data */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, part->specific.lua_specific.cbref);
+ }
+ else {
+ part->part_type = RSPAMD_MIME_PART_CUSTOM_LUA;
+ lua_pushnil(L);
+ }
+
+ /* Now, we push argument on the position 2 and save its reference */
+ lua_pushvalue(L, 2);
+ part->specific.lua_specific.cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ /* Now stack has just a return value as luaL_ref removes value from stack */
+
+ gint ltype = lua_type(L, 2);
+
+ switch (ltype) {
+ case LUA_TTABLE:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_TABLE;
+ break;
+ case LUA_TSTRING:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_STRING;
+ break;
+ case LUA_TUSERDATA:
+ if (rspamd_lua_check_udata_maybe(L, 2, "rspamd{text}")) {
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_TEXT;
+ }
+ else {
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN;
+ }
+ break;
+ case LUA_TFUNCTION:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_FUNCTION;
+ break;
+ default:
+ part->specific.lua_specific.type = RSPAMD_LUA_PART_UNKNOWN;
+ break;
+ }
+
+ return 1;
+}
+
+void luaopen_textpart(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{textpart}", textpartlib_m);
+ lua_pop(L, 1);
+}
+
+void luaopen_mimepart(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{mimepart}", mimepartlib_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_parsers.c b/src/lua/lua_parsers.c
new file mode 100644
index 0000000..1fc71db
--- /dev/null
+++ b/src/lua/lua_parsers.c
@@ -0,0 +1,410 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "tokenizers/tokenizers.h"
+#include "contrib/uthash/utlist.h"
+#include "libserver/html/html.h"
+#include "libmime/email_addr.h"
+#include "libmime/content_type.h"
+#include "libmime/mime_headers.h"
+#include "libmime/smtp_parsers.h"
+#include "lua_parsers.h"
+
+/***
+ * @module rspamd_parsers
+ * This module contains Lua-C interfaces to Rspamd parsers of different kind.
+ */
+
+/***
+ * @function parsers.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+
+
+/***
+ * @function parsers.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+
+/***
+ * @function parsers.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+
+/***
+ * @function parsers.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+
+/***
+ * @function parsers.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+
+static const struct luaL_reg parserslib_f[] = {
+ LUA_INTERFACE_DEF(parsers, tokenize_text),
+ LUA_INTERFACE_DEF(parsers, parse_html),
+ LUA_INTERFACE_DEF(parsers, parse_mail_address),
+ LUA_INTERFACE_DEF(parsers, parse_content_type),
+ LUA_INTERFACE_DEF(parsers, parse_smtp_date),
+
+ {NULL, NULL}};
+
+gint lua_parsers_tokenize_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *in = NULL;
+ gsize len = 0, pos, ex_len, i;
+ GList *exceptions = NULL, *cur;
+ struct rspamd_lua_text *t;
+ struct rspamd_process_exception *ex;
+ UText utxt = UTEXT_INITIALIZER;
+ GArray *res;
+ rspamd_stat_token_t *w;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ in = luaL_checklstring(L, 1, &len);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t) {
+ in = t->start;
+ len = t->len;
+ }
+ }
+
+ if (in == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ if (lua_gettop(L) > 1 && lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_rawgeti(L, -1, 1);
+ pos = luaL_checknumber(L, -1);
+ lua_pop(L, 1);
+ lua_rawgeti(L, -1, 2);
+ ex_len = luaL_checknumber(L, -1);
+ lua_pop(L, 1);
+
+ if (ex_len > 0) {
+ ex = g_malloc0(sizeof(*ex));
+ ex->pos = pos;
+ ex->len = ex_len;
+ ex->type = RSPAMD_EXCEPTION_GENERIC;
+ exceptions = g_list_prepend(exceptions, ex);
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (exceptions) {
+ exceptions = g_list_reverse(exceptions);
+ }
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+ utext_openUTF8(&utxt,
+ in,
+ len,
+ &uc_err);
+
+ res = rspamd_tokenize_text((gchar *) in, len,
+ &utxt,
+ RSPAMD_TOKENIZE_UTF, NULL,
+ exceptions,
+ NULL, NULL, NULL);
+
+ if (res == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, res->len, 0);
+
+ for (i = 0; i < res->len; i++) {
+ w = &g_array_index(res, rspamd_stat_token_t, i);
+ lua_pushlstring(L, w->original.begin, w->original.len);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ cur = exceptions;
+ while (cur) {
+ ex = cur->data;
+ g_free(ex);
+ cur = g_list_next(cur);
+ }
+
+ g_list_free(exceptions);
+ utext_close(&utxt);
+
+ return 1;
+}
+
+gint lua_parsers_parse_html(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *start = NULL;
+ gsize len;
+ GByteArray *in;
+ rspamd_mempool_t *pool;
+ void *hc;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ start = t->start;
+ len = t->len;
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TSTRING) {
+ start = luaL_checklstring(L, 1, &len);
+ }
+
+ if (start != NULL) {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0);
+ in = g_byte_array_sized_new(len);
+ g_byte_array_append(in, start, len);
+
+ hc = rspamd_html_process_part(pool, in);
+
+ rspamd_ftok_t res;
+ rspamd_html_get_parsed_content(hc, &res);
+ lua_new_text(L, res.begin, res.len, TRUE);
+
+ g_byte_array_free(in, TRUE);
+ rspamd_mempool_delete(pool);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+gint lua_parsers_parse_mail_address(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GPtrArray *addrs;
+ gsize len;
+ const gchar *str = luaL_checklstring(L, 1, &len);
+ gint max_addrs = luaL_optinteger(L, 3, 10240);
+ rspamd_mempool_t *pool;
+ gboolean own_pool = FALSE;
+
+ if (str) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ pool = rspamd_lua_check_mempool(L, 2);
+
+ if (pool == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else {
+ pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "lua parsers", 0);
+ own_pool = TRUE;
+ }
+
+ addrs = rspamd_email_address_from_mime(pool, str, len, NULL, max_addrs);
+
+ if (addrs == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_push_emails_address_list(L, addrs, 0);
+ }
+
+ if (own_pool) {
+ rspamd_mempool_delete(pool);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+gint lua_parsers_parse_content_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len;
+ const gchar *ct_str = luaL_checklstring(L, 1, &len);
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 2);
+ struct rspamd_content_type *ct;
+
+ if (!ct_str || !pool) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ ct = rspamd_content_type_parse(ct_str, len, pool);
+
+ if (ct == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ GHashTableIter it;
+ gpointer k, v;
+
+ lua_createtable(L, 0, 4 + (ct->attrs ? g_hash_table_size(ct->attrs) : 0));
+
+ if (ct->type.len > 0) {
+ lua_pushstring(L, "type");
+ lua_pushlstring(L, ct->type.begin, ct->type.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->subtype.len > 0) {
+ lua_pushstring(L, "subtype");
+ lua_pushlstring(L, ct->subtype.begin, ct->subtype.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->charset.len > 0) {
+ lua_pushstring(L, "charset");
+ lua_pushlstring(L, ct->charset.begin, ct->charset.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->orig_boundary.len > 0) {
+ lua_pushstring(L, "boundary");
+ lua_pushlstring(L, ct->orig_boundary.begin, ct->orig_boundary.len);
+ lua_settable(L, -3);
+ }
+
+ if (ct->attrs) {
+ g_hash_table_iter_init(&it, ct->attrs);
+
+ while (g_hash_table_iter_next(&it, &k, &v)) {
+ struct rspamd_content_type_param *param =
+ (struct rspamd_content_type_param *) v,
+ *cur;
+ guint i = 1;
+
+ lua_pushlstring(L, param->name.begin, param->name.len);
+ lua_createtable(L, 1, 0);
+
+ DL_FOREACH(param, cur)
+ {
+ lua_pushlstring(L, cur->value.begin, cur->value.len);
+ lua_rawseti(L, -2, i++);
+ }
+
+ lua_settable(L, -3);
+ }
+ }
+ }
+
+ return 1;
+}
+
+int lua_parsers_parse_smtp_date(lua_State *L)
+{
+ gsize slen;
+ const gchar *str = lua_tolstring(L, 1, &slen);
+ GError *err = NULL;
+
+ if (str == NULL) {
+ return luaL_argerror(L, 1, "invalid argument");
+ }
+
+ time_t tt = rspamd_parse_smtp_date(str, slen, &err);
+
+ if (err == NULL) {
+ if (lua_isboolean(L, 2) && !!lua_toboolean(L, 2)) {
+ struct tm t;
+
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ tt = mktime(&t);
+ }
+
+ lua_pushnumber(L, tt);
+ }
+ else {
+ lua_pushnil(L);
+ lua_pushstring(L, err->message);
+ g_error_free(err);
+
+ return 2;
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_parsers(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, parserslib_f);
+
+ return 1;
+}
+
+void luaopen_parsers(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_parsers", lua_load_parsers);
+} \ No newline at end of file
diff --git a/src/lua/lua_parsers.h b/src/lua/lua_parsers.h
new file mode 100644
index 0000000..2466938
--- /dev/null
+++ b/src/lua/lua_parsers.h
@@ -0,0 +1,88 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LUA_PARSERS_H
+#define RSPAMD_LUA_PARSERS_H
+
+#include "lua_common.h"
+
+/***
+ * @function parsers.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, tokenize_text);
+
+/***
+ * @function parsers.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_html);
+
+/***
+ * @function parsers.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_mail_address);
+
+/***
+ * @function parsers.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_content_type);
+
+/***
+ * @function parsers.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+LUA_PUBLIC_FUNCTION_DEF(parsers, parse_smtp_date);
+
+
+#endif//RSPAMD_LUA_PARSERS_H
diff --git a/src/lua/lua_redis.c b/src/lua/lua_redis.c
new file mode 100644
index 0000000..1ad3b3d
--- /dev/null
+++ b/src/lua/lua_redis.c
@@ -0,0 +1,1662 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+
+#include "contrib/hiredis/hiredis.h"
+#include "contrib/hiredis/async.h"
+
+#define REDIS_DEFAULT_TIMEOUT 1.0
+
+static const gchar *M = "rspamd lua redis";
+static void *redis_null;
+
+/***
+ * @module rspamd_redis
+ * This module implements redis asynchronous client for rspamd LUA API.
+ * Here is an example of using of this module:
+ * @example
+local rspamd_redis = require "rspamd_redis"
+local rspamd_logger = require "rspamd_logger"
+
+local function symbol_callback(task)
+ local redis_key = 'some_key'
+ local function redis_cb(err, data)
+ if not err then
+ rspamd_logger.infox('redis returned %1=%2', redis_key, data)
+ end
+ end
+
+ rspamd_redis.make_request(task, "127.0.0.1:6379", redis_cb,
+ 'GET', {redis_key})
+ -- or in table form:
+ -- rspamd_redis.make_request({task=task, host="127.0.0.1:6379,
+ -- callback=redis_cb, timeout=2.0, cmd='GET', args={redis_key}})
+end
+ */
+
+LUA_FUNCTION_DEF(redis, make_request);
+LUA_FUNCTION_DEF(redis, make_request_sync);
+LUA_FUNCTION_DEF(redis, connect);
+LUA_FUNCTION_DEF(redis, connect_sync);
+LUA_FUNCTION_DEF(redis, add_cmd);
+LUA_FUNCTION_DEF(redis, exec);
+LUA_FUNCTION_DEF(redis, gc);
+
+static const struct luaL_reg redislib_f[] = {
+ LUA_INTERFACE_DEF(redis, make_request),
+ LUA_INTERFACE_DEF(redis, make_request_sync),
+ LUA_INTERFACE_DEF(redis, connect),
+ LUA_INTERFACE_DEF(redis, connect_sync),
+ {NULL, NULL}};
+
+static const struct luaL_reg redislib_m[] = {
+ LUA_INTERFACE_DEF(redis, add_cmd),
+ LUA_INTERFACE_DEF(redis, exec),
+ {"__gc", lua_redis_gc},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#undef REDIS_DEBUG_REFS
+#ifdef REDIS_DEBUG_REFS
+#define REDIS_RETAIN(x) \
+ do { \
+ msg_err("retain ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RETAIN(x); \
+ } while (0)
+
+#define REDIS_RELEASE(x) \
+ do { \
+ msg_err("release ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RELEASE(x); \
+ } while (0)
+#else
+#define REDIS_RETAIN REF_RETAIN
+#define REDIS_RELEASE REF_RELEASE
+#endif
+
+struct lua_redis_request_specific_userdata;
+/**
+ * Struct for userdata representation
+ */
+struct lua_redis_userdata {
+ redisAsyncContext *ctx;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct ev_loop *event_loop;
+ struct rspamd_config *cfg;
+ struct rspamd_redis_pool *pool;
+ gchar *server;
+ gchar log_tag[RSPAMD_LOG_ID_LEN + 1];
+ struct lua_redis_request_specific_userdata *specific;
+ gdouble timeout;
+ guint16 port;
+ guint16 terminated;
+};
+
+#define msg_debug_lua_redis(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_lua_redis_log_id, "lua_redis", ud->log_tag, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+INIT_LOG_MODULE(lua_redis)
+
+#define LUA_REDIS_SPECIFIC_REPLIED (1 << 0)
+/* session was finished */
+#define LUA_REDIS_SPECIFIC_FINISHED (1 << 1)
+#define LUA_REDIS_ASYNC (1 << 0)
+#define LUA_REDIS_TEXTDATA (1 << 1)
+#define LUA_REDIS_TERMINATED (1 << 2)
+#define LUA_REDIS_NO_POOL (1 << 3)
+#define LUA_REDIS_SUBSCRIBED (1 << 4)
+#define IS_ASYNC(ctx) ((ctx)->flags & LUA_REDIS_ASYNC)
+
+struct lua_redis_request_specific_userdata {
+ gint cbref;
+ guint nargs;
+ gchar **args;
+ gsize *arglens;
+ struct lua_redis_userdata *c;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_request_specific_userdata *next;
+ ev_timer timeout_ev;
+ guint flags;
+};
+
+struct lua_redis_ctx {
+ guint flags;
+ struct lua_redis_userdata async;
+ guint cmds_pending;
+ ref_entry_t ref;
+ GQueue *replies; /* for sync connection only */
+ GQueue *events_cleanup; /* for sync connection only */
+ struct thread_entry *thread; /* for sync mode, set only if there was yield */
+};
+
+struct lua_redis_result {
+ gboolean is_error;
+ gint result_ref;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct rspamd_task *task;
+ struct lua_redis_request_specific_userdata *sp_ud;
+};
+
+static struct lua_redis_ctx *
+lua_check_redis(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{redis}");
+ luaL_argcheck(L, ud != NULL, pos, "'redis' expected");
+ return ud ? *((struct lua_redis_ctx **) ud) : NULL;
+}
+
+static void
+lua_redis_free_args(char **args, gsize *arglens, guint nargs)
+{
+ guint i;
+
+ if (args) {
+ for (i = 0; i < nargs; i++) {
+ g_free(args[i]);
+ }
+
+ g_free(args);
+ g_free(arglens);
+ }
+}
+
+static void
+lua_redis_dtor(struct lua_redis_ctx *ctx)
+{
+ struct lua_redis_userdata *ud;
+ struct lua_redis_request_specific_userdata *cur, *tmp;
+ gboolean is_successful = TRUE;
+ struct redisAsyncContext *ac;
+
+ ud = &ctx->async;
+ msg_debug_lua_redis("destructing %p", ctx);
+
+ if (ud->ctx) {
+
+ LL_FOREACH_SAFE(ud->specific, cur, tmp)
+ {
+ ev_timer_stop(ud->event_loop, &cur->timeout_ev);
+
+ if (!(cur->flags & LUA_REDIS_SPECIFIC_REPLIED)) {
+ is_successful = FALSE;
+ }
+
+ cur->flags |= LUA_REDIS_SPECIFIC_FINISHED;
+ }
+
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ ud->terminated = 1;
+ ac = ud->ctx;
+ ud->ctx = NULL;
+
+ if (!is_successful) {
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+ else {
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ (ctx->flags & LUA_REDIS_NO_POOL) ? RSPAMD_REDIS_RELEASE_ENFORCE : RSPAMD_REDIS_RELEASE_DEFAULT);
+ }
+ }
+
+ LL_FOREACH_SAFE(ud->specific, cur, tmp)
+ {
+ lua_redis_free_args(cur->args, cur->arglens, cur->nargs);
+
+ if (cur->cbref != -1) {
+ luaL_unref(ud->cfg->lua_state, LUA_REGISTRYINDEX, cur->cbref);
+ }
+
+ g_free(cur);
+ }
+
+ if (ctx->events_cleanup) {
+ g_queue_free(ctx->events_cleanup);
+ ctx->events_cleanup = NULL;
+ }
+ if (ctx->replies) {
+ g_queue_free(ctx->replies);
+ ctx->replies = NULL;
+ }
+
+ g_free(ctx);
+}
+
+static gint
+lua_redis_gc(lua_State *L)
+{
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+
+ if (ctx) {
+ REDIS_RELEASE(ctx);
+ }
+
+ return 0;
+}
+
+static void
+lua_redis_fin(void *arg)
+{
+ struct lua_redis_request_specific_userdata *sp_ud = arg;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(sp_ud->ctx->async.event_loop, &sp_ud->timeout_ev);
+ }
+
+ msg_debug_lua_redis("finished redis query %p from session %p; refcount=%d",
+ sp_ud, ctx, ctx->ref.refcount);
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_FINISHED;
+
+ REDIS_RELEASE(ctx);
+}
+
+/**
+ * Push error of redis request to lua callback
+ * @param code
+ * @param ud
+ */
+static void
+lua_redis_push_error(const gchar *err,
+ struct lua_redis_ctx *ctx,
+ struct lua_redis_request_specific_userdata *sp_ud,
+ gboolean connected)
+{
+ struct lua_redis_userdata *ud = sp_ud->c;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (!(sp_ud->flags & (LUA_REDIS_SPECIFIC_REPLIED | LUA_REDIS_SPECIFIC_FINISHED))) {
+ if (sp_ud->cbref != -1) {
+
+ lua_thread_pool_prepare_callback(ud->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(L);
+ /* Push error */
+ lua_rawgeti(cbs.L, LUA_REGISTRYINDEX, sp_ud->cbref);
+
+ /* String of error */
+ lua_pushstring(cbs.L, err);
+ /* Data is nil */
+ lua_pushnil(cbs.L);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ if (lua_pcall(cbs.L, 2, 0, err_idx) != 0) {
+ msg_info("call to callback failed: %s", lua_tostring(cbs.L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ lua_thread_pool_restore_callback(&cbs);
+ }
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (connected && ud->s) {
+ if (ud->item) {
+ rspamd_symcache_item_async_dec_check(ud->task, ud->item, M);
+ }
+
+ rspamd_session_remove_event(ud->s, lua_redis_fin, sp_ud);
+ }
+ else {
+ lua_redis_fin(sp_ud);
+ }
+ }
+}
+
+static void
+lua_redis_push_reply(lua_State *L, const redisReply *r, gboolean text_data)
+{
+ guint i;
+ struct rspamd_lua_text *t;
+
+ switch (r->type) {
+ case REDIS_REPLY_INTEGER:
+ lua_pushinteger(L, r->integer);
+ break;
+ case REDIS_REPLY_NIL:
+ lua_getfield(L, LUA_REGISTRYINDEX, "redis.null");
+ break;
+ case REDIS_REPLY_STRING:
+ case REDIS_REPLY_STATUS:
+ if (text_data) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->start = r->str;
+ t->len = r->len;
+ }
+ else {
+ lua_pushlstring(L, r->str, r->len);
+ }
+ break;
+ case REDIS_REPLY_ARRAY:
+ lua_createtable(L, r->elements, 0);
+ for (i = 0; i < r->elements; ++i) {
+ lua_redis_push_reply(L, r->element[i], text_data);
+ lua_rawseti(L, -2, i + 1); /* Store sub-reply */
+ }
+ break;
+ default: /* should not happen */
+ msg_info("unknown reply type: %d", r->type);
+ break;
+ }
+}
+
+/**
+ * Push data of redis request to lua callback
+ * @param r redis reply data
+ * @param ud
+ */
+static void
+lua_redis_push_data(const redisReply *r, struct lua_redis_ctx *ctx,
+ struct lua_redis_request_specific_userdata *sp_ud)
+{
+ struct lua_redis_userdata *ud = sp_ud->c;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (!(sp_ud->flags & (LUA_REDIS_SPECIFIC_REPLIED | LUA_REDIS_SPECIFIC_FINISHED)) ||
+ (sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (sp_ud->cbref != -1) {
+ lua_thread_pool_prepare_callback(ud->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ int err_idx = lua_gettop(L);
+ /* Push error */
+ lua_rawgeti(cbs.L, LUA_REGISTRYINDEX, sp_ud->cbref);
+ /* Error is nil */
+ lua_pushnil(cbs.L);
+ /* Data */
+ lua_redis_push_reply(cbs.L, r, ctx->flags & LUA_REDIS_TEXTDATA);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ gint ret = lua_pcall(cbs.L, 2, 0, err_idx);
+
+ if (ret != 0) {
+ msg_info("call to lua_redis callback failed (%d): %s",
+ ret, lua_tostring(cbs.L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ lua_thread_pool_restore_callback(&cbs);
+ }
+
+ if (sp_ud->flags & LUA_REDIS_SUBSCRIBED) {
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_REPLIED)) {
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(sp_ud->ctx->async.event_loop,
+ &sp_ud->timeout_ev);
+ }
+ }
+ }
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (!(sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (ud->s) {
+ if (ud->item) {
+ rspamd_symcache_item_async_dec_check(ud->task,
+ ud->item, M);
+ }
+
+ rspamd_session_remove_event(ud->s, lua_redis_fin, sp_ud);
+ }
+ else {
+ lua_redis_fin(sp_ud);
+ }
+ }
+ }
+}
+
+/**
+ * Callback for redis replies
+ * @param c context of redis connection
+ * @param r redis reply
+ * @param priv userdata
+ */
+static void
+lua_redis_callback(redisAsyncContext *c, gpointer r, gpointer priv)
+{
+ redisReply *reply = r;
+ struct lua_redis_request_specific_userdata *sp_ud = priv;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ redisAsyncContext *ac;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ if (ud->terminated || !rspamd_lua_is_initialised()) {
+ /* We are already at the termination stage, just go out */
+ return;
+ }
+
+ msg_debug_lua_redis("got reply from redis %p for query %p", sp_ud->c->ctx,
+ sp_ud);
+
+ REDIS_RETAIN(ctx);
+
+ /* If session is finished, we cannot call lua callbacks */
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) ||
+ (sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ if (c->err == 0) {
+ if (r != NULL) {
+ if (reply->type != REDIS_REPLY_ERROR) {
+ lua_redis_push_data(reply, ctx, sp_ud);
+ }
+ else {
+ lua_redis_push_error(reply->str, ctx, sp_ud, TRUE);
+ }
+ }
+ else {
+ lua_redis_push_error("received no data from server", ctx, sp_ud, TRUE);
+ }
+ }
+ else {
+ if (c->err == REDIS_ERR_IO) {
+ lua_redis_push_error(strerror(errno), ctx, sp_ud, TRUE);
+ }
+ else {
+ lua_redis_push_error(c->errstr, ctx, sp_ud, TRUE);
+ }
+ }
+ }
+
+ if (!(sp_ud->flags & LUA_REDIS_SUBSCRIBED)) {
+ ctx->cmds_pending--;
+
+ if (ctx->cmds_pending == 0 && !ud->terminated) {
+ /* Disconnect redis early as we don't need it anymore */
+ ud->terminated = 1;
+ ac = ud->ctx;
+ ud->ctx = NULL;
+
+ if (ac) {
+ msg_debug_lua_redis("release redis connection ud=%p; ctx=%p; refcount=%d",
+ ud, ctx, ctx->ref.refcount);
+ rspamd_redis_pool_release_connection(ud->pool, ac,
+ (ctx->flags & LUA_REDIS_NO_POOL) ? RSPAMD_REDIS_RELEASE_ENFORCE : RSPAMD_REDIS_RELEASE_DEFAULT);
+ }
+ }
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+static gint
+lua_redis_push_results(struct lua_redis_ctx *ctx, lua_State *L)
+{
+ gint results = g_queue_get_length(ctx->replies);
+ gint i;
+ gboolean can_use_lua = TRUE;
+
+ results = g_queue_get_length(ctx->replies);
+
+ if (!lua_checkstack(L, (results * 2) + 1)) {
+ luaL_error(L, "cannot resize stack to fit %d commands",
+ ctx->cmds_pending);
+
+ can_use_lua = FALSE;
+ }
+
+ for (i = 0; i < results; i++) {
+ struct lua_redis_result *result = g_queue_pop_head(ctx->replies);
+
+ if (can_use_lua) {
+ lua_pushboolean(L, !result->is_error);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, result->result_ref);
+ }
+
+ luaL_unref(L, LUA_REGISTRYINDEX, result->result_ref);
+
+ g_queue_push_tail(ctx->events_cleanup, result);
+ }
+
+ return can_use_lua ? results * 2 : 0;
+}
+
+static void
+lua_redis_cleanup_events(struct lua_redis_ctx *ctx)
+{
+ REDIS_RETAIN(ctx); /* To avoid preliminary destruction */
+
+ while (!g_queue_is_empty(ctx->events_cleanup)) {
+ struct lua_redis_result *result = g_queue_pop_head(ctx->events_cleanup);
+
+ if (result->item) {
+ rspamd_symcache_item_async_dec_check(result->task, result->item, M);
+ }
+
+ if (result->s) {
+ rspamd_session_remove_event(result->s, lua_redis_fin, result->sp_ud);
+ }
+ else {
+ lua_redis_fin(result->sp_ud);
+ }
+
+ g_free(result);
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+/**
+ * Callback for redis replies
+ * @param c context of redis connection
+ * @param r redis reply
+ * @param priv userdata
+ */
+static void
+lua_redis_callback_sync(redisAsyncContext *ac, gpointer r, gpointer priv)
+{
+ redisReply *reply = r;
+
+ struct lua_redis_request_specific_userdata *sp_ud = priv;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ struct thread_entry *thread;
+ gint results;
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+ lua_State *L = ctx->async.cfg->lua_state;
+
+ sp_ud->flags |= LUA_REDIS_SPECIFIC_REPLIED;
+
+ if (ud->terminated) {
+ /* We are already at the termination stage, just go out */
+ /* TODO:
+ if somebody is waiting for us (ctx->thread), return result,
+ otherwise, indeed, ignore
+ */
+ return;
+ }
+
+ if (ev_can_stop(&sp_ud->timeout_ev)) {
+ ev_timer_stop(ud->event_loop, &sp_ud->timeout_ev);
+ }
+
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) {
+ msg_debug_lua_redis("got reply from redis: %p for query %p", ac, sp_ud);
+
+ struct lua_redis_result *result = g_malloc0(sizeof *result);
+
+ if (ac->err == 0) {
+ if (r != NULL) {
+ if (reply->type != REDIS_REPLY_ERROR) {
+ result->is_error = FALSE;
+ lua_redis_push_reply(L, reply, ctx->flags & LUA_REDIS_TEXTDATA);
+ }
+ else {
+ result->is_error = TRUE;
+ lua_pushstring(L, reply->str);
+ }
+ }
+ else {
+ result->is_error = TRUE;
+ lua_pushliteral(L, "received no data from server");
+ }
+ }
+ else {
+ result->is_error = TRUE;
+ if (ac->err == REDIS_ERR_IO) {
+ lua_pushstring(L, strerror(errno));
+ }
+ else {
+ lua_pushstring(L, ac->errstr);
+ }
+ }
+
+ /* if error happened, we should terminate the connection,
+ and release it */
+
+ if (result->is_error && sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+
+ result->result_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ result->s = ud->s;
+ result->item = ud->item;
+ result->task = ud->task;
+ result->sp_ud = sp_ud;
+
+ g_queue_push_tail(ctx->replies, result);
+ }
+
+ ctx->cmds_pending--;
+
+ if (ctx->cmds_pending == 0) {
+ if (ctx->thread) {
+ if (!(sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED)) {
+ /* somebody yielded and waits for results */
+ thread = ctx->thread;
+ ctx->thread = NULL;
+
+ results = lua_redis_push_results(ctx, thread->lua_state);
+
+ if (ud->item) {
+ rspamd_symcache_set_cur_item(ud->task, ud->item);
+ }
+
+ lua_thread_resume(thread, results);
+ lua_redis_cleanup_events(ctx);
+ }
+ else {
+ /* We cannot resume the thread as the associated task has gone */
+ lua_thread_pool_terminate_entry_full(ud->cfg->lua_thread_pool,
+ ctx->thread, G_STRLOC, true);
+ ctx->thread = NULL;
+ }
+ }
+ }
+}
+
+static void
+lua_redis_timeout_sync(EV_P_ ev_timer *w, int revents)
+{
+ struct lua_redis_request_specific_userdata *sp_ud =
+ (struct lua_redis_request_specific_userdata *) w->data;
+ struct lua_redis_ctx *ctx;
+ struct lua_redis_userdata *ud;
+ redisAsyncContext *ac;
+
+ if (sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) {
+ return;
+ }
+
+ ud = sp_ud->c;
+ ctx = sp_ud->ctx;
+ msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud,
+ sp_ud->c->ctx);
+
+ if (sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ac->err = REDIS_ERR_IO;
+ errno = ETIMEDOUT;
+ ctx->flags |= LUA_REDIS_TERMINATED;
+
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+}
+
+static void
+lua_redis_timeout(EV_P_ ev_timer *w, int revents)
+{
+ struct lua_redis_request_specific_userdata *sp_ud =
+ (struct lua_redis_request_specific_userdata *) w->data;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx;
+ redisAsyncContext *ac;
+
+ if (sp_ud->flags & LUA_REDIS_SPECIFIC_FINISHED) {
+ return;
+ }
+
+ ctx = sp_ud->ctx;
+ ud = sp_ud->c;
+
+ REDIS_RETAIN(ctx);
+ msg_debug_lua_redis("timeout while querying redis server: %p, redis: %p", sp_ud,
+ sp_ud->c->ctx);
+ lua_redis_push_error("timeout while connecting the server", ctx, sp_ud, TRUE);
+
+ if (sp_ud->c->ctx) {
+ ac = sp_ud->c->ctx;
+ /* Set to NULL to avoid double free in dtor */
+ sp_ud->c->ctx = NULL;
+ ac->err = REDIS_ERR_IO;
+ errno = ETIMEDOUT;
+ /*
+ * This will call all callbacks pending so the entire context
+ * will be destructed
+ */
+ rspamd_redis_pool_release_connection(sp_ud->c->pool, ac,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ }
+
+ REDIS_RELEASE(ctx);
+}
+
+
+static void
+lua_redis_parse_args(lua_State *L, gint idx, const gchar *cmd,
+ gchar ***pargs, gsize **parglens, guint *nargs)
+{
+ gchar **args = NULL;
+ gsize *arglens;
+ gint top;
+
+ if (idx != 0 && lua_type(L, idx) == LUA_TTABLE) {
+ /* Get all arguments */
+ lua_pushvalue(L, idx);
+ lua_pushnil(L);
+ top = 0;
+
+ while (lua_next(L, -2) != 0) {
+ gint type = lua_type(L, -1);
+
+ if (type == LUA_TNUMBER || type == LUA_TSTRING ||
+ type == LUA_TUSERDATA) {
+ top++;
+ }
+ lua_pop(L, 1);
+ }
+
+ args = g_malloc((top + 1) * sizeof(gchar *));
+ arglens = g_malloc((top + 1) * sizeof(gsize));
+ arglens[0] = strlen(cmd);
+ args[0] = g_malloc(arglens[0]);
+ memcpy(args[0], cmd, arglens[0]);
+ top = 1;
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ gint type = lua_type(L, -1);
+
+ if (type == LUA_TSTRING) {
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &arglens[top]);
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], s, arglens[top]);
+ top++;
+ }
+ else if (type == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text(L, -1);
+
+ if (t && t->start) {
+ arglens[top] = t->len;
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], t->start, arglens[top]);
+ top++;
+ }
+ }
+ else if (type == LUA_TNUMBER) {
+ gdouble val = lua_tonumber(L, -1);
+ gint r;
+ gchar numbuf[64];
+
+ if (val == (gdouble) ((gint64) val)) {
+ r = rspamd_snprintf(numbuf, sizeof(numbuf), "%L",
+ (gint64) val);
+ }
+ else {
+ r = rspamd_snprintf(numbuf, sizeof(numbuf), "%f",
+ val);
+ }
+
+ arglens[top] = r;
+ args[top] = g_malloc(arglens[top]);
+ memcpy(args[top], numbuf, arglens[top]);
+ top++;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ /* Use merely cmd */
+
+ args = g_malloc(sizeof(gchar *));
+ arglens = g_malloc(sizeof(gsize));
+ arglens[0] = strlen(cmd);
+ args[0] = g_malloc(arglens[0]);
+ memcpy(args[0], cmd, arglens[0]);
+ top = 1;
+ }
+
+ *pargs = args;
+ *parglens = arglens;
+ *nargs = top;
+}
+
+static struct lua_redis_ctx *
+rspamd_lua_redis_prepare_connection(lua_State *L, gint *pcbref, gboolean is_async)
+{
+ struct lua_redis_ctx *ctx = NULL;
+ rspamd_inet_addr_t *ip = NULL;
+ struct lua_redis_userdata *ud = NULL;
+ struct rspamd_lua_ip *addr = NULL;
+ struct rspamd_task *task = NULL;
+ const gchar *host = NULL;
+ const gchar *username = NULL, *password = NULL, *dbname = NULL, *log_tag = NULL;
+ gint cbref = -1;
+ struct rspamd_config *cfg = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct ev_loop *ev_base = NULL;
+ gboolean ret = FALSE;
+ guint flags = 0;
+
+ if (lua_istable(L, 1)) {
+ /* Table version */
+ lua_pushvalue(L, 1);
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task_maybe(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!task) {
+ /* We need to get ev_base, config and session separately */
+ lua_pushstring(L, "config");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ cfg = lua_check_config(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ session = lua_check_session(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ ev_base = lua_check_ev_base(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (cfg && ev_base) {
+ ret = TRUE;
+ }
+ else if (!cfg) {
+ msg_err_task_check("config is not passed");
+ }
+ else {
+ msg_err_task_check("ev_base is not set");
+ }
+ }
+ else {
+ cfg = task->cfg;
+ session = task->s;
+ ev_base = task->event_loop;
+ log_tag = task->task_pool->tag.uid;
+ ret = TRUE;
+ }
+
+ if (pcbref) {
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ /* This also pops function from the stack */
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ *pcbref = cbref;
+ }
+ else {
+ *pcbref = -1;
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ addr = lua_check_ip(L, -1);
+ host = rspamd_inet_address_to_string_pretty(addr->addr);
+ }
+ else if (lua_type(L, -1) == LUA_TSTRING) {
+ host = lua_tostring(L, -1);
+
+ if (rspamd_parse_inet_address(&ip,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = ip;
+
+ if (rspamd_inet_address_get_port(ip) == 0) {
+ rspamd_inet_address_set_port(ip, 6379);
+ }
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "username");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ username = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "password");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ password = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "dbname");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ dbname = lua_tostring(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_data");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_TEXTDATA;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "no_pool");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_NO_POOL;
+ }
+ lua_pop(L, 1);
+
+ lua_pop(L, 1); /* table */
+
+ if (session && rspamd_session_blocked(session)) {
+ msg_err_task_check("Session is being destroying");
+ ret = FALSE;
+ }
+
+ if (ret && addr != NULL) {
+ ctx = g_malloc0(sizeof(struct lua_redis_ctx));
+ REF_INIT_RETAIN(ctx, lua_redis_dtor);
+ if (is_async) {
+ ctx->flags |= flags | LUA_REDIS_ASYNC;
+ ud = &ctx->async;
+ }
+ else {
+ ud = &ctx->async;
+ ctx->replies = g_queue_new();
+ ctx->events_cleanup = g_queue_new();
+ }
+
+ ud->s = session;
+ ud->cfg = cfg;
+ ud->pool = cfg->redis_pool;
+ ud->event_loop = ev_base;
+ ud->task = task;
+
+ if (log_tag) {
+ rspamd_strlcpy(ud->log_tag, log_tag, sizeof(ud->log_tag));
+ }
+ else {
+ /* Use pointer itself as a tag */
+ rspamd_snprintf(ud->log_tag, sizeof(ud->log_tag),
+ "%ud",
+ (int) rspamd_cryptobox_fast_hash(&ud, sizeof(ud), 0));
+ }
+
+ if (task) {
+ ud->item = rspamd_symcache_get_cur_item(task);
+ }
+
+ ret = TRUE;
+ }
+ else {
+ if (cbref != -1) {
+ luaL_unref(L, LUA_REGISTRYINDEX, cbref);
+ }
+
+ msg_err_task_check("incorrect function invocation");
+ ret = FALSE;
+ }
+ }
+
+ if (ret) {
+ ud->terminated = 0;
+ ud->ctx = rspamd_redis_pool_connect(ud->pool,
+ dbname, username, password,
+ rspamd_inet_address_to_string(addr->addr),
+ rspamd_inet_address_get_port(addr->addr));
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ if (ud->ctx == NULL || ud->ctx->err) {
+ if (ud->ctx) {
+ msg_err_task_check("cannot connect to redis: %s",
+ ud->ctx->errstr);
+ rspamd_redis_pool_release_connection(ud->pool, ud->ctx,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ ud->ctx = NULL;
+ }
+ else {
+ msg_err_task_check("cannot connect to redis (OS error): %s",
+ strerror(errno));
+ }
+
+ REDIS_RELEASE(ctx);
+
+ return NULL;
+ }
+
+ msg_debug_lua_redis("opened redis connection host=%s; ctx=%p; ud=%p",
+ host, ctx, ud);
+
+ return ctx;
+ }
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ return NULL;
+}
+
+/***
+ * @function rspamd_redis.make_request({params})
+ * Make request to redis server, params is a table of key=value arguments in any order
+ * @param {task} task worker task object
+ * @param {ip|string} host server address
+ * @param {function} callback callback to be called in form `function (task, err, data)`
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args numeric array of strings used as redis arguments
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean} `true` if a request has been scheduled
+ */
+static int
+lua_redis_make_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_request_specific_userdata *sp_ud;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx, **pctx;
+ const gchar *cmd = NULL;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ gint cbref = -1;
+ gboolean ret = FALSE;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, &cbref, TRUE);
+
+ if (ctx) {
+ ud = &ctx->async;
+ sp_ud = g_malloc0(sizeof(*sp_ud));
+ sp_ud->cbref = cbref;
+ sp_ud->c = ud;
+ sp_ud->ctx = ctx;
+
+ lua_pushstring(L, "cmd");
+ lua_gettable(L, -2);
+ cmd = lua_tostring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+ ud->timeout = timeout;
+
+
+ lua_pushstring(L, "args");
+ lua_gettable(L, 1);
+ lua_redis_parse_args(L, -1, cmd, &sp_ud->args, &sp_ud->arglens,
+ &sp_ud->nargs);
+ lua_pop(L, 1);
+ LL_PREPEND(ud->specific, sp_ud);
+
+ ret = redisAsyncCommandArgv(ud->ctx,
+ lua_redis_callback,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+
+ if (ret == REDIS_OK) {
+ if (ud->s) {
+ rspamd_session_add_event(ud->s,
+ lua_redis_fin, sp_ud,
+ M);
+
+ if (ud->item) {
+ rspamd_symcache_item_async_inc(ud->task, ud->item, M);
+ }
+ }
+
+ REDIS_RETAIN(ctx); /* Cleared by fin event */
+ ctx->cmds_pending++;
+
+ if (ud->ctx->c.flags & REDIS_SUBSCRIBED) {
+ msg_debug_lua_redis("subscribe command, never unref/timeout");
+ sp_ud->flags |= LUA_REDIS_SUBSCRIBED;
+ }
+
+ sp_ud->timeout_ev.data = sp_ud;
+ ev_now_update_if_cheap((struct ev_loop *) ud->event_loop);
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout, timeout, 0.0);
+ ev_timer_start(ud->event_loop, &sp_ud->timeout_ev);
+
+ ret = TRUE;
+ }
+ else {
+ msg_info("call to redis failed: %s", ud->ctx->errstr);
+ rspamd_redis_pool_release_connection(ud->pool, ud->ctx,
+ RSPAMD_REDIS_RELEASE_FATAL);
+ ud->ctx = NULL;
+ REDIS_RELEASE(ctx);
+ ret = FALSE;
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ lua_pushboolean(L, ret);
+
+ if (ret) {
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 2;
+}
+
+/***
+ * @function rspamd_redis.make_request_sync({params})
+ * Make blocking request to redis server, params is a table of key=value arguments in any order
+ * @param {ip|string} host server address
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args numeric array of strings used as redis arguments
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean + result} `true` and a result if a request has been successful
+ */
+static int
+lua_redis_make_request_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_ip *addr = NULL;
+ rspamd_inet_addr_t *ip = NULL;
+ const gchar *cmd = NULL, *host;
+ struct timeval tv;
+ gboolean ret = FALSE;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ gchar **args = NULL;
+ gsize *arglens = NULL;
+ guint nargs = 0, flags = 0;
+ redisContext *ctx;
+ redisReply *r;
+
+ if (lua_istable(L, 1)) {
+ lua_pushvalue(L, 1);
+
+ lua_pushstring(L, "cmd");
+ lua_gettable(L, -2);
+ cmd = lua_tostring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ addr = lua_check_ip(L, -1);
+ }
+ else if (lua_type(L, -1) == LUA_TSTRING) {
+ host = lua_tostring(L, -1);
+ if (rspamd_parse_inet_address(&ip,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ addr = g_alloca(sizeof(*addr));
+ addr->addr = ip;
+
+ if (rspamd_inet_address_get_port(ip) == 0) {
+ rspamd_inet_address_set_port(ip, 6379);
+ }
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "opaque_data");
+ lua_gettable(L, -2);
+ if (!!lua_toboolean(L, -1)) {
+ flags |= LUA_REDIS_TEXTDATA;
+ }
+ lua_pop(L, 1);
+
+
+ if (cmd) {
+ lua_pushstring(L, "args");
+ lua_gettable(L, -2);
+ lua_redis_parse_args(L, -1, cmd, &args, &arglens, &nargs);
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+
+ if (addr && cmd) {
+ ret = TRUE;
+ }
+ }
+
+ if (ret) {
+ double_to_tv(timeout, &tv);
+
+ if (rspamd_inet_address_get_af(addr->addr) == AF_UNIX) {
+ ctx = redisConnectUnixWithTimeout(
+ rspamd_inet_address_to_string(addr->addr), tv);
+ }
+ else {
+ ctx = redisConnectWithTimeout(
+ rspamd_inet_address_to_string(addr->addr),
+ rspamd_inet_address_get_port(addr->addr), tv);
+ }
+
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+
+ if (ctx == NULL || ctx->err) {
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+
+ r = redisCommandArgv(ctx,
+ nargs,
+ (const gchar **) args,
+ arglens);
+
+ if (r != NULL) {
+ if (r->type != REDIS_REPLY_ERROR) {
+ lua_pushboolean(L, TRUE);
+ lua_redis_push_reply(L, r, flags & LUA_REDIS_TEXTDATA);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, r->str);
+ }
+
+ freeReplyObject(r);
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+
+ return 2;
+ }
+ else {
+ msg_info("call to redis failed: %s", ctx->errstr);
+ redisFree(ctx);
+ lua_redis_free_args(args, arglens, nargs);
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ if (ip) {
+ rspamd_inet_address_free(ip);
+ }
+ msg_err("bad arguments for redis request");
+ lua_redis_free_args(args, arglens, nargs);
+
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_redis.connect({params})
+ * Make request to redis server, params is a table of key=value arguments in any order
+ * @param {task} task worker task object
+ * @param {ip|string} host server address
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {boolean,redis} new connection object or nil if connection failed
+ */
+static int
+lua_redis_connect(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_userdata *ud;
+ struct lua_redis_ctx *ctx, **pctx;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, NULL, TRUE);
+
+ if (ctx) {
+ ud = &ctx->async;
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+
+ lua_pop(L, 1);
+ ud->timeout = timeout;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushnil(L);
+
+ return 2;
+ }
+
+ lua_pushboolean(L, TRUE);
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+
+ return 2;
+}
+
+/***
+ * @function rspamd_redis.connect_sync({params})
+ * Make blocking request to redis server, params is a table of key=value arguments in any order
+ * @param {ip|string} host server address
+ * @param {number} timeout timeout in seconds for request (1.0 by default)
+ * @return {redis} redis object if a request has been successful
+ */
+static int
+lua_redis_connect_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble timeout = REDIS_DEFAULT_TIMEOUT;
+ struct lua_redis_ctx *ctx, **pctx;
+
+ ctx = rspamd_lua_redis_prepare_connection(L, NULL, FALSE);
+
+ if (ctx) {
+ if (lua_istable(L, 1)) {
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, 1);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+ }
+
+ ctx->async.timeout = timeout;
+
+ lua_pushboolean(L, TRUE);
+ pctx = lua_newuserdata(L, sizeof(ctx));
+ *pctx = ctx;
+ rspamd_lua_setclass(L, "rspamd{redis}", -1);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, "bad arguments for redis request");
+ return 2;
+ }
+
+ return 2;
+}
+
+/***
+ * @method rspamd_redis:add_cmd(cmd, {args})
+ * Append new cmd to redis pipeline
+ * @param {string} cmd command to be sent to redis
+ * @param {table} args array of strings used as redis arguments
+ * @return {boolean} `true` if a request has been successful
+ */
+static int
+lua_redis_add_cmd(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+ struct lua_redis_request_specific_userdata *sp_ud;
+ struct lua_redis_userdata *ud;
+ const gchar *cmd = NULL;
+ gint args_pos = 2;
+ gint cbref = -1, ret;
+
+ if (ctx) {
+ if (ctx->flags & LUA_REDIS_TERMINATED) {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, "Connection is terminated");
+
+ return 2;
+ }
+
+ /* Async version */
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* No callback version */
+ cmd = lua_tostring(L, 2);
+ args_pos = 3;
+ }
+ else if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cmd = lua_tostring(L, 3);
+ args_pos = 4;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sp_ud = g_malloc0(sizeof(*sp_ud));
+ if (IS_ASYNC(ctx)) {
+ sp_ud->c = &ctx->async;
+ ud = &ctx->async;
+ sp_ud->cbref = cbref;
+ }
+ else {
+ sp_ud->c = &ctx->async;
+ ud = &ctx->async;
+ }
+ sp_ud->ctx = ctx;
+
+ lua_redis_parse_args(L, args_pos, cmd, &sp_ud->args,
+ &sp_ud->arglens, &sp_ud->nargs);
+
+ LL_PREPEND(sp_ud->c->specific, sp_ud);
+
+ if (ud->s && rspamd_session_blocked(ud->s)) {
+ lua_pushboolean(L, 0);
+ lua_pushstring(L, "session is terminating");
+
+ return 2;
+ }
+
+ if (IS_ASYNC(ctx)) {
+ ret = redisAsyncCommandArgv(sp_ud->c->ctx,
+ lua_redis_callback,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+ }
+ else {
+ ret = redisAsyncCommandArgv(sp_ud->c->ctx,
+ lua_redis_callback_sync,
+ sp_ud,
+ sp_ud->nargs,
+ (const gchar **) sp_ud->args,
+ sp_ud->arglens);
+ }
+
+ if (ret == REDIS_OK) {
+ if (ud->s) {
+ rspamd_session_add_event(ud->s,
+ lua_redis_fin,
+ sp_ud,
+ M);
+
+ if (ud->item) {
+ rspamd_symcache_item_async_inc(ud->task, ud->item, M);
+ }
+ }
+
+ sp_ud->timeout_ev.data = sp_ud;
+
+ if (IS_ASYNC(ctx)) {
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout,
+ sp_ud->c->timeout, 0.0);
+ }
+ else {
+ ev_timer_init(&sp_ud->timeout_ev, lua_redis_timeout_sync,
+ sp_ud->c->timeout, 0.0);
+ }
+
+ ev_timer_start(ud->event_loop, &sp_ud->timeout_ev);
+ REDIS_RETAIN(ctx);
+ ctx->cmds_pending++;
+ }
+ else {
+ msg_info("call to redis failed: %s",
+ sp_ud->c->ctx->errstr);
+ lua_pushboolean(L, 0);
+ lua_pushstring(L, sp_ud->c->ctx->errstr);
+
+ return 2;
+ }
+ }
+
+ lua_pushboolean(L, true);
+
+ return 1;
+}
+
+/***
+ * @method rspamd_redis:exec()
+ * Executes pending commands (suitable for blocking IO only for now)
+ * @return {boolean}, {table}, ...: pairs in format [bool, result] for each request pending
+ */
+static int
+lua_redis_exec(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_redis_ctx *ctx = lua_check_redis(L, 1);
+
+ if (ctx == NULL) {
+ lua_error(L);
+
+ return 1;
+ }
+
+ if (IS_ASYNC(ctx)) {
+ lua_pushstring(L, "Async redis pipelining is not implemented");
+ lua_error(L);
+ return 0;
+ }
+ else {
+ if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) == 0) {
+ lua_pushstring(L, "No pending commands to execute");
+ lua_error(L);
+ }
+ if (ctx->cmds_pending == 0 && g_queue_get_length(ctx->replies) > 0) {
+ gint results = lua_redis_push_results(ctx, L);
+ return results;
+ }
+ else {
+ ctx->thread = lua_thread_pool_get_running_entry(ctx->async.cfg->lua_thread_pool);
+ return lua_thread_yield(ctx->thread, 0);
+ }
+ }
+}
+
+static gint
+lua_load_redis(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, redislib_f);
+
+ return 1;
+}
+
+static gint
+lua_redis_null_idx(lua_State *L)
+{
+ lua_pushnil(L);
+
+ return 1;
+}
+
+static void
+lua_redis_null_mt(lua_State *L)
+{
+ luaL_newmetatable(L, "redis{null}");
+
+ lua_pushcfunction(L, lua_redis_null_idx);
+ lua_setfield(L, -2, "__index");
+ lua_pushcfunction(L, lua_redis_null_idx);
+ lua_setfield(L, -2, "__tostring");
+
+ lua_pop(L, 1);
+}
+
+/**
+ * Open redis library
+ * @param L lua stack
+ * @return
+ */
+void luaopen_redis(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{redis}", redislib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_redis", lua_load_redis);
+
+ /* Set null element */
+ lua_redis_null_mt(L);
+ redis_null = lua_newuserdata(L, 0);
+ luaL_getmetatable(L, "redis{null}");
+ lua_setmetatable(L, -2);
+ lua_setfield(L, LUA_REGISTRYINDEX, "redis.null");
+}
diff --git a/src/lua/lua_regexp.c b/src/lua/lua_regexp.c
new file mode 100644
index 0000000..7e638ca
--- /dev/null
+++ b/src/lua/lua_regexp.c
@@ -0,0 +1,858 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+/***
+ * @module rspamd_regexp
+ * Rspamd regexp is an utility module that handles rspamd perl compatible
+ * regular expressions
+ * @example
+ * local rspamd_regexp = require "rspamd_regexp"
+ *
+ * local re = rspamd_regexp.create_cached('/^\\s*some_string\\s*$/i')
+ * re:match('some_string')
+ * local re = rspamd_regexp.create_cached('/\\s+/i')
+ * re:split('word word word') -- returns ['word', 'word', 'word']
+ */
+
+LUA_FUNCTION_DEF(regexp, create);
+LUA_FUNCTION_DEF(regexp, import_glob);
+LUA_FUNCTION_DEF(regexp, import_plain);
+LUA_FUNCTION_DEF(regexp, create_cached);
+LUA_FUNCTION_DEF(regexp, get_cached);
+LUA_FUNCTION_DEF(regexp, get_pattern);
+LUA_FUNCTION_DEF(regexp, set_limit);
+LUA_FUNCTION_DEF(regexp, set_max_hits);
+LUA_FUNCTION_DEF(regexp, get_max_hits);
+LUA_FUNCTION_DEF(regexp, search);
+LUA_FUNCTION_DEF(regexp, match);
+LUA_FUNCTION_DEF(regexp, matchn);
+LUA_FUNCTION_DEF(regexp, split);
+LUA_FUNCTION_DEF(regexp, destroy);
+LUA_FUNCTION_DEF(regexp, gc);
+
+static const struct luaL_reg regexplib_m[] = {
+ LUA_INTERFACE_DEF(regexp, get_pattern),
+ LUA_INTERFACE_DEF(regexp, set_limit),
+ LUA_INTERFACE_DEF(regexp, set_max_hits),
+ LUA_INTERFACE_DEF(regexp, get_max_hits),
+ LUA_INTERFACE_DEF(regexp, match),
+ LUA_INTERFACE_DEF(regexp, matchn),
+ LUA_INTERFACE_DEF(regexp, search),
+ LUA_INTERFACE_DEF(regexp, split),
+ LUA_INTERFACE_DEF(regexp, destroy),
+ {"__tostring", lua_regexp_get_pattern},
+ {"__gc", lua_regexp_gc},
+ {NULL, NULL}};
+static const struct luaL_reg regexplib_f[] = {
+ LUA_INTERFACE_DEF(regexp, create),
+ LUA_INTERFACE_DEF(regexp, import_glob),
+ LUA_INTERFACE_DEF(regexp, import_plain),
+ LUA_INTERFACE_DEF(regexp, get_cached),
+ LUA_INTERFACE_DEF(regexp, create_cached),
+ {NULL, NULL}};
+
+#define LUA_REGEXP_FLAG_DESTROYED (1 << 0)
+#define IS_DESTROYED(re) ((re)->re_flags & LUA_REGEXP_FLAG_DESTROYED)
+
+rspamd_mempool_t *regexp_static_pool = NULL;
+
+struct rspamd_lua_regexp *
+lua_check_regexp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{regexp}");
+
+ luaL_argcheck(L, ud != NULL, pos, "'regexp' expected");
+ return ud ? *((struct rspamd_lua_regexp **) ud) : NULL;
+}
+
+/***
+ * @function rspamd_regexp.create(pattern[, flags])
+ * Creates new rspamd_regexp
+ * @param {string} pattern pattern to build regexp. If this pattern is enclosed in `//` then it is possible to specify flags after it
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.create('/^test.*[0-9]\\s*$/i')
+ */
+static int
+lua_regexp_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ GError *err = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_new(string, flags_str, &err);
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.import_glob(glob_pattern[, flags])
+ * Creates new rspamd_regexp from glob
+ * @param {string} pattern pattern to build regexp.
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.import_glob('ab*', 'i')
+ */
+static int
+lua_regexp_import_glob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ gchar *escaped;
+ gsize pat_len;
+ GError *err = NULL;
+
+ string = luaL_checklstring(L, 1, &pat_len);
+
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ escaped = rspamd_str_regexp_escape(string, pat_len, NULL,
+ RSPAMD_REGEXP_ESCAPE_GLOB | RSPAMD_REGEXP_ESCAPE_UTF);
+
+ re = rspamd_regexp_new(escaped, flags_str, &err);
+
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ g_free(escaped);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = escaped;
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.import_plain(plain_string[, flags])
+ * Creates new rspamd_regexp from plain string (escaping specials)
+ * @param {string} pattern pattern to build regexp.
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.import_plain('exact_string_with*', 'i')
+ */
+static int
+lua_regexp_import_plain(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ gchar *escaped;
+ gsize pat_len;
+ GError *err = NULL;
+
+ string = luaL_checklstring(L, 1, &pat_len);
+
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ escaped = rspamd_str_regexp_escape(string, pat_len, NULL,
+ RSPAMD_REGEXP_ESCAPE_ASCII);
+
+ re = rspamd_regexp_new(escaped, flags_str, &err);
+
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ g_free(escaped);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = re;
+ new->re_pattern = escaped;
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.get_cached(pattern)
+ * This function gets cached and pre-compiled regexp created by either `create`
+ * or `create_cached` methods. If no cached regexp is found then `nil` is returned.
+ *
+ * @param {string} pattern regexp pattern
+ * @return {regexp} cached regexp structure or `nil`
+ */
+static int
+lua_regexp_get_cached(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_cache_query(NULL, string, flags_str);
+
+ if (re) {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_regexp.create_cached(pattern[, flags])
+ * This function is similar to `create` but it tries to search for regexp in the
+ * cache first.
+ * @param {string} pattern pattern to build regexp. If this pattern is enclosed in `//` then it is possible to specify flags after it
+ * @param {string} flags optional flags to create regular expression
+ * @return {regexp} regexp argument that is *not* automatically destroyed
+ * @example
+ * local regexp = require "rspamd_regexp"
+ *
+ * local re = regexp.create_cached('/^test.*[0-9]\\s*$/i')
+ * ...
+ * -- This doesn't create new regexp object
+ * local other_re = regexp.create_cached('/^test.*[0-9]\\s*$/i')
+ */
+static int
+lua_regexp_create_cached(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_regexp_t *re;
+ struct rspamd_lua_regexp *new, **pnew;
+ const gchar *string, *flags_str = NULL;
+ GError *err = NULL;
+
+ string = luaL_checkstring(L, 1);
+ if (lua_gettop(L) == 2) {
+ flags_str = luaL_checkstring(L, 2);
+ }
+
+ if (string) {
+ re = rspamd_regexp_cache_query(NULL, string, flags_str);
+
+ if (re) {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ else {
+ re = rspamd_regexp_cache_create(NULL, string, flags_str, &err);
+ if (re == NULL) {
+ lua_pushnil(L);
+ msg_info("cannot parse regexp: %s, error: %s",
+ string,
+ err == NULL ? "undefined" : err->message);
+ g_error_free(err);
+ }
+ else {
+ new = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ new->re = rspamd_regexp_ref(re);
+ new->re_pattern = g_strdup(string);
+ new->module = rspamd_lua_get_module_name(L);
+ pnew = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pnew = new;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:get_pattern()
+ * Get a pattern for specified regexp object
+ * @return {string} pattern line
+ */
+static int
+lua_regexp_get_pattern(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushstring(L, rspamd_regexp_get_pattern(re->re));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:set_limit(lim)
+ * Set maximum size of text length to be matched with this regexp (if `lim` is
+ * less or equal to zero then all texts are checked)
+ * @param {number} lim limit in bytes
+ */
+static int
+lua_regexp_set_limit(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ gint64 lim;
+
+ lim = lua_tointeger(L, 2);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ if (lim > 0) {
+ rspamd_regexp_set_match_limit(re->re, lim);
+ }
+ else {
+ rspamd_regexp_set_match_limit(re->re, 0);
+ }
+ }
+
+ return 0;
+}
+
+/***
+ * @method re:set_max_hits(lim)
+ * Set maximum number of hits returned by a regexp
+ * @param {number} lim limit in hits count
+ * @return {number} old number of max hits
+ */
+static int
+lua_regexp_set_max_hits(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ guint lim;
+
+ lim = luaL_checkinteger(L, 2);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushinteger(L, rspamd_regexp_set_maxhits(re->re, lim));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:get_max_hits(lim)
+ * Get maximum number of hits returned by a regexp
+ * @return {number} number of max hits
+ */
+static int
+lua_regexp_get_max_hits(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+
+ if (re && re->re && !IS_DESTROYED(re)) {
+ lua_pushinteger(L, rspamd_regexp_get_maxhits(re->re));
+ }
+ else {
+ lua_pushinteger(L, 1);
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:search(line[, raw[, capture]])
+ * Search line in regular expression object. If line matches then this
+ * function returns the table of captured strings. Otherwise, nil is returned.
+ * If `raw` is specified, then input is treated as raw data not encoded in `utf-8`.
+ * If `capture` is true, then this function saves all captures to the table of
+ * values, so the first element is the whole matched string and the
+ * subsequent elements are ordered captures defined within pattern.
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {bool} match raw regexp instead of utf8 one
+ * @param {bool} capture perform subpatterns capturing
+ * @return {table or nil} table of strings or tables (if `capture` is true) or nil if not matched
+ * @example
+ * local re = regexp.create_cached('/^\s*([0-9]+)\s*$/')
+ * -- returns nil
+ * local m1 = re:search('blah')
+ * local m2 = re:search(' 190 ')
+ * -- prints ' 190 '
+ * print(m2[1])
+ *
+ * local m3 = re:search(' 100500 ')
+ * -- prints ' 100500 '
+ * print(m3[1][1])
+ * -- prints '100500' capture
+ * print(m3[1][2])
+ */
+static int
+lua_regexp_search(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ const gchar *data = NULL;
+ struct rspamd_lua_text *t;
+ const gchar *start = NULL, *end = NULL;
+ gint i;
+ gsize len = 0, capn;
+ gboolean matched = FALSE, capture = FALSE, raw = FALSE;
+ GArray *captures = NULL;
+ struct rspamd_re_capture *cap;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ if (lua_gettop(L) >= 3) {
+ raw = lua_toboolean(L, 3);
+ }
+
+ if (data && len > 0) {
+ if (lua_gettop(L) >= 4 && lua_toboolean(L, 4)) {
+ capture = TRUE;
+ captures = g_array_new(FALSE, TRUE,
+ sizeof(struct rspamd_re_capture));
+ }
+
+ lua_newtable(L);
+ i = 0;
+
+ while (rspamd_regexp_search(re->re, data, len, &start, &end, raw,
+ captures)) {
+
+ if (capture) {
+ lua_createtable(L, captures->len, 0);
+
+ for (capn = 0; capn < captures->len; capn++) {
+ cap = &g_array_index(captures, struct rspamd_re_capture,
+ capn);
+ lua_pushlstring(L, cap->p, cap->len);
+ lua_rawseti(L, -2, capn + 1);
+ }
+
+ lua_rawseti(L, -2, ++i);
+ }
+ else {
+ lua_pushlstring(L, start, end - start);
+ lua_rawseti(L, -2, ++i);
+ }
+
+ matched = TRUE;
+ }
+
+ if (!matched) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+
+ if (capture) {
+ g_array_free(captures, TRUE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:match(line[, raw_match])
+ * Matches line against the regular expression and return true if line matches
+ * (partially or completely)
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {bool} match raw regexp instead of utf8 one
+ * @return {bool} true if `line` matches
+ */
+static int
+lua_regexp_match(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ struct rspamd_lua_text *t;
+ const gchar *data = NULL;
+ gsize len = 0;
+ gboolean raw = FALSE;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ if (lua_gettop(L) == 3) {
+ raw = lua_toboolean(L, 3);
+ }
+
+ if (data && len > 0) {
+ if (rspamd_regexp_search(re->re, data, len, NULL, NULL, raw, NULL)) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method re:matchn(line, max_matches, [, raw_match])
+ * Matches line against the regular expression and return number of matches if line matches
+ * (partially or completely). This process stop when `max_matches` is reached.
+ * If `max_matches` is zero, then only a single match is counted which is equal to
+ * @see re:match If `max_matches` is negative, then all matches are considered.
+ *
+ * @param {string} line match the specified line against regexp object
+ * @param {number} max_matches maximum number of matches
+ * @param {bool} match raw regexp instead of utf8 one
+ * @return {number} number of matches found in the `line` argument
+ */
+static int
+lua_regexp_matchn(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ struct rspamd_lua_text *t;
+ const gchar *data = NULL, *start = NULL, *end = NULL;
+ gint max_matches, matches;
+ gsize len = 0;
+ gboolean raw = FALSE;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+ if (t != NULL) {
+ data = t->start;
+ len = t->len;
+ }
+ }
+
+ max_matches = lua_tointeger(L, 3);
+ matches = 0;
+
+ if (lua_gettop(L) == 4) {
+ raw = lua_toboolean(L, 4);
+ }
+
+ if (data && len > 0) {
+ for (;;) {
+ if (rspamd_regexp_search(re->re, data, len, &start, &end, raw,
+ NULL)) {
+ matches++;
+ }
+ else {
+ break;
+ }
+
+ if (max_matches >= 0 && matches >= max_matches) {
+ break;
+ }
+ }
+ }
+
+ lua_pushinteger(L, matches);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+/***
+ * @method re:split(line)
+ * Split line using the specified regular expression.
+ * Breaks the string on the pattern, and returns an array of the tokens.
+ * If the pattern contains capturing parentheses, then the text for each
+ * of the substrings will also be returned. If the pattern does not match
+ * anywhere in the string, then the whole string is returned as the first
+ * token.
+ * @param {string/text} line line to split
+ * @return {table} table of split line portions (if text was the input, then text is used for return parts)
+ */
+static int
+lua_regexp_split(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *re = lua_check_regexp(L, 1);
+ const gchar *data = NULL;
+ struct rspamd_lua_text *t;
+ gboolean matched = FALSE, is_text = FALSE;
+ gsize len = 0;
+ const gchar *start = NULL, *end = NULL, *old_start;
+ gint i;
+
+ if (re && !IS_DESTROYED(re)) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ data = luaL_checklstring(L, 2, &len);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (t == NULL) {
+ lua_error(L);
+ return 0;
+ }
+
+ data = t->start;
+ len = t->len;
+ is_text = TRUE;
+ }
+
+ if (data && len > 0) {
+ lua_newtable(L);
+ i = 0;
+ old_start = data;
+
+ while (rspamd_regexp_search(re->re, data, len, &start, &end, FALSE,
+ NULL)) {
+ if (start - old_start > 0) {
+ if (!is_text) {
+ lua_pushlstring(L, old_start, start - old_start);
+ }
+ else {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = old_start;
+ t->len = start - old_start;
+ t->flags = 0;
+ }
+
+ lua_rawseti(L, -2, ++i);
+ matched = TRUE;
+ }
+ else if (start == end) {
+ break;
+ }
+ old_start = end;
+ }
+
+ if (len > 0 && (end == NULL || end < data + len)) {
+ if (end == NULL) {
+ end = data;
+ }
+
+ if (!is_text) {
+ lua_pushlstring(L, end, (data + len) - end);
+ }
+ else {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = end;
+ t->len = (data + len) - end;
+ t->flags = 0;
+ }
+
+ lua_rawseti(L, -2, ++i);
+ matched = TRUE;
+ }
+
+ if (!matched) {
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+ return 1;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method re:destroy()
+ * Destroy regexp from caches if needed (the pointer is removed by garbage collector)
+ */
+static gint
+lua_regexp_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *to_del = lua_check_regexp(L, 1);
+
+ if (to_del) {
+ rspamd_regexp_cache_remove(NULL, to_del->re);
+ rspamd_regexp_unref(to_del->re);
+ to_del->re = NULL;
+ to_del->re_flags |= LUA_REGEXP_FLAG_DESTROYED;
+ }
+
+ return 0;
+}
+
+static gint
+lua_regexp_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_regexp *to_del = lua_check_regexp(L, 1);
+
+ if (to_del) {
+ if (!IS_DESTROYED(to_del)) {
+ rspamd_regexp_unref(to_del->re);
+ }
+
+ g_free(to_del->re_pattern);
+ g_free(to_del->module);
+ g_free(to_del);
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_regexp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, regexplib_f);
+
+ return 1;
+}
+
+void luaopen_regexp(lua_State *L)
+{
+ if (!regexp_static_pool) {
+ regexp_static_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "regexp_lua_pool", 0);
+ }
+
+ rspamd_lua_new_class(L, "rspamd{regexp}", regexplib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_regexp", lua_load_regexp);
+}
+
+RSPAMD_DESTRUCTOR(lua_re_static_pool_dtor)
+{
+ if (regexp_static_pool) {
+ rspamd_mempool_delete(regexp_static_pool);
+ }
+} \ No newline at end of file
diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c
new file mode 100644
index 0000000..ae5acc8
--- /dev/null
+++ b/src/lua/lua_rsa.c
@@ -0,0 +1,867 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file lua_rsa.c
+ * This module exports routines to load rsa keys, check inline or external
+ * rsa signatures. It assumes sha256 based signatures.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/sha.h>
+#include <openssl/rsa.h>
+
+LUA_FUNCTION_DEF(rsa_pubkey, load);
+LUA_FUNCTION_DEF(rsa_pubkey, create);
+LUA_FUNCTION_DEF(rsa_pubkey, gc);
+LUA_FUNCTION_DEF(rsa_pubkey, tostring);
+
+LUA_FUNCTION_DEF(rsa_privkey, load_file);
+LUA_FUNCTION_DEF(rsa_privkey, load_pem);
+LUA_FUNCTION_DEF(rsa_privkey, load_raw);
+LUA_FUNCTION_DEF(rsa_privkey, load_base64);
+LUA_FUNCTION_DEF(rsa_privkey, create);
+LUA_FUNCTION_DEF(rsa_privkey, gc);
+LUA_FUNCTION_DEF(rsa_privkey, save);
+
+LUA_FUNCTION_DEF(rsa_signature, create);
+LUA_FUNCTION_DEF(rsa_signature, load);
+LUA_FUNCTION_DEF(rsa_signature, save);
+LUA_FUNCTION_DEF(rsa_signature, base64);
+LUA_FUNCTION_DEF(rsa_signature, gc);
+
+LUA_FUNCTION_DEF(rsa, verify_memory);
+LUA_FUNCTION_DEF(rsa, sign_memory);
+LUA_FUNCTION_DEF(rsa, keypair);
+
+static const struct luaL_reg rsalib_f[] = {
+ LUA_INTERFACE_DEF(rsa, verify_memory),
+ LUA_INTERFACE_DEF(rsa, sign_memory),
+ LUA_INTERFACE_DEF(rsa, keypair),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsapubkeylib_f[] = {
+ LUA_INTERFACE_DEF(rsa_pubkey, load),
+ LUA_INTERFACE_DEF(rsa_pubkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsapubkeylib_m[] = {
+ {"__tostring", lua_rsa_pubkey_tostring},
+ {"__gc", lua_rsa_pubkey_gc},
+ {NULL, NULL}};
+
+static const struct luaL_reg rsaprivkeylib_f[] = {
+ LUA_INTERFACE_DEF(rsa_privkey, load_file),
+ LUA_INTERFACE_DEF(rsa_privkey, load_pem),
+ LUA_INTERFACE_DEF(rsa_privkey, load_raw),
+ LUA_INTERFACE_DEF(rsa_privkey, load_base64),
+ LUA_INTERFACE_DEF(rsa_privkey, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsaprivkeylib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_rsa_privkey_gc},
+ LUA_INTERFACE_DEF(rsa_privkey, save),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsasignlib_f[] = {
+ LUA_INTERFACE_DEF(rsa_signature, load),
+ LUA_INTERFACE_DEF(rsa_signature, create),
+ {NULL, NULL}};
+
+static const struct luaL_reg rsasignlib_m[] = {
+ LUA_INTERFACE_DEF(rsa_signature, save),
+ LUA_INTERFACE_DEF(rsa_signature, base64),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_rsa_signature_gc},
+ {NULL, NULL}};
+
+static RSA *
+lua_check_rsa_pubkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_pubkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_pubkey' expected");
+ return ud ? *((RSA **) ud) : NULL;
+}
+
+static RSA *
+lua_check_rsa_privkey(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_privkey}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_privkey' expected");
+ return ud ? *((RSA **) ud) : NULL;
+}
+
+static rspamd_fstring_t *
+lua_check_rsa_sign(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{rsa_signature}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'rsa_signature' expected");
+ return ud ? *((rspamd_fstring_t **) ud) : NULL;
+}
+
+static gint
+lua_rsa_pubkey_load(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *filename;
+ FILE *f;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ msg_err("cannot open pubkey from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (!PEM_read_RSA_PUBKEY(f, &rsa, NULL, NULL)) {
+ msg_err("cannot open pubkey from file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = rsa;
+ }
+ fclose(f);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_save(lua_State *L)
+{
+ const gchar *filename;
+ const gchar *type = "pem";
+ FILE *f;
+ int ret;
+
+ RSA *rsa = lua_check_rsa_privkey(L, 1);
+
+ filename = luaL_checkstring(L, 2);
+ if (lua_gettop(L) > 2) {
+ type = luaL_checkstring(L, 3);
+ }
+
+ if (rsa != NULL && filename != NULL) {
+ if (strcmp(filename, "-") == 0) {
+ f = stdout;
+ }
+ else {
+ f = fopen(filename, "wb");
+ }
+ if (f == NULL) {
+ msg_err("cannot save privkey to file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ if (f != stdout) {
+ /* Set secure permissions for the private key file */
+ chmod(filename, S_IRUSR | S_IWUSR);
+ }
+
+ if (strcmp(type, "der") == 0) {
+ ret = i2d_RSAPrivateKey_fp(f, rsa);
+ }
+ else {
+ ret = PEM_write_RSAPrivateKey(f, rsa, NULL, NULL, 0, NULL, NULL);
+ }
+
+ if (!ret) {
+ msg_err("cannot save privkey to file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ if (f != stdout) {
+ fclose(f);
+ }
+ else {
+ fflush(f);
+ }
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_rsa_pubkey_create(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *buf;
+ BIO *bp;
+
+ buf = luaL_checkstring(L, 1);
+ if (buf != NULL) {
+ bp = BIO_new_mem_buf((void *) buf, -1);
+
+ if (!PEM_read_bio_RSA_PUBKEY(bp, &rsa, NULL, NULL)) {
+ msg_err("cannot parse pubkey: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = rsa;
+ }
+ BIO_free(bp);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_pubkey_gc(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_pubkey(L, 1);
+
+ if (rsa != NULL) {
+ RSA_free(rsa);
+ }
+
+ return 0;
+}
+
+static gint
+lua_rsa_pubkey_tostring(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_pubkey(L, 1);
+
+ if (rsa != NULL) {
+ BIO *pubout = BIO_new(BIO_s_mem());
+ const gchar *pubdata;
+ gsize publen;
+ int rc = i2d_RSA_PUBKEY_bio(pubout, rsa);
+
+ if (rc != 1) {
+ BIO_free(pubout);
+
+ return luaL_error(L, "i2d_RSA_PUBKEY_bio failed");
+ }
+
+ publen = BIO_get_mem_data(pubout, &pubdata);
+ lua_pushlstring(L, pubdata, publen);
+ BIO_free(pubout);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_file(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *filename;
+ FILE *f;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ f = fopen(filename, "r");
+ if (f == NULL) {
+ msg_err("cannot open private key from file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (!PEM_read_RSAPrivateKey(f, &rsa, NULL, NULL)) {
+ msg_err("cannot open private key from file: %s, %s", filename,
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+ fclose(f);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_pem(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ b = BIO_new_mem_buf(data, len);
+
+ if (!PEM_read_bio_RSAPrivateKey(b, &rsa, NULL, NULL)) {
+ msg_err("cannot open private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ BIO_free(b);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_raw(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ gsize len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ b = BIO_new_mem_buf(data, len);
+ rsa = d2i_RSAPrivateKey_bio(b, NULL);
+
+ if (rsa == NULL) {
+ msg_err("cannot open private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ BIO_free(b);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_load_base64(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ BIO *b;
+ EVP_PKEY *evp = NULL;
+ struct rspamd_lua_text *t;
+ const gchar *data;
+ guchar *decoded;
+ gsize len, dec_len;
+
+ if (lua_isuserdata(L, 1)) {
+ t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ len = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 1, &len);
+ }
+
+ if (data != NULL) {
+ decoded = g_malloc(len);
+
+ if (!rspamd_cryptobox_base64_decode(data, len, decoded, &dec_len)) {
+ g_free(decoded);
+
+ return luaL_error(L, "invalid base64 encoding");
+ }
+
+ b = BIO_new_mem_buf(decoded, dec_len);
+
+ if (d2i_PrivateKey_bio(b, &evp) != NULL) {
+ rsa = EVP_PKEY_get1_RSA(evp);
+
+ if (rsa == NULL) {
+ msg_err("cannot open RSA private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+
+ EVP_PKEY_free(evp);
+ }
+ else {
+ msg_err("cannot open EVP private key from data, %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+
+ BIO_free(b);
+ g_free(decoded);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_create(lua_State *L)
+{
+ RSA *rsa = NULL, **prsa;
+ const gchar *buf;
+ BIO *bp;
+
+ buf = luaL_checkstring(L, 1);
+ if (buf != NULL) {
+ bp = BIO_new_mem_buf((void *) buf, -1);
+
+ if (!PEM_read_bio_RSAPrivateKey(bp, &rsa, NULL, NULL)) {
+ msg_err("cannot parse private key: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ lua_pushnil(L);
+ }
+ else {
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = rsa;
+ }
+ BIO_free(bp);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_privkey_gc(lua_State *L)
+{
+ RSA *rsa = lua_check_rsa_privkey(L, 1);
+
+ if (rsa != NULL) {
+ RSA_free(rsa);
+ }
+
+ return 0;
+}
+
+static gint
+lua_rsa_signature_load(lua_State *L)
+{
+ rspamd_fstring_t *sig, **psig;
+ const gchar *filename;
+ gpointer data;
+ int fd;
+ struct stat st;
+
+ filename = luaL_checkstring(L, 1);
+ if (filename != NULL) {
+ fd = open(filename, O_RDONLY);
+ if (fd == -1) {
+ msg_err("cannot open signature file: %s, %s", filename,
+ strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (fstat(fd, &st) == -1 ||
+ (data =
+ mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+ msg_err("cannot mmap file %s: %s", filename, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ sig = rspamd_fstring_new_init(data, st.st_size);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = sig;
+ munmap(data, st.st_size);
+ }
+ close(fd);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+static gint
+lua_rsa_signature_save(lua_State *L)
+{
+ rspamd_fstring_t *sig;
+ gint fd, flags;
+ const gchar *filename;
+ gboolean forced = FALSE, res = TRUE;
+
+ sig = lua_check_rsa_sign(L, 1);
+ filename = luaL_checkstring(L, 2);
+ if (lua_gettop(L) > 2) {
+ forced = lua_toboolean(L, 3);
+ }
+
+ if (sig != NULL && filename != NULL) {
+ flags = O_WRONLY | O_CREAT;
+ if (forced) {
+ flags |= O_TRUNC;
+ }
+ else {
+ flags |= O_EXCL;
+ }
+ fd = open(filename, flags, 00644);
+ if (fd == -1) {
+ msg_err("cannot create a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ lua_pushboolean(L, FALSE);
+ }
+ else {
+ while (write(fd, sig->str, sig->len) == -1) {
+ if (errno == EINTR) {
+ continue;
+ }
+ msg_err("cannot write to a signature file: %s, %s",
+ filename,
+ strerror(errno));
+ res = FALSE;
+ break;
+ }
+ lua_pushboolean(L, res);
+ close(fd);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_signature_create(lua_State *L)
+{
+ rspamd_fstring_t *sig, **psig;
+ const gchar *data;
+ gsize dlen;
+
+ data = luaL_checklstring(L, 1, &dlen);
+ if (data != NULL) {
+ sig = rspamd_fstring_new_init(data, dlen);
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = sig;
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_signature_gc(lua_State *L)
+{
+ rspamd_fstring_t *sig = lua_check_rsa_sign(L, 1);
+
+ rspamd_fstring_free(sig);
+
+ return 0;
+}
+
+static gint
+lua_rsa_signature_base64(lua_State *L)
+{
+ rspamd_fstring_t *sig = lua_check_rsa_sign(L, 1);
+ guint boundary = 0;
+ gchar *b64;
+ gsize outlen;
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_isnumber(L, 2)) {
+ boundary = lua_tonumber(L, 2);
+ }
+
+ if (lua_isstring(L, 3)) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (strcmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (strcmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else {
+ how = RSPAMD_TASK_NEWLINES_CRLF;
+ }
+ }
+
+ b64 = rspamd_encode_base64_fold(sig->str, sig->len, boundary, &outlen, how);
+
+ if (b64) {
+ lua_pushlstring(L, b64, outlen);
+ g_free(b64);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/**
+ * Check memory using specified rsa key and signature
+ *
+ * arguments:
+ * (rsa_pubkey, rsa_signature, string)
+ *
+ * returns:
+ * true - if string match rsa signature
+ * false - otherwise
+ */
+static gint
+lua_rsa_verify_memory(lua_State *L)
+{
+ RSA *rsa;
+ rspamd_fstring_t *signature;
+ const gchar *data;
+ gsize sz;
+ gint ret;
+
+ rsa = lua_check_rsa_pubkey(L, 1);
+ signature = lua_check_rsa_sign(L, 2);
+ data = luaL_checklstring(L, 3, &sz);
+
+ if (rsa != NULL && signature != NULL && data != NULL) {
+ ret = RSA_verify(NID_sha256, data, sz,
+ signature->str, signature->len, rsa);
+
+ if (ret == 0) {
+ lua_pushboolean(L, FALSE);
+ lua_pushstring(L, ERR_error_string(ERR_get_error(), NULL));
+
+ return 2;
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/**
+ * Sign memory using specified rsa key and signature
+ *
+ * arguments:
+ * (rsa_privkey, string)
+ *
+ * returns:
+ * rspamd_signature object
+ * nil - otherwise
+ */
+static gint
+lua_rsa_sign_memory(lua_State *L)
+{
+ RSA *rsa;
+ rspamd_fstring_t *signature, **psig;
+ const gchar *data;
+ gsize sz;
+ gint ret;
+
+ rsa = lua_check_rsa_privkey(L, 1);
+ data = luaL_checklstring(L, 2, &sz);
+
+ if (rsa != NULL && data != NULL) {
+ signature = rspamd_fstring_sized_new(RSA_size(rsa));
+
+ guint siglen = signature->len;
+ ret = RSA_sign(NID_sha256, data, sz,
+ signature->str, &siglen, rsa);
+
+ if (ret != 1) {
+ rspamd_fstring_free(signature);
+
+ return luaL_error(L, "cannot sign: %s",
+ ERR_error_string(ERR_get_error(), NULL));
+ }
+ else {
+ signature->len = siglen;
+ psig = lua_newuserdata(L, sizeof(rspamd_fstring_t *));
+ rspamd_lua_setclass(L, "rspamd{rsa_signature}", -1);
+ *psig = signature;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_rsa_keypair(lua_State *L)
+{
+ BIGNUM *e;
+ RSA *rsa, *pub_rsa, *priv_rsa, **prsa;
+ gint bits = lua_gettop(L) > 0 ? lua_tointeger(L, 1) : 1024;
+
+ if (bits > 4096 || bits < 512) {
+ return luaL_error(L, "invalid bits count");
+ }
+
+ e = BN_new();
+ rsa = RSA_new();
+ g_assert(BN_set_word(e, RSA_F4) == 1);
+ g_assert(RSA_generate_key_ex(rsa, bits, e, NULL) == 1);
+
+ priv_rsa = RSAPrivateKey_dup(rsa);
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_privkey}", -1);
+ *prsa = priv_rsa;
+
+ pub_rsa = RSAPublicKey_dup(rsa);
+ prsa = lua_newuserdata(L, sizeof(RSA *));
+ rspamd_lua_setclass(L, "rspamd{rsa_pubkey}", -1);
+ *prsa = pub_rsa;
+
+ RSA_free(rsa);
+ BN_free(e);
+
+ return 2;
+}
+
+static gint
+lua_load_pubkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsapubkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_privkey(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsaprivkeylib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_signature(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsasignlib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_rsa(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rsalib_f);
+
+ return 1;
+}
+
+void luaopen_rsa(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{rsa_pubkey}", rsapubkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_pubkey", lua_load_pubkey);
+
+ rspamd_lua_new_class(L, "rspamd{rsa_privkey}", rsaprivkeylib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_privkey", lua_load_privkey);
+
+ rspamd_lua_new_class(L, "rspamd{rsa_signature}", rsasignlib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_rsa_signature", lua_load_signature);
+
+ rspamd_lua_add_preload(L, "rspamd_rsa", lua_load_rsa);
+
+ lua_settop(L, 0);
+}
diff --git a/src/lua/lua_spf.c b/src/lua/lua_spf.c
new file mode 100644
index 0000000..a67a267
--- /dev/null
+++ b/src/lua/lua_spf.c
@@ -0,0 +1,620 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file lua_spf.c
+ * This module exports spf functions to Lua
+ */
+
+#include "lua_common.h"
+#include "libserver/spf.h"
+#include "libutil/ref.h"
+
+#define SPF_RECORD_CLASS "rspamd{spf_record}"
+
+LUA_FUNCTION_DEF(spf, resolve);
+LUA_FUNCTION_DEF(spf, config);
+
+LUA_FUNCTION_DEF(spf_record, check_ip);
+LUA_FUNCTION_DEF(spf_record, dtor);
+LUA_FUNCTION_DEF(spf_record, get_domain);
+LUA_FUNCTION_DEF(spf_record, get_elts);
+LUA_FUNCTION_DEF(spf_record, get_ttl);
+LUA_FUNCTION_DEF(spf_record, get_timestamp);
+LUA_FUNCTION_DEF(spf_record, get_digest);
+
+static luaL_reg rspamd_spf_f[] = {
+ LUA_INTERFACE_DEF(spf, resolve),
+ LUA_INTERFACE_DEF(spf, config),
+ {NULL, NULL},
+};
+
+static luaL_reg rspamd_spf_record_m[] = {
+ LUA_INTERFACE_DEF(spf_record, check_ip),
+ LUA_INTERFACE_DEF(spf_record, get_domain),
+ LUA_INTERFACE_DEF(spf_record, get_ttl),
+ LUA_INTERFACE_DEF(spf_record, get_digest),
+ LUA_INTERFACE_DEF(spf_record, get_elts),
+ LUA_INTERFACE_DEF(spf_record, get_timestamp),
+ {"__gc", lua_spf_record_dtor},
+ {NULL, NULL},
+};
+
+struct rspamd_lua_spf_cbdata {
+ struct rspamd_task *task;
+ lua_State *L;
+ struct rspamd_symcache_dynamic_item *item;
+ gint cbref;
+ ref_entry_t ref;
+};
+
+static gint
+lua_load_spf(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Create integer arguments to check SPF results */
+ lua_newtable(L);
+ lua_pushinteger(L, SPF_FAIL);
+ lua_setfield(L, -2, "fail");
+ lua_pushinteger(L, SPF_PASS);
+ lua_setfield(L, -2, "pass");
+ lua_pushinteger(L, SPF_NEUTRAL);
+ lua_setfield(L, -2, "neutral");
+ lua_pushinteger(L, SPF_SOFT_FAIL);
+ lua_setfield(L, -2, "soft_fail");
+
+ lua_setfield(L, -2, "policy");
+
+ /* Flags stuff */
+ lua_newtable(L);
+
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_TEMP_FAILED);
+ lua_setfield(L, -2, "temp_fail");
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_NA);
+ lua_setfield(L, -2, "na");
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PERM_FAILED);
+ lua_setfield(L, -2, "perm_fail");
+ lua_pushinteger(L, RSPAMD_SPF_FLAG_CACHED);
+ lua_setfield(L, -2, "cached");
+
+ lua_setfield(L, -2, "flags");
+
+ luaL_register(L, NULL, rspamd_spf_f);
+
+ return 1;
+}
+
+void luaopen_spf(lua_State *L)
+{
+ rspamd_lua_new_class(L, SPF_RECORD_CLASS, rspamd_spf_record_m);
+ lua_pop(L, 1); /* No need in metatable... */
+
+ rspamd_lua_add_preload(L, "rspamd_spf", lua_load_spf);
+ lua_settop(L, 0);
+}
+
+static void
+lua_spf_push_result(struct rspamd_lua_spf_cbdata *cbd, gint code_flags,
+ struct spf_resolved *resolved, const gchar *err)
+{
+ g_assert(cbd != NULL);
+ REF_RETAIN(cbd);
+
+ lua_pushcfunction(cbd->L, &rspamd_lua_traceback);
+ gint err_idx = lua_gettop(cbd->L);
+
+ lua_rawgeti(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ if (resolved) {
+ struct spf_resolved **presolved;
+
+ presolved = lua_newuserdata(cbd->L, sizeof(*presolved));
+ rspamd_lua_setclass(cbd->L, SPF_RECORD_CLASS, -1);
+ *presolved = spf_record_ref(resolved);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ lua_pushinteger(cbd->L, code_flags);
+
+ if (err) {
+ lua_pushstring(cbd->L, err);
+ }
+ else {
+ lua_pushnil(cbd->L);
+ }
+
+ if (lua_pcall(cbd->L, 3, 0, err_idx) != 0) {
+ struct rspamd_task *task = cbd->task;
+
+ msg_err_task("cannot call callback function for spf: %s",
+ lua_tostring(cbd->L, -1));
+ }
+
+ lua_settop(cbd->L, err_idx - 1);
+
+ REF_RELEASE(cbd);
+}
+
+static void
+lua_spf_dtor(struct rspamd_lua_spf_cbdata *cbd)
+{
+ if (cbd) {
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item,
+ "lua_spf");
+ }
+ }
+}
+
+static void
+spf_lua_lib_callback(struct spf_resolved *record, struct rspamd_task *task,
+ gpointer ud)
+{
+ struct rspamd_lua_spf_cbdata *cbd = (struct rspamd_lua_spf_cbdata *) ud;
+
+ if (record) {
+ if ((record->flags & RSPAMD_SPF_RESOLVED_NA)) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_NA, NULL,
+ "no SPF record");
+ }
+ else if (record->elts->len == 0) {
+ if (record->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "bad SPF record");
+ }
+ else if ((record->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED)) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED, NULL,
+ "temporary DNS error");
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "empty SPF record");
+ }
+ }
+ else if (record->domain) {
+ spf_record_ref(record);
+ lua_spf_push_result(cbd, record->flags, record, NULL);
+ spf_record_unref(record);
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "internal error: non empty record for no domain");
+ }
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_PERM_FAILED, NULL,
+ "internal error: no record");
+ }
+
+ REF_RELEASE(cbd);
+}
+
+/***
+ * @function rspamd_spf.resolve(task, callback)
+ * Resolves SPF credentials for a task
+ * @param {rspamd_task} task task
+ * @param {function} callback callback that is called on spf resolution
+*/
+gint lua_spf_resolve(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task && lua_isfunction(L, 2)) {
+ struct rspamd_lua_spf_cbdata *cbd = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*cbd));
+ struct rspamd_spf_cred *spf_cred;
+
+ cbd->task = task;
+ cbd->L = L;
+ lua_pushvalue(L, 2);
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ /* TODO: make it as an optional parameter */
+ spf_cred = rspamd_spf_get_cred(task);
+ cbd->item = rspamd_symcache_get_cur_item(task);
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_inc(task, cbd->item, "lua_spf");
+ }
+ REF_INIT_RETAIN(cbd, lua_spf_dtor);
+
+ if (!rspamd_spf_resolve(task, spf_lua_lib_callback, cbd, spf_cred)) {
+ msg_info_task("cannot make spf request for %s",
+ spf_cred ? spf_cred->domain : "empty domain");
+ if (spf_cred) {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_TEMP_FAILED,
+ NULL, "DNS failed");
+ }
+ else {
+ lua_spf_push_result(cbd, RSPAMD_SPF_RESOLVED_NA,
+ NULL, "No domain");
+ }
+ REF_RELEASE(cbd);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_spf_record_dtor(lua_State *L)
+{
+ struct spf_resolved *record;
+
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ spf_record_unref(record);
+ }
+
+ return 0;
+}
+
+static void
+lua_spf_push_spf_addr(lua_State *L, struct spf_addr *addr)
+{
+ gchar *addr_mask;
+
+ lua_createtable(L, 0, 4);
+
+ lua_pushinteger(L, addr->mech);
+ lua_setfield(L, -2, "result");
+ lua_pushinteger(L, addr->flags);
+ lua_setfield(L, -2, "flags");
+
+ if (addr->spf_string) {
+ lua_pushstring(L, addr->spf_string);
+ lua_setfield(L, -2, "str");
+ }
+
+ addr_mask = spf_addr_mask_to_string(addr);
+
+ if (addr_mask) {
+ lua_pushstring(L, addr_mask);
+ lua_setfield(L, -2, "addr");
+ g_free(addr_mask);
+ }
+}
+
+static gint
+spf_check_element(lua_State *L, struct spf_resolved *rec, struct spf_addr *addr,
+ struct rspamd_lua_ip *ip)
+{
+ gboolean res = FALSE;
+ const guint8 *s, *d;
+ guint af, mask, bmask, addrlen;
+
+
+ if (addr->flags & RSPAMD_SPF_FLAG_TEMPFAIL) {
+ /* Ignore failed addresses */
+
+ return -1;
+ }
+
+ af = rspamd_inet_address_get_af(ip->addr);
+ /* Basic comparing algorithm */
+ if (((addr->flags & RSPAMD_SPF_FLAG_IPV6) && af == AF_INET6) ||
+ ((addr->flags & RSPAMD_SPF_FLAG_IPV4) && af == AF_INET)) {
+ d = rspamd_inet_address_get_hash_key(ip->addr, &addrlen);
+
+ if (af == AF_INET6) {
+ s = (const guint8 *) addr->addr6;
+ mask = addr->m.dual.mask_v6;
+ }
+ else {
+ s = (const guint8 *) addr->addr4;
+ mask = addr->m.dual.mask_v4;
+ }
+
+ /* Compare the first bytes */
+ bmask = mask / CHAR_BIT;
+ if (mask > addrlen * CHAR_BIT) {
+ /* XXX: add logging */
+ }
+ else if (memcmp(s, d, bmask) == 0) {
+ if (bmask * CHAR_BIT < mask) {
+ /* Compare the remaining bits */
+ s += bmask;
+ d += bmask;
+ mask = (0xff << (CHAR_BIT - (mask - bmask * 8))) & 0xff;
+
+ if ((*s & mask) == (*d & mask)) {
+ res = TRUE;
+ }
+ }
+ else {
+ res = TRUE;
+ }
+ }
+ }
+ else {
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ res = TRUE;
+ }
+ else {
+ res = FALSE;
+ }
+ }
+
+ if (res) {
+ if (addr->flags & RSPAMD_SPF_FLAG_ANY) {
+ if (rec->flags & RSPAMD_SPF_RESOLVED_PERM_FAILED) {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_PERM_FAILED);
+ lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
+ }
+ else if (rec->flags & RSPAMD_SPF_RESOLVED_TEMP_FAILED) {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, RSPAMD_SPF_RESOLVED_TEMP_FAILED);
+ lua_pushfstring(L, "%cany", spf_mech_char(addr->mech));
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, addr->mech);
+ lua_spf_push_spf_addr(L, addr);
+ }
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, addr->mech);
+ lua_spf_push_spf_addr(L, addr);
+ }
+
+ return 3;
+ }
+
+ return -1;
+}
+
+/***
+ * @method rspamd_spf_record:check_ip(ip)
+ * Checks the processed record versus a specific IP address. This function
+ * returns 3 values normally:
+ * 1. Boolean check result
+ * 2. If result is `false` then the second value is the error flag (e.g. rspamd_spf.flags.temp_fail), otherwise it will be an SPF method
+ * 3. If result is `false` then this will be an error string, otherwise - an SPF string (e.g. `mx` or `ip4:x.y.z.1`)
+ * @param {rspamd_ip|string} ip address
+ * @return {result,flag_or_policy,error_or_addr} - triplet
+*/
+static gint
+lua_spf_record_check_ip(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+ struct rspamd_lua_ip *ip = NULL;
+ gint nres = 0;
+ gboolean need_free_ip = FALSE;
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ ip = lua_check_ip(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *ip_str;
+ gsize iplen;
+
+ ip = g_malloc0(sizeof(struct rspamd_lua_ip));
+ ip_str = lua_tolstring(L, 2, &iplen);
+
+ if (!rspamd_parse_inet_address(&ip->addr,
+ ip_str, iplen, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ g_free(ip);
+ ip = NULL;
+ }
+ else {
+ need_free_ip = TRUE;
+ }
+ }
+
+ if (record && ip && ip->addr) {
+ for (guint i = 0; i < record->elts->len; i++) {
+ struct spf_addr *addr = &g_array_index(record->elts, struct spf_addr, i);
+ if ((nres = spf_check_element(L, record, addr, ip)) > 0) {
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ return nres;
+ }
+ }
+ }
+ else {
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (need_free_ip) {
+ g_free(ip);
+ }
+
+ /* If we are here it means that there is no ALL record */
+ /*
+ * According to https://tools.ietf.org/html/rfc7208#section-4.7 it means
+ * SPF neutral
+ */
+ struct spf_addr fake_all;
+
+ fake_all.mech = SPF_NEUTRAL;
+ fake_all.flags = RSPAMD_SPF_FLAG_ANY;
+ fake_all.spf_string = "all";
+
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, SPF_NEUTRAL);
+ lua_spf_push_spf_addr(L, &fake_all);
+
+ return 3;
+}
+
+/***
+ * @method rspamd_spf_record:get_domain()
+ * Returns domain for the specific spf record
+*/
+static gint
+lua_spf_record_get_domain(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushstring(L, record->domain);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_ttl()
+ * Returns ttl for the specific spf record
+*/
+static gint
+lua_spf_record_get_ttl(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushinteger(L, record->ttl);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_timestamp()
+ * Returns ttl for the specific spf record
+*/
+static gint
+lua_spf_record_get_timestamp(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ lua_pushnumber(L, record->timestamp);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_digest()
+ * Returns string hex representation of the record digest (fast hash function)
+*/
+static gint
+lua_spf_record_get_digest(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ gchar hexbuf[64];
+
+ rspamd_snprintf(hexbuf, sizeof(hexbuf), "%xuL", record->digest);
+ lua_pushstring(L, hexbuf);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method rspamd_spf_record:get_elts()
+ * Returns a list of all elements in an SPF record. Each element is a table with the
+ * following fields:
+ *
+ * - result - mech flag from rspamd_spf.results
+ * - flags - all flags
+ * - addr - address and mask as a string
+ * - str - string representation (if available)
+*/
+static gint
+lua_spf_record_get_elts(lua_State *L)
+{
+ struct spf_resolved *record;
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, 1, SPF_RECORD_CLASS,
+ struct spf_resolved,
+ record);
+
+ if (record) {
+ guint i;
+ struct spf_addr *addr;
+
+ lua_createtable(L, record->elts->len, 0);
+
+ for (i = 0; i < record->elts->len; i++) {
+ addr = (struct spf_addr *) &g_array_index(record->elts,
+ struct spf_addr, i);
+ lua_spf_push_spf_addr(L, addr);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function rspamd_spf.config(object)
+ * Configures SPF library according to the UCL config
+ * @param {table} object configuration object
+*/
+gint lua_spf_config(lua_State *L)
+{
+ ucl_object_t *config_obj = ucl_object_lua_import(L, 1);
+
+ if (config_obj) {
+ spf_library_config(config_obj);
+ ucl_object_unref(config_obj); /* As we copy data all the time */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+} \ No newline at end of file
diff --git a/src/lua/lua_sqlite3.c b/src/lua/lua_sqlite3.c
new file mode 100644
index 0000000..be7a9ae
--- /dev/null
+++ b/src/lua/lua_sqlite3.c
@@ -0,0 +1,379 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "sqlite_utils.h"
+
+/***
+ * @module rspamd_sqlite3
+ * This module provides routines to query sqlite3 databases
+@example
+local sqlite3 = require "rspamd_sqlite3"
+
+local db = sqlite3.open("/tmp/db.sqlite")
+
+if db then
+ db:exec([[ CREATE TABLE x (id INT, value TEXT); ]])
+
+ db:exec([[ INSERT INTO x VALUES (?1, ?2); ]], 1, 'test')
+
+ for row in db:rows([[ SELECT * FROM x ]]) do
+ print(string.format('%d -> %s', row.id, row.value))
+ end
+end
+ */
+
+LUA_FUNCTION_DEF(sqlite3, open);
+LUA_FUNCTION_DEF(sqlite3, sql);
+LUA_FUNCTION_DEF(sqlite3, rows);
+LUA_FUNCTION_DEF(sqlite3, close);
+LUA_FUNCTION_DEF(sqlite3_stmt, close);
+
+static const struct luaL_reg sqlitelib_f[] = {
+ LUA_INTERFACE_DEF(sqlite3, open),
+ {NULL, NULL}};
+
+static const struct luaL_reg sqlitelib_m[] = {
+ LUA_INTERFACE_DEF(sqlite3, sql),
+ {"query", lua_sqlite3_sql},
+ {"exec", lua_sqlite3_sql},
+ LUA_INTERFACE_DEF(sqlite3, rows),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_sqlite3_close},
+ {NULL, NULL}};
+
+static const struct luaL_reg sqlitestmtlib_m[] = {
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_sqlite3_stmt_close},
+ {NULL, NULL}};
+
+static void lua_sqlite3_push_row(lua_State *L, sqlite3_stmt *stmt);
+
+static sqlite3 *
+lua_check_sqlite3(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{sqlite3}");
+ luaL_argcheck(L, ud != NULL, pos, "'sqlite3' expected");
+ return ud ? *((sqlite3 **) ud) : NULL;
+}
+
+static sqlite3_stmt *
+lua_check_sqlite3_stmt(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{sqlite3_stmt}");
+ luaL_argcheck(L, ud != NULL, pos, "'sqlite3_stmt' expected");
+ return ud ? *((sqlite3_stmt **) ud) : NULL;
+}
+
+
+/***
+ * @function rspamd_sqlite3.open(path)
+ * Opens sqlite3 database at the specified path. DB is created if not exists.
+ * @param {string} path path to db
+ * @return {sqlite3} sqlite3 handle
+ */
+static gint
+lua_sqlite3_open(lua_State *L)
+{
+ const gchar *path = luaL_checkstring(L, 1);
+ sqlite3 *db, **pdb;
+ GError *err = NULL;
+
+ if (path == NULL) {
+ lua_pushnil(L);
+ return 1;
+ }
+
+ db = rspamd_sqlite3_open_or_create(NULL, path, NULL, 0, &err);
+
+ if (db == NULL) {
+ if (err) {
+ msg_err("cannot open db: %e", err);
+ g_error_free(err);
+ }
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ pdb = lua_newuserdata(L, sizeof(db));
+ *pdb = db;
+ rspamd_lua_setclass(L, "rspamd{sqlite3}", -1);
+
+ return 1;
+}
+
+static void
+lua_sqlite3_bind_statements(lua_State *L, gint start, gint end,
+ sqlite3_stmt *stmt)
+{
+ gint i, type, num = 1;
+ const gchar *str;
+ gsize slen;
+ gdouble n;
+
+ g_assert(start <= end && start > 0 && end > 0);
+
+ for (i = start; i <= end; i++) {
+ type = lua_type(L, i);
+
+ switch (type) {
+ case LUA_TNUMBER:
+ n = lua_tonumber(L, i);
+
+ if (n == (gdouble) ((gint64) n)) {
+ sqlite3_bind_int64(stmt, num, n);
+ }
+ else {
+ sqlite3_bind_double(stmt, num, n);
+ }
+ num++;
+ break;
+ case LUA_TSTRING:
+ str = lua_tolstring(L, i, &slen);
+ sqlite3_bind_text(stmt, num, str, slen, SQLITE_TRANSIENT);
+ num++;
+ break;
+ default:
+ msg_err("invalid type at position %d: %s", i, lua_typename(L, type));
+ break;
+ }
+ }
+}
+
+/***
+ * @function rspamd_sqlite3:sql(query[, args..])
+ * Performs sqlite3 query replacing '?1', '?2' and so on with the subsequent args
+ * of the function
+ *
+ * @param {string} query SQL query
+ * @param {string|number} args... variable number of arguments
+ * @return {boolean} `true` if a statement has been successfully executed
+ */
+static gint
+lua_sqlite3_sql(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+ const gchar *query = luaL_checkstring(L, 2);
+ sqlite3_stmt *stmt;
+ gboolean ret = FALSE;
+ gint top = 1, rc;
+
+ if (db && query) {
+ if (sqlite3_prepare_v2(db, query, -1, &stmt, NULL) != SQLITE_OK) {
+ msg_err("cannot prepare query %s: %s", query, sqlite3_errmsg(db));
+ return luaL_error(L, sqlite3_errmsg(db));
+ }
+ else {
+ top = lua_gettop(L);
+
+ if (top > 2) {
+ /* Push additional arguments to sqlite3 */
+ lua_sqlite3_bind_statements(L, 3, top, stmt);
+ }
+
+ rc = sqlite3_step(stmt);
+ top = 1;
+
+ if (rc == SQLITE_ROW || rc == SQLITE_OK || rc == SQLITE_DONE) {
+ ret = TRUE;
+
+ if (rc == SQLITE_ROW) {
+ lua_sqlite3_push_row(L, stmt);
+ top = 2;
+ }
+ }
+ else {
+ msg_warn("sqlite3 error: %s", sqlite3_errmsg(db));
+ }
+
+ sqlite3_finalize(stmt);
+ }
+ }
+
+ lua_pushboolean(L, ret);
+
+ return top;
+}
+
+static void
+lua_sqlite3_push_row(lua_State *L, sqlite3_stmt *stmt)
+{
+ const gchar *str;
+ gsize slen;
+ gint64 num;
+ gchar numbuf[32];
+ gint nresults, i, type;
+
+ nresults = sqlite3_column_count(stmt);
+ lua_createtable(L, 0, nresults);
+
+ for (i = 0; i < nresults; i++) {
+ lua_pushstring(L, sqlite3_column_name(stmt, i));
+ type = sqlite3_column_type(stmt, i);
+
+ switch (type) {
+ case SQLITE_INTEGER:
+ /*
+ * XXX: we represent int64 as strings, as we can nothing else to do
+ * about it portably
+ */
+ num = sqlite3_column_int64(stmt, i);
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", num);
+ lua_pushstring(L, numbuf);
+ break;
+ case SQLITE_FLOAT:
+ lua_pushnumber(L, sqlite3_column_double(stmt, i));
+ break;
+ case SQLITE_TEXT:
+ slen = sqlite3_column_bytes(stmt, i);
+ str = sqlite3_column_text(stmt, i);
+ lua_pushlstring(L, str, slen);
+ break;
+ case SQLITE_BLOB:
+ slen = sqlite3_column_bytes(stmt, i);
+ str = sqlite3_column_blob(stmt, i);
+ lua_pushlstring(L, str, slen);
+ break;
+ default:
+ lua_pushboolean(L, 0);
+ break;
+ }
+
+ lua_settable(L, -3);
+ }
+}
+
+static gint
+lua_sqlite3_next_row(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3_stmt *stmt = *(sqlite3_stmt **) lua_touserdata(L, lua_upvalueindex(1));
+ gint rc;
+
+ if (stmt != NULL) {
+ rc = sqlite3_step(stmt);
+
+ if (rc == SQLITE_ROW) {
+ lua_sqlite3_push_row(L, stmt);
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+
+ return 1;
+}
+
+/***
+ * @function rspamd_sqlite3:rows(query[, args..])
+ * Performs sqlite3 query replacing '?1', '?2' and so on with the subsequent args
+ * of the function. This function returns iterator suitable for loop construction:
+ *
+ * @param {string} query SQL query
+ * @param {string|number} args... variable number of arguments
+ * @return {function} iterator to get all rows
+@example
+for row in db:rows([[ SELECT * FROM x ]]) do
+ print(string.format('%d -> %s', row.id, row.value))
+end
+ */
+static gint
+lua_sqlite3_rows(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+ const gchar *query = luaL_checkstring(L, 2);
+ sqlite3_stmt *stmt, **pstmt;
+ gint top;
+
+ if (db && query) {
+ if (sqlite3_prepare_v2(db, query, -1, &stmt, NULL) != SQLITE_OK) {
+ msg_err("cannot prepare query %s: %s", query, sqlite3_errmsg(db));
+ lua_pushstring(L, sqlite3_errmsg(db));
+ return lua_error(L);
+ }
+ else {
+ top = lua_gettop(L);
+
+ if (top > 2) {
+ /* Push additional arguments to sqlite3 */
+ lua_sqlite3_bind_statements(L, 3, top, stmt);
+ }
+
+ /* Create C closure */
+ pstmt = lua_newuserdata(L, sizeof(stmt));
+ *pstmt = stmt;
+ rspamd_lua_setclass(L, "rspamd{sqlite3_stmt}", -1);
+
+ lua_pushcclosure(L, lua_sqlite3_next_row, 1);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_sqlite3_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ sqlite3 *db = lua_check_sqlite3(L, 1);
+
+ if (db) {
+ sqlite3_close(db);
+ }
+
+ return 0;
+}
+
+static gint
+lua_sqlite3_stmt_close(lua_State *L)
+{
+ sqlite3_stmt *stmt = lua_check_sqlite3_stmt(L, 1);
+
+ if (stmt) {
+ sqlite3_finalize(stmt);
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_sqlite3(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, sqlitelib_f);
+
+ return 1;
+}
+/**
+ * Open redis library
+ * @param L lua stack
+ * @return
+ */
+void luaopen_sqlite3(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{sqlite3}", sqlitelib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_new_class(L, "rspamd{sqlite3_stmt}", sqlitestmtlib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_sqlite3", lua_load_sqlite3);
+}
diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c
new file mode 100644
index 0000000..7278602
--- /dev/null
+++ b/src/lua/lua_task.c
@@ -0,0 +1,7295 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_url.h"
+
+#include "message.h"
+#include "images.h"
+#include "archives.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include "libmime/smtp_parsers.h"
+#include "libserver/mempool_vars_internal.h"
+#include "libserver/dkim.h"
+#include "libserver/task.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result_private.h"
+#include "libstat/stat_api.h"
+#include "libserver/maps/map_helpers.h"
+
+#include <math.h>
+#include "libmime/received.h"
+
+/***
+ * @module rspamd_task
+ * This module provides routines for tasks manipulation in rspamd. Tasks usually
+ * represent messages being scanned, and this API provides access to such elements
+ * as headers, symbols, metrics and so on and so forth. Normally, task objects
+ * are passed to the lua callbacks allowing to check specific properties of messages
+ * and add the corresponding symbols to the scan's results.
+@example
+rspamd_config.DATE_IN_PAST = function(task)
+ local dm = task:get_date{format = 'message', gmt = true}
+ local dt = task:get_date{format = 'connect', gmt = true}
+ -- A day
+ if dt - dm > 86400 then
+ return true
+ end
+
+ return false
+end
+ */
+
+/* Task methods */
+
+/***
+ * @function rspamd_task.create([cfg])
+ * Create a new empty task
+ * @return {rspamd_task} new task
+ */
+LUA_FUNCTION_DEF(task, create);
+/***
+ * @function rspamd_task.load_from_file(filename[, cfg])
+ * Loads a message from specific file
+ * @return {boolean,rspamd_task|error} status + new task or error message
+ */
+LUA_FUNCTION_DEF(task, load_from_file);
+/***
+ * @function rspamd_task.load_from_string(message[, cfg])
+ * Loads a message from specific file
+ * @return {boolean,rspamd_task|error} status + new task or error message
+ */
+LUA_FUNCTION_DEF(task, load_from_string);
+/***
+ * @method task:get_message()
+ * Returns task raw message content as opaque text
+ * @return {rspamd_text} task raw content
+ */
+LUA_FUNCTION_DEF(task, get_message);
+/***
+ * @method task:set_message(msg)
+ * Updates task message with another message; It also parses a message to
+ * fill the internal structures.
+ * Input might be a string, a lua_text or a table of the former stuff.
+ * @param {string/text/table} msg new message to set
+ * @return {boolean,number} if a message has been set + its raw size
+ */
+LUA_FUNCTION_DEF(task, set_message);
+/***
+ * @method task:process_message()
+ * Parses message
+ */
+LUA_FUNCTION_DEF(task, process_message);
+/***
+ * @method task:get_cfg()
+ * Get configuration object for a task.
+ * @return {rspamd_config} (config.md)[configuration object] for the task
+ */
+LUA_FUNCTION_DEF(task, get_cfg);
+LUA_FUNCTION_DEF(task, set_cfg);
+LUA_FUNCTION_DEF(task, destroy);
+/***
+ * @method task:get_mempool()
+ * Returns memory pool valid for a lifetime of task. It is used internally by
+ * many rspamd routines.
+ * @return {rspamd_mempool} memory pool object
+ */
+LUA_FUNCTION_DEF(task, get_mempool);
+/***
+ * @method task:get_session()
+ * Returns asynchronous session object that is used by many rspamd asynchronous
+ * utilities internally.
+ * @return {rspamd_session} session object
+ */
+LUA_FUNCTION_DEF(task, get_session);
+/***
+ * @method task:set_session(session)
+ * Sets new async session for a task
+ */
+LUA_FUNCTION_DEF(task, set_session);
+/***
+ * @method task:get_ev_base()
+ * Return asynchronous event base for using in callbacks and resolver.
+ * @return {rspamd_ev_base} event base
+ */
+LUA_FUNCTION_DEF(task, get_ev_base);
+/***
+ * @method task:get_worker()
+ * Returns a worker object associated with the task
+ * @return {rspamd_worker} worker object
+ */
+LUA_FUNCTION_DEF(task, get_worker);
+/***
+ * @method task:insert_result([enforce_symbol,]symbol, weight[, option1, ...])
+ * Insert specific symbol to the tasks scanning results assigning the initial
+ * weight to it.
+ * @param {boolean} enforce_symbol if represented and true, then insert symbol even if it is not registered in the metric
+ * @param {string} symbol symbol to insert
+ * @param {number} weight initial weight (this weight is multiplied by the metric weight)
+ * @param {string} options list of optional options attached to a symbol inserted
+@example
+local function cb(task)
+ if task:get_header('Some header') then
+ task:insert_result('SOME_HEADER', 1.0, 'Got some header')
+ end
+end
+ */
+LUA_FUNCTION_DEF(task, insert_result);
+/***
+ * @method task:insert_result_named(shadow_result, [enforce_symbol,]symbol, weight[, option1, ...])
+ * Insert specific symbol to the tasks scanning results assigning the initial
+ * weight to it.
+ * @param {string} shadow_result name of shadow result
+ * @param {boolean} enforce_symbol if represented and true, then insert symbol even if it is not registered in the metric
+ * @param {string} symbol symbol to insert
+ * @param {number} weight initial weight (this weight is multiplied by the metric weight)
+ * @param {string} options list of optional options attached to a symbol inserted
+ */
+LUA_FUNCTION_DEF(task, insert_result_named);
+
+/***
+ * @method task:adjust_result(symbol, score[, option1, ...])
+ * Alters the existing symbol's score to a new score. It is not affected by
+ * metric score or grow factor. You can also add new options
+ * using this method. Symbol must be already inserted into metric or an error
+ * will be emitted.
+ * @param {string} symbol symbol to adjust
+ * @param {number} score this value is NOT multiplied by the metric score
+ * @param {string/table} options list of optional options attached to a symbol adjusted
+ */
+LUA_FUNCTION_DEF(task, adjust_result);
+
+/***
+ * @method task:remove_result(symbol[, shadow_result])
+ * Removes the symbol from a named or unnamed/default result
+ * @param {string} symbol symbol to remove
+ * @param {string} shadow_result name of shadow result
+ * @return {boolean} true if a symbol has been removed
+ */
+LUA_FUNCTION_DEF(task, remove_result);
+/***
+ * @method task:set_pre_result(action, [message, [module], [score], [priority], [flags])
+ * Sets pre-result for a task. It is used in pre-filters to specify early result
+ * of the task scanned. If a pre-filter sets some result, then further processing
+ * may be skipped. For selecting action it is possible to use global table
+ * `rspamd_actions` or a string value:
+ *
+ * - `reject`: reject message permanently
+ * - `add header`: add spam header
+ * - `rewrite subject`: rewrite subject to spam subject
+ * - `greylist`: greylist message
+ * - `accept` or `no action`: whitelist message
+ *
+ * This function also accepts a table from Rspamd 2.6 with the following keys:
+ * - action: string required
+ * - message: string
+ * - module: string
+ * - score: number
+ * - priority: integer
+ * - flags: flags string
+ * - result: named result if needed
+ *
+ * @param {rspamd_action or string} action a numeric or string action value
+ * @param {string} message action message
+ * @param {string} module optional module name
+ * @param {number/nil} score optional explicit score
+ * @param {number/nil} priority optional explicit priority
+ * @param {string/nil} flags optional flags (e.g. 'least' for least action)
+@example
+local function cb(task)
+ local gr = task:get_header('Greylist')
+ if gr and gr == 'greylist' then
+ task:set_pre_result('soft reject', 'Greylisting required')
+ end
+end
+ */
+LUA_FUNCTION_DEF(task, set_pre_result);
+
+/***
+ * @method task:has_pre_result()
+ * Returns true if task has some pre-result being set.
+ * If result has been set this function also returns pre result action,
+ * message and module as strings in this order.
+ *
+ * @return {boolean,[string,string,string]} true if task has some pre-result being set
+ */
+LUA_FUNCTION_DEF(task, has_pre_result);
+/***
+ * @method task:append_message(message, [category])
+ * Adds a message to scanning output.
+ * @param {string} message
+ * @param {category} message category
+@example
+local function cb(task)
+ task:append_message('Example message')
+end
+ */
+LUA_FUNCTION_DEF(task, append_message);
+/***
+ * @method task:get_urls([need_emails|list_protos][, need_images])
+ * Get all URLs found in a message. Telephone urls and emails are not included unless explicitly asked in `list_protos`
+ * @param {boolean} need_emails if `true` then return also email urls, this can be a comma separated string of protocols desired or a table (e.g. `mailto` or `telephone`)
+ * @param {boolean} need_images return urls from images (<img src=...>) as well
+ * @return {table rspamd_url} list of all urls found
+@example
+local function phishing_cb(task)
+ local urls = task:get_urls({'https', 'http'});
+
+ if urls then
+ for _,url in ipairs(urls) do
+ if url:is_phished() then
+ return true
+ end
+ end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_urls);
+/***
+ * @method task:get_urls_filtered([{flags_include}, [{flags_exclude}]], [{protocols_mask}])
+ * Get urls managed by either exclude or include flags list
+ * - If flags include are nil then all but excluded urls are returned
+ * - If flags exclude are nil then only included explicitly urls are returned
+ * - If both parameters are nil then all urls are included
+ * @param {table} flags_include included flags
+ * @param {table} flags_exclude excluded flags
+ * @param {table} protocols_mask include only specific protocols
+ * @return {table rspamd_url} list of urls matching conditions
+ */
+LUA_FUNCTION_DEF(task, get_urls_filtered);
+/***
+ * @method task:has_urls([need_emails])
+ * Returns 'true' if a task has urls listed
+ * @param {boolean} need_emails if `true` then return also email urls
+ * @return {boolean} true if a task has urls (urls or emails if `need_emails` is true)
+ */
+LUA_FUNCTION_DEF(task, has_urls);
+/***
+ * @method task:inject_url(url)
+ * Inserts an url into a task (useful for redirected urls)
+ * @param {lua_url} url url to inject
+ */
+LUA_FUNCTION_DEF(task, inject_url);
+/***
+ * @method task:get_content()
+ * Get raw content for the specified task
+ * @return {text} the data contained in the task
+ */
+LUA_FUNCTION_DEF(task, get_content);
+
+/***
+ * @method task:get_filename()
+ * Returns filename for a specific task
+ * @return {string|nil} filename or nil if unknown
+ */
+LUA_FUNCTION_DEF(task, get_filename);
+
+/***
+ * @method task:get_rawbody()
+ * Get raw body for the specified task
+ * @return {text} the data contained in the task
+ */
+LUA_FUNCTION_DEF(task, get_rawbody);
+
+/***
+ * @method task:get_emails()
+ * Get all email addresses found in a message.
+ * @return {table rspamd_url} list of all email addresses found
+ */
+LUA_FUNCTION_DEF(task, get_emails);
+/***
+ * @method task:get_text_parts()
+ * Get all text (and HTML) parts found in a message
+ * @return {table rspamd_text_part} list of text parts
+ */
+LUA_FUNCTION_DEF(task, get_text_parts);
+/***
+ * @method task:get_parts()
+ * Get all mime parts found in a message
+ * @return {table rspamd_mime_part} list of mime parts
+ */
+LUA_FUNCTION_DEF(task, get_parts);
+/***
+ * @method task:get_meta_words([how='stem'])
+ * Get meta words from task (subject and displayed names)
+ * - `stem`: stemmed words (default)
+ * - `norm`: normalised words (utf normalised + lowercased)
+ * - `raw`: raw words in utf (if possible)
+ * - `full`: list of tables, each table has the following fields:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ */
+LUA_FUNCTION_DEF(task, get_meta_words);
+/***
+ * @method task:get_request_header(name)
+ * Get value of a HTTP request header.
+ * @param {string} name name of header to get
+ * @return {rspamd_text} value of an HTTP header
+ */
+LUA_FUNCTION_DEF(task, get_request_header);
+/***
+ * @method task:set_request_header(name, value)
+ * Set value of a HTTP request header. If value is omitted, then a header is removed
+ * @param {string} name name of header to get
+ * @param {rspamd_text/string} value new header's value
+ */
+LUA_FUNCTION_DEF(task, set_request_header);
+/***
+ * @method task:get_subject()
+ * Returns task subject (either from the protocol override or from a header)
+ * @return {string} value of a subject (decoded)
+ */
+LUA_FUNCTION_DEF(task, get_subject);
+/***
+ * @method task:get_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} decoded value of a header
+ */
+LUA_FUNCTION_DEF(task, get_header);
+/***
+ * @method task:has_header(name[, case_sensitive])
+ * Get decoded value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in the case insensitive matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {boolean} true if header exists
+ */
+LUA_FUNCTION_DEF(task, has_header);
+/***
+ * @method task:get_header_raw(name[, case_sensitive])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {string} raw value of a header
+ */
+LUA_FUNCTION_DEF(task, get_header_raw);
+/***
+ * @method task:get_header_full(name[, case_sensitive[, need_modified]])
+ * Get raw value of a header specified with optional case_sensitive flag.
+ * By default headers are searched in caseless matter. This method returns more
+ * information about the header as a list of tables with the following structure:
+ *
+ * - `name` - name of a header
+ * - `value` - raw value of a header
+ * - `decoded` - decoded value of a header
+ * - `tab_separated` - `true` if a header and a value are separated by `tab` character
+ * - `empty_separator` - `true` if there are no separator between a header and a value
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @param {boolean} need_modified return a modified value of a header if presented
+ * @return {list of tables} all values of a header as specified above
+@example
+function check_header_delimiter_tab(task, header_name)
+ for _,rh in ipairs(task:get_header_full(header_name)) do
+ if rh['tab_separated'] then return true end
+ end
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_header_full);
+/***
+ * @method task:get_header_count(name[, case_sensitive])
+ * Lightweight version if you need just a header's count
+ * * By default headers are searched in caseless matter.
+ * @param {string} name name of header to get
+ * @param {boolean} case_sensitive case sensitiveness flag to search for a header
+ * @return {number} number of header's occurrences or 0 if not found
+ */
+LUA_FUNCTION_DEF(task, get_header_count);
+/***
+ * @method task:get_raw_headers()
+ * Get all undecoded headers of a message as a string
+ * @return {rspamd_text} all raw headers for a message as opaque text
+ */
+LUA_FUNCTION_DEF(task, get_raw_headers);
+
+/***
+ * @method task:get_headers([need_modified=false])
+ * Get all headers of a message in the same format as get_header_full
+ * @return {table of headers data} all headers for a message
+ */
+LUA_FUNCTION_DEF(task, get_headers);
+
+/***
+ * @method task:modify_header(name, mods)
+ * Modify an existing or non-existing header with the name `name`
+ * Mods is a table with the following structure:
+ * {
+ * "add" = { {order, value}, {order, value} },
+ * "remove" = {order, order, order}
+ * }
+ * Modifications are evaluated in order: remove, add, so headers are first
+ * removed (if any) and then added
+ * Order in remove starts from 1, where 0 means 'remove all', and negative value means
+ * remove from the end
+ * Order in addition means addition from the top: 0 means the most top header, 1 one after, etc
+ * negative order means addition to the end, e.g. -1 is appending header.
+ * @return {bool} true if header could be modified (always true unless we don't have an unparsed message)
+ */
+LUA_FUNCTION_DEF(task, modify_header);
+
+/***
+ * @method task:get_received_headers()
+ * Returns a list of tables of parsed received headers. A tables returned have
+ * the following structure:
+ *
+ * - `from_hostname` - string that represents hostname provided by a peer
+ * - `from_ip` - string representation of sending IP address
+ * - `real_hostname` - hostname as resolved by MTA
+ * - `real_ip` - rspamd_ip object representing sending IP address
+ * - `by_hostname` - MTA hostname
+ * - `proto` - protocol, e.g. ESMTP or ESMTPS
+ * - `timestamp` - received timestamp
+ * - `for` - for value (unparsed mailbox)
+ *
+ * Please note that in some situations rspamd cannot parse all the fields of received headers.
+ * In that case you should check all strings for validity.
+ * @return {table of tables} list of received headers described above
+ */
+LUA_FUNCTION_DEF(task, get_received_headers);
+/***
+ * @method task:get_queue_id()
+ * Returns queue ID of the message being processed.
+ */
+LUA_FUNCTION_DEF(task, get_queue_id);
+/***
+ * @method task:get_uid()
+ * Returns ID of the task being processed.
+ */
+LUA_FUNCTION_DEF(task, get_uid);
+/***
+ * @method task:get_resolver()
+ * Returns ready to use rspamd_resolver object suitable for making asynchronous DNS requests.
+ * @return {rspamd_resolver} resolver object associated with the task's session
+ * @example
+local logger = require "rspamd_logger"
+
+local function task_cb(task)
+ local function dns_cb(resolver, to_resolve, results, err)
+ -- task object is available due to closure
+ task:inc_dns_req()
+ if results then
+ logger.info(string.format('<%s> [%s] resolved for symbol: %s',
+ task:get_message_id(), to_resolve, 'EXAMPLE_SYMBOL'))
+ task:insert_result('EXAMPLE_SYMBOL', 1)
+ end
+ end
+ local r = task:get_resolver()
+ r:resolve_a(task:get_session(), task:get_mempool(), 'example.com', dns_cb)
+end
+ */
+LUA_FUNCTION_DEF(task, get_resolver);
+/***
+ * @method task:set_resolver(resolver)
+ * Sets rspamd_resolver for a specified task.
+ */
+LUA_FUNCTION_DEF(task, set_resolver);
+/***
+* @method task:inc_dns_req()
+* Increment number of DNS requests for the task. Is used just for logging purposes.
+*/
+LUA_FUNCTION_DEF(task, inc_dns_req);
+/***
+ * @method task:get_dns_req()
+ * Get number of dns requests being sent in the task
+ * @return {number} number of DNS requests
+ */
+LUA_FUNCTION_DEF(task, get_dns_req);
+
+/***
+ * @method task:has_recipients([type])
+ * Return true if there are SMTP or MIME recipients for a task.
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {bool,integer} `true` if there are recipients of the following type and a number of such a recipients excluding artificial ones
+ */
+LUA_FUNCTION_DEF(task, has_recipients);
+
+/***
+ * @method task:get_recipients([type])
+ * Return SMTP or MIME recipients for a task. This function returns list of internet addresses each one is a table with the following structure:
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {list of addresses} list of recipients or `nil`
+ */
+LUA_FUNCTION_DEF(task, get_recipients);
+
+/***
+ * @method task:get_principal_recipient()
+ * Returns a single string with so called `principal recipient` for a message. The order
+ * of check is the following:
+ *
+ * - deliver-to request header
+ * - the first recipient (envelope)
+ * - the first recipient (mime)
+ * @return {string} principal recipient
+ */
+LUA_FUNCTION_DEF(task, get_principal_recipient);
+/***
+ * @method task:get_reply_sender()
+ * Returns a single string with address that should be used to reply on a message
+ *
+ * - reply-to header
+ * - from header
+ * - smtp from as a last resort
+ * @return {address} email address
+ */
+LUA_FUNCTION_DEF(task, get_reply_sender);
+
+/***
+ * @method task:set_recipients([type], {rcpt1, rcpt2...}, [how='add'])
+ * Sets recipients for a task. This function accepts table that will be converted to the address.
+ * If some fields are missing they are subsequently reconstructed by this function. E.g. if you
+ * specify 'user' and 'domain', then address and raw string will be reconstructed
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @param {list of tables} recipients recipients to set
+ * @param {string} how define how to set recipients: `rewrite` - rewrite existing recipients, `alias` - treat existing recipients as aliased recipients, `add` - add new recipients
+ * @return {boolean} result of the operation
+ */
+LUA_FUNCTION_DEF(task, set_recipients);
+
+/***
+ * @method task:has_from([type])
+ * Return true if there is SMTP or MIME sender for a task.
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP recipients and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP recipients and `2` or `mime` means MIME recipients only
+ * @return {bool} `true` if there is sender of the following type
+ */
+LUA_FUNCTION_DEF(task, has_from);
+
+/***
+ * @method task:get_from([type])
+ * Return SMTP or MIME sender for a task. This function returns an internet address which one is a table with the following structure:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP sender and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP sender and `2` or `mime` means MIME `From:` only
+ * @return {address} sender or `nil`
+ */
+LUA_FUNCTION_DEF(task, get_from);
+
+/***
+ * @method task:set_from(type, addr)
+ * Sets sender for a task. This function accepts table that will be converted to the address.
+ * If some fields are missing they are subsequently reconstructed by this function. E.g. if you
+ * specify 'user' and 'domain', then address and raw string will be reconstructed
+ *
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * @param {integer|string} type if specified has the following meaning: `0` or `any` means try SMTP sender and fallback to MIME if failed, `1` or `smtp` means checking merely SMTP sender and `2` or `mime` means MIME `From:` only
+ * @param {table
+ * @return {boolean} success or not
+ */
+LUA_FUNCTION_DEF(task, set_from);
+/***
+ * @method task:get_user()
+ * Returns authenticated user name for this task if specified by an MTA.
+ * @return {string} username or nil
+ */
+LUA_FUNCTION_DEF(task, get_user);
+/***
+ * @method task:set_user([username])
+ * Sets or resets (if username is not specified) authenticated user name for this task.
+ * @return {string} the previously set username or nil
+ */
+LUA_FUNCTION_DEF(task, set_user);
+/***
+ * @method task:get_from_ip()
+ * Returns [ip_addr](ip.md) object of a sender that is provided by MTA
+ * @return {rspamd_ip} ip address object
+ */
+LUA_FUNCTION_DEF(task, get_from_ip);
+/***
+ * @method task:set_from_ip(str)
+ * Set tasks's IP address based on the passed string
+ * @param {string} str string representation of ip
+ */
+LUA_FUNCTION_DEF(task, set_from_ip);
+LUA_FUNCTION_DEF(task, get_from_ip_num);
+/***
+ * @method task:get_client_ip()
+ * Returns [ip_addr](ip.md) object of a client connected to rspamd (normally, it is an IP address of MTA)
+ * @return {rspamd_ip} ip address object
+ */
+LUA_FUNCTION_DEF(task, get_client_ip);
+/***
+ * @method task:get_helo()
+ * Returns the value of SMTP helo provided by MTA.
+ * @return {string} HELO value
+ */
+LUA_FUNCTION_DEF(task, get_helo);
+LUA_FUNCTION_DEF(task, set_helo);
+/***
+ * @method task:get_hostname()
+ * Returns the value of sender's hostname provided by MTA
+ * @return {string} hostname value
+ */
+LUA_FUNCTION_DEF(task, get_hostname);
+LUA_FUNCTION_DEF(task, set_hostname);
+/***
+ * @method task:get_images()
+ * Returns list of all images found in a task as a table of `rspamd_image`.
+ * Each image has the following methods:
+ *
+ * * `get_width` - return width of an image in pixels
+ * * `get_height` - return height of an image in pixels
+ * * `get_type` - return string representation of image's type (e.g. 'jpeg')
+ * * `get_filename` - return string with image's file name
+ * * `get_size` - return size in bytes
+ * @return {list of rspamd_image} images found in a message
+ */
+LUA_FUNCTION_DEF(task, get_images);
+/***
+ * @method task:get_archives()
+ * Returns list of all archives found in a task as a table of `rspamd_archive`.
+ * Each archive has the following methods available:
+ *
+ * * `get_files` - return list of strings with filenames inside archive
+ * * `get_files_full` - return list of tables with all information about files
+ * * `is_encrypted` - return true if an archive is encrypted
+ * * `get_type` - return string representation of image's type (e.g. 'zip')
+ * * `get_filename` - return string with archive's file name
+ * * `get_size` - return size in bytes
+ * @return {list of rspamd_archive} archives found in a message
+ */
+LUA_FUNCTION_DEF(task, get_archives);
+/***
+ * @method task:get_dkim_results()
+ * Returns list of all dkim check results as table of maps. Callee must ensure that
+ * dkim checks have been completed by adding dependency on `DKIM_TRACE` symbol.
+ * Fields in map:
+ *
+ * * `result` - string result of check:
+ * - `reject`
+ * - `allow`
+ * - `tempfail`
+ * - `permfail`
+ * - `not found`
+ * - `bad record`
+ * * `domain` - dkim domain
+ * * `selector` - dkim selector
+ * * `bhash` - short version of b tag (8 base64 symbols)
+ * * `fail_reason` - reason of failure (if applicable)
+ * @return {list of maps} dkim check results
+ */
+LUA_FUNCTION_DEF(task, get_dkim_results);
+/***
+ * @method task:get_symbol(name, [shadow_result_name])
+ * Searches for a symbol `name` in all metrics results and returns a list of tables
+ * one per metric that describes the symbol inserted.
+ * Please note, that for using this function you need to ensure that the symbol
+ * being queried is already checked. This is guaranteed if there is a dependency
+ * between the caller symbol and the checked symbol (either virtual or real).
+ * Please check `rspamd_config:register_dependency` method for details.
+ * The symbols are returned as the list of the following tables:
+ *
+ * - `metric` - name of metric
+ * - `score` - score of a symbol in that metric
+ * - `options` - a table of strings representing options of a symbol
+ * - `group` - a group of symbol (or 'ungrouped')
+ * @param {string} name symbol's name
+ * @return {list of tables} list of tables or nil if symbol was not found
+ */
+LUA_FUNCTION_DEF(task, get_symbol);
+/***
+ * @method task:get_symbols_all()
+ * Returns array of symbols matched in default metric with all metadata
+ * @return {table} table of tables formatted as in `task:get_symbol()` except that `metric` is absent and `name` is added
+ */
+LUA_FUNCTION_DEF(task, get_symbols_all);
+/***
+ * @method task:get_symbols([shadow_result_name])
+ * Returns array of all symbols matched for this task
+ * @return {table, table} table of strings with symbols names + table of theirs scores
+ */
+LUA_FUNCTION_DEF(task, get_symbols);
+
+/***
+ * @method task:get_groups([need_private])
+ * Returns a map [group -> group_score] for matched group. If `need_private` is
+ * unspecified, then the global option `public_groups_only` is used for default.
+ * @return {table, number} a map [group -> group_score]
+ */
+LUA_FUNCTION_DEF(task, get_groups);
+
+/***
+ * @method task:get_symbols_numeric()
+ * Returns array of all symbols matched for this task
+ * @return {table|number, table|number} table of numbers with symbols ids + table of theirs scores
+ */
+LUA_FUNCTION_DEF(task, get_symbols_numeric);
+
+/***
+ * @method task:get_symbols_tokens()
+ * Returns array of all symbols as statistical tokens
+ * @return {table|number} table of numbers
+ */
+LUA_FUNCTION_DEF(task, get_symbols_tokens);
+
+/***
+ * @method task:process_ann_tokens(symbols, ann_tokens, offset, [min])
+ * Processes ann tokens
+ * @param {table|string} symbols list of symbols in this profile
+ * @param {table|number} ann_tokens list of tokens (including metatokens)
+ * @param {integer} offset offset for symbols token (#metatokens)
+ * @param {number} min minimum value for symbols found (e.g. for 0 score symbols)
+ * @return nothing
+ */
+LUA_FUNCTION_DEF(task, process_ann_tokens);
+
+/***
+ * @method task:has_symbol(name, [shadow_result_name])
+ * Fast path to check if a specified symbol is in the task's results.
+ * Please note, that for using this function you need to ensure that the symbol
+ * being queried is already checked. This is guaranteed if there is a dependency
+ * between the caller symbol and the checked symbol (either virtual or real).
+ * Please check `rspamd_config:register_dependency` method for details.
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, has_symbol);
+/***
+ * @method task:enable_symbol(name)
+ * Enable specified symbol for this particular task
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, enable_symbol);
+/***
+ * @method task:disable_symbol(name)
+ * Disable specified symbol for this particular task
+ * @param {string} name symbol's name
+ * @return {boolean} `true` if symbol has been found
+ */
+LUA_FUNCTION_DEF(task, disable_symbol);
+/***
+ * @method task:get_date(type[, gmt])
+ * Returns timestamp for a connection or for a MIME message. This function can be called with a
+ * single table arguments with the following fields:
+ *
+ * * `format` - a format of date returned:
+ * - `message` - returns a mime date as integer (unix timestamp)
+ * - `connect` - returns a unix timestamp of a connection to rspamd
+ * * `gmt` - returns date in `GMT` timezone (normal for unix timestamps)
+ *
+ * By default this function returns connection time in numeric format.
+ * @param {string} type date format as described above
+ * @param {boolean} gmt gmt flag as described above
+ * @return {string/number} date representation according to format
+ * @example
+rspamd_config.DATE_IN_PAST = function(task)
+ local dm = task:get_date{format = 'message', gmt = true}
+ local dt = task:get_date{format = 'connect', gmt = true}
+ -- A day
+ if dt - dm > 86400 then
+ return true
+ end
+
+ return false
+end
+ */
+LUA_FUNCTION_DEF(task, get_date);
+/***
+ * @method task:get_message_id()
+ * Returns message identifier from the `Message-ID` header. Angle brackets (`<>`)
+ * are stripped off if present. If a Message-ID header is missing `undef` is
+ * returned.
+ * @return {string} ID of the message
+ */
+LUA_FUNCTION_DEF(task, get_message_id);
+/***
+ * @method task:get_timeval([raw])
+ * Returns the timestamp for a task start processing time.
+ * @param {boolean} raw if true then two float numbers are returned: task start timestamp and timeout event timestamp
+ * @return {table} table with fields as described in `struct timeval` in C
+ */
+LUA_FUNCTION_DEF(task, get_timeval);
+/***
+ * @method task:get_scan_time([set])
+ * Returns 2 floating point numbers: scan real time and scan virtual time.
+ * If `set` is `true`, then the finishing time is also set (enabled by default).
+ * This function should be normally called on idempotent phase.
+ * @return {number,number} real and virtual times in seconds with floating point
+ */
+LUA_FUNCTION_DEF(task, get_scan_time);
+/***
+ * @method task:get_metric_result()
+ * Get full result of a metric as a table:
+ * - `score`: current score
+ * - `action`: current action as a string
+ * - `nnegative`: number of negative rules matched
+ * - `npositive`: number of positive rules matched
+ * - `positive_score`: total score for positive rules
+ * - `negative_score`: total score for negative rules
+ * - `passthrough`: set to true if message has a passthrough result
+ * @return {table} metric result
+ */
+LUA_FUNCTION_DEF(task, get_metric_result);
+/***
+ * @method task:get_metric_score(name)
+ * Get the current score of metric `name` (must be nil or 'default') . Should be used in idempotent filters only.
+ * @param {string} name name of a metric
+ * @return {number,number} 2 numbers containing the current score and required score of the metric
+ */
+LUA_FUNCTION_DEF(task, get_metric_score);
+/***
+ * @method task:get_metric_action(name)
+ * Get the current action of metric `name` (must be nil or 'default'). Should be used in idempotent filters only.
+ * @param {string} name name of a metric
+ * @return {string} the current action of the metric as a string
+ */
+LUA_FUNCTION_DEF(task, get_metric_action);
+/***
+ * @method task:set_metric_score(name, score)
+ * Set the current score of metric `name`. Should be used in post-filters only.
+ * @param {string} name name of a metric
+ * @param {number} score the current score of the metric
+ */
+LUA_FUNCTION_DEF(task, set_metric_score);
+/***
+ * @method task:set_metric_subject(subject)
+ * Set the subject in the default metric
+ * @param {string} subject subject to set
+ */
+LUA_FUNCTION_DEF(task, set_metric_subject);
+
+/***
+ * @method task:learn(is_spam[, classifier)
+ * Learn classifier `classifier` with the task. If `is_spam` is true then message
+ * is learnt as spam. Otherwise HAM is learnt. By default, this function learns
+ * `bayes` classifier.
+ * @param {boolean} is_spam learn spam or ham
+ * @param {string} classifier classifier's name
+ * @return {boolean} `true` if classifier has been learnt successfully
+ */
+LUA_FUNCTION_DEF(task, learn);
+/***
+ * @method task:set_settings(obj)
+ * Set users settings object for a task. The format of this object is described
+ * [here](https://rspamd.com/doc/configuration/settings.html).
+ * @param {any} obj any lua object that corresponds to the settings format
+ */
+LUA_FUNCTION_DEF(task, set_settings);
+
+/***
+ * @method task:set_settings_id(id)
+ * Set users settings id for a task (must be registered previously)
+ * @available 2.0+
+ * @param {number} id numeric id
+ * @return {boolean} true if settings id has been replaced from the existing one
+ */
+LUA_FUNCTION_DEF(task, set_settings_id);
+
+/***
+ * @method task:get_settings()
+ * Gets users settings object for a task. The format of this object is described
+ * [here](https://rspamd.com/doc/configuration/settings.html).
+ * @return {lua object} lua object generated from UCL
+ */
+LUA_FUNCTION_DEF(task, get_settings);
+
+/***
+ * @method task:lookup_settings(key)
+ * Gets users settings object with the specified key for a task.
+ * @param {string} key key to lookup
+ * @return {lua object} lua object generated from UCL
+ */
+LUA_FUNCTION_DEF(task, lookup_settings);
+
+/***
+ * @method task:get_settings_id()
+ * Get numeric hash of settings id if specified for this task. 0 is returned otherwise.
+ * @return {number} settings-id hash
+ */
+LUA_FUNCTION_DEF(task, get_settings_id);
+
+/***
+ * @method task:set_milter_reply(obj)
+ * Set special reply for milter
+ * @param {any} obj any lua object that corresponds to the settings format
+ * @example
+task:set_milter_reply({
+ add_headers = {['X-Lua'] = 'test'},
+ -- 1 is the position of header to remove
+ remove_headers = {['DKIM-Signature'] = 1},
+})
+ */
+LUA_FUNCTION_DEF(task, set_milter_reply);
+
+/***
+ * @method task:process_re(params)
+ * Processes the specified regexp and returns number of captures (cached or new)
+ * Params is the table with the following fields (mandatory fields are marked with `*`):
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `rawmime`: raw mime regexp
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `strong`: case sensitive match for headers
+ * @return {number} number of regexp occurrences in the task (limited by 255 so far)
+ */
+LUA_FUNCTION_DEF(task, process_regexp);
+
+/***
+ * @method task:cache_set(key, value)
+ * Store some value to the task cache
+ * @param {string} key key to use
+ * @param {any} value any value (including functions and tables)
+ */
+LUA_FUNCTION_DEF(task, cache_set);
+/***
+ * @method task:cache_get(key)
+ * Returns cached value or nil if nothing is cached
+ * @param {string} key key to use
+ * @return {any} cached value
+ */
+LUA_FUNCTION_DEF(task, cache_get);
+
+/***
+ * @method task:get_size()
+ * Returns size of the task in bytes (that includes headers + parts size)
+ * @return {number} size in bytes
+ */
+LUA_FUNCTION_DEF(task, get_size);
+
+/***
+ * @method task:set_flag(flag_name[, set])
+ * Set specific flag for task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * @param {string} flag to set
+ * @param {boolean} set set or clear flag (default is set)
+@example
+--[[
+For messages with undefined queue ID (scanned with rspamc or WebUI)
+do not include results into statistics and do not log task summary
+(it will not appear in the WebUI history as well).
+]]--
+
+-- Callback function to set flags
+local function no_log_stat_cb(task)
+ if not task:get_queue_id() then
+ task:set_flag('no_log')
+ task:set_flag('no_stat')
+ end
+end
+
+rspamd_config:register_symbol({
+ name = 'LOCAL_NO_LOG_STAT',
+ type = 'postfilter',
+ callback = no_log_stat_cb
+})
+ */
+LUA_FUNCTION_DEF(task, set_flag);
+
+
+/***
+ * @method task:has_flag(flag_name)
+ * Checks for a specific flag in task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * @param {string} flag to check
+ * @return {boolean} true if flags is set
+ */
+LUA_FUNCTION_DEF(task, has_flag);
+
+
+/***
+ * @method task:get_flags()
+ * Get list of flags for task:
+ *
+ * - `no_log`: do not log task summary
+ * - `no_stat`: do not include task into scanned stats
+ * - `pass_all`: check all filters for task
+ * - `extended_urls`: output extended info about urls
+ * - `skip`: skip task processing
+ * - `learn_spam`: learn message as spam
+ * - `learn_ham`: learn message as ham
+ * - `broken_headers`: header data is broken for a message
+ * - `milter`: task is initiated by milter connection
+ * @return {array of strings} table with all flags as strings
+ */
+LUA_FUNCTION_DEF(task, get_flags);
+
+/***
+ * @method task:get_digest()
+ * Returns message's unique digest (32 hex symbols)
+ * @return {string} hex digest
+ */
+LUA_FUNCTION_DEF(task, get_digest);
+
+/***
+ * @method task:store_in_file([mode|table])
+ * If task was loaded using file scan, then this method just returns its name,
+ * otherwise, a fresh temporary file is created and its name is returned. Default
+ * mode is 0600. To convert lua number to the octal mode you can use the following
+ * trick: `tonumber("0644", 8)`. The file is automatically removed when task is
+ * destroyed.
+ *
+ * If table argument is specified, the following extra fields are allowed:
+ *
+ * - `mode`: same as mode argument
+ * - `force_new`: always create a new file
+ * - `filename`: use specific filename instead of a temporary one
+ * - `tmpmask`: use specific tempmask, e.g. '/tmp/file-XXXXX', where XXXX will be replaced by some random letters
+ * - `keep`: do not remove file after task is processed
+ *
+ * @param {number} mode mode for new file
+ * @return {string} file name with task content
+ */
+LUA_FUNCTION_DEF(task, store_in_file);
+
+/***
+ * @method task:get_protocol_reply([flags])
+ * This method being called from a **postfilter** will return reply for a message
+ * as it is returned to a client. This method returns the Lua table corresponding
+ * to the UCL object. Flags is a table that specify which information should be
+ * there in a reply:
+ *
+ * - `basic`: basic info, such as message-id
+ * - `metrics`: metrics and symbols
+ * - `messages`: messages
+ * - `dkim`: dkim signature
+ * - `milter`: milter control block
+ * - `extra`: extra data, such as profiling
+ * - `urls`: list of all urls in a message
+ *
+ * @param {table} flags table of flags (default is all flags but `urls`)
+ * @return {table} ucl object corresponding to the reply
+ */
+LUA_FUNCTION_DEF(task, get_protocol_reply);
+
+/***
+ * @method task:headers_foreach(callback, [params])
+ * This method calls `callback` for each header that satisfies some condition.
+ * By default, all headers are iterated unless `callback` returns `true`. Nil or
+ * false means continue of iterations.
+ * Params could be as following:
+ *
+ * - `full`: header value is full table of all attributes @see task:get_header_full for details
+ * - `regexp`: return headers that satisfies the specified regexp
+ * @param {function} callback function from header name and header value
+ * @param {table} params optional parameters
+ */
+LUA_FUNCTION_DEF(task, headers_foreach);
+
+/***
+ * @method task:disable_action(action)
+ * Disables some action for this task (e.g. 'greylist')
+ *
+ * @param {string} action action to disable
+ * @return {boolean} true if an action was enabled and is disabled after the method call
+ */
+LUA_FUNCTION_DEF(task, disable_action);
+
+/***
+ * @method task:get_newlines_type()
+ * Returns the most frequent newlines type met in a task
+ *
+ * @return {string} "cr" for \r, "lf" for \n, "crlf" for \r\n
+ */
+LUA_FUNCTION_DEF(task, get_newlines_type);
+
+/***
+ * @method task:get_stat_tokens()
+ * Returns list of tables the statistical tokens:
+ * - `data`: 64 bit number encoded as a string
+ * - `t1`: the first token (if any)
+ * - `t2`: the second token (if any)
+ * - `win`: window index
+ * - `flag`: table of strings:
+ * - `text`: text token
+ * - `meta`: meta token
+ * - `lua`: lua meta token
+ * - `exception`: exception
+ * - `subject`: subject token
+ * - `unigram`: unigram token
+ *
+ * @return {table of tables}
+ */
+LUA_FUNCTION_DEF(task, get_stat_tokens);
+
+/***
+ * @method task:lookup_words(map, function({o, n, s, f}) ... end)
+ * Matches words in a task (including meta words) against some map (set, regexp and so on)
+ * and call the specified function with a table containing 4 values:
+ * - [1] - stemmed word
+ * - [2] - normalised word
+ * - [3] - raw word
+ * - [4] - flags (table of strings)
+ */
+LUA_FUNCTION_DEF(task, lookup_words);
+
+/**
+ * @method task:topointer()
+ *
+ * Returns raw C pointer (lightuserdata) associated with task. This might be
+ * broken with luajit and GC64, use with caution.
+ */
+LUA_FUNCTION_DEF(task, topointer);
+
+/**
+ * @method task:add_named_result(name, symbol_control_function)
+ *
+ * Adds a new named result for this task. symbol_control_function is a callback
+ * called with 3 parameters:
+ * `function(task, symbol, result_name)` and it should return boolean that
+ * specifies if this symbol should be added to this named result.
+ * @param {string} name for this result
+ * @param {function} symbol_control_function predicate for symbols
+ */
+LUA_FUNCTION_DEF(task, add_named_result);
+
+/**
+ * @method task:get_all_named_results()
+ *
+ * Returns all named results registered for the task as a table of strings
+ * @return {table|string} all named results starting from `default`
+ */
+LUA_FUNCTION_DEF(task, get_all_named_results);
+
+/***
+ * @method task:get_dns_req()
+ * Get number of dns requests being sent in the task
+ * @return {number} number of DNS requests
+ */
+LUA_FUNCTION_DEF(task, get_dns_req);
+
+static const struct luaL_reg tasklib_f[] = {
+ LUA_INTERFACE_DEF(task, create),
+ LUA_INTERFACE_DEF(task, load_from_file),
+ LUA_INTERFACE_DEF(task, load_from_string),
+ {NULL, NULL}};
+
+static const struct luaL_reg tasklib_m[] = {
+ LUA_INTERFACE_DEF(task, get_message),
+ LUA_INTERFACE_DEF(task, set_message),
+ LUA_INTERFACE_DEF(task, destroy),
+ LUA_INTERFACE_DEF(task, process_message),
+ LUA_INTERFACE_DEF(task, set_cfg),
+ LUA_INTERFACE_DEF(task, get_cfg),
+ LUA_INTERFACE_DEF(task, get_mempool),
+ LUA_INTERFACE_DEF(task, get_session),
+ LUA_INTERFACE_DEF(task, set_session),
+ LUA_INTERFACE_DEF(task, get_ev_base),
+ LUA_INTERFACE_DEF(task, get_worker),
+ LUA_INTERFACE_DEF(task, insert_result),
+ LUA_INTERFACE_DEF(task, insert_result_named),
+ LUA_INTERFACE_DEF(task, adjust_result),
+ LUA_INTERFACE_DEF(task, remove_result),
+ LUA_INTERFACE_DEF(task, set_pre_result),
+ LUA_INTERFACE_DEF(task, has_pre_result),
+ LUA_INTERFACE_DEF(task, append_message),
+ LUA_INTERFACE_DEF(task, has_urls),
+ LUA_INTERFACE_DEF(task, get_urls),
+ LUA_INTERFACE_DEF(task, get_urls_filtered),
+ LUA_INTERFACE_DEF(task, inject_url),
+ LUA_INTERFACE_DEF(task, get_content),
+ LUA_INTERFACE_DEF(task, get_filename),
+ LUA_INTERFACE_DEF(task, get_rawbody),
+ LUA_INTERFACE_DEF(task, get_emails),
+ LUA_INTERFACE_DEF(task, get_text_parts),
+ LUA_INTERFACE_DEF(task, get_parts),
+ LUA_INTERFACE_DEF(task, get_request_header),
+ LUA_INTERFACE_DEF(task, set_request_header),
+ LUA_INTERFACE_DEF(task, get_header),
+ LUA_INTERFACE_DEF(task, has_header),
+ LUA_INTERFACE_DEF(task, get_header_raw),
+ LUA_INTERFACE_DEF(task, get_header_full),
+ LUA_INTERFACE_DEF(task, get_header_count),
+ LUA_INTERFACE_DEF(task, get_raw_headers),
+ LUA_INTERFACE_DEF(task, get_headers),
+ LUA_INTERFACE_DEF(task, modify_header),
+ LUA_INTERFACE_DEF(task, get_received_headers),
+ LUA_INTERFACE_DEF(task, get_queue_id),
+ LUA_INTERFACE_DEF(task, get_uid),
+ LUA_INTERFACE_DEF(task, get_resolver),
+ LUA_INTERFACE_DEF(task, set_resolver),
+ LUA_INTERFACE_DEF(task, inc_dns_req),
+ LUA_INTERFACE_DEF(task, get_dns_req),
+ LUA_INTERFACE_DEF(task, has_recipients),
+ LUA_INTERFACE_DEF(task, get_recipients),
+ LUA_INTERFACE_DEF(task, set_recipients),
+ LUA_INTERFACE_DEF(task, get_principal_recipient),
+ LUA_INTERFACE_DEF(task, get_reply_sender),
+ LUA_INTERFACE_DEF(task, has_from),
+ LUA_INTERFACE_DEF(task, get_from),
+ LUA_INTERFACE_DEF(task, set_from),
+ LUA_INTERFACE_DEF(task, get_user),
+ LUA_INTERFACE_DEF(task, set_user),
+ {"get_addr", lua_task_get_from_ip},
+ {"get_ip", lua_task_get_from_ip},
+ {"get_from_addr", lua_task_get_from_ip},
+ LUA_INTERFACE_DEF(task, get_from_ip),
+ LUA_INTERFACE_DEF(task, set_from_ip),
+ LUA_INTERFACE_DEF(task, get_from_ip_num),
+ LUA_INTERFACE_DEF(task, get_client_ip),
+ LUA_INTERFACE_DEF(task, get_subject),
+ LUA_INTERFACE_DEF(task, get_helo),
+ LUA_INTERFACE_DEF(task, set_helo),
+ LUA_INTERFACE_DEF(task, get_hostname),
+ LUA_INTERFACE_DEF(task, set_hostname),
+ LUA_INTERFACE_DEF(task, get_images),
+ LUA_INTERFACE_DEF(task, get_archives),
+ LUA_INTERFACE_DEF(task, get_dkim_results),
+ LUA_INTERFACE_DEF(task, get_symbol),
+ LUA_INTERFACE_DEF(task, get_symbols),
+ LUA_INTERFACE_DEF(task, get_symbols_all),
+ LUA_INTERFACE_DEF(task, get_symbols_numeric),
+ LUA_INTERFACE_DEF(task, get_symbols_tokens),
+ LUA_INTERFACE_DEF(task, get_groups),
+ LUA_INTERFACE_DEF(task, process_ann_tokens),
+ LUA_INTERFACE_DEF(task, has_symbol),
+ LUA_INTERFACE_DEF(task, enable_symbol),
+ LUA_INTERFACE_DEF(task, disable_symbol),
+ LUA_INTERFACE_DEF(task, get_date),
+ LUA_INTERFACE_DEF(task, get_message_id),
+ LUA_INTERFACE_DEF(task, get_timeval),
+ LUA_INTERFACE_DEF(task, get_scan_time),
+ LUA_INTERFACE_DEF(task, get_metric_result),
+ LUA_INTERFACE_DEF(task, get_metric_score),
+ LUA_INTERFACE_DEF(task, get_metric_action),
+ LUA_INTERFACE_DEF(task, set_metric_score),
+ LUA_INTERFACE_DEF(task, set_metric_subject),
+ LUA_INTERFACE_DEF(task, learn),
+ LUA_INTERFACE_DEF(task, set_settings),
+ LUA_INTERFACE_DEF(task, get_settings),
+ LUA_INTERFACE_DEF(task, lookup_settings),
+ LUA_INTERFACE_DEF(task, get_settings_id),
+ LUA_INTERFACE_DEF(task, set_settings_id),
+ LUA_INTERFACE_DEF(task, cache_get),
+ LUA_INTERFACE_DEF(task, cache_set),
+ LUA_INTERFACE_DEF(task, process_regexp),
+ LUA_INTERFACE_DEF(task, get_size),
+ LUA_INTERFACE_DEF(task, set_flag),
+ LUA_INTERFACE_DEF(task, get_flags),
+ LUA_INTERFACE_DEF(task, has_flag),
+ {"set_rmilter_reply", lua_task_set_milter_reply},
+ LUA_INTERFACE_DEF(task, set_milter_reply),
+ LUA_INTERFACE_DEF(task, get_digest),
+ LUA_INTERFACE_DEF(task, store_in_file),
+ LUA_INTERFACE_DEF(task, get_protocol_reply),
+ LUA_INTERFACE_DEF(task, headers_foreach),
+ LUA_INTERFACE_DEF(task, disable_action),
+ LUA_INTERFACE_DEF(task, get_newlines_type),
+ LUA_INTERFACE_DEF(task, get_stat_tokens),
+ LUA_INTERFACE_DEF(task, get_meta_words),
+ LUA_INTERFACE_DEF(task, lookup_words),
+ LUA_INTERFACE_DEF(task, add_named_result),
+ LUA_INTERFACE_DEF(task, get_all_named_results),
+ LUA_INTERFACE_DEF(task, topointer),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Image methods */
+LUA_FUNCTION_DEF(image, get_width);
+LUA_FUNCTION_DEF(image, get_height);
+LUA_FUNCTION_DEF(image, get_type);
+LUA_FUNCTION_DEF(image, get_filename);
+LUA_FUNCTION_DEF(image, get_size);
+
+static const struct luaL_reg imagelib_m[] = {
+ LUA_INTERFACE_DEF(image, get_width),
+ LUA_INTERFACE_DEF(image, get_height),
+ LUA_INTERFACE_DEF(image, get_type),
+ LUA_INTERFACE_DEF(image, get_filename),
+ LUA_INTERFACE_DEF(image, get_size),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Archive methods */
+LUA_FUNCTION_DEF(archive, get_type);
+LUA_FUNCTION_DEF(archive, get_files);
+LUA_FUNCTION_DEF(archive, get_files_full);
+/* TODO: Export archive flags as integers to use bitops for that */
+LUA_FUNCTION_DEF(archive, is_encrypted);
+LUA_FUNCTION_DEF(archive, is_obfuscated);
+LUA_FUNCTION_DEF(archive, is_unreadable);
+LUA_FUNCTION_DEF(archive, get_filename);
+LUA_FUNCTION_DEF(archive, get_size);
+
+static const struct luaL_reg archivelib_m[] = {
+ LUA_INTERFACE_DEF(archive, get_type),
+ LUA_INTERFACE_DEF(archive, get_files),
+ LUA_INTERFACE_DEF(archive, get_files_full),
+ LUA_INTERFACE_DEF(archive, is_encrypted),
+ LUA_INTERFACE_DEF(archive, is_obfuscated),
+ LUA_INTERFACE_DEF(archive, is_unreadable),
+ LUA_INTERFACE_DEF(archive, get_filename),
+ LUA_INTERFACE_DEF(archive, get_size),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/* Utility functions */
+struct rspamd_task *
+lua_check_task(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{task}");
+ luaL_argcheck(L, ud != NULL, pos, "'task' expected");
+ return ud ? *((struct rspamd_task **) ud) : NULL;
+}
+
+struct rspamd_task *
+lua_check_task_maybe(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata_maybe(L, pos, "rspamd{task}");
+
+ return ud ? *((struct rspamd_task **) ud) : NULL;
+}
+
+static struct rspamd_image *
+lua_check_image(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{image}");
+ luaL_argcheck(L, ud != NULL, 1, "'image' expected");
+ return ud ? *((struct rspamd_image **) ud) : NULL;
+}
+
+static struct rspamd_archive *
+lua_check_archive(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{archive}");
+ luaL_argcheck(L, ud != NULL, 1, "'archive' expected");
+ return ud ? *((struct rspamd_archive **) ud) : NULL;
+}
+
+static void
+lua_task_set_cached(lua_State *L, struct rspamd_task *task, const gchar *key,
+ gint pos)
+{
+ LUA_TRACE_POINT;
+ khiter_t k;
+
+ lua_pushvalue(L, pos);
+
+ k = kh_get(rspamd_task_lua_cache, &task->lua_cache, (char *) key);
+
+ if (G_UNLIKELY(k != kh_end(&task->lua_cache))) {
+ /* Unref previous value */
+ luaL_unref(L, LUA_REGISTRYINDEX, kh_value(&task->lua_cache, k).ref);
+ }
+ else {
+ int r;
+
+ k = kh_put(rspamd_task_lua_cache, &task->lua_cache, rspamd_mempool_strdup(task->task_pool, key), &r);
+ }
+
+ kh_value(&task->lua_cache, k).ref = luaL_ref(L, LUA_REGISTRYINDEX);
+ kh_value(&task->lua_cache, k).id = GPOINTER_TO_UINT(task->message);
+}
+
+
+static gboolean
+lua_task_get_cached(lua_State *L, struct rspamd_task *task, const gchar *key)
+{
+ LUA_TRACE_POINT;
+ khiter_t k;
+
+ k = kh_get(rspamd_task_lua_cache, &task->lua_cache, (char *) key);
+
+ if (k != kh_end(&task->lua_cache) && (kh_value(&task->lua_cache, k).id == GPOINTER_TO_UINT(task->message))) {
+ lua_rawgeti(L, LUA_REGISTRYINDEX, kh_value(&task->lua_cache, k).ref);
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+/* Task methods */
+static int
+lua_task_process_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean enforce = FALSE;
+
+ if (task != NULL) {
+ if (task->msg.len > 0) {
+ if (lua_isboolean(L, 2)) {
+ enforce = lua_toboolean(L, 2);
+ }
+
+ if (rspamd_message_parse(task)) {
+ if (enforce ||
+ (!(task->flags & RSPAMD_TASK_FLAG_SKIP_PROCESS) &&
+ !(task->processed_stages & RSPAMD_TASK_STAGE_PROCESS_MESSAGE))) {
+
+ lua_pushboolean(L, TRUE);
+ rspamd_message_process(task);
+ task->processed_stages |= RSPAMD_TASK_STAGE_PROCESS_MESSAGE;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_get_cfg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_config **pcfg;
+
+ if (task) {
+ pcfg = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = task->cfg;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_set_cfg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ void *ud = rspamd_lua_check_udata(L, 2, "rspamd{config}");
+
+ if (task) {
+ luaL_argcheck(L, ud != NULL, 1, "'config' expected");
+ task->cfg = ud ? *((struct rspamd_config **) ud) : NULL;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static int
+lua_task_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ rspamd_task_free(task);
+ }
+
+ return 0;
+}
+
+static int
+lua_task_get_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->flags = 0;
+ t->start = task->msg.begin;
+ t->len = task->msg.len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_set_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean message_set = FALSE;
+
+ if (task) {
+ gsize final_len = 0;
+ gchar *buf = NULL;
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ /* Piecewise construct */
+ guint vec_len = rspamd_lua_table_size(L, 2);
+
+
+ for (guint i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize l;
+
+ (void) lua_tolstring(L, -1, &l);
+ final_len += l;
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ final_len += t->len;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (final_len > 0) {
+ gchar *pos;
+
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ pos = buf;
+
+ for (guint i = 0; i < vec_len; i++) {
+ lua_rawgeti(L, 2, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize l;
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &l);
+ memcpy(pos, s, l);
+ pos += l;
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ memcpy(pos, t->start, t->len);
+ pos += t->len;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ task->flags |= RSPAMD_TASK_FLAG_MESSAGE_REWRITE;
+ task->msg.begin = buf;
+ task->msg.len = final_len;
+ message_set = TRUE;
+ }
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *s;
+
+ s = lua_tolstring(L, -1, &final_len);
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ memcpy(buf, s, final_len);
+ }
+ else {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ final_len = t->len;
+ buf = rspamd_mempool_alloc(task->task_pool, final_len);
+ memcpy(buf, t->start, final_len);
+ }
+ }
+
+ if (buf) {
+ task->msg.begin = buf;
+ task->msg.len = final_len;
+ task->flags |= RSPAMD_TASK_FLAG_MESSAGE_REWRITE;
+ message_set = TRUE;
+ }
+ }
+
+ if (message_set) {
+ if (rspamd_message_parse(task)) {
+ rspamd_message_process(task);
+ lua_pushboolean(L, TRUE);
+ lua_pushinteger(L, final_len);
+
+ return 2;
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static void
+lua_task_unmap_dtor(gpointer p)
+{
+ struct rspamd_task *task = (struct rspamd_task *) p;
+
+ if (task->msg.begin) {
+ munmap((gpointer) task->msg.begin, task->msg.len);
+ }
+}
+
+static void
+lua_task_free_dtor(gpointer p)
+{
+ g_free(p);
+}
+
+static gint
+lua_task_load_from_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ const gchar *fname = luaL_checkstring(L, 1), *err = NULL;
+ struct rspamd_config *cfg = NULL;
+ gboolean res = FALSE;
+ gpointer map;
+ gsize sz;
+
+ if (fname) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ if (strcmp(fname, "-") == 0) {
+ /* Read from stdin */
+ gint fd = STDIN_FILENO;
+ GString *data = g_string_sized_new(BUFSIZ);
+ gchar buf[BUFSIZ];
+ gssize r;
+
+ for (;;) {
+ r = read(fd, buf, sizeof(buf));
+
+ if (r == -1) {
+ err = strerror(errno);
+ break;
+ }
+ else if (r == 0) {
+ break;
+ }
+ else {
+ g_string_append_len(data, buf, r);
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = data->str;
+ task->msg.len = data->len;
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_task_free_dtor, data->str);
+ res = TRUE;
+ g_string_free(data, FALSE); /* Buffer is still valid */
+ }
+ else {
+ map = rspamd_file_xmap(fname, PROT_READ, &sz, TRUE);
+
+ if (!map) {
+ err = strerror(errno);
+ }
+ else {
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = map;
+ task->msg.len = sz;
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_task_unmap_dtor, task);
+ res = TRUE;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, res);
+
+ if (res) {
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ }
+ else {
+ if (err) {
+ lua_pushstring(L, err);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_load_from_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ const gchar *str_message;
+ gsize message_len;
+ struct rspamd_config *cfg = NULL;
+
+ str_message = luaL_checklstring(L, 1, &message_len);
+
+ if (str_message) {
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, NULL, FALSE);
+ task->msg.begin = g_malloc(message_len);
+ memcpy((gchar *) task->msg.begin, str_message, message_len);
+ task->msg.len = message_len;
+ rspamd_mempool_add_destructor(task->task_pool, lua_task_free_dtor,
+ (gpointer) task->msg.begin);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, true);
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ return 2;
+}
+
+static gint
+lua_task_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = NULL, **ptask;
+ struct rspamd_config *cfg = NULL;
+ struct ev_loop *ev_base = NULL;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 1, "rspamd{config}");
+
+ if (p) {
+ cfg = *(struct rspamd_config **) p;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ gpointer p;
+ p = rspamd_lua_check_udata_maybe(L, 2, "rspamd{ev_base}");
+
+ if (p) {
+ ev_base = *(struct ev_loop **) p;
+ }
+ }
+
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, ev_base, FALSE);
+ task->flags |= RSPAMD_TASK_FLAG_EMPTY;
+
+ ptask = lua_newuserdata(L, sizeof(*ptask));
+ *ptask = task;
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+ return 1;
+}
+
+static int
+lua_task_get_mempool(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t **ppool;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ ppool = lua_newuserdata(L, sizeof(rspamd_mempool_t *));
+ rspamd_lua_setclass(L, "rspamd{mempool}", -1);
+ *ppool = task->task_pool;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static int
+lua_task_get_session(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session **psession;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ psession = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{session}", -1);
+ *psession = task->s;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_set_session(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_async_session *session = lua_check_session(L, 2);
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL && session != NULL) {
+ task->s = session;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_get_ev_base(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop **pbase;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ pbase = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pbase = task->event_loop;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+static int
+lua_task_get_worker(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_worker **pworker;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ if (task->worker) {
+ pworker = lua_newuserdata(L, sizeof(struct rspamd_worker *));
+ rspamd_lua_setclass(L, "rspamd{worker}", -1);
+ *pworker = task->worker;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ return 1;
+}
+
+
+static gint
+lua_task_insert_result_common(lua_State *L, struct rspamd_scan_result *result,
+ gint common_args_pos)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name;
+ double weight;
+ struct rspamd_symbol_result *s;
+ enum rspamd_symbol_insert_flags flags = RSPAMD_SYMBOL_INSERT_DEFAULT;
+ gint i, top, args_start;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, common_args_pos)) {
+ args_start = common_args_pos + 1;
+
+ if (lua_toboolean(L, common_args_pos)) {
+ flags |= RSPAMD_SYMBOL_INSERT_ENFORCE;
+ }
+ }
+ else {
+ args_start = common_args_pos;
+ }
+
+ symbol_name = rspamd_mempool_strdup(task->task_pool,
+ luaL_checkstring(L, args_start));
+ weight = luaL_checknumber(L, args_start + 1);
+ top = lua_gettop(L);
+ s = rspamd_task_insert_result_full(task, symbol_name, weight,
+ NULL, flags, result);
+
+ /* Get additional options */
+ if (s) {
+ if (s->sym == NULL) {
+ /* Unknown symbol, print traceback */
+ lua_pushfstring(L, "unknown symbol %s", symbol_name);
+ rspamd_lua_traceback(L);
+
+ msg_info_task("symbol insertion issue: %s", lua_tostring(L, -1));
+
+ lua_pop(L, 1); /* Traceback string */
+ }
+ for (i = args_start + 2; i <= top; i++) {
+ gint ltype = lua_type(L, i);
+
+ if (ltype == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (ltype == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (ltype == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ else {
+ return luaL_error(L, "not rspamd_text option in a table "
+ "when adding symbol %s: %s type",
+ s->name);
+ }
+ }
+ else {
+ const gchar *tname = lua_typename(L, lua_type(L, -1));
+ lua_pop(L, 2);
+
+ return luaL_error(L, "not a string option in a table "
+ "when adding symbol %s: %s type",
+ s->name, tname);
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else if (ltype == LUA_TNIL) {
+ /* We have received a NULL option, it is not good but not a fatal error */
+ msg_info_task("nil option when adding symbol %s at pos %d",
+ s->name, i);
+ continue;
+ }
+ else {
+ const gchar *tname = lua_typename(L, ltype);
+
+ return luaL_error(L, "not a string/table option "
+ "when adding symbol %s: %s type",
+ s->name, tname);
+ }
+ }
+ }
+ else if (task->settings == NULL && task->settings_elt == NULL) {
+ lua_pushfstring(L, "insertion failed for %s", symbol_name);
+ rspamd_lua_traceback(L);
+
+ msg_info_task("symbol insertion issue: %s", lua_tostring(L, -1));
+
+ lua_pop(L, 2); /* Traceback string + error string */
+ }
+ else {
+ /* Usually denied by settings */
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_insert_result(lua_State *L)
+{
+ return lua_task_insert_result_common(L, NULL, 2);
+}
+
+static gint
+lua_task_insert_result_named(lua_State *L)
+{
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *named_result = luaL_checkstring(L, 2);
+ struct rspamd_scan_result *res;
+
+ if (task && named_result) {
+ res = rspamd_find_metric_result(task, named_result);
+
+ if (res == NULL) {
+ return luaL_error(L, "invalid arguments: bad named result: %s",
+ named_result);
+ }
+
+ return lua_task_insert_result_common(L, res, 3);
+ }
+
+ return luaL_error(L, "invalid arguments");
+}
+
+static gint
+lua_task_adjust_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name;
+ struct rspamd_scan_result *metric_res;
+ struct rspamd_symbol_result *s = NULL;
+ double weight;
+ gint i, top;
+
+ if (task != NULL) {
+
+ symbol_name = luaL_checkstring(L, 2);
+ weight = luaL_checknumber(L, 3);
+ top = lua_gettop(L);
+ metric_res = task->result;
+
+ if (metric_res) {
+ s = rspamd_task_find_symbol_result(task, symbol_name, NULL);
+ }
+ else {
+ return luaL_error(L, "no metric result");
+ }
+
+ if (s) {
+ if (!isnan(weight)) {
+ metric_res->score -= s->score;
+ s->score = weight;
+ metric_res->score += s->score;
+ }
+ }
+ else {
+ return luaL_error(L, "symbol not found: %s", symbol_name);
+ }
+
+ /* Get additional options */
+ if (s) {
+ for (i = 4; i <= top; i++) {
+ if (lua_type(L, i) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, i, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, i) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, i);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+ else if (lua_type(L, i) == LUA_TTABLE) {
+ gsize objlen = rspamd_lua_table_size(L, i);
+
+ for (guint j = 1; j <= objlen; j++) {
+ lua_rawgeti(L, i, j);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gsize optlen;
+ const char *opt = lua_tolstring(L, -1, &optlen);
+
+ rspamd_task_add_result_option(task, s, opt, optlen);
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, -1);
+
+ if (t) {
+ rspamd_task_add_result_option(task, s, t->start,
+ t->len);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_remove_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol_name = luaL_checkstring(L, 2);
+ struct rspamd_scan_result *metric_res;
+ const gchar *named_result = luaL_optstring(L, 3, NULL);
+
+ if (task != NULL) {
+ metric_res = rspamd_find_metric_result(task, named_result);
+
+ if (metric_res == NULL) {
+ return luaL_error(L, "invalid arguments: bad named result: %s",
+ named_result);
+ }
+
+ lua_pushboolean(L, (rspamd_task_remove_symbol_result(task, symbol_name,
+ metric_res)) != NULL);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_pre_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *message = NULL, *module = NULL, *fl_str = NULL, *act_str = NULL,
+ *res_name = NULL;
+ gdouble score = NAN;
+ struct rspamd_action *action;
+ guint priority = RSPAMD_PASSTHROUGH_NORMAL, flags = 0;
+
+ if (task != NULL) {
+
+ if (RSPAMD_TASK_IS_SKIPPED(task)) {
+ /* Do not set pre-result for a skipped task */
+ return 0;
+ }
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ GError *err = NULL;
+
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*action=S;message=S;module=S;score=D;priority=i;flags=S;result=S",
+ &act_str, &message, &module, &score, &priority, &fl_str, &res_name)) {
+ gint ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ act_str = lua_tostring(L, 2);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ message = lua_tostring(L, 3);
+ }
+
+ if (lua_type(L, 4) == LUA_TSTRING) {
+ module = lua_tostring(L, 4);
+ }
+
+ if (lua_type(L, 5) == LUA_TNUMBER) {
+ score = lua_tonumber(L, 5);
+ }
+
+ if (lua_type(L, 6) == LUA_TNUMBER) {
+ priority = lua_tointeger(L, 6);
+ }
+
+ if (lua_type(L, 7) == LUA_TSTRING) {
+ fl_str = lua_tostring(L, 7);
+ }
+ }
+
+ enum rspamd_action_type internal_type;
+
+ if (strcmp(act_str, "accept") == 0) {
+ /* Compatibility! */
+ act_str = "no action";
+ }
+ else if (rspamd_action_from_str(act_str, &internal_type)) {
+ /* Compatibility! */
+ act_str = rspamd_action_to_str(internal_type);
+ }
+
+ action = rspamd_config_get_action(task->cfg, act_str);
+
+ if (action == NULL) {
+ return luaL_error(L, "unknown action %s", act_str);
+ }
+
+ if (module == NULL) {
+ module = "Unknown lua";
+ }
+
+ if (message == NULL) {
+ message = "unknown reason";
+ flags |= RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE;
+ }
+
+ if (fl_str != NULL) {
+ /*
+ * TODO: convert to a set of string and split by `,` + add table support
+ * once this legacy code is migrated to C++
+ */
+ if (strstr(fl_str, "least") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_LEAST;
+ }
+ else if (strstr(fl_str, "no_smtp_message") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE;
+ }
+ else if (strstr(fl_str, "process_all") != NULL) {
+ flags |= RSPAMD_PASSTHROUGH_PROCESS_ALL;
+ }
+ }
+
+
+ rspamd_add_passthrough_result(task,
+ action,
+ priority,
+ score,
+ rspamd_mempool_strdup(task->task_pool, message),
+ rspamd_mempool_strdup(task->task_pool, module),
+ flags,
+ rspamd_find_metric_result(task, res_name));
+
+ /* Don't classify or filter message if pre-filter sets results */
+
+ if (res_name == NULL && !(flags & (RSPAMD_PASSTHROUGH_LEAST | RSPAMD_PASSTHROUGH_PROCESS_ALL))) {
+ task->processed_stages |= (RSPAMD_TASK_STAGE_CLASSIFIERS |
+ RSPAMD_TASK_STAGE_CLASSIFIERS_PRE |
+ RSPAMD_TASK_STAGE_CLASSIFIERS_POST);
+ rspamd_symcache_disable_all_symbols(task, task->cfg->cache,
+ SYMBOL_TYPE_IDEMPOTENT | SYMBOL_TYPE_IGNORE_PASSTHROUGH);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_has_pre_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint nret = 1;
+
+ if (task) {
+ if (task->result->passthrough_result) {
+ struct rspamd_passthrough_result *pr = task->result->passthrough_result;
+
+ lua_pushboolean(L, true);
+ nret = 4;
+ /* bool, action, message, module */
+
+ if (pr->action) {
+ lua_pushstring(L, rspamd_action_to_str(pr->action->action_type));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (pr->message) {
+ lua_pushstring(L, pr->message);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ if (pr->module) {
+ lua_pushstring(L, pr->module);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return nret;
+}
+
+static gint
+lua_task_append_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *category;
+
+ if (task != NULL) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ category = luaL_checkstring(L, 3);
+ }
+ else {
+ category = "unknown";
+ }
+
+ ucl_object_insert_key(task->messages,
+ ucl_object_lua_import(L, 2),
+ category, 0,
+ true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+
+static gint
+lua_task_get_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0;
+
+ if (task) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (task->message == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ /* Exclude RSPAMD_URL_FLAG_CONTENT to preserve backward compatibility */
+ if (!lua_url_cbdata_fill(L, 2, &cb, default_protocols_mask,
+ ~(RSPAMD_URL_FLAG_CONTENT | RSPAMD_URL_FLAG_IMAGE),
+ max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ if (cb.sort) {
+ struct rspamd_url **urls_sorted;
+ gint i = 0;
+
+ urls_sorted = g_new0(struct rspamd_url *, sz);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (i < sz) {
+ urls_sorted[i] = u;
+ i++;
+ }
+ });
+
+ qsort(urls_sorted, i, sizeof(struct rspamd_url *), rspamd_url_cmp_qsort);
+
+ for (int j = 0; j < i; j++) {
+ lua_tree_url_callback(urls_sorted[j], urls_sorted[j], &cb);
+ }
+
+ g_free(urls_sorted);
+ }
+ else {
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+ }
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, no task");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_urls_filtered(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ static const gint default_protocols_mask = PROTOCOL_HTTP | PROTOCOL_HTTPS |
+ PROTOCOL_FILE | PROTOCOL_FTP;
+ gsize sz, max_urls = 0;
+
+ if (task) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (task->message == NULL) {
+ lua_newtable(L);
+
+ return 1;
+ }
+
+ if (!lua_url_cbdata_fill_exclude_include(L, 2, &cb, default_protocols_mask, max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ if (cb.sort) {
+ struct rspamd_url **urls_sorted;
+ gint i = 0;
+
+ urls_sorted = g_new0(struct rspamd_url *, sz);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (i < sz) {
+ urls_sorted[i] = u;
+ i++;
+ }
+ });
+
+ qsort(urls_sorted, i, sizeof(struct rspamd_url *), rspamd_url_cmp_qsort);
+
+ for (int j = 0; j < i; j++) {
+ lua_tree_url_callback(urls_sorted[j], urls_sorted[j], &cb);
+ }
+
+ g_free(urls_sorted);
+ }
+ else {
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+ }
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, no task");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_has_urls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ bool need_emails = false;
+ gboolean ret = FALSE;
+ gsize sz = 0;
+
+ if (task) {
+ if (task->message) {
+ if (lua_gettop(L) >= 2) {
+ need_emails = lua_toboolean(L, 2);
+ }
+
+ if (need_emails) {
+ /* Simplified check */
+ if (kh_size(MESSAGE_FIELD(task, urls)) > 0) {
+ sz += kh_size(MESSAGE_FIELD(task, urls));
+ ret = TRUE;
+ }
+ }
+ else {
+ /* Linear scan */
+ struct rspamd_url *u;
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ if (u->protocol != PROTOCOL_MAILTO) {
+ sz++;
+ ret = TRUE;
+ }
+ });
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ lua_pushinteger(L, sz);
+
+ return 2;
+}
+
+static gint
+lua_task_inject_url(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_url *url = lua_check_url(L, 2);
+ struct rspamd_mime_part *mpart = NULL;
+
+ if (lua_isuserdata(L, 3)) {
+ /* We also have a mime part there */
+ mpart = *((struct rspamd_mime_part **) rspamd_lua_check_udata_maybe(L,
+ 3, "rspamd{mimepart}"));
+ }
+
+ if (task && task->message && url && url->url) {
+ if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url->url, false)) {
+ if (mpart && mpart->urls) {
+ /* Also add url to the mime part */
+ g_ptr_array_add(mpart->urls, url->url);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_content(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->len = task->msg.len;
+ t->start = task->msg.begin;
+ t->flags = 0;
+
+ if (lua_is_text_binary(t)) {
+ t->flags |= RSPAMD_TEXT_FLAG_BINARY;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->msg.fpath) {
+ lua_pushstring(L, task->msg.fpath);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_rawbody(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task) {
+ if (task->message != NULL) {
+
+
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ g_assert(MESSAGE_FIELD(task, raw_headers_content).len <= task->msg.len);
+ t = lua_new_text_task(L, task, task->msg.begin + MESSAGE_FIELD(task, raw_headers_content).len,
+ task->msg.len - MESSAGE_FIELD(task, raw_headers_content).len, FALSE);
+ }
+ else {
+ t = lua_new_text_task(L, task, task->msg.begin,
+ task->msg.len, FALSE);
+ }
+
+ t->flags = 0;
+ }
+ else {
+ /* Push body it it is there */
+ if (task->msg.len > 0 && task->msg.begin != NULL) {
+ lua_new_text_task(L, task, task->msg.begin, task->msg.len, FALSE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_emails(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct lua_tree_cb_data cb;
+ struct rspamd_url *u;
+ gsize max_urls = 0, sz;
+
+ if (task) {
+ if (task->message) {
+ if (task->cfg) {
+ max_urls = task->cfg->max_lua_urls;
+ }
+
+ if (!lua_url_cbdata_fill(L, 2, &cb, PROTOCOL_MAILTO,
+ ~(RSPAMD_URL_FLAG_CONTENT | RSPAMD_URL_FLAG_IMAGE),
+ max_urls)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ sz = kh_size(MESSAGE_FIELD(task, urls));
+ sz = lua_url_adjust_skip_prob(task->task_timestamp,
+ MESSAGE_FIELD(task, digest), &cb, sz);
+
+ lua_createtable(L, sz, 0);
+
+ kh_foreach_key(MESSAGE_FIELD(task, urls), u, {
+ lua_tree_url_callback(u, u, &cb);
+ });
+
+ lua_url_cbdata_dtor(&cb);
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_text_parts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint i;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_text_part *part, **ppart;
+
+ if (task != NULL) {
+
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "text_parts")) {
+ lua_createtable(L, MESSAGE_FIELD(task, text_parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ ppart = lua_newuserdata(L, sizeof(struct rspamd_mime_text_part *));
+ *ppart = part;
+ rspamd_lua_setclass(L, "rspamd{textpart}", -1);
+ /* Make it array */
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_task_set_cached(L, task, "text_parts", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_parts(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint i;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_part *part, **ppart;
+
+ if (task != NULL) {
+ if (task->message) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ ppart = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+ *ppart = part;
+ rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+ /* Make it array */
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_request_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_ftok_t *hdr;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *s;
+ struct rspamd_lua_text *t;
+
+ s = luaL_checkstring(L, 2);
+
+ if (s && task) {
+ hdr = rspamd_task_get_request_header(task, s);
+
+ if (hdr) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = hdr->begin;
+ t->len = hdr->len;
+ t->flags = 0;
+
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_request_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *s, *v = NULL;
+ rspamd_fstring_t *buf;
+ struct rspamd_lua_text *t;
+ rspamd_ftok_t *hdr, *new_name;
+ gsize len, vlen = 0;
+
+ s = luaL_checklstring(L, 2, &len);
+
+ if (s && task) {
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ v = luaL_checklstring(L, 3, &vlen);
+ }
+ else if (lua_type(L, 3) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 3);
+
+ if (t != NULL) {
+ v = t->start;
+ vlen = t->len;
+ }
+ }
+
+ if (v != NULL) {
+ buf = rspamd_fstring_new_init(v, vlen);
+ hdr = rspamd_ftok_map(buf);
+ buf = rspamd_fstring_new_init(s, len);
+ new_name = rspamd_ftok_map(buf);
+
+ rspamd_task_add_request_header(task, new_name, hdr);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 0;
+}
+
+
+gint rspamd_lua_push_header(lua_State *L, struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how)
+{
+ LUA_TRACE_POINT;
+ switch (how) {
+ case RSPAMD_TASK_HEADER_PUSH_FULL:
+ /* Create new associated table for a header */
+ lua_createtable(L, 0, 7);
+ rspamd_lua_table_set(L, "name", rh->name);
+
+ if (rh->value) {
+ rspamd_lua_table_set(L, "value", rh->value);
+ }
+
+ if (rh->raw_len > 0) {
+ lua_pushstring(L, "raw");
+ lua_pushlstring(L, rh->raw_value, rh->raw_len);
+ lua_settable(L, -3);
+ }
+
+ if (rh->decoded) {
+ rspamd_lua_table_set(L, "decoded", rh->decoded);
+ }
+
+ lua_pushstring(L, "tab_separated");
+ lua_pushboolean(L, rh->flags & RSPAMD_HEADER_TAB_SEPARATED);
+ lua_settable(L, -3);
+ lua_pushstring(L, "empty_separator");
+ lua_pushboolean(L, rh->flags & RSPAMD_HEADER_EMPTY_SEPARATOR);
+ lua_settable(L, -3);
+ rspamd_lua_table_set(L, "separator", rh->separator);
+ lua_pushstring(L, "order");
+ lua_pushinteger(L, rh->order);
+ lua_settable(L, -3);
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_RAW:
+ if (rh->value) {
+ lua_pushstring(L, rh->value);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_SIMPLE:
+ if (rh->decoded) {
+ lua_pushstring(L, rh->decoded);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ break;
+ case RSPAMD_TASK_HEADER_PUSH_COUNT:
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ return 1;
+}
+
+gint rspamd_lua_push_header_array(lua_State *L,
+ const gchar *name,
+ struct rspamd_mime_header *rh,
+ enum rspamd_lua_task_header_type how,
+ gboolean strong)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_mime_header *cur;
+ guint i;
+ gint nret = 1;
+
+ if (rh == NULL) {
+ if (how == RSPAMD_TASK_HEADER_PUSH_HAS) {
+ lua_pushboolean(L, false);
+ nret = 1;
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_COUNT) {
+ lua_pushnumber(L, 0);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return nret;
+ }
+
+ if (how == RSPAMD_TASK_HEADER_PUSH_FULL) {
+ lua_createtable(L, 0, 0);
+ i = 0;
+
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ rspamd_lua_push_header(L, cur, how);
+ lua_rawseti(L, -2, ++i);
+ }
+ }
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_COUNT) {
+ i = 0;
+
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ i++;
+ }
+ }
+
+ lua_pushinteger(L, i);
+ }
+ else if (how == RSPAMD_TASK_HEADER_PUSH_HAS) {
+ nret = 1;
+ bool found = false;
+
+ if (strong) {
+ /* We still have to check all headers in the chain */
+ DL_FOREACH(rh, cur)
+ {
+ if (strcmp(name, cur->name) == 0) {
+ found = true;
+ break;
+ }
+ }
+ }
+ else {
+ found = true;
+ }
+
+ lua_pushboolean(L, found);
+ }
+ else {
+ DL_FOREACH(rh, cur)
+ {
+ if (!strong || strcmp(name, cur->name) == 0) {
+ return rspamd_lua_push_header(L, cur, how);
+ }
+ }
+
+ /* Not found with this case */
+ lua_pushnil(L);
+ }
+
+ return nret;
+}
+
+static gint
+lua_task_get_header_common(lua_State *L, enum rspamd_lua_task_header_type how)
+{
+ LUA_TRACE_POINT;
+ gboolean strong = FALSE, need_modified = FALSE;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *rh;
+ const gchar *name;
+
+ name = luaL_checkstring(L, 2);
+
+ if (name && task) {
+ if (lua_gettop(L) >= 3) {
+ strong = lua_toboolean(L, 3);
+ if (lua_isboolean(L, 4)) {
+ need_modified = lua_toboolean(L, 4);
+ }
+ }
+
+
+ rh = rspamd_message_get_header_array(task, name, need_modified);
+
+ return rspamd_lua_push_header_array(L, name, rh, how, strong);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+}
+
+static gint
+lua_task_get_header_full(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_FULL);
+}
+
+static gint
+lua_task_get_header(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_SIMPLE);
+}
+
+static gint
+lua_task_get_header_raw(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_RAW);
+}
+
+static gint
+lua_task_get_header_count(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_COUNT);
+}
+
+static gint
+lua_task_has_header(lua_State *L)
+{
+ return lua_task_get_header_common(L, RSPAMD_TASK_HEADER_PUSH_HAS);
+}
+
+static gint
+lua_task_get_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ bool need_modified = lua_isnoneornil(L, 2) ? false : lua_toboolean(L, 2);
+
+ if (task && task->message) {
+ struct rspamd_mime_header *cur;
+ int i = 1;
+
+ lua_createtable(L, rspamd_mime_headers_count(MESSAGE_FIELD(task, raw_headers)), 0);
+ LL_FOREACH2(MESSAGE_FIELD(task, headers_order), cur, ord_next)
+ {
+ if (need_modified && cur->modified_chain) {
+ struct rspamd_mime_header *cur_modified;
+
+ LL_FOREACH(cur->modified_chain, cur_modified)
+ {
+ rspamd_lua_push_header(L, cur_modified, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ else {
+ rspamd_lua_push_header(L, cur, RSPAMD_TASK_HEADER_PUSH_FULL);
+ lua_rawseti(L, -2, i++);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_task_get_raw_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_text *t;
+
+ if (task && task->message) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = MESSAGE_FIELD(task, raw_headers_content).begin;
+ t->len = MESSAGE_FIELD(task, raw_headers_content).len;
+ t->flags = 0;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_task_get_received_headers(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (!task->message) {
+ /* No message - no received */
+ lua_newtable(L);
+ return 1;
+ }
+
+ if (!lua_task_get_cached(L, task, "received")) {
+
+ if (rspamd_received_export_to_lua(task, L)) {
+ lua_task_set_cached(L, task, "received", -1);
+ }
+ else {
+ /* no received, preserve compatibility */
+ lua_newtable(L);
+ return 1;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_queue_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->queue_id != NULL && strcmp(task->queue_id, "undef") != 0) {
+ lua_pushstring(L, task->queue_id);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_uid(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ lua_pushstring(L, task->task_pool->tag.uid);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_dns_resolver **presolver;
+
+ if (task != NULL && task->resolver != NULL) {
+ presolver = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{resolver}", -1);
+ *presolver = task->resolver;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_resolver(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_dns_resolver *resolver = lua_check_dns_resolver(L, 2);
+
+ if (task != NULL && resolver != NULL) {
+ task->resolver = resolver;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_inc_dns_req(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ static guint warning_shown = 0;
+
+ if (warning_shown < 100) {
+ warning_shown++;
+ msg_warn_task_check("task:inc_dns_req is deprecated and should not be used");
+ }
+
+ if (task != NULL) {
+ /* Deprecation: already done in rspamd_dns_resolver_request */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_dns_req(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ lua_pushinteger(L, task->dns_requests);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+enum lua_email_address_type {
+ LUA_ADDRESS_ANY = 0u,
+ LUA_ADDRESS_SMTP = 1,
+ LUA_ADDRESS_MIME = 2,
+ LUA_ADDRESS_MASK = 0x3FF,
+ LUA_ADDRESS_RAW = (1u << 10),
+ LUA_ADDRESS_ORIGINAL = (1u << 11),
+ LUA_ADDRESS_MAX = LUA_ADDRESS_MASK,
+};
+
+/*
+ * Convert element at the specified position to the type
+ * for get_from/get_recipients
+ */
+static enum lua_email_address_type
+lua_task_str_to_get_type(lua_State *L, struct rspamd_task *task, gint pos, gint last_pos)
+{
+ const gchar *type = NULL;
+ gint ret = LUA_ADDRESS_ANY;
+ guint64 h;
+ gsize sz;
+
+ /* Get what value */
+ do {
+ if (lua_type(L, pos) == LUA_TNUMBER) {
+ ret = lua_tonumber(L, pos);
+
+ if (ret >= LUA_ADDRESS_ANY && ret < LUA_ADDRESS_MAX) {
+ return ret;
+ }
+
+ return LUA_ADDRESS_ANY;
+ }
+ else if (lua_type(L, pos) == LUA_TSTRING) {
+ type = lua_tolstring(L, pos, &sz);
+
+ if (type && sz > 0) {
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ type, sz, 0xdeadbabe);
+
+ switch (h) {
+ case 0xDA081341FB600389ULL: /* mime */
+ ret = LUA_ADDRESS_MIME;
+ break;
+ case 0xEEC8A7832F8C43ACULL: /* any */
+ ret = LUA_ADDRESS_ANY;
+ break;
+ case 0x472274D5193B2A80ULL: /* smtp */
+ case 0xEFE0F586CC9F14A9ULL: /* envelope */
+ ret = LUA_ADDRESS_SMTP;
+ break;
+ default:
+ msg_err_task("invalid email type: %*s", (gint) sz, type);
+ break;
+ }
+ }
+ }
+ else if (lua_type(L, pos) == LUA_TTABLE) {
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ type = lua_tolstring(L, -1, &sz);
+
+ if (type && sz > 0) {
+ h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+ type, sz, 0xdeadbabe);
+
+ switch (h) {
+ case 0xDA081341FB600389ULL: /* mime */
+ ret |= LUA_ADDRESS_MIME;
+ break;
+ case 0xEEC8A7832F8C43ACULL: /* any */
+ ret |= LUA_ADDRESS_ANY;
+ break;
+ case 0x472274D5193B2A80ULL: /* smtp */
+ case 0xEFE0F586CC9F14A9ULL: /* envelope */
+ ret |= LUA_ADDRESS_SMTP;
+ break;
+ case 0xAF4DE083D9AD0132: /* raw */
+ ret |= LUA_ADDRESS_RAW;
+ break;
+ case 0xC7AB6C7B7B0F5A8A: /* orig */
+ case 0x1778AE905589E431: /* original */
+ ret |= LUA_ADDRESS_ORIGINAL;
+ break;
+ default:
+ msg_err_task("invalid email type: %*s", (gint) sz, type);
+ break;
+ }
+ }
+ }
+ }
+ pos++;
+ } while (pos <= last_pos);
+
+ return ret;
+}
+
+#define EMAIL_CHECK_FLAG(fl, str) \
+ do { \
+ if (addr->flags & (fl)) { \
+ lua_pushstring(L, (str)); \
+ lua_pushboolean(L, true); \
+ lua_settable(L, -3); \
+ } \
+ } while (0)
+
+static void
+lua_push_email_address(lua_State *L, struct rspamd_email_address *addr)
+{
+ if (addr) {
+ lua_createtable(L, 0, 5);
+
+ if (addr->raw_len > 0) {
+ lua_pushstring(L, "raw");
+ lua_pushlstring(L, addr->raw, addr->raw_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "raw");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->addr_len > 0) {
+ lua_pushstring(L, "addr");
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "addr");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->domain_len > 0) {
+ lua_pushstring(L, "domain");
+ lua_pushlstring(L, addr->domain, addr->domain_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "domain");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+ if (addr->user_len > 0) {
+ lua_pushstring(L, "user");
+ lua_pushlstring(L, addr->user, addr->user_len);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "user");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+
+ if (addr->name) {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, addr->name);
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, "");
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 7);
+
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_VALID, "valid");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_IP, "ip");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_BRACED, "braced");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_QUOTED, "quoted");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_EMPTY, "empty");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_HAS_BACKSLASH, "backslash");
+ EMAIL_CHECK_FLAG(RSPAMD_EMAIL_ADDR_HAS_8BIT, "8bit");
+
+ lua_settable(L, -3);
+ }
+}
+
+void lua_push_emails_address_list(lua_State *L, GPtrArray *addrs, int flags)
+{
+ struct rspamd_email_address *addr;
+ guint i, pos = 1;
+
+ lua_createtable(L, addrs->len, 0);
+
+ for (i = 0; i < addrs->len; i++) {
+ addr = g_ptr_array_index(addrs, i);
+
+ if (addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL) {
+ if (flags & LUA_ADDRESS_ORIGINAL) {
+ lua_push_email_address(L, addr);
+ lua_rawseti(L, -2, pos);
+ pos++;
+ }
+ }
+ else {
+ lua_push_email_address(L, addr);
+ lua_rawseti(L, -2, pos);
+ pos++;
+ }
+ }
+}
+
+static gboolean
+lua_import_email_address(lua_State *L, struct rspamd_task *task,
+ gint pos,
+ struct rspamd_email_address **paddr)
+{
+ struct rspamd_email_address *addr;
+ const gchar *p;
+ gchar *dst;
+ gsize len;
+
+ g_assert(paddr != NULL);
+
+ if (!lua_istable(L, pos)) {
+ return FALSE;
+ }
+
+ addr = g_malloc0(sizeof(*addr));
+
+ lua_pushstring(L, "name");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+ rspamd_strlcpy(dst, p, len + 1);
+ addr->name = dst;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "user");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->user = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->user, p, len);
+ addr->user_len = len;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "domain");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->domain = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->domain, p, len);
+ addr->domain_len = len;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "addr");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ p = lua_tolstring(L, -1, &len);
+ addr->addr = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ memcpy((gchar *) addr->addr, p, len);
+ addr->addr_len = len;
+ }
+ else {
+ /* Construct addr */
+ len = addr->domain_len + addr->user_len + 1;
+ addr->addr = (const gchar *) rspamd_mempool_alloc(task->task_pool, len);
+ addr->addr_len = rspamd_snprintf((gchar *) addr->addr, len, "%*s@%*s",
+ (int) addr->user_len, addr->user,
+ (int) addr->domain_len, addr->domain);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, pos);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ gchar *cpy;
+ p = lua_tolstring(L, -1, &len);
+ cpy = rspamd_mempool_alloc(task->task_pool, len + 1);
+ memcpy(cpy, p, len);
+ cpy[len] = '\0';
+ addr->raw_len = len;
+ addr->raw = cpy;
+ }
+ else {
+ /* Construct raw addr */
+ len = addr->addr_len + 3;
+
+ if (addr->name) {
+ len += strlen(addr->name) + 1;
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+
+ addr->raw_len = rspamd_snprintf(dst, len, "%s <%*s>",
+ addr->name,
+ (int) addr->addr_len, addr->addr);
+ }
+ else {
+ dst = rspamd_mempool_alloc(task->task_pool, len + 1);
+
+ addr->raw_len = rspamd_snprintf(dst, len, "<%*s@%*s>",
+ (int) addr->user_len, addr->user,
+ (int) addr->domain_len, addr->domain);
+ }
+
+ addr->raw = dst;
+ }
+
+ lua_pop(L, 1);
+ addr->flags = RSPAMD_EMAIL_ADDR_VALID;
+
+ *paddr = addr;
+
+ return TRUE;
+}
+
+static gint
+lua_task_get_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *ptrs = NULL;
+ gint what = 0;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ ptrs = task->rcpt_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ }
+ break;
+ }
+ if (ptrs) {
+ lua_push_emails_address_list(L, ptrs, what & ~LUA_ADDRESS_MASK);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *ptrs = NULL;
+ struct rspamd_email_address *addr = NULL;
+ gint what = 0, pos = 3;
+ const gchar *how = "add";
+ gboolean need_update_digest = FALSE;
+
+ if (task && lua_gettop(L) >= 3) {
+
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, -1);
+
+ if (lua_isstring(L, 4)) {
+ how = lua_tostring(L, 4);
+ }
+
+ switch (what) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = g_ptr_array_new();
+ task->rcpt_envelope = ptrs;
+ }
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ need_update_digest = TRUE;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->rcpt_envelope) {
+ if (task->rcpt_envelope) {
+ ptrs = task->rcpt_envelope;
+ }
+ else {
+ ptrs = g_ptr_array_new();
+ task->rcpt_envelope = ptrs;
+ }
+ }
+ else {
+ ptrs = MESSAGE_FIELD_CHECK(task, rcpt_mime);
+ need_update_digest = TRUE;
+ }
+ break;
+ }
+ if (ptrs) {
+ guint i, flags_existing = RSPAMD_EMAIL_ADDR_ORIGINAL, flags_add = 0;
+ struct rspamd_email_address *tmp;
+
+ if (strcmp(how, "alias") == 0) {
+ flags_add |= RSPAMD_EMAIL_ADDR_ALIASED;
+ }
+ else if (strcmp(how, "rewrite") == 0) {
+ /* Clear old addresses */
+ PTR_ARRAY_FOREACH(ptrs, i, tmp)
+ {
+ rspamd_email_address_free(addr);
+ }
+
+ g_ptr_array_set_size(ptrs, 0);
+ }
+
+ PTR_ARRAY_FOREACH(ptrs, i, tmp)
+ {
+ tmp->flags |= flags_existing;
+ }
+
+ lua_pushvalue(L, pos);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_import_email_address(L, task, lua_gettop(L), &addr)) {
+
+ if (need_update_digest) {
+ rspamd_message_update_digest(task->message,
+ addr->addr, addr->addr_len);
+ }
+
+ addr->flags |= flags_add;
+ g_ptr_array_add(ptrs, addr);
+ }
+ }
+
+ lua_pop(L, 1);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+#define CHECK_EMAIL_ADDR(addr) \
+ do { \
+ if (addr == NULL) { \
+ ret = 0; \
+ } \
+ else { \
+ ret = addr->flags & RSPAMD_EMAIL_ADDR_VALID; \
+ } \
+ } while (0)
+
+#define CHECK_EMAIL_ADDR_LIST(addr) \
+ do { \
+ if (addr == NULL) { \
+ ret = 0; \
+ } \
+ else { \
+ ret = addr->len > 0; \
+ nrcpt = addr->len; \
+ } \
+ } while (0)
+
+static gint
+lua_task_has_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint what = 0, nrcpt = 0;
+ gboolean ret = FALSE;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ CHECK_EMAIL_ADDR(task->from_envelope);
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ CHECK_EMAIL_ADDR_LIST(MESSAGE_FIELD_CHECK(task, from_mime));
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ CHECK_EMAIL_ADDR(task->from_envelope);
+
+ if (!ret) {
+ CHECK_EMAIL_ADDR_LIST(MESSAGE_FIELD_CHECK(task, from_mime));
+ }
+ break;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ (void) nrcpt; /* Silence warning */
+
+ return 1;
+}
+
+static inline int
+rspamd_check_real_recipients_array_size(GPtrArray *ar)
+{
+ gint ret = 0, i;
+ struct rspamd_email_address *addr;
+
+ PTR_ARRAY_FOREACH(ar, i, addr)
+ {
+ if (!(addr->flags & RSPAMD_EMAIL_ADDR_ORIGINAL)) {
+ ret++;
+ }
+ }
+
+ return ret;
+}
+
+static gint
+lua_task_has_recipients(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint what = 0, nrcpt = 0;
+ gboolean ret = FALSE;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ nrcpt = rspamd_check_real_recipients_array_size(task->rcpt_envelope);
+ ret = nrcpt > 0;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ nrcpt = rspamd_check_real_recipients_array_size(MESSAGE_FIELD_CHECK(task, rcpt_mime));
+ ret = nrcpt > 0;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ nrcpt = rspamd_check_real_recipients_array_size(task->rcpt_envelope);
+ ret = nrcpt > 0;
+
+ if (!ret) {
+ nrcpt = rspamd_check_real_recipients_array_size(MESSAGE_FIELD_CHECK(task, rcpt_mime));
+ ret = nrcpt > 0;
+ }
+ break;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, ret);
+ lua_pushinteger(L, nrcpt);
+
+ return 2;
+}
+
+static gint
+lua_task_get_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ GPtrArray *addrs = NULL;
+ struct rspamd_email_address *addr = NULL;
+ gint what = 0;
+
+ if (task) {
+ if (lua_gettop(L) == 2) {
+ /* Get what value */
+ what = lua_task_str_to_get_type(L, task, 2, lua_gettop(L));
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ addr = task->from_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->from_envelope) {
+ addr = task->from_envelope;
+ }
+ else {
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ }
+ break;
+ }
+
+ if (addrs && addrs->len > 0) {
+ lua_push_emails_address_list(L, addrs, what & ~LUA_ADDRESS_MASK);
+ }
+ else if (addr) {
+ /* Create table to preserve compatibility */
+ if (addr->addr) {
+ lua_createtable(L, 1, 0);
+ if (what & LUA_ADDRESS_ORIGINAL) {
+ if (task->from_envelope_orig) {
+ lua_push_email_address(L, task->from_envelope_orig);
+ }
+ else {
+ lua_push_email_address(L, addr);
+ }
+ }
+ else {
+ lua_push_email_address(L, addr);
+ }
+
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_from(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *how = "rewrite";
+ GPtrArray *addrs = NULL;
+ struct rspamd_email_address **paddr = NULL, *addr;
+ gboolean need_update_digest = FALSE;
+ gint what = 0;
+
+ if (task && lua_gettop(L) >= 3) {
+ what = lua_task_str_to_get_type(L, task, 2, -1);
+
+ if (lua_isstring(L, 4)) {
+ how = lua_tostring(L, 4);
+ }
+
+ switch (what & LUA_ADDRESS_MASK) {
+ case LUA_ADDRESS_SMTP:
+ /* Here we check merely envelope rcpt */
+ paddr = &task->from_envelope;
+ break;
+ case LUA_ADDRESS_MIME:
+ /* Here we check merely mime rcpt */
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ need_update_digest = TRUE;
+ break;
+ case LUA_ADDRESS_ANY:
+ default:
+ if (task->from_envelope) {
+ paddr = &task->from_envelope;
+ }
+ else {
+ addrs = MESSAGE_FIELD_CHECK(task, from_mime);
+ need_update_digest = TRUE;
+ }
+ break;
+ }
+
+ if (addrs) {
+ if (lua_import_email_address(L, task, 3, &addr)) {
+ guint i, flags_add = RSPAMD_EMAIL_ADDR_ORIGINAL;
+ struct rspamd_email_address *tmp;
+
+ if (strcmp(how, "alias") == 0) {
+ flags_add |= RSPAMD_EMAIL_ADDR_ALIASED;
+ }
+
+ PTR_ARRAY_FOREACH(addrs, i, tmp)
+ {
+ tmp->flags |= flags_add;
+ }
+
+ if (need_update_digest) {
+ rspamd_message_update_digest(task->message,
+ addr->addr, addr->addr_len);
+ }
+
+ g_ptr_array_add(addrs, addr);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else if (paddr) {
+ /* SMTP from case */
+ if (lua_import_email_address(L, task, 3, &addr)) {
+ task->from_envelope_orig = *paddr;
+ task->from_envelope = addr;
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_principal_recipient(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *r;
+
+ if (task) {
+ r = rspamd_task_get_principal_recipient(task);
+ if (r != NULL) {
+ lua_pushstring(L, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_reply_sender(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *rh;
+
+ if (task) {
+
+ rh = rspamd_message_get_header_array(task, "Reply-To", FALSE);
+
+ if (rh) {
+ GPtrArray *addrs;
+
+ addrs = rspamd_email_address_from_mime(task->task_pool, rh->decoded,
+ strlen(rh->decoded), NULL, -1);
+
+ if (addrs == NULL || addrs->len == 0) {
+ lua_pushnil(L);
+ }
+ else {
+ struct rspamd_email_address *addr;
+
+ addr = (struct rspamd_email_address *) g_ptr_array_index(addrs, 0);
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ }
+ }
+ else if (MESSAGE_FIELD_CHECK(task, from_mime) &&
+ MESSAGE_FIELD(task, from_mime)->len >= 1) {
+ struct rspamd_email_address *addr;
+
+ addr = (struct rspamd_email_address *) g_ptr_array_index(
+ MESSAGE_FIELD(task, from_mime), 0);
+
+ lua_pushlstring(L, addr->addr, addr->addr_len);
+ }
+ else if (task->from_envelope) {
+ lua_pushlstring(L, task->from_envelope->addr,
+ task->from_envelope->addr_len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->auth_user != NULL) {
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_user;
+
+ if (task) {
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ new_user = lua_tostring(L, 2);
+
+ if (task->auth_user) {
+ /* Push old user */
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ task->auth_user = rspamd_mempool_strdup(task->task_pool, new_user);
+ }
+ else {
+ /* Reset user */
+ if (task->auth_user) {
+ /* Push old user */
+ lua_pushstring(L, task->auth_user);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ task->auth_user = NULL;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_from_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->from_addr) {
+ rspamd_lua_ip_push(L, task->from_addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_from_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ rspamd_inet_addr_t *addr = NULL;
+
+ if (!task) {
+ return luaL_error(L, "no task");
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ gsize len;
+ const gchar *ip_str = lua_tolstring(L, 2, &len);
+
+ if (!rspamd_parse_inet_address(&addr,
+ ip_str,
+ len,
+ RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ return luaL_error(L, "invalid IP string: %s", ip_str);
+ }
+ else {
+ if (task->from_addr) {
+ rspamd_inet_address_free(task->from_addr);
+ }
+
+ task->from_addr = addr;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ struct rspamd_lua_ip *ip = lua_check_ip(L, 2);
+
+ if (ip && ip->addr) {
+ if (task->from_addr) {
+ rspamd_inet_address_free(task->from_addr);
+ }
+
+ task->from_addr = rspamd_inet_address_copy(ip->addr, NULL);
+ }
+ else {
+ return luaL_error(L, "invalid IP object");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid IP argument type: %s", lua_typename(L, lua_type(L, 2)));
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_from_ip_num(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ msg_err("this function is deprecated and should no longer be used");
+ lua_pushnil(L);
+ return 1;
+}
+
+static gint
+lua_task_get_client_ip(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->client_addr) {
+ rspamd_lua_ip_push(L, task->client_addr);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_helo(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->helo != NULL) {
+ lua_pushstring(L, task->helo);
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (MESSAGE_FIELD_CHECK(task, subject) != NULL) {
+ lua_pushstring(L, MESSAGE_FIELD(task, subject));
+ return 1;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_helo(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_helo;
+
+ if (task) {
+ new_helo = luaL_checkstring(L, 2);
+ if (new_helo) {
+ task->helo = rspamd_mempool_strdup(task->task_pool, new_helo);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->hostname != NULL) {
+ /* Check whether it looks like an IP address */
+ if (*task->hostname == '[') {
+ /*
+ * From the milter documentation:
+ * If the reverse lookup fails or if none of the IP
+ * addresses of the resolved host name matches the
+ * original IP address, hostname will contain the
+ * message sender's IP address enclosed in square
+ * brackets (e.g. `[a.b.c.d]')
+ */
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushstring(L, task->hostname);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *new_hostname;
+
+ if (task) {
+ new_hostname = luaL_checkstring(L, 2);
+ if (new_hostname) {
+ task->hostname = rspamd_mempool_strdup(task->task_pool,
+ new_hostname);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_images(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_mime_part *part;
+ struct rspamd_image **pimg;
+
+ if (task) {
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "images")) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_IMAGE) {
+ pimg = lua_newuserdata(L, sizeof(struct rspamd_image *));
+ rspamd_lua_setclass(L, "rspamd{image}", -1);
+ *pimg = part->specific.img;
+ lua_rawseti(L, -2, ++nelt);
+ }
+ }
+
+ lua_task_set_cached(L, task, "images", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_archives(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_mime_part *part;
+ struct rspamd_archive **parch;
+
+ if (task) {
+ if (task->message) {
+ if (!lua_task_get_cached(L, task, "archives")) {
+ lua_createtable(L, MESSAGE_FIELD(task, parts)->len, 0);
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+ {
+ if (part->part_type == RSPAMD_MIME_PART_ARCHIVE) {
+ parch = lua_newuserdata(L, sizeof(struct rspamd_archive *));
+ rspamd_lua_setclass(L, "rspamd{archive}", -1);
+ *parch = part->specific.arch;
+ lua_rawseti(L, -2, ++nelt);
+ }
+ }
+
+ lua_task_set_cached(L, task, "archives", -1);
+ }
+ }
+ else {
+ lua_newtable(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_dkim_results(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint nelt = 0, i;
+ struct rspamd_dkim_check_result **pres, **cur;
+
+ if (task) {
+ if (!lua_task_get_cached(L, task, "dkim_results")) {
+ pres = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_DKIM_CHECK_RESULTS);
+
+ if (pres == NULL) {
+ lua_newtable(L);
+ }
+ else {
+ for (cur = pres; *cur != NULL; cur++) {
+ nelt++;
+ }
+
+ lua_createtable(L, nelt, 0);
+
+ for (i = 0; i < nelt; i++) {
+ struct rspamd_dkim_check_result *res = pres[i];
+ const gchar *result_str = "unknown";
+
+ lua_createtable(L, 0, 4);
+
+ switch (res->rcode) {
+ case DKIM_CONTINUE:
+ result_str = "allow";
+ break;
+ case DKIM_REJECT:
+ result_str = "reject";
+ break;
+ case DKIM_TRYAGAIN:
+ result_str = "tempfail";
+ break;
+ case DKIM_NOTFOUND:
+ result_str = "not found";
+ break;
+ case DKIM_RECORD_ERROR:
+ result_str = "bad record";
+ break;
+ case DKIM_PERM_ERROR:
+ result_str = "permanent error";
+ break;
+ default:
+ break;
+ }
+
+ rspamd_lua_table_set(L, "result", result_str);
+
+ if (res->domain) {
+ rspamd_lua_table_set(L, "domain", res->domain);
+ }
+
+ if (res->selector) {
+ rspamd_lua_table_set(L, "selector", res->selector);
+ }
+
+ if (res->short_b) {
+ rspamd_lua_table_set(L, "bhash", res->short_b);
+ }
+
+ if (res->fail_reason) {
+ rspamd_lua_table_set(L, "fail_reason", res->fail_reason);
+ }
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+
+ lua_task_set_cached(L, task, "dkim_results", -1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline gboolean
+lua_push_symbol_result(lua_State *L,
+ struct rspamd_task *task,
+ const gchar *symbol,
+ struct rspamd_symbol_result *symbol_result,
+ struct rspamd_scan_result *metric_res,
+ gboolean add_metric,
+ gboolean add_name)
+{
+
+ struct rspamd_symbol_result *s = NULL;
+ struct rspamd_symbol_option *opt;
+ struct rspamd_symbols_group *sym_group;
+ guint i;
+ gint j = 1, table_fields_cnt = 4;
+
+ if (!metric_res) {
+ metric_res = task->result;
+ }
+
+ if (!symbol_result) {
+ s = rspamd_task_find_symbol_result(task, symbol, metric_res);
+ }
+ else {
+ s = symbol_result;
+ }
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ if (add_metric) {
+ table_fields_cnt++;
+ }
+ if (add_name) {
+ table_fields_cnt++;
+ }
+
+ lua_createtable(L, 0, table_fields_cnt);
+
+ if (add_name) {
+ lua_pushstring(L, "name");
+ lua_pushstring(L, symbol);
+ lua_settable(L, -3);
+ }
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, s->score);
+ lua_settable(L, -3);
+
+ if (s->sym && s->sym->gr) {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, s->sym->gr->name);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "groups");
+ lua_createtable(L, s->sym->groups->len, 0);
+
+ PTR_ARRAY_FOREACH(s->sym->groups, i, sym_group)
+ {
+ lua_pushstring(L, sym_group->name);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushstring(L, "group");
+ lua_pushstring(L, "ungrouped");
+ lua_settable(L, -3);
+ }
+
+ if (s->options) {
+ lua_pushstring(L, "options");
+ lua_createtable(L, kh_size(s->options), 0);
+
+ DL_FOREACH(s->opts_head, opt)
+ {
+ lua_pushlstring(L, opt->option, opt->optlen);
+ lua_rawseti(L, -2, j++);
+ }
+
+ lua_settable(L, -3);
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static gint
+lua_task_get_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ struct rspamd_scan_result *sres = NULL;
+
+ if (lua_isstring(L, 3)) {
+ sres = rspamd_find_metric_result(task, lua_tostring(L, 3));
+
+ if (sres == NULL) {
+ return luaL_error(L, "invalid scan result: %s",
+ lua_tostring(L, 3));
+ }
+ }
+
+ /* Always push as a table for compatibility :( */
+ lua_createtable(L, 1, 0);
+
+ if ((found = lua_push_symbol_result(L, task, symbol,
+ NULL, sres, TRUE, FALSE))) {
+ lua_rawseti(L, -2, 1);
+ }
+ else {
+ /* Pop table */
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!found) {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_has_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_symbol_result *s;
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ if (lua_isstring(L, 3)) {
+ s = rspamd_task_find_symbol_result(task, symbol,
+ rspamd_find_metric_result(task, lua_tostring(L, 3)));
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ found = TRUE;
+ }
+ }
+ else {
+ s = rspamd_task_find_symbol_result(task, symbol, NULL);
+
+ if (s && !(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ found = TRUE;
+ }
+ }
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_enable_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ found = rspamd_symcache_enable_symbol(task, task->cfg->cache, symbol);
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_disable_symbol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *symbol;
+ gboolean found = FALSE;
+
+ symbol = luaL_checkstring(L, 2);
+
+ if (task && symbol) {
+ found = rspamd_symcache_disable_symbol(task, task->cfg->cache, symbol);
+ lua_pushboolean(L, found);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_symbols(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ gint i = 1;
+ struct rspamd_symbol_result *s;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ lua_createtable(L, kh_size(mres->symbols), 0);
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ lua_pushstring(L, s->name);
+ lua_rawseti(L, -3, i);
+ lua_pushnumber(L, s->score);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ });
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_createtable(L, 0, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_symbols_all(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ struct rspamd_symbol_result *s;
+ gboolean found = FALSE;
+ gint i = 1;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ found = TRUE;
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ lua_push_symbol_result(L, task, s->name, s, mres, FALSE, TRUE);
+ lua_rawseti(L, -2, i++);
+ }
+ });
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!found) {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_task_get_symbols_numeric(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *mres;
+ gint i = 1, id;
+ struct rspamd_symbol_result *s;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres) {
+ lua_createtable(L, kh_size(mres->symbols), 0);
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ lua_createtable(L, kh_size(mres->symbols), 0);
+
+ kh_foreach_value(mres->symbols, s, {
+ if (!(s->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+ id = rspamd_symcache_find_symbol(task->cfg->cache,
+ s->name);
+ lua_pushinteger(L, id);
+ lua_rawseti(L, -3, i);
+ lua_pushnumber(L, s->score);
+ lua_rawseti(L, -2, i);
+ i++;
+ }
+ });
+ }
+ else {
+ lua_createtable(L, 0, 0);
+ lua_createtable(L, 0, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_groups(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean need_private;
+ struct rspamd_scan_result *mres;
+ struct rspamd_symbols_group *gr;
+ gdouble gr_score;
+
+ if (task) {
+ mres = task->result;
+
+ if (lua_isboolean(L, 2)) {
+ need_private = lua_toboolean(L, 2);
+ }
+ else {
+ need_private = !(task->cfg->public_groups_only);
+ }
+
+ if (lua_isstring(L, 3)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 3));
+ }
+
+ if (mres == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ lua_createtable(L, 0, kh_size(mres->sym_groups));
+
+ kh_foreach(mres->sym_groups, gr, gr_score, {
+ if (!(gr->flags & RSPAMD_SYMBOL_GROUP_PUBLIC)) {
+ if (!need_private) {
+ continue;
+ }
+ }
+
+ lua_pushnumber(L, gr_score);
+ lua_setfield(L, -2, gr->name);
+ });
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct tokens_foreach_cbdata {
+ struct rspamd_task *task;
+ lua_State *L;
+ gint idx;
+ gboolean normalize;
+};
+
+static void
+tokens_foreach_cb(struct rspamd_symcache_item *item, gpointer ud)
+{
+ struct tokens_foreach_cbdata *cbd = ud;
+ struct rspamd_symbol_result *s;
+ gint flags;
+ const gchar *sym;
+
+ sym = rspamd_symcache_item_name(item);
+ flags = rspamd_symcache_item_flags(item);
+
+ if (flags & SYMBOL_TYPE_NOSTAT) {
+ return;
+ }
+
+ if ((s = rspamd_task_find_symbol_result(cbd->task, sym, NULL)) != NULL) {
+ if (s->flags & RSPAMD_SYMBOL_RESULT_IGNORED) {
+ lua_pushnumber(cbd->L, 0.0);
+ }
+ else {
+ if (cbd->normalize) {
+ lua_pushnumber(cbd->L, tanh(s->score));
+ }
+ else {
+ lua_pushnumber(cbd->L, s->score);
+ }
+ }
+ }
+ else {
+ lua_pushnumber(cbd->L, 0.0);
+ }
+
+ lua_rawseti(cbd->L, -2, cbd->idx++);
+}
+
+static gint
+lua_task_get_symbols_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct tokens_foreach_cbdata cbd;
+
+ if (task) {
+ cbd.task = task;
+ cbd.L = L;
+ cbd.idx = 1;
+ cbd.normalize = TRUE;
+
+ if (lua_type(L, 2) == LUA_TBOOLEAN) {
+ cbd.normalize = lua_toboolean(L, 2);
+ }
+ else {
+ cbd.normalize = TRUE;
+ }
+
+ lua_createtable(L,
+ rspamd_symcache_stats_symbols_count(task->cfg->cache), 0);
+ rspamd_symcache_foreach(task->cfg->cache, tokens_foreach_cb, &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ /* Return type is table created */
+ return 1;
+}
+
+static gint
+lua_task_process_ann_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint offset = luaL_checkinteger(L, 4);
+ gdouble min_score = 0.0;
+
+ if (task && lua_istable(L, 2) && lua_istable(L, 3)) {
+ guint symlen = rspamd_lua_table_size(L, 2);
+ if (lua_isnumber(L, 5)) {
+ min_score = lua_tonumber(L, 5);
+ }
+
+ for (guint i = 1; i <= symlen; i++, offset++) {
+ const gchar *sym;
+ struct rspamd_symbol_result *sres;
+
+ lua_rawgeti(L, 2, i);
+ sym = lua_tostring(L, -1);
+
+ /*
+ * TODO: this cycle involves one hash lookup per symbol in a profile
+ * Basically, in a common case that would be a table of all symbols
+ * So we need to do N_symbols hash lookups which is not optimal
+ * The optimal solution is to convert [sym1, sym2, ... symn] profile
+ * to a set {sym1 = true, sym2 = true, ...} and then for each
+ * resulting symbol check this table.
+ *
+ * That would lead to N_results lookups which is usually MUCH smaller
+ */
+ sres = rspamd_task_find_symbol_result(task, sym, NULL);
+
+ if (sres && !(sres->flags & RSPAMD_SYMBOL_RESULT_IGNORED)) {
+
+ if (!isnan(sres->score) && !isinf(sres->score) &&
+ (!sres->sym ||
+ !(rspamd_symcache_item_flags(sres->sym->cache_item) & SYMBOL_TYPE_NOSTAT))) {
+
+ gdouble norm_score;
+
+ if (sres->sym && !isnan(sres->sym->score)) {
+ if (sres->sym->score == 0) {
+
+ if (sres->score == 0) {
+ /* Binary symbol */
+ norm_score = 1.0;
+ }
+ else {
+ norm_score = fabs(tanh(sres->score));
+ }
+ }
+ else {
+ /* Get dynamic weight */
+ norm_score = fabs(sres->score / sres->sym->score);
+
+ if (norm_score > 1.0) {
+ /* Multiple hits, we assume them as a single one */
+ norm_score = 1.0;
+ }
+ }
+ }
+ else {
+ norm_score = fabs(tanh(sres->score));
+ }
+
+ lua_pushnumber(L, MAX(min_score, norm_score));
+ lua_rawseti(L, 3, offset + 1);
+ }
+ }
+
+ lua_pop(L, 1); /* Symbol name */
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+enum lua_date_type {
+ DATE_CONNECT = 0,
+ DATE_MESSAGE,
+ DATE_INVALID
+};
+
+static enum lua_date_type
+lua_task_detect_date_type(struct rspamd_task *task,
+ lua_State *L, gint idx, gboolean *gmt)
+{
+ enum lua_date_type type = DATE_CONNECT;
+
+ if (lua_type(L, idx) == LUA_TNUMBER) {
+ gint num = lua_tonumber(L, idx);
+ if (num >= DATE_CONNECT && num < DATE_INVALID) {
+ return num;
+ }
+ }
+ else if (lua_type(L, idx) == LUA_TTABLE) {
+ const gchar *str;
+
+ lua_pushvalue(L, idx);
+ lua_pushstring(L, "format");
+ lua_gettable(L, -2);
+
+ str = lua_tostring(L, -1);
+
+ if (str) {
+ if (g_ascii_strcasecmp(str, "message") == 0) {
+ type = DATE_MESSAGE;
+ }
+ }
+ else {
+ msg_warn_task("date format has not been specified");
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "gmt");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ *gmt = lua_toboolean(L, -1);
+ }
+
+ /* Value and table */
+ lua_pop(L, 2);
+ }
+
+ return type;
+}
+
+static gint
+lua_task_get_date(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_mime_header *h;
+ gdouble tim;
+ enum lua_date_type type = DATE_CONNECT;
+ gboolean gmt = TRUE;
+
+ if (task != NULL) {
+ if (lua_gettop(L) > 1) {
+ type = lua_task_detect_date_type(task, L, 2, &gmt);
+ }
+ /* Get GMT date and store it to time_t */
+ if (type == DATE_CONNECT) {
+ tim = task->task_timestamp;
+
+ if (!gmt) {
+ struct tm t;
+ time_t tt;
+
+ tt = tim;
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ /* Preserve fractional part as Lua is aware of it */
+ tim = mktime(&t) + (tim - tt);
+ }
+ }
+ else {
+ h = rspamd_message_get_header_array(task, "Date", FALSE);
+
+ if (h) {
+ time_t tt;
+ struct tm t;
+ GError *err = NULL;
+
+ tt = rspamd_parse_smtp_date(h->decoded, strlen(h->decoded),
+ &err);
+
+ if (err == NULL) {
+ if (!gmt) {
+ rspamd_localtime(tt, &t);
+#if !defined(__sun)
+ t.tm_gmtoff = 0;
+#endif
+ t.tm_isdst = 0;
+ tim = mktime(&t);
+ }
+ else {
+ tim = tt;
+ }
+ }
+ else {
+ g_error_free(err);
+ tim = 0.0;
+ }
+ }
+ else {
+ tim = 0.0;
+ }
+ }
+
+ lua_pushnumber(L, tim);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_message_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ if (MESSAGE_FIELD_CHECK(task, message_id) != NULL) {
+ lua_pushstring(L, MESSAGE_FIELD(task, message_id));
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_timeval(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct timeval tv;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, 2) && !!lua_toboolean(L, 2)) {
+ lua_pushnumber(L, task->task_timestamp);
+ }
+ else {
+ double_to_tv(task->task_timestamp, &tv);
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "tv_sec");
+ lua_pushinteger(L, (lua_Integer) tv.tv_sec);
+ lua_settable(L, -3);
+ lua_pushstring(L, "tv_usec");
+ lua_pushinteger(L, (lua_Integer) tv.tv_usec);
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_scan_time(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean set = TRUE;
+
+ if (task != NULL) {
+ if (lua_isboolean(L, 2)) {
+ set = lua_toboolean(L, 2);
+ }
+
+ rspamd_task_set_finish_time(task);
+ gdouble diff = task->time_real_finish - task->task_timestamp;
+ lua_pushnumber(L, diff);
+ lua_pushnumber(L, diff);
+
+ if (!set) {
+ /* Reset to nan to allow further calcs in rspamd_task_set_finish_time */
+ task->time_real_finish = NAN;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_task_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+ lua_pushinteger(L, task->msg.len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/**
+* - `no_log`: do not log task summary
+* - `no_stat`: do not include task into scanned stats
+* - `pass_all`: check all filters for task
+* - `extended_urls`: output extended info about urls
+* - `skip`: skip task processing
+*/
+
+#define LUA_TASK_FLAG_WRITE(flag, set) \
+ do { \
+ task->flags = (set) ? (task->flags | (flag)) : (task->flags & ~(flag)); \
+ } while (0)
+
+#define LUA_TASK_SET_FLAG(flag, strname, macro, set) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_FLAG_WRITE((macro), set); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+#define LUA_TASK_FLAG_READ(flag) \
+ do { \
+ lua_pushboolean(L, !!(task->flags & (flag))); \
+ } while (0)
+
+#define LUA_TASK_GET_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+#define LUA_TASK_PROTOCOL_FLAG_READ(flag) \
+ do { \
+ lua_pushboolean(L, !!(task->protocol_flags & (flag))); \
+ } while (0)
+
+#define LUA_TASK_GET_PROTOCOL_FLAG(flag, strname, macro) \
+ do { \
+ if (!found && strcmp((flag), strname) == 0) { \
+ LUA_TASK_PROTOCOL_FLAG_READ((macro)); \
+ found = TRUE; \
+ } \
+ } while (0)
+
+static gint
+lua_task_set_flag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *flag = luaL_checkstring(L, 2);
+ gboolean set = TRUE, found = FALSE;
+
+ if (lua_gettop(L) >= 3) {
+ set = lua_toboolean(L, 3);
+ }
+
+ if (task != NULL && flag != NULL) {
+ LUA_TASK_SET_FLAG(flag, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL, set);
+ LUA_TASK_SET_FLAG(flag, "no_log", RSPAMD_TASK_FLAG_NO_LOG, set);
+ LUA_TASK_SET_FLAG(flag, "no_stat", RSPAMD_TASK_FLAG_NO_STAT, set);
+ LUA_TASK_SET_FLAG(flag, "skip", RSPAMD_TASK_FLAG_SKIP, set);
+ LUA_TASK_SET_FLAG(flag, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM, set);
+ LUA_TASK_SET_FLAG(flag, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM, set);
+ LUA_TASK_SET_FLAG(flag, "broken_headers",
+ RSPAMD_TASK_FLAG_BROKEN_HEADERS, set);
+ LUA_TASK_SET_FLAG(flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED, set);
+ LUA_TASK_SET_FLAG(flag, "skip_process", RSPAMD_TASK_FLAG_SKIP_PROCESS, set);
+ LUA_TASK_SET_FLAG(flag, "message_rewrite", RSPAMD_TASK_FLAG_MESSAGE_REWRITE, set);
+
+ if (!found) {
+ msg_warn_task("unknown flag requested: %s", flag);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_has_flag(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *flag = luaL_checkstring(L, 2);
+ gboolean found = FALSE;
+
+ if (task != NULL && flag != NULL) {
+ LUA_TASK_GET_FLAG(flag, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+ LUA_TASK_GET_FLAG(flag, "no_log", RSPAMD_TASK_FLAG_NO_LOG);
+ LUA_TASK_GET_FLAG(flag, "no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+ LUA_TASK_GET_FLAG(flag, "skip", RSPAMD_TASK_FLAG_SKIP);
+ LUA_TASK_GET_FLAG(flag, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM);
+ LUA_TASK_GET_FLAG(flag, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM);
+ LUA_TASK_GET_FLAG(flag, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
+ LUA_TASK_GET_FLAG(flag, "broken_headers",
+ RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+ LUA_TASK_GET_FLAG(flag, "skip_process",
+ RSPAMD_TASK_FLAG_SKIP_PROCESS);
+ LUA_TASK_GET_FLAG(flag, "bad_unicode",
+ RSPAMD_TASK_FLAG_BAD_UNICODE);
+ LUA_TASK_GET_FLAG(flag, "mime",
+ RSPAMD_TASK_FLAG_MIME);
+ LUA_TASK_GET_FLAG(flag, "message_rewrite",
+ RSPAMD_TASK_FLAG_MESSAGE_REWRITE);
+ LUA_TASK_GET_PROTOCOL_FLAG(flag, "milter",
+ RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
+
+ if (!found) {
+ msg_warn_task("unknown flag requested: %s", flag);
+ lua_pushboolean(L, 0);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gint idx = 1;
+ guint flags, bit, i;
+
+ if (task) {
+ lua_createtable(L, 8, 0);
+
+ flags = task->flags;
+
+ for (i = 0; i <= RSPAMD_TASK_FLAG_MAX_SHIFT; i++) {
+ bit = (1U << i);
+
+ if (flags & bit) {
+ switch (bit) {
+ case RSPAMD_TASK_FLAG_PASS_ALL:
+ lua_pushstring(L, "pass_all");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_NO_LOG:
+ lua_pushstring(L, "no_log");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_NO_STAT:
+ lua_pushstring(L, "no_stat");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_SKIP:
+ lua_pushstring(L, "skip");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_BROKEN_HEADERS:
+ lua_pushstring(L, "broken_headers");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_LEARN_SPAM:
+ lua_pushstring(L, "learn_spam");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_LEARN_HAM:
+ lua_pushstring(L, "learn_ham");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_GREYLISTED:
+ lua_pushstring(L, "greylisted");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_SKIP_PROCESS:
+ lua_pushstring(L, "skip_process");
+ lua_rawseti(L, -2, idx++);
+ break;
+ case RSPAMD_TASK_FLAG_MESSAGE_REWRITE:
+ lua_pushstring(L, "message_rewrite");
+ lua_rawseti(L, -2, idx++);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) {
+ lua_pushstring(L, "milter");
+ lua_rawseti(L, -2, idx++);
+ }
+ if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_BODY_BLOCK) {
+ lua_pushstring(L, "body_block");
+ lua_rawseti(L, -2, idx++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_digest(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gchar hexbuf[sizeof(MESSAGE_FIELD(task, digest)) * 2 + 1];
+ gint r;
+
+ if (task) {
+ if (task->message) {
+ r = rspamd_encode_hex_buf(MESSAGE_FIELD(task, digest),
+ sizeof(MESSAGE_FIELD(task, digest)),
+ hexbuf, sizeof(hexbuf) - 1);
+
+ if (r > 0) {
+ hexbuf[r] = '\0';
+ lua_pushstring(L, hexbuf);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_learn(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean is_spam = FALSE;
+ const gchar *clname = NULL;
+ GError *err = NULL;
+ int ret = 1;
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ is_spam = lua_toboolean(L, 2);
+ if (lua_gettop(L) > 2) {
+ clname = luaL_checkstring(L, 3);
+ }
+
+ if (!rspamd_learn_task_spam(task, is_spam, clname, &err)) {
+ lua_pushboolean(L, FALSE);
+ if (err != NULL) {
+ lua_pushstring(L, err->message);
+ ret = 2;
+ }
+ }
+ else {
+ lua_pushboolean(L, TRUE);
+ }
+
+ return ret;
+}
+
+static gint
+lua_task_set_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ ucl_object_t *settings;
+ const ucl_object_t *act, *metric_elt, *vars, *cur;
+ ucl_object_iter_t it = NULL;
+ struct rspamd_scan_result *mres;
+ guint i;
+
+ settings = ucl_object_lua_import(L, 2);
+
+ if (settings != NULL && task != NULL) {
+
+ if (task->settings) {
+ /* Do not allow to set settings on top of the existing ones */
+ ucl_object_unref(settings);
+
+ return luaL_error(L, "invalid invocation: settings has been already set");
+ }
+
+ metric_elt = ucl_object_lookup(settings, DEFAULT_METRIC);
+
+ if (metric_elt) {
+ task->settings = ucl_object_ref(metric_elt);
+ ucl_object_unref(settings);
+ }
+ else {
+ task->settings = settings;
+ }
+
+ act = ucl_object_lookup(task->settings, "actions");
+
+ if (act && ucl_object_type(act) == UCL_OBJECT) {
+ /* Adjust desired actions */
+ mres = task->result;
+
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(act, &it, true)) != NULL) {
+ const gchar *act_name = ucl_object_key(cur);
+ struct rspamd_action_config *action_config = NULL;
+ double act_score;
+ enum rspamd_action_type act_type;
+
+ if (!rspamd_action_from_str(act_name, &act_type)) {
+ act_type = -1;
+ }
+
+ for (i = 0; i < mres->nactions; i++) {
+ struct rspamd_action_config *cur_act = &mres->actions_config[i];
+
+ if (cur_act->action->action_type == METRIC_ACTION_CUSTOM &&
+ act_type == -1) {
+ /* Compare by name */
+ if (g_ascii_strcasecmp(act_name, cur_act->action->name) == 0) {
+ action_config = cur_act;
+ break;
+ }
+ }
+ else {
+ if (cur_act->action->action_type == act_type) {
+ action_config = cur_act;
+ break;
+ }
+ }
+ }
+
+ if (!action_config) {
+ act_score = ucl_object_todouble(cur);
+ if (!isnan(act_score)) {
+ struct rspamd_action *new_act;
+
+ new_act = rspamd_config_get_action(task->cfg, act_name);
+
+ if (new_act == NULL) {
+ /* New action! */
+ msg_info_task("added new action %s with threshold %.2f "
+ "due to settings",
+ act_name,
+ act_score);
+ new_act = rspamd_mempool_alloc0(task->task_pool,
+ sizeof(*new_act));
+ new_act->name = rspamd_mempool_strdup(task->task_pool, act_name);
+ new_act->action_type = METRIC_ACTION_CUSTOM;
+ new_act->threshold = act_score;
+ }
+ else {
+ /* A disabled action that is enabled */
+ msg_info_task("enabled disabled action %s with threshold %.2f "
+ "due to settings",
+ act_name,
+ act_score);
+ }
+
+ /* Insert it to the mres structure */
+ gsize new_actions_cnt = mres->nactions + 1;
+ struct rspamd_action_config *old_actions = mres->actions_config;
+
+ mres->actions_config = rspamd_mempool_alloc(task->task_pool,
+ sizeof(struct rspamd_action_config) * new_actions_cnt);
+ memcpy(mres->actions_config, old_actions,
+ sizeof(struct rspamd_action_config) * mres->nactions);
+ mres->actions_config[mres->nactions].action = new_act;
+ mres->actions_config[mres->nactions].cur_limit = act_score;
+ mres->nactions++;
+ }
+ /* Disabled/missing action is disabled one more time, not an error */
+ }
+ else {
+ /* Found the existing configured action */
+ if (ucl_object_type(cur) == UCL_NULL) {
+ /* Disable action completely */
+ action_config->flags |= RSPAMD_ACTION_RESULT_DISABLED;
+ msg_info_task("disabled action %s due to settings",
+ action_config->action->name);
+ }
+ else {
+ act_score = ucl_object_todouble(cur);
+ if (isnan(act_score)) {
+ msg_info_task("disabled action %s threshold (was %.2f) due to settings",
+ action_config->action->name,
+ action_config->cur_limit);
+ action_config->flags |= RSPAMD_ACTION_RESULT_NO_THRESHOLD;
+ }
+ else {
+ action_config->cur_limit = act_score;
+ msg_debug_task("adjusted action %s: %.2f -> %.2f",
+ act_name,
+ action_config->cur_limit,
+ act_score);
+ }
+ }
+ }
+ }
+ }
+
+ vars = ucl_object_lookup(task->settings, "variables");
+ if (vars && ucl_object_type(vars) == UCL_OBJECT) {
+ /* Set memory pool variables */
+ it = NULL;
+
+ while ((cur = ucl_object_iterate(vars, &it, true)) != NULL) {
+ if (ucl_object_type(cur) == UCL_STRING) {
+ rspamd_mempool_set_variable(task->task_pool,
+ ucl_object_key(cur), rspamd_mempool_strdup(task->task_pool, ucl_object_tostring(cur)), NULL);
+ }
+ }
+ }
+
+ rspamd_symcache_process_settings(task, task->cfg->cache);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_set_milter_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ ucl_object_t *reply, *prev;
+
+ reply = ucl_object_lua_import(L, 2);
+
+ if (reply != NULL && task != NULL) {
+ prev = rspamd_mempool_get_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MILTER_REPLY);
+
+ if (prev) {
+ /*
+ * We need to be very special about the add_headers part
+ * If we want to insert some existing object, such as
+ * add_headers = {
+ * hdr = {value = val1, order = 1},
+ * }
+ *
+ * and new header has something similar:
+ * add_headers = {
+ * hdr = {value = val2, order = 1},
+ * }
+ *
+ * then we need to convert it to an array...
+ *
+ * add_headers = {
+ * hdr = [{value = val1, order = 1}, {value = val2, order = 1}],
+ * }
+ *
+ * UCL itself cannot do it directly. So the trick is to extract the
+ * original object, pack it into an array and then insert it back.
+ *
+ * I wish there was a simpler way to do it...
+ */
+ const ucl_object_t *add_hdrs = ucl_object_lookup(prev, "add_headers");
+ const ucl_object_t *nadd_hdrs = ucl_object_lookup(reply, "add_headers");
+
+ if (add_hdrs && nadd_hdrs) {
+ ucl_object_iter_t it = NULL;
+ const ucl_object_t *cur;
+
+ while ((cur = ucl_object_iterate(nadd_hdrs, &it, true)) != NULL) {
+ gsize klen;
+ const gchar *key = ucl_object_keyl(cur, &klen);
+ const ucl_object_t *existing;
+
+ existing = ucl_object_lookup_len(add_hdrs, key, klen);
+
+ if (existing && ucl_object_type(existing) != UCL_ARRAY) {
+ ucl_object_t *ar = ucl_object_typed_new(UCL_ARRAY);
+
+ ucl_array_append(ar, ucl_object_ref(existing));
+ /* Avoid double refcount */
+ key = ucl_object_keyl(existing, &klen);
+ ucl_object_delete_keyl((ucl_object_t *) add_hdrs, key, klen);
+ ucl_object_insert_key((ucl_object_t *) add_hdrs,
+ ar, key, klen, false);
+ }
+ }
+ }
+
+ if (!ucl_object_merge(prev, reply, false)) {
+ msg_err_task("internal error: cannot merge two objects when setting milter reply!");
+ }
+ ucl_object_unref(reply);
+ }
+ else {
+ rspamd_mempool_set_variable(task->task_pool,
+ RSPAMD_MEMPOOL_MILTER_REPLY,
+ reply,
+ (rspamd_mempool_destruct_t) ucl_object_unref);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+
+ if (task->settings) {
+ return ucl_object_push_lua(L, task->settings, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_lookup_settings(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = NULL;
+ const ucl_object_t *elt;
+
+ if (task != NULL) {
+
+ if (lua_isstring(L, 2)) {
+ key = lua_tostring(L, 2);
+ }
+
+ if (task->settings) {
+ if (key == NULL) {
+ return ucl_object_push_lua(L, task->settings, true);
+ }
+ else {
+ elt = ucl_object_lookup(task->settings, key);
+
+ if (elt) {
+ return ucl_object_push_lua(L, elt, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task != NULL) {
+
+ if (task->settings_elt) {
+ lua_pushinteger(L, task->settings_elt->id);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_settings_id(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint32 id = lua_tointeger(L, 2);
+
+ if (task != NULL && id != 0) {
+
+ struct rspamd_config_settings_elt *selt =
+ rspamd_config_find_settings_id_ref(task->cfg, id);
+
+ if (selt == NULL) {
+ return luaL_error(L, "settings id %f is unknown", (lua_Number) id);
+ }
+ if (task->settings_elt) {
+ /* Overwrite existing settings from Lua */
+ REF_RELEASE(task->settings_elt);
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ task->settings_elt = selt;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_cache_get(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = luaL_checkstring(L, 2);
+
+ if (task && key) {
+ if (!lua_task_get_cached(L, task, key)) {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_cache_set(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *key = luaL_checkstring(L, 2);
+
+ if (task && key && lua_gettop(L) >= 3) {
+ lua_task_set_cached(L, task, key, 3);
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+struct lua_file_cbdata {
+ gchar *fname;
+ gint fd;
+ gboolean keep;
+};
+
+static void
+lua_tmp_file_dtor(gpointer p)
+{
+ struct lua_file_cbdata *cbdata = p;
+
+ if (!cbdata->keep) {
+ unlink(cbdata->fname);
+ }
+
+ close(cbdata->fd);
+}
+
+static gint
+lua_task_store_in_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gboolean force_new = FALSE, keep = FALSE;
+ gchar fpath[PATH_MAX];
+ const gchar *tmpmask = NULL, *fname = NULL;
+ guint mode = 00600;
+ gint fd;
+ struct lua_file_cbdata *cbdata;
+ GError *err = NULL;
+
+ if (task) {
+ if (lua_istable(L, 2)) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "filename=S;tmpmask=S;mode=I;force_new=B;keep=B",
+ &fname, &tmpmask, &mode, &force_new, &keep)) {
+ msg_err_task("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ if (!force_new && (task->flags & RSPAMD_TASK_FLAG_FILE) &&
+ task->msg.fpath) {
+ lua_pushstring(L, task->msg.fpath);
+ }
+ else {
+ if (fname == NULL) {
+ if (tmpmask == NULL) {
+ rspamd_snprintf(fpath, sizeof(fpath), "%s%c%s",
+ task->cfg->temp_dir,
+ G_DIR_SEPARATOR, "rmsg-XXXXXXXXXX");
+ }
+ else {
+ rspamd_snprintf(fpath, sizeof(fpath), "%s", tmpmask);
+ }
+
+ fd = g_mkstemp_full(fpath, O_WRONLY | O_CREAT | O_EXCL, mode);
+ fname = fpath;
+
+ if (fd != -1) {
+ fchmod(fd, mode);
+ }
+ }
+ else {
+ fd = rspamd_file_xopen(fname, O_WRONLY | O_CREAT | O_EXCL,
+ (guint) mode, FALSE);
+ }
+
+ if (fd == -1) {
+ msg_err_task("cannot save file: %s", strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ if (write(fd, task->msg.begin, task->msg.len) == -1) {
+ msg_err_task("cannot write file %s: %s", fpath,
+ strerror(errno));
+ unlink(fname);
+ close(fd);
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ cbdata = rspamd_mempool_alloc(task->task_pool, sizeof(*cbdata));
+ cbdata->fd = fd;
+ cbdata->fname = rspamd_mempool_strdup(task->task_pool, fname);
+ cbdata->keep = keep;
+ lua_pushstring(L, cbdata->fname);
+ rspamd_mempool_add_destructor(task->task_pool,
+ lua_tmp_file_dtor, cbdata);
+ }
+ }
+ }
+ else {
+ luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_process_regexp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_regexp *re = NULL;
+ gboolean strong = FALSE;
+ const gchar *type_str = NULL, *header_str = NULL;
+ gsize header_len = 0;
+ GError *err = NULL;
+ gint ret = 0;
+ enum rspamd_re_type type = RSPAMD_RE_BODY;
+
+ /*
+ * - `re`* : regular expression object
+ * - `type`*: type of regular expression:
+ * + `mime`: mime regexp
+ * + `rawmime`: raw mime regexp
+ * + `header`: header regexp
+ * + `rawheader`: raw header expression
+ * + `body`: raw body regexp
+ * + `url`: url regexp
+ * - `header`: for header and rawheader regexp means the name of header
+ * - `strong`: case sensitive match for headers
+ */
+ if (task != NULL) {
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "*re=U{regexp};*type=S;header=V;strong=B",
+ &re, &type_str, &header_len, &header_str,
+ &strong)) {
+ msg_err_task("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ type = rspamd_re_cache_type_from_string(type_str);
+
+ if ((type == RSPAMD_RE_HEADER || type == RSPAMD_RE_RAWHEADER) && header_str == NULL) {
+ msg_err_task(
+ "header argument is mandatory for header/rawheader regexps");
+ }
+ else {
+ ret = rspamd_re_cache_process(task, re->re, type,
+ (gpointer) header_str, header_len, strong);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *metric_res;
+ struct rspamd_action *action;
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 2)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 2));
+
+ if (metric_res == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+ }
+
+ /* Fields added:
+ * - `score`: current score
+ * - `action`: current action as a string
+ * - `nnegative`: number of negative rules matched
+ * - `npositive`: number of positive rules matched
+ * - `positive_score`: total score for positive rules
+ * - `negative_score`: total score for negative rules
+ * - `passthrough`: set to true if message has a passthrough result
+ */
+ lua_createtable(L, 0, 7);
+
+ lua_pushstring(L, "score");
+ lua_pushnumber(L, metric_res->score);
+ lua_settable(L, -3);
+
+ action = rspamd_check_action_metric(task, NULL, metric_res);
+
+ if (action) {
+ lua_pushstring(L, "action");
+ lua_pushstring(L, action->name);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "nnegative");
+ lua_pushnumber(L, metric_res->nnegative);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "npositive");
+ lua_pushnumber(L, metric_res->npositive);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "positive_score");
+ lua_pushnumber(L, metric_res->positive_score);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "negative_score");
+ lua_pushnumber(L, metric_res->negative_score);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "passthrough");
+ lua_pushboolean(L, !!(metric_res->passthrough_result != NULL));
+ lua_settable(L, -3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_score(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ gdouble rs;
+ struct rspamd_scan_result *metric_res;
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 2)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (metric_res != NULL) {
+ lua_createtable(L, 2, 0);
+ lua_pushnumber(L, isnan(metric_res->score) ? 0.0 : metric_res->score);
+ rs = rspamd_task_get_required_score(task, metric_res);
+ lua_rawseti(L, -2, 1);
+ lua_pushnumber(L, rs);
+ lua_rawseti(L, -2, 2);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_metric_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_action *action;
+
+ if (task) {
+ struct rspamd_scan_result *mres = task->result;
+
+ if (lua_isstring(L, 2)) {
+ mres = rspamd_find_metric_result(task, lua_tostring(L, 2));
+ }
+
+ if (mres == NULL) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ action = rspamd_check_action_metric(task, NULL, mres);
+ lua_pushstring(L, action->name);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_metric_score(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_scan_result *metric_res;
+ gdouble nscore;
+
+ if (lua_isnumber(L, 2)) {
+ nscore = luaL_checknumber(L, 2);
+ }
+ else {
+ nscore = luaL_checknumber(L, 3);
+ }
+
+ if (task) {
+ metric_res = task->result;
+
+ if (lua_isstring(L, 4)) {
+ metric_res = rspamd_find_metric_result(task, lua_tostring(L, 4));
+ }
+
+ if (metric_res != NULL) {
+ msg_debug_task("set metric score from %.2f to %.2f",
+ metric_res->score, nscore);
+ metric_res->score = nscore;
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_disable_action(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *action_name;
+ struct rspamd_action_config *action_res;
+
+ action_name = luaL_checkstring(L, 2);
+
+ if (task && action_name) {
+
+ for (guint i = 0; i < task->result->nactions; i++) {
+ action_res = &task->result->actions_config[i];
+
+ if (strcmp(action_name, action_res->action->name) == 0) {
+ if (isnan(action_res->cur_limit)) {
+ lua_pushboolean(L, false);
+ }
+ else {
+ action_res->cur_limit = NAN;
+ lua_pushboolean(L, true);
+ }
+
+ break;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_newlines_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ if (task->message) {
+ switch (MESSAGE_FIELD(task, nlines_type)) {
+ case RSPAMD_TASK_NEWLINES_CR:
+ lua_pushstring(L, "cr");
+ break;
+ case RSPAMD_TASK_NEWLINES_LF:
+ lua_pushstring(L, "lf");
+ break;
+ case RSPAMD_TASK_NEWLINES_CRLF:
+ default:
+ lua_pushstring(L, "crlf");
+ break;
+ }
+ }
+ else {
+ lua_pushstring(L, "crlf");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static void
+lua_push_stat_token(lua_State *L, rspamd_token_t *tok)
+{
+ gchar numbuf[64];
+
+ /* Table values
+ * - `data`: 64 bit number encoded as a string
+ * - `t1`: the first token (if any)
+ * - `t2`: the second token (if any)
+ * - `win`: window index
+ * - `flag`: table of strings:
+ * - `text`: text token
+ * - `meta`: meta token
+ * - `lua`: lua meta token
+ * - `exception`: exception
+ * - `subject`: subject token
+ * - `unigram`: unigram token
+ */
+ lua_createtable(L, 0, 5);
+
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%uL", tok->data);
+ lua_pushstring(L, "data");
+ lua_pushstring(L, numbuf);
+ lua_settable(L, -3);
+
+ if (tok->t1) {
+ lua_pushstring(L, "t1");
+ lua_pushlstring(L, tok->t1->stemmed.begin, tok->t1->stemmed.len);
+ lua_settable(L, -3);
+ }
+
+ if (tok->t2) {
+ lua_pushstring(L, "t2");
+ lua_pushlstring(L, tok->t2->stemmed.begin, tok->t2->stemmed.len);
+ lua_settable(L, -3);
+ }
+
+ lua_pushstring(L, "win");
+ lua_pushinteger(L, tok->window_idx);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "flags");
+ lua_createtable(L, 0, 5);
+
+ /* Flags */
+ {
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT) {
+ lua_pushstring(L, "text");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_META) {
+ lua_pushstring(L, "meta");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_LUA_META) {
+ lua_pushstring(L, "lua");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION) {
+ lua_pushstring(L, "exception");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ if (tok->flags & RSPAMD_STAT_TOKEN_FLAG_HEADER) {
+ lua_pushstring(L, "header");
+ lua_pushboolean(L, true);
+ lua_settable(L, -3);
+ }
+ }
+ lua_settable(L, -3);
+}
+
+static gint
+lua_task_get_stat_tokens(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint i;
+ rspamd_token_t *tok;
+
+ if (task) {
+ if (!task->tokens) {
+ rspamd_stat_process_tokenize(NULL, task);
+ }
+
+ if (!task->tokens) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_createtable(L, task->tokens->len, 0);
+
+ PTR_ARRAY_FOREACH(task->tokens, i, tok)
+ {
+ lua_push_stat_token(L, tok);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_set_metric_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *subject;
+
+ subject = luaL_checkstring(L, 2);
+
+ if (task && subject) {
+ rspamd_mempool_set_variable(task->task_pool, "metric_subject",
+ rspamd_mempool_strdup(task->task_pool, subject), NULL);
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_protocol_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ guint flags = 0;
+ ucl_object_t *obj;
+
+ if (!task) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (!(task->processed_stages & (RSPAMD_TASK_STAGE_POST_FILTERS >> 1))) {
+ return luaL_error(L, "must not be called before post-filters");
+ }
+
+ if (lua_istable(L, 2)) {
+ for (lua_pushnil(L); lua_next(L, 2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ const gchar *str = lua_tostring(L, -1);
+
+ if (strcmp(str, "default") == 0) {
+ flags |= RSPAMD_PROTOCOL_DEFAULT;
+ }
+ else if (strcmp(str, "basic") == 0) {
+ flags |= RSPAMD_PROTOCOL_BASIC;
+ }
+ else if (strcmp(str, "metrics") == 0) {
+ flags |= RSPAMD_PROTOCOL_METRICS;
+ }
+ else if (strcmp(str, "messages") == 0) {
+ flags |= RSPAMD_PROTOCOL_MESSAGES;
+ }
+ else if (strcmp(str, "rmilter") == 0) {
+ flags |= RSPAMD_PROTOCOL_RMILTER;
+ }
+ else if (strcmp(str, "dkim") == 0) {
+ flags |= RSPAMD_PROTOCOL_DKIM;
+ }
+ else if (strcmp(str, "extra") == 0) {
+ flags |= RSPAMD_PROTOCOL_EXTRA;
+ }
+ else {
+ msg_err_task("invalid protocol flag: %s", str);
+ }
+ }
+ }
+ }
+ else {
+ flags = RSPAMD_PROTOCOL_DEFAULT;
+ }
+
+ obj = rspamd_protocol_write_ucl(task, flags);
+
+ if (obj) {
+ ucl_object_push_lua(L, obj, true);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_headers_foreach(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ enum rspamd_lua_task_header_type how = RSPAMD_TASK_HEADER_PUSH_SIMPLE;
+ struct rspamd_lua_regexp *re = NULL;
+ struct rspamd_mime_header *hdr, *cur;
+ gint old_top;
+
+ if (task && lua_isfunction(L, 2)) {
+ if (task->message) {
+ if (lua_istable(L, 3)) {
+ lua_pushstring(L, "full");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_FULL;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "raw");
+ lua_gettable(L, 3);
+
+ if (lua_isboolean(L, -1) && lua_toboolean(L, -1)) {
+ how = RSPAMD_TASK_HEADER_PUSH_RAW;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "regexp");
+ lua_gettable(L, 3);
+
+ if (lua_isuserdata(L, -1)) {
+ RSPAMD_LUA_CHECK_UDATA_PTR_OR_RETURN(L, -1, "rspamd{regexp}",
+ struct rspamd_lua_regexp, re);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ if (MESSAGE_FIELD(task, headers_order)) {
+ hdr = MESSAGE_FIELD(task, headers_order);
+
+ LL_FOREACH2(hdr, cur, ord_next)
+ {
+ if (re && re->re) {
+ if (!rspamd_regexp_match(re->re, cur->name,
+ strlen(cur->name), FALSE)) {
+ continue;
+ }
+ }
+
+ old_top = lua_gettop(L);
+ lua_pushvalue(L, 2);
+ lua_pushstring(L, cur->name);
+ rspamd_lua_push_header(L, cur, how);
+
+ if (lua_pcall(L, 2, LUA_MULTRET, 0) != 0) {
+ msg_err("call to header_foreach failed: %s",
+ lua_tostring(L, -1));
+ lua_settop(L, old_top);
+ break;
+ }
+ else {
+ if (lua_gettop(L) > old_top) {
+ if (lua_isboolean(L, old_top + 1)) {
+ if (lua_toboolean(L, old_top + 1)) {
+ lua_settop(L, old_top);
+ break;
+ }
+ }
+ }
+ }
+
+ lua_settop(L, old_top);
+ }
+ }
+ } /* if (task->message) */
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_modify_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *hname = luaL_checkstring(L, 2);
+
+ if (hname && task && lua_type(L, 3) == LUA_TTABLE) {
+ if (task->message) {
+ ucl_object_t *mods = ucl_object_lua_import(L, 3);
+
+ rspamd_message_set_modified_header(task,
+ MESSAGE_FIELD(task, raw_headers),
+ hname,
+ mods,
+ &(MESSAGE_FIELD(task, headers_order)));
+ ucl_object_unref(mods);
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_get_meta_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ enum rspamd_lua_words_type how = RSPAMD_LUA_WORDS_STEM;
+
+ if (task == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (task->meta_words == NULL) {
+ lua_createtable(L, 0, 0);
+ }
+ else {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 2);
+
+ if (strcmp(how_str, "stem") == 0) {
+ how = RSPAMD_LUA_WORDS_STEM;
+ }
+ else if (strcmp(how_str, "norm") == 0) {
+ how = RSPAMD_LUA_WORDS_NORM;
+ }
+ else if (strcmp(how_str, "raw") == 0) {
+ how = RSPAMD_LUA_WORDS_RAW;
+ }
+ else if (strcmp(how_str, "full") == 0) {
+ how = RSPAMD_LUA_WORDS_FULL;
+ }
+ else {
+ return luaL_error(L, "unknown words type: %s", how_str);
+ }
+ }
+
+ return rspamd_lua_push_words(L, task->meta_words, how);
+ }
+
+ return 1;
+}
+
+static guint
+lua_lookup_words_array(lua_State *L,
+ gint cbpos,
+ struct rspamd_task *task,
+ struct rspamd_lua_map *map,
+ GArray *words)
+{
+ rspamd_stat_token_t *tok;
+ guint i, nmatched = 0;
+ gint err_idx;
+ gboolean matched;
+ const gchar *key;
+ gsize keylen;
+
+ for (i = 0; i < words->len; i++) {
+ tok = &g_array_index(words, rspamd_stat_token_t, i);
+
+ matched = FALSE;
+
+ if (tok->normalized.len == 0) {
+ continue;
+ }
+
+ key = tok->normalized.begin;
+ keylen = tok->normalized.len;
+
+ switch (map->type) {
+ case RSPAMD_LUA_MAP_SET:
+ case RSPAMD_LUA_MAP_HASH:
+ /* We know that tok->normalized is zero terminated in fact */
+ if (rspamd_match_hash_map(map->data.hash, key, keylen)) {
+ matched = TRUE;
+ }
+ break;
+ case RSPAMD_LUA_MAP_REGEXP:
+ case RSPAMD_LUA_MAP_REGEXP_MULTIPLE:
+ if (rspamd_match_regexp_map_single(map->data.re_map, key,
+ keylen)) {
+ matched = TRUE;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ break;
+ }
+
+ if (matched) {
+ nmatched++;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_pushvalue(L, cbpos); /* Function */
+ rspamd_lua_push_full_word(L, tok);
+
+ if (lua_pcall(L, 1, 0, err_idx) != 0) {
+ msg_err_task("cannot call callback function for lookup words: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+ }
+ }
+
+ return nmatched;
+}
+
+static gint
+lua_task_lookup_words(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ struct rspamd_lua_map *map = lua_check_map(L, 2);
+ struct rspamd_mime_text_part *tp;
+
+ guint i, matches = 0;
+
+ if (task == NULL || map == NULL || task->message == NULL || lua_type(L, 3) != LUA_TFUNCTION) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (map->type != RSPAMD_LUA_MAP_SET &&
+ map->type != RSPAMD_LUA_MAP_REGEXP &&
+ map->type != RSPAMD_LUA_MAP_HASH &&
+ map->type != RSPAMD_LUA_MAP_REGEXP_MULTIPLE) {
+ return luaL_error(L, "invalid map type");
+ }
+
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+ {
+ if (tp->utf_words) {
+ matches += lua_lookup_words_array(L, 3, task, map, tp->utf_words);
+ }
+ }
+
+ if (task->meta_words) {
+ matches += lua_lookup_words_array(L, 3, task, map, task->meta_words);
+ }
+
+ lua_pushinteger(L, matches);
+
+ return 1;
+}
+
+static gint
+lua_task_topointer(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ /* XXX: this might cause issues on arm64 and LuaJIT */
+ lua_pushlightuserdata(L, task);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_task_add_named_result(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+ const gchar *name = luaL_checkstring(L, 2);
+ gint cbref;
+
+ if (task && name && lua_isfunction(L, 3)) {
+ lua_pushvalue(L, 3);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ rspamd_create_metric_result(task, name, cbref);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_task_get_all_named_results(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_task *task = lua_check_task(L, 1);
+
+ if (task) {
+ gint n = 0;
+ struct rspamd_scan_result *res;
+
+ DL_COUNT(task->result, res, n);
+ lua_createtable(L, n, 0);
+ n = 1;
+
+ DL_FOREACH(task->result, res)
+ {
+ if (res->name != NULL) {
+ lua_pushstring(L, res->name);
+ }
+ else {
+ lua_pushstring(L, DEFAULT_METRIC);
+ }
+
+ lua_rawseti(L, -2, n++);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+/* Image functions */
+static gint
+lua_image_get_width(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->width);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_height(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->height);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushstring(L, rspamd_image_type_str(img->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ lua_pushinteger(L, img->data->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_image_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_image *img = lua_check_image(L);
+
+ if (img != NULL) {
+ if (img->filename != NULL) {
+ lua_pushlstring(L, img->filename->begin, img->filename->len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Archive methods */
+static gint
+lua_archive_get_type(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushstring(L, rspamd_archive_type_str(arch->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_files(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+ guint i, max_files = 0;
+ struct rspamd_archive_file *f;
+
+ if (arch != NULL) {
+ if (lua_isnumber(L, 2)) {
+ max_files = lua_tointeger(L, 2);
+ max_files = MIN(arch->files->len, max_files);
+ }
+ else {
+ max_files = arch->files->len;
+ }
+
+ lua_createtable(L, max_files, 0);
+
+ for (i = 0; i < max_files; i++) {
+ f = g_ptr_array_index(arch->files, i);
+
+ lua_pushlstring(L, f->fname->str, f->fname->len);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_files_full(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+ guint i, max_files = 0;
+ struct rspamd_archive_file *f;
+
+ if (arch != NULL) {
+ if (lua_isnumber(L, 2)) {
+ max_files = lua_tointeger(L, 2);
+ max_files = MIN(arch->files->len, max_files);
+ }
+ else {
+ max_files = arch->files->len;
+ }
+
+ lua_createtable(L, max_files, 0);
+
+ for (i = 0; i < max_files; i++) {
+ f = g_ptr_array_index(arch->files, i);
+
+ lua_createtable(L, 0, 4);
+
+ lua_pushstring(L, "name");
+ lua_pushlstring(L, f->fname->str, f->fname->len);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "compressed_size");
+ lua_pushinteger(L, f->compressed_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "uncompressed_size");
+ lua_pushinteger(L, f->uncompressed_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "encrypted");
+ lua_pushboolean(L, (f->flags & RSPAMD_ARCHIVE_FILE_ENCRYPTED) ? true : false);
+ lua_settable(L, -3);
+
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_encrypted(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L, (arch->flags & RSPAMD_ARCHIVE_ENCRYPTED) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_obfuscated(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L,
+ (arch->flags & RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_is_unreadable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushboolean(L, (arch->flags & RSPAMD_ARCHIVE_CANNOT_READ) ? true : false);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_size(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushinteger(L, arch->size);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_archive_get_filename(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_archive *arch = lua_check_archive(L);
+
+ if (arch != NULL) {
+ lua_pushlstring(L, arch->archive_name->begin, arch->archive_name->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Init part */
+
+static gint
+lua_load_task(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, tasklib_f);
+
+ return 1;
+}
+
+static void
+luaopen_archive(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{archive}", archivelib_m);
+ lua_pop(L, 1);
+}
+
+void luaopen_task(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{task}", tasklib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_task", lua_load_task);
+
+ luaopen_archive(L);
+}
+
+void luaopen_image(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{image}", imagelib_m);
+ lua_pop(L, 1);
+}
+
+void rspamd_lua_task_push(lua_State *L, struct rspamd_task *task)
+{
+ struct rspamd_task **ptask;
+
+ ptask = lua_newuserdata(L, sizeof(gpointer));
+ rspamd_lua_setclass(L, "rspamd{task}", -1);
+ *ptask = task;
+}
diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c
new file mode 100644
index 0000000..45faa79
--- /dev/null
+++ b/src/lua/lua_tcp.c
@@ -0,0 +1,2566 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "libserver/ssl_util.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include <math.h>
+
+static const gchar *M = "rspamd lua tcp";
+
+/***
+ * @module rspamd_tcp
+ * Rspamd TCP module represents generic TCP asynchronous client available from LUA code.
+ * This module hides all complexity: DNS resolving, sessions management, zero-copy
+ * text transfers and so on under the hood. It can work in partial or complete modes:
+ *
+ * - partial mode is used when you need to call a continuation routine each time data is available for read
+ * - complete mode calls for continuation merely when all data is read from socket (e.g. when a server sends reply and closes a connection)
+ * @example
+local logger = require "rspamd_logger"
+local tcp = require "rspamd_tcp"
+
+rspamd_config.SYM = function(task)
+
+ local function cb(err, data)
+ logger.infox('err: %1, data: %2', err, tostring(data))
+ end
+
+ tcp.request({
+ task = task,
+ host = "google.com",
+ port = 80,
+ data = {"GET / HTTP/1.0\r\n", "Host: google.com\r\n", "\r\n"},
+ callback = cb})
+end
+
+-- New TCP syntax test
+rspamd_config:register_symbol({
+ name = 'TCP_TEST',
+ type = "normal",
+ callback = function(task)
+ local logger = require "rspamd_logger"
+ local function rcpt_done_cb(err, data, conn)
+ logger.errx(task, 'RCPT: got reply: %s, error: %s', data, err)
+ conn:close()
+ end
+ local function rcpt_cb(err, conn)
+ logger.errx(task, 'written rcpt, error: %s', err)
+ conn:add_read(rcpt_done_cb, '\r\n')
+ end
+ local function from_done_cb(err, data, conn)
+ logger.errx(task, 'FROM: got reply: %s, error: %s', data, err)
+ conn:add_write(rcpt_cb, 'RCPT TO: <test@yandex.ru>\r\n')
+ end
+ local function from_cb(err, conn)
+ logger.errx(task, 'written from, error: %s', err)
+ conn:add_read(from_done_cb, '\r\n')
+ end
+ local function hello_done_cb(err, data, conn)
+ logger.errx(task, 'HELO: got reply: %s, error: %s', data, err)
+ conn:add_write(from_cb, 'MAIL FROM: <>\r\n')
+ end
+ local function hello_cb(err, conn)
+ logger.errx(task, 'written hello, error: %s', err)
+ conn:add_read(hello_done_cb, '\r\n')
+ end
+ local function init_cb(err, data, conn)
+ logger.errx(task, 'got reply: %s, error: %s', data, err)
+ conn:add_write(hello_cb, 'HELO example.com\r\n')
+ end
+ tcp.request{
+ task = task,
+ callback = init_cb,
+ stop_pattern = '\r\n',
+ host = 'mx.yandex.ru',
+ port = 25
+ }
+ end,
+ priority = 10,
+})
+ */
+
+LUA_FUNCTION_DEF(tcp, request);
+
+/***
+ * @function rspamd_tcp.connect_sync()
+ *
+ * Creates pseudo-synchronous TCP connection.
+ * Each method of the connection requiring IO, becomes a yielding point,
+ * i.e. current thread Lua thread is get suspended and resumes as soon as IO is done
+ *
+ * This class represents low-level API, using of "lua_tcp_sync" module is recommended.
+ *
+ * @example
+
+local rspamd_tcp = require "rspamd_tcp"
+local logger = require "rspamd_logger"
+
+local function http_simple_tcp_symbol(task)
+
+ local err
+ local is_ok, connection = rspamd_tcp.connect_sync {
+ task = task,
+ host = '127.0.0.1',
+ timeout = 20,
+ port = 18080,
+ ssl = false, -- If SSL connection is needed
+ ssl_verify = true, -- set to false if verify is not needed
+ }
+
+ is_ok, err = connection:write('GET /request_sync HTTP/1.1\r\nConnection: keep-alive\r\n\r\n')
+
+ logger.errx(task, 'write %1, %2', is_ok, err)
+ if not is_ok then
+ logger.errx(task, 'write error: %1', err)
+ end
+
+ local data
+ is_ok, data = connection:read_once();
+
+ logger.errx(task, 'read_once: is_ok: %1, data: %2', is_ok, data)
+
+ is_ok, err = connection:write("POST /request2 HTTP/1.1\r\n\r\n")
+ logger.errx(task, 'write[2] %1, %2', is_ok, err)
+
+ is_ok, data = connection:read_once();
+ logger.errx(task, 'read_once[2]: is_ok %1, data: %2', is_ok, data)
+
+ connection:close()
+end
+
+rspamd_config:register_symbol({
+ name = 'SIMPLE_TCP_TEST',
+ score = 1.0,
+ callback = http_simple_tcp_symbol,
+ no_squeeze = true
+})
+ *
+ */
+LUA_FUNCTION_DEF(tcp, connect_sync);
+
+/***
+ * @method tcp:close()
+ *
+ * Closes TCP connection
+ */
+LUA_FUNCTION_DEF(tcp, close);
+
+/***
+ * @method tcp:add_read(callback, [pattern])
+ *
+ * Adds new read event to the tcp connection
+ * @param {function} callback to be called when data is read
+ * @param {string} pattern optional stop pattern
+ */
+LUA_FUNCTION_DEF(tcp, add_read);
+
+/***
+ * @method tcp:add_write(callback, data)
+ *
+ * Adds new write event to the tcp connection
+ * @param {function} optional callback to be called when data is completely written
+ * @param {table/string/text} data to send to a remote server
+ */
+LUA_FUNCTION_DEF(tcp, add_write);
+
+/***
+ * @method tcp:shift_callback()
+ *
+ * Shifts the current callback and go to the next one (if any)
+ */
+LUA_FUNCTION_DEF(tcp, shift_callback);
+
+/***
+ * @method tcp:starttls([no_verify])
+ *
+ * Starts tls connection
+ * @param {boolean} no_verify used to skip ssl verification
+ */
+LUA_FUNCTION_DEF(tcp, starttls);
+
+static const struct luaL_reg tcp_libf[] = {
+ LUA_INTERFACE_DEF(tcp, request),
+ {"new", lua_tcp_request},
+ {"connect", lua_tcp_request},
+ {"connect_sync", lua_tcp_connect_sync},
+ {NULL, NULL}};
+
+static const struct luaL_reg tcp_libm[] = {
+ LUA_INTERFACE_DEF(tcp, close),
+ LUA_INTERFACE_DEF(tcp, add_read),
+ LUA_INTERFACE_DEF(tcp, add_write),
+ LUA_INTERFACE_DEF(tcp, shift_callback),
+ LUA_INTERFACE_DEF(tcp, starttls),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+/***
+ * @method tcp:close()
+ *
+ * Closes TCP connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, close);
+
+/***
+ * @method read_once()
+ *
+ * Performs one read operation. If syscall returned with EAGAIN/EINT,
+ * restarts the operation, so it always returns either data or error.
+ */
+LUA_FUNCTION_DEF(tcp_sync, read_once);
+
+/***
+ * @method eof()
+ *
+ * True if last IO operation ended with EOF, i.e. endpoint closed connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, eof);
+
+/***
+ * @method shutdown()
+ *
+ * Half-shutdown TCP connection
+ */
+LUA_FUNCTION_DEF(tcp_sync, shutdown);
+
+/***
+ * @method write()
+ *
+ * Writes data into the stream. If syscall returned with EAGAIN/EINT
+ * restarts the operation. If performs write() until all the passed
+ * data is written completely.
+ */
+LUA_FUNCTION_DEF(tcp_sync, write);
+
+LUA_FUNCTION_DEF(tcp_sync, gc);
+
+static void lua_tcp_sync_session_dtor(gpointer ud);
+
+static const struct luaL_reg tcp_sync_libm[] = {
+ LUA_INTERFACE_DEF(tcp_sync, close),
+ LUA_INTERFACE_DEF(tcp_sync, read_once),
+ LUA_INTERFACE_DEF(tcp_sync, write),
+ LUA_INTERFACE_DEF(tcp_sync, eof),
+ LUA_INTERFACE_DEF(tcp_sync, shutdown),
+ {"__gc", lua_tcp_sync_gc},
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+struct lua_tcp_read_handler {
+ gchar *stop_pattern;
+ guint plen;
+ gint cbref;
+};
+
+struct lua_tcp_write_handler {
+ struct iovec *iov;
+ guint iovlen;
+ gint cbref;
+ gsize pos;
+ gsize total_bytes;
+};
+
+enum lua_tcp_handler_type {
+ LUA_WANT_WRITE = 0,
+ LUA_WANT_READ,
+ LUA_WANT_CONNECT
+};
+
+struct lua_tcp_handler {
+ union {
+ struct lua_tcp_read_handler r;
+ struct lua_tcp_write_handler w;
+ } h;
+ enum lua_tcp_handler_type type;
+};
+
+struct lua_tcp_dtor {
+ rspamd_mempool_destruct_t dtor;
+ void *data;
+ struct lua_tcp_dtor *next;
+};
+
+#define LUA_TCP_FLAG_PARTIAL (1u << 0u)
+#define LUA_TCP_FLAG_SHUTDOWN (1u << 2u)
+#define LUA_TCP_FLAG_CONNECTED (1u << 3u)
+#define LUA_TCP_FLAG_FINISHED (1u << 4u)
+#define LUA_TCP_FLAG_SYNC (1u << 5u)
+#define LUA_TCP_FLAG_RESOLVED (1u << 6u)
+#define LUA_TCP_FLAG_SSL (1u << 7u)
+#define LUA_TCP_FLAG_SSL_NOVERIFY (1u << 8u)
+
+#undef TCP_DEBUG_REFS
+#ifdef TCP_DEBUG_REFS
+#define TCP_RETAIN(x) \
+ do { \
+ msg_info("retain ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RETAIN(x); \
+ } while (0)
+
+#define TCP_RELEASE(x) \
+ do { \
+ msg_info("release ref %p, refcount: %d", (x), (x)->ref.refcount); \
+ REF_RELEASE(x); \
+ } while (0)
+#else
+#define TCP_RETAIN(x) REF_RETAIN(x)
+#define TCP_RELEASE(x) REF_RELEASE(x)
+#endif
+
+struct lua_tcp_cbdata {
+ struct rspamd_async_session *session;
+ struct rspamd_async_event *async_ev;
+ struct ev_loop *event_loop;
+ rspamd_inet_addr_t *addr;
+ GByteArray *in;
+ GQueue *handlers;
+ gint fd;
+ gint connect_cb;
+ guint port;
+ guint flags;
+ gchar tag[7];
+ struct rspamd_io_ev ev;
+ struct lua_tcp_dtor *dtors;
+ ref_entry_t ref;
+ struct rspamd_task *task;
+ struct rspamd_symcache_dynamic_item *item;
+ struct thread_entry *thread;
+ struct rspamd_config *cfg;
+ struct rspamd_ssl_connection *ssl_conn;
+ gchar *hostname;
+ struct upstream *up;
+ gboolean eof;
+};
+
+#define IS_SYNC(c) (((c)->flags & LUA_TCP_FLAG_SYNC) != 0)
+
+#define msg_debug_tcp(...) rspamd_conditional_debug_fast(NULL, cbd->addr, \
+ rspamd_lua_tcp_log_id, "lua_tcp", cbd->tag, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_tcp)
+
+static void lua_tcp_handler(int fd, short what, gpointer ud);
+static void lua_tcp_plan_handler_event(struct lua_tcp_cbdata *cbd,
+ gboolean can_read, gboolean can_write);
+static void lua_tcp_unregister_event(struct lua_tcp_cbdata *cbd);
+
+static void
+lua_tcp_void_finalyser(gpointer arg)
+{
+}
+
+static const gdouble default_tcp_timeout = 5.0;
+
+static struct rspamd_dns_resolver *
+lua_tcp_global_resolver(struct ev_loop *ev_base,
+ struct rspamd_config *cfg)
+{
+ static struct rspamd_dns_resolver *global_resolver;
+
+ if (cfg && cfg->dns_resolver) {
+ return cfg->dns_resolver;
+ }
+
+ if (global_resolver == NULL) {
+ global_resolver = rspamd_dns_resolver_init(NULL, ev_base, cfg);
+ }
+
+ return global_resolver;
+}
+
+static gboolean
+lua_tcp_shift_handler(struct lua_tcp_cbdata *cbd)
+{
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_pop_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ /* We are done */
+ return FALSE;
+ }
+
+ if (hdl->type == LUA_WANT_READ) {
+ msg_debug_tcp("switch from read handler %d", hdl->h.r.cbref);
+ if (hdl->h.r.cbref && hdl->h.r.cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, hdl->h.r.cbref);
+ }
+
+ if (hdl->h.r.stop_pattern) {
+ g_free(hdl->h.r.stop_pattern);
+ }
+ }
+ else if (hdl->type == LUA_WANT_WRITE) {
+ msg_debug_tcp("switch from write handler %d", hdl->h.r.cbref);
+ if (hdl->h.w.cbref && hdl->h.w.cbref != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, hdl->h.w.cbref);
+ }
+
+ if (hdl->h.w.iov) {
+ g_free(hdl->h.w.iov);
+ }
+ }
+ else {
+ msg_debug_tcp("removing connect handler");
+ /* LUA_WANT_CONNECT: it doesn't allocate anything, nothing to do here */
+ }
+
+ g_free(hdl);
+
+ return TRUE;
+}
+
+static void
+lua_tcp_fin(gpointer arg)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) arg;
+ struct lua_tcp_dtor *dtor, *dttmp;
+
+ if (IS_SYNC(cbd) && cbd->task) {
+ /*
+ pointer is now becoming invalid, we should remove registered destructor,
+ all the necessary steps are done here
+ */
+ rspamd_mempool_replace_destructor(cbd->task->task_pool,
+ lua_tcp_sync_session_dtor, cbd, NULL);
+ }
+
+ msg_debug_tcp("finishing TCP %s connection", IS_SYNC(cbd) ? "sync" : "async");
+
+ if (cbd->connect_cb != -1) {
+ luaL_unref(cbd->cfg->lua_state, LUA_REGISTRYINDEX, cbd->connect_cb);
+ }
+
+ if (cbd->ssl_conn) {
+ /* TODO: postpone close in case ssl is used ! */
+ rspamd_ssl_connection_free(cbd->ssl_conn);
+ }
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ }
+
+ while (lua_tcp_shift_handler(cbd)) {}
+ g_queue_free(cbd->handlers);
+
+ LL_FOREACH_SAFE(cbd->dtors, dtor, dttmp)
+ {
+ dtor->dtor(dtor->data);
+ g_free(dtor);
+ }
+
+ g_byte_array_unref(cbd->in);
+ g_free(cbd->hostname);
+ g_free(cbd);
+}
+
+static struct lua_tcp_cbdata *
+lua_check_tcp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{tcp}");
+ luaL_argcheck(L, ud != NULL, pos, "'tcp' expected");
+ return ud ? *((struct lua_tcp_cbdata **) ud) : NULL;
+}
+
+static void
+lua_tcp_maybe_free(struct lua_tcp_cbdata *cbd)
+{
+ if (IS_SYNC(cbd)) {
+ /*
+ * in this mode, we don't remove object, we only remove the event
+ * Object is owned by lua and will be destroyed on __gc()
+ */
+
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->session, lua_tcp_void_finalyser, cbd);
+ }
+
+ cbd->async_ev = NULL;
+ }
+ else {
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->session, lua_tcp_fin, cbd);
+ }
+ else {
+ lua_tcp_fin(cbd);
+ }
+ }
+}
+
+#ifdef __GNUC__
+static void
+lua_tcp_push_error(struct lua_tcp_cbdata *cbd, gboolean is_fatal,
+ const char *err, ...) __attribute__((format(printf, 3, 4)));
+#endif
+
+static void lua_tcp_resume_thread_error_argp(struct lua_tcp_cbdata *cbd, const gchar *error, va_list argp);
+
+static void
+lua_tcp_push_error(struct lua_tcp_cbdata *cbd, gboolean is_fatal,
+ const char *err, ...)
+{
+ va_list ap, ap_copy;
+ struct lua_tcp_cbdata **pcbd;
+ struct lua_tcp_handler *hdl;
+ gint cbref, top;
+ struct lua_callback_state cbs;
+ lua_State *L;
+ gboolean callback_called = FALSE;
+
+ if (is_fatal && cbd->up) {
+ rspamd_upstream_fail(cbd->up, false, err);
+ }
+
+ if (cbd->thread) {
+ va_start(ap, err);
+ lua_tcp_resume_thread_error_argp(cbd, err, ap);
+ va_end(ap);
+
+ return;
+ }
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ va_start(ap, err);
+
+ for (;;) {
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ break;
+ }
+
+ if (hdl->type == LUA_WANT_READ) {
+ cbref = hdl->h.r.cbref;
+ }
+ else {
+ cbref = hdl->h.w.cbref;
+ }
+
+ if (cbref != -1) {
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+
+ /* Error message */
+ va_copy(ap_copy, ap);
+ lua_pushvfstring(L, err, ap_copy);
+ va_end(ap_copy);
+
+ /* Body */
+ lua_pushnil(L);
+ /* Connection */
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+ TCP_RETAIN(cbd);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 3, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+
+ TCP_RELEASE(cbd);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+
+ callback_called = TRUE;
+ }
+
+ if (!is_fatal) {
+ if (callback_called) {
+ /* Stop on the first callback found */
+ break;
+ }
+ else {
+ /* Shift to another callback to inform about non fatal error */
+ msg_debug_tcp("non fatal error find matching callback");
+ lua_tcp_shift_handler(cbd);
+ continue;
+ }
+ }
+ else {
+ msg_debug_tcp("fatal error rollback all handlers");
+ lua_tcp_shift_handler(cbd);
+ }
+ }
+
+ va_end(ap);
+
+ lua_thread_pool_restore_callback(&cbs);
+}
+
+static void lua_tcp_resume_thread(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len);
+
+static void
+lua_tcp_push_data(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len)
+{
+ struct rspamd_lua_text *t;
+ struct lua_tcp_cbdata **pcbd;
+ struct lua_tcp_handler *hdl;
+ gint cbref, arg_cnt, top;
+ struct lua_callback_state cbs;
+ lua_State *L;
+
+ if (cbd->thread) {
+ lua_tcp_resume_thread(cbd, str, len);
+ return;
+ }
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL);
+
+ if (hdl->type == LUA_WANT_READ) {
+ cbref = hdl->h.r.cbref;
+ }
+ else {
+ cbref = hdl->h.w.cbref;
+ }
+
+ if (cbref != -1) {
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbref);
+ /* Error */
+ lua_pushnil(L);
+ /* Body */
+
+ if (hdl->type == LUA_WANT_READ) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = (const gchar *) str;
+ t->len = len;
+ t->flags = 0;
+ arg_cnt = 3;
+ }
+ else {
+ arg_cnt = 2;
+ }
+ /* Connection */
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+
+ TCP_RETAIN(cbd);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, arg_cnt, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ TCP_RELEASE(cbd);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_thread_pool_restore_callback(&cbs);
+}
+
+static void
+lua_tcp_resume_thread_error_argp(struct lua_tcp_cbdata *cbd, const gchar *error, va_list argp)
+{
+ struct thread_entry *thread = cbd->thread;
+ lua_State *L = thread->lua_state;
+
+ lua_pushboolean(L, FALSE);
+ lua_pushvfstring(L, error, argp);
+
+ lua_tcp_shift_handler(cbd);
+ // lua_tcp_unregister_event (cbd);
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool, cbd->thread);
+ lua_thread_resume(thread, 2);
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_resume_thread(struct lua_tcp_cbdata *cbd, const guint8 *str, gsize len)
+{
+ /*
+ * typical call returns:
+ *
+ * read:
+ * error:
+ * (nil, error message)
+ * got data:
+ * (true, data)
+ * write/connect:
+ * error:
+ * (nil, error message)
+ * wrote
+ * (true)
+ */
+
+ lua_State *L = cbd->thread->lua_state;
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ lua_pushboolean(L, TRUE);
+ if (hdl->type == LUA_WANT_READ) {
+ lua_pushlstring(L, str, len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ lua_tcp_shift_handler(cbd);
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool,
+ cbd->thread);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ lua_thread_resume(cbd->thread, 2);
+
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_plan_read(struct lua_tcp_cbdata *cbd)
+{
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+}
+
+static void
+lua_tcp_connect_helper(struct lua_tcp_cbdata *cbd)
+{
+ /* This is used for sync mode only */
+ lua_State *L = cbd->thread->lua_state;
+
+ struct lua_tcp_cbdata **pcbd;
+
+ lua_pushboolean(L, TRUE);
+
+ lua_thread_pool_set_running_entry(cbd->cfg->lua_thread_pool, cbd->thread);
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ rspamd_lua_setclass(L, "rspamd{tcp_sync}", -1);
+ msg_debug_tcp("tcp connected");
+
+ lua_tcp_shift_handler(cbd);
+
+ // lua_tcp_unregister_event (cbd);
+ lua_thread_resume(cbd->thread, 2);
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_write_helper(struct lua_tcp_cbdata *cbd)
+{
+ struct iovec *start;
+ guint niov, i;
+ gint flags = 0;
+ bool allocated_iov = false;
+ gsize remain;
+ gssize r;
+ struct iovec *cur_iov;
+ struct lua_tcp_handler *hdl;
+ struct lua_tcp_write_handler *wh;
+ struct msghdr msg;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL && hdl->type == LUA_WANT_WRITE);
+ wh = &hdl->h.w;
+
+ if (wh->pos == wh->total_bytes) {
+ goto call_finish_handler;
+ }
+
+ start = &wh->iov[0];
+ niov = wh->iovlen;
+ remain = wh->pos;
+ /* We know that niov is small enough for that */
+
+ if (niov < 1024) {
+ cur_iov = g_alloca(niov * sizeof(struct iovec));
+ }
+ else {
+ cur_iov = g_malloc0(niov * sizeof(struct iovec));
+ allocated_iov = true;
+ }
+
+ memcpy(cur_iov, wh->iov, niov * sizeof(struct iovec));
+
+ for (i = 0; i < wh->iovlen && remain > 0; i++) {
+ /* Find out the first iov required */
+ start = &cur_iov[i];
+ if (start->iov_len <= remain) {
+ remain -= start->iov_len;
+ start = &cur_iov[i + 1];
+ niov--;
+ }
+ else {
+ start->iov_base = (void *) ((char *) start->iov_base + remain);
+ start->iov_len -= remain;
+ remain = 0;
+ }
+ }
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = start;
+ msg.msg_iovlen = MIN(IOV_MAX, niov);
+ g_assert(niov > 0);
+#ifdef MSG_NOSIGNAL
+ flags = MSG_NOSIGNAL;
+#endif
+
+ msg_debug_tcp("want write %d io vectors of %d", (int) msg.msg_iovlen,
+ (int) niov);
+
+ if (cbd->ssl_conn) {
+ r = rspamd_ssl_writev(cbd->ssl_conn, msg.msg_iov, msg.msg_iovlen);
+ }
+ else {
+ r = sendmsg(cbd->fd, &msg, flags);
+ }
+
+ if (allocated_iov) {
+ g_free(cur_iov);
+ }
+
+ if (r == -1) {
+ if (!(cbd->ssl_conn)) {
+ if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
+ msg_debug_tcp("got temporary failure, retry write");
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ return;
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE,
+ "IO write error while trying to write %d bytes: %s",
+ (gint) remain, strerror(errno));
+
+ msg_debug_tcp("write error, terminate connection");
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ return;
+ }
+ else {
+ wh->pos += r;
+ }
+
+ msg_debug_tcp("written %z bytes: %z/%z", r,
+ wh->pos, wh->total_bytes);
+
+ if (wh->pos >= wh->total_bytes) {
+ goto call_finish_handler;
+ }
+ else {
+ /* Want to write more */
+ if (r > 0) {
+ /* XXX: special case: we know that we want to write more data
+ * than it is available in iov function.
+ *
+ * Hence, we need to check if we can write more at some point...
+ */
+ lua_tcp_write_helper(cbd);
+ }
+ }
+
+ return;
+
+call_finish_handler:
+
+ msg_debug_tcp("finishing TCP write, calling TCP handler");
+
+ if ((cbd->flags & LUA_TCP_FLAG_SHUTDOWN)) {
+ /* Half close the connection */
+ shutdown(cbd->fd, SHUT_WR);
+ cbd->flags &= ~LUA_TCP_FLAG_SHUTDOWN;
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_ok(cbd->up);
+ }
+
+ lua_tcp_push_data(cbd, NULL, 0);
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+}
+
+static gboolean
+lua_tcp_process_read_handler(struct lua_tcp_cbdata *cbd,
+ struct lua_tcp_read_handler *rh, gboolean eof)
+{
+ guint slen;
+ goffset pos;
+
+ if (rh->stop_pattern) {
+ slen = rh->plen;
+
+ if (cbd->in->len >= slen) {
+ if ((pos = rspamd_substring_search(cbd->in->data, cbd->in->len,
+ rh->stop_pattern, slen)) != -1) {
+ msg_debug_tcp("found TCP stop pattern");
+ lua_tcp_push_data(cbd, cbd->in->data, pos);
+
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ }
+ if (pos + slen < cbd->in->len) {
+ /* We have a leftover */
+ memmove(cbd->in->data, cbd->in->data + pos + slen,
+ cbd->in->len - (pos + slen));
+ cbd->in->len = cbd->in->len - (pos + slen);
+ }
+ else {
+ cbd->in->len = 0;
+ }
+
+ return TRUE;
+ }
+ else {
+ /* Plan new read */
+ msg_debug_tcp("NOT found TCP stop pattern");
+
+ if (!cbd->eof) {
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ /* Got session finished but no stop pattern */
+ lua_tcp_push_error(cbd, TRUE,
+ "IO read error: connection terminated");
+ }
+ }
+ }
+ }
+ else {
+ msg_debug_tcp("read TCP partial data %d bytes", cbd->in->len);
+ slen = cbd->in->len;
+
+ /* we have eaten all the data, handler should not know that there is something */
+ cbd->in->len = 0;
+ lua_tcp_push_data(cbd, cbd->in->data, slen);
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ }
+
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void
+lua_tcp_process_read(struct lua_tcp_cbdata *cbd,
+ guchar *in, gssize r)
+{
+ struct lua_tcp_handler *hdl;
+ struct lua_tcp_read_handler *rh;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ g_assert(hdl != NULL && hdl->type == LUA_WANT_READ);
+ rh = &hdl->h.r;
+
+ if (r > 0) {
+ if (cbd->flags & LUA_TCP_FLAG_PARTIAL) {
+ lua_tcp_push_data(cbd, in, r);
+ /* Plan next event */
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ g_byte_array_append(cbd->in, in, r);
+
+ if (!lua_tcp_process_read_handler(cbd, rh, FALSE)) {
+ /* Plan more read */
+ lua_tcp_plan_read(cbd);
+ }
+ else {
+ /* Go towards the next handler */
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+ }
+ }
+ }
+ else if (r == 0) {
+ /* EOF */
+ cbd->eof = TRUE;
+ if (cbd->in->len > 0) {
+ /* We have some data to process */
+ lua_tcp_process_read_handler(cbd, rh, TRUE);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "IO read error: connection terminated");
+
+ if ((cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_tcp_plan_handler_event(cbd, FALSE, FALSE);
+ }
+ else {
+ /* An error occurred */
+ if (errno == EAGAIN || errno == EINTR) {
+ /* Restart call */
+ lua_tcp_plan_read(cbd);
+
+ return;
+ }
+
+ /* Fatal error */
+ cbd->eof = TRUE;
+ if (cbd->in->len > 0) {
+ /* We have some data to process */
+ lua_tcp_process_read_handler(cbd, rh, TRUE);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE,
+ "IO read error while trying to read data: %s",
+ strerror(errno));
+
+ if ((cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+
+ lua_tcp_plan_handler_event(cbd, FALSE, FALSE);
+ }
+}
+
+static void
+lua_tcp_handler(int fd, short what, gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = ud;
+ guchar inbuf[8192];
+ gssize r;
+ gint so_error = 0;
+ socklen_t so_len = sizeof(so_error);
+ struct lua_callback_state cbs;
+ lua_State *L;
+ enum lua_tcp_handler_type event_type;
+ TCP_RETAIN(cbd);
+
+ msg_debug_tcp("processed TCP event: %d", what);
+
+ struct lua_tcp_handler *rh = g_queue_peek_head(cbd->handlers);
+ event_type = rh->type;
+
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+
+ if (what == EV_READ) {
+ if (cbd->ssl_conn) {
+ r = rspamd_ssl_read(cbd->ssl_conn, inbuf, sizeof(inbuf));
+ }
+ else {
+ r = read(cbd->fd, inbuf, sizeof(inbuf));
+ }
+
+ lua_tcp_process_read(cbd, inbuf, r);
+ }
+ else if (what == EV_WRITE) {
+
+ if (!(cbd->flags & LUA_TCP_FLAG_CONNECTED)) {
+ if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &so_error, &so_len) == -1) {
+ lua_tcp_push_error(cbd, TRUE, "Cannot get socket error: %s",
+ strerror(errno));
+ TCP_RELEASE(cbd);
+ goto out;
+ }
+ else if (so_error != 0) {
+ lua_tcp_push_error(cbd, TRUE, "Socket error detected: %s",
+ strerror(so_error));
+ TCP_RELEASE(cbd);
+ goto out;
+ }
+ else {
+ cbd->flags |= LUA_TCP_FLAG_CONNECTED;
+
+ if (cbd->connect_cb != -1) {
+ struct lua_tcp_cbdata **pcbd;
+ gint top;
+
+ lua_thread_pool_prepare_callback(cbd->cfg->lua_thread_pool, &cbs);
+ L = cbs.L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->connect_cb);
+ pcbd = lua_newuserdata(L, sizeof(*pcbd));
+ *pcbd = cbd;
+ TCP_RETAIN(cbd);
+ rspamd_lua_setclass(L, "rspamd{tcp}", -1);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 1, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ TCP_RELEASE(cbd);
+ lua_thread_pool_restore_callback(&cbs);
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+ }
+ }
+
+ if (event_type == LUA_WANT_WRITE) {
+ lua_tcp_write_helper(cbd);
+ }
+ else if (event_type == LUA_WANT_CONNECT) {
+ lua_tcp_connect_helper(cbd);
+ }
+ else {
+ g_assert_not_reached();
+ }
+
+ if ((cbd->flags & (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) ==
+ (LUA_TCP_FLAG_FINISHED | LUA_TCP_FLAG_CONNECTED)) {
+ /* A callback has called `close` method, so we need to release a refcount */
+ TCP_RELEASE(cbd);
+ }
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "IO timeout");
+ TCP_RELEASE(cbd);
+ }
+
+out:
+ TCP_RELEASE(cbd);
+}
+
+static void
+lua_tcp_plan_handler_event(struct lua_tcp_cbdata *cbd, gboolean can_read,
+ gboolean can_write)
+{
+ struct lua_tcp_handler *hdl;
+
+ hdl = g_queue_peek_head(cbd->handlers);
+
+ if (hdl == NULL) {
+ if (!(cbd->flags & LUA_TCP_FLAG_FINISHED)) {
+ /* We are finished with a connection */
+ msg_debug_tcp("no handlers left, finish session");
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+ TCP_RELEASE(cbd);
+ }
+ }
+ else {
+ if (hdl->type == LUA_WANT_READ) {
+
+ /* We need to check if we have some leftover in the buffer */
+ if (cbd->in->len > 0) {
+ msg_debug_tcp("process read buffer leftover");
+ if (lua_tcp_process_read_handler(cbd, &hdl->h.r, FALSE)) {
+ if (!IS_SYNC(cbd)) {
+ /* We can go to the next handler */
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ else {
+ if (can_read) {
+ /* We need to plan a new event */
+ msg_debug_tcp("plan new read");
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_READ);
+ }
+ else {
+ /* Cannot read more */
+ msg_debug_tcp("cannot read more");
+ lua_tcp_push_error(cbd, FALSE, "EOF, cannot read more data");
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ }
+ else if (hdl->type == LUA_WANT_WRITE) {
+ /*
+ * We need to plan write event if there is something in the
+ * write request
+ */
+
+ if (hdl->h.w.pos < hdl->h.w.total_bytes) {
+ msg_debug_tcp("plan new write");
+ if (can_write) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_WRITE);
+ }
+ else {
+ /* Cannot write more */
+ lua_tcp_push_error(cbd, FALSE, "EOF, cannot write more data");
+ if (!IS_SYNC(cbd)) {
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, can_read, can_write);
+ }
+ }
+ }
+ else {
+ /* We shouldn't have empty write handlers */
+ g_assert_not_reached();
+ }
+ }
+ else { /* LUA_WANT_CONNECT */
+ msg_debug_tcp("plan new connect");
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev,
+ EV_WRITE);
+ }
+ }
+}
+
+static gboolean
+lua_tcp_register_event(struct lua_tcp_cbdata *cbd)
+{
+ if (cbd->session) {
+ event_finalizer_t fin = IS_SYNC(cbd) ? lua_tcp_void_finalyser : lua_tcp_fin;
+
+ if (cbd->item) {
+ cbd->async_ev = rspamd_session_add_event_full(cbd->session, fin, cbd, M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ cbd->async_ev = rspamd_session_add_event(cbd->session, fin, cbd, M);
+ }
+
+ if (!cbd->async_ev) {
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+static void
+lua_tcp_register_watcher(struct lua_tcp_cbdata *cbd)
+{
+ if (cbd->item && cbd->task) {
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+}
+
+static void
+lua_tcp_ssl_on_error(gpointer ud, GError *err)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) ud;
+
+ if (err) {
+ lua_tcp_push_error(cbd, TRUE, "ssl error: %s", err->message);
+ }
+ else {
+ lua_tcp_push_error(cbd, TRUE, "ssl error: unknown error");
+ }
+
+ TCP_RELEASE(cbd);
+}
+
+static gboolean
+lua_tcp_make_connection(struct lua_tcp_cbdata *cbd)
+{
+ int fd;
+
+ rspamd_inet_address_set_port(cbd->addr, cbd->port);
+ fd = rspamd_inet_address_connect(cbd->addr, SOCK_STREAM, TRUE);
+
+ if (fd == -1) {
+ if (cbd->session) {
+ rspamd_mempool_t *pool = rspamd_session_mempool(cbd->session);
+ msg_info_pool("cannot connect to %s (%s): %s",
+ rspamd_inet_address_to_string(cbd->addr),
+ cbd->hostname,
+ strerror(errno));
+ }
+ else {
+ msg_info("cannot connect to %s (%s): %s",
+ rspamd_inet_address_to_string(cbd->addr),
+ cbd->hostname,
+ strerror(errno));
+ }
+
+ return FALSE;
+ }
+
+ cbd->fd = fd;
+
+#if 0
+ if (!(cbd->flags & LUA_TCP_FLAG_RESOLVED)) {
+ /* We come here without resolving, so we need to add a watcher */
+ lua_tcp_register_watcher (cbd);
+ }
+ else {
+ cbd->flags |= LUA_TCP_FLAG_RESOLVED;
+ }
+#endif
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL) {
+ gpointer ssl_ctx;
+ gboolean verify_peer;
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL_NOVERIFY) {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx_noverify;
+ verify_peer = FALSE;
+ }
+ else {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx;
+ verify_peer = TRUE;
+ }
+
+ cbd->ssl_conn = rspamd_ssl_connection_new(ssl_ctx,
+ cbd->event_loop,
+ verify_peer,
+ cbd->tag);
+
+ if (!rspamd_ssl_connect_fd(cbd->ssl_conn, fd, cbd->hostname, &cbd->ev,
+ cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "ssl connection failed: %s",
+ strerror(errno));
+
+ return FALSE;
+ }
+ else {
+ lua_tcp_register_event(cbd);
+ }
+ }
+ else {
+ rspamd_ev_watcher_init(&cbd->ev, cbd->fd, EV_WRITE,
+ lua_tcp_handler, cbd);
+ lua_tcp_register_event(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+ }
+
+
+ return TRUE;
+}
+
+static void
+lua_tcp_dns_handler(struct rdns_reply *reply, gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = (struct lua_tcp_cbdata *) ud;
+ const struct rdns_request_name *rn;
+
+ if (reply->code != RDNS_RC_NOERROR) {
+ rn = rdns_request_get_name(reply->request, NULL);
+ lua_tcp_push_error(cbd, TRUE, "unable to resolve host: %s",
+ rn->name);
+ TCP_RELEASE(cbd);
+ }
+ else {
+ /*
+ * We set this flag as it means that we have already registered the watcher
+ * when started DNS query
+ */
+ struct rdns_reply_entry *entry;
+
+ DL_FOREACH(reply->entries, entry)
+ {
+ if (entry->type == RDNS_REQUEST_A) {
+ cbd->addr = rspamd_inet_address_new(AF_INET,
+ &entry->content.a.addr);
+ break;
+ }
+ else if (entry->type == RDNS_REQUEST_AAAA) {
+ cbd->addr = rspamd_inet_address_new(AF_INET6,
+ &entry->content.aaa.addr);
+ break;
+ }
+ }
+
+ if (cbd->addr == NULL) {
+ rn = rdns_request_get_name(reply->request, NULL);
+ lua_tcp_push_error(cbd, TRUE, "unable to resolve host: %s; no records with this name",
+ rn->name);
+ TCP_RELEASE(cbd);
+ return;
+ }
+
+ cbd->flags |= LUA_TCP_FLAG_RESOLVED;
+ rspamd_inet_address_set_port(cbd->addr, cbd->port);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "unable to make connection to the host %s",
+ rspamd_inet_address_to_string(cbd->addr));
+ TCP_RELEASE(cbd);
+ }
+ }
+}
+
+static gboolean
+lua_tcp_arg_toiovec(lua_State *L, gint pos, struct lua_tcp_cbdata *cbd,
+ struct iovec *vec)
+{
+ struct rspamd_lua_text *t;
+ gsize len;
+ const gchar *str;
+ struct lua_tcp_dtor *dtor;
+
+ if (lua_type(L, pos) == LUA_TUSERDATA) {
+ t = lua_check_text(L, pos);
+
+ if (t) {
+ vec->iov_base = (void *) t->start;
+ vec->iov_len = t->len;
+
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ /* Steal ownership */
+ t->flags = 0;
+ dtor = g_malloc0(sizeof(*dtor));
+ dtor->dtor = g_free;
+ dtor->data = (void *) t->start;
+ LL_PREPEND(cbd->dtors, dtor);
+ }
+ }
+ else {
+ msg_err("bad userdata argument at position %d", pos);
+ return FALSE;
+ }
+ }
+ else if (lua_type(L, pos) == LUA_TSTRING) {
+ str = luaL_checklstring(L, pos, &len);
+ vec->iov_base = g_malloc(len);
+ dtor = g_malloc0(sizeof(*dtor));
+ dtor->dtor = g_free;
+ dtor->data = vec->iov_base;
+ LL_PREPEND(cbd->dtors, dtor);
+ memcpy(vec->iov_base, str, len);
+ vec->iov_len = len;
+ }
+ else {
+ msg_err("bad argument at position %d", pos);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+/***
+ * @function rspamd_tcp.request({params})
+ * This function creates and sends TCP request to the specified host and port,
+ * resolves hostname (if needed) and invokes continuation callback upon data received
+ * from the remote peer. This function accepts table of arguments with the following
+ * attributes
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `resolver`: DNS resolver (no task)
+ * - `session`: events session (no task)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use
+ * - `data`: a table of strings or `rspamd_text` objects that contains data pieces
+ * - `callback`: continuation function (required)
+ * - `on_connect`: callback called on connection success
+ * - `timeout`: floating point value that specifies timeout for IO operations in **seconds**
+ * - `partial`: boolean flag that specifies that callback should be called on any data portion received
+ * - `stop_pattern`: stop reading on finding a certain pattern (e.g. \r\n.\r\n for smtp)
+ * - `shutdown`: half-close socket after writing (boolean: default false)
+ * - `read`: read response after sending request (boolean: default true)
+ * - `upstream`: optional upstream object that would be used to get an address
+ * @return {boolean} true if request has been sent
+ */
+static gint
+lua_tcp_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ gchar *stop_pattern = NULL;
+ guint port;
+ gint cbref, tp, conn_cbref = -1;
+ gsize plen = 0;
+ struct ev_loop *event_loop = NULL;
+ struct lua_tcp_cbdata *cbd;
+ struct rspamd_dns_resolver *resolver = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_task *task = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct iovec *iov = NULL;
+ struct upstream *up = NULL;
+ guint niov = 0, total_out;
+ guint64 h;
+ gdouble timeout = default_tcp_timeout;
+ gboolean partial = FALSE, do_shutdown = FALSE, do_read = TRUE,
+ ssl = FALSE, ssl_noverify = FALSE;
+
+ if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+ host = luaL_checkstring(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "port");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ port = lua_tointeger(L, -1);
+ }
+ else {
+ /* We assume that it is a unix socket */
+ port = 0;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (host == NULL || lua_type(L, -1) != LUA_TFUNCTION) {
+ lua_pop(L, 1);
+ msg_err("tcp request has bad params");
+ lua_pushboolean(L, FALSE);
+ return 1;
+ }
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ cbd = g_malloc0(sizeof(*cbd));
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+ event_loop = task->event_loop;
+ resolver = task->resolver;
+ session = task->s;
+ cfg = task->cfg;
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ event_loop = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ g_free(cbd);
+
+ return luaL_error(L, "event loop is required");
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "config");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{config}")) {
+ cfg = *(struct rspamd_config **) lua_touserdata(L, -1);
+ }
+ else {
+ cfg = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "resolver");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{resolver}")) {
+ resolver = *(struct rspamd_dns_resolver **) lua_touserdata(L, -1);
+ }
+ else {
+ resolver = lua_tcp_global_resolver(event_loop, cfg);
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "stop_pattern");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *p;
+
+ p = lua_tolstring(L, -1, &plen);
+
+ if (p && plen > 0) {
+ stop_pattern = g_malloc(plen);
+ memcpy(stop_pattern, p, plen);
+ }
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "partial");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ partial = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "shutdown");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ do_shutdown = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "read");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ do_read = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ssl");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl = lua_toboolean(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "ssl_noverify");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl_noverify = lua_toboolean(L, -1);
+ lua_pop(L, 1);
+ }
+ else {
+ lua_pop(L, 1); /* Previous nil... */
+ /* Similar to lua http, meh... */
+ lua_pushstring(L, "no_ssl_verify");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TBOOLEAN) {
+ ssl_noverify = lua_toboolean(L, -1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "on_connect");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ conn_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "upstream");
+ lua_gettable(L, 1);
+
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_upstream *lup = lua_check_upstream(L, -1);
+
+ if (lup) {
+ /* Preserve pointer in case if lup is destructed */
+ up = lup->up;
+ }
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "data");
+ lua_gettable(L, -2);
+ total_out = 0;
+
+ tp = lua_type(L, -1);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, iov)) {
+ lua_pop(L, 1);
+ msg_err("tcp request has bad data argument");
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ lua_pop(L, 2);
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "tcp request has bad params");
+ }
+
+ if (resolver == NULL && cfg == NULL && task == NULL) {
+ g_free(cbd);
+ g_free(iov);
+
+ return luaL_error(L, "tcp request has bad params: one of "
+ "{resolver,task,config} should be set");
+ }
+
+ cbd->task = task;
+
+ if (task) {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+ }
+
+ cbd->cfg = cfg;
+ h = rspamd_random_uint64_fast();
+ rspamd_snprintf(cbd->tag, sizeof(cbd->tag), "%uxL", h);
+ cbd->handlers = g_queue_new();
+ cbd->hostname = g_strdup(host);
+
+ if (total_out > 0) {
+ struct lua_tcp_handler *wh;
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ /* Cannot set write handler here */
+ wh->h.w.cbref = -1;
+
+ if (cbref != -1 && !do_read) {
+ /* We have write only callback */
+ wh->h.w.cbref = cbref;
+ }
+ else {
+ /* We have simple client callback */
+ wh->h.w.cbref = -1;
+ }
+
+ g_queue_push_tail(cbd->handlers, wh);
+ }
+
+ cbd->event_loop = event_loop;
+ cbd->fd = -1;
+ cbd->port = port;
+ cbd->ev.timeout = timeout;
+
+ if (ssl) {
+ cbd->flags |= LUA_TCP_FLAG_SSL;
+
+ if (ssl_noverify) {
+ cbd->flags |= LUA_TCP_FLAG_SSL_NOVERIFY;
+ }
+ }
+
+ if (do_read) {
+ cbd->in = g_byte_array_sized_new(8192);
+ }
+ else {
+ /* Save some space... */
+ cbd->in = g_byte_array_new();
+ }
+
+ if (partial) {
+ cbd->flags |= LUA_TCP_FLAG_PARTIAL;
+ }
+
+ if (do_shutdown) {
+ cbd->flags |= LUA_TCP_FLAG_SHUTDOWN;
+ }
+
+ if (do_read) {
+ struct lua_tcp_handler *rh;
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = cbref;
+ rh->h.r.stop_pattern = stop_pattern;
+ rh->h.r.plen = plen;
+ g_queue_push_tail(cbd->handlers, rh);
+ }
+
+ cbd->connect_cb = conn_cbref;
+ REF_INIT_RETAIN(cbd, lua_tcp_maybe_free);
+
+ if (up) {
+ cbd->up = rspamd_upstream_ref(up);
+ }
+
+ if (session) {
+ cbd->session = session;
+
+ if (rspamd_session_blocked(session)) {
+ lua_tcp_push_error(cbd, TRUE, "async session is the blocked state");
+ TCP_RELEASE(cbd);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+ lua_pushboolean(L, FALSE);
+
+ return 1;
+ }
+ }
+
+ if (cbd->up) {
+ /* Use upstream to get addr */
+ cbd->addr = rspamd_inet_address_copy(rspamd_upstream_addr_next(cbd->up), NULL);
+
+ /* Host is numeric IP, no need to resolve */
+ lua_tcp_register_watcher(cbd);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot connect to the host: %s", host);
+ lua_pushboolean(L, FALSE);
+
+ rspamd_upstream_fail(cbd->up, true, "failed to connect");
+
+ /* No reset of the item as watcher has been registered */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ }
+ else if (rspamd_parse_inet_address(&cbd->addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_inet_address_set_port(cbd->addr, port);
+ /* Host is numeric IP, no need to resolve */
+ lua_tcp_register_watcher(cbd);
+
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot connect to the host: %s", host);
+ lua_pushboolean(L, FALSE);
+
+ /* No reset of the item as watcher has been registered */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ }
+ else {
+ if (task == NULL) {
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot resolve host: %s", host);
+ lua_pushboolean(L, FALSE);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ else {
+ if (!rspamd_dns_resolver_request_task(task, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_tcp_push_error(cbd, TRUE, "cannot resolve host: %s", host);
+ lua_pushboolean(L, FALSE);
+ cbd->item = NULL; /* To avoid decrease with no watcher */
+
+ TCP_RELEASE(cbd);
+
+ return 1;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ }
+
+ lua_pushboolean(L, TRUE);
+ return 1;
+}
+
+/***
+ * @function rspamd_tcp.connect_sync({params})
+ * Creates new pseudo-synchronous connection to the specific address:port
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session`, `ev_base` and `resolver` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `resolver`: DNS resolver (no task)
+ * - `session`: events session (no task)
+ * - `config`: config (no task)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use
+ * - `timeout`: floating point value that specifies timeout for IO operations in **seconds**
+ * @return {boolean} true if request has been sent
+ */
+static gint
+lua_tcp_connect_sync(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ GError *err = NULL;
+
+ gint64 port = -1;
+ gdouble timeout = default_tcp_timeout;
+ const gchar *host = NULL;
+ gint ret;
+ guint64 h;
+
+ struct rspamd_task *task = NULL;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_dns_resolver *resolver = NULL;
+ struct rspamd_config *cfg = NULL;
+ struct ev_loop *ev_base = NULL;
+ struct lua_tcp_cbdata *cbd;
+
+
+ int arguments_validated = rspamd_lua_parse_table_arguments(L, 1, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "task=U{task};session=U{session};resolver=U{resolver};ev_base=U{ev_base};"
+ "*host=S;*port=I;timeout=D;config=U{config}",
+ &task, &session, &resolver, &ev_base,
+ &host, &port, &timeout, &cfg);
+
+ if (!arguments_validated) {
+ if (err) {
+ ret = luaL_error(L, "invalid arguments: %s", err->message);
+ g_error_free(err);
+
+ return ret;
+ }
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (0 > port || port > 65535) {
+ return luaL_error(L, "invalid port given (correct values: 1..65535)");
+ }
+
+ if (task == NULL && (cfg == NULL || ev_base == NULL || session == NULL)) {
+ return luaL_error(L, "invalid arguments: either task or config+ev_base+session should be set");
+ }
+
+ if (isnan(timeout)) {
+ /* rspamd_lua_parse_table_arguments() sets missing N field to zero */
+ timeout = default_tcp_timeout;
+ }
+
+ cbd = g_new0(struct lua_tcp_cbdata, 1);
+
+ if (task) {
+ static const gchar hexdigests[16] = "0123456789abcdef";
+
+ cfg = task->cfg;
+ ev_base = task->event_loop;
+ session = task->s;
+ /* Make a readable tag */
+ memcpy(cbd->tag, task->task_pool->tag.uid, sizeof(cbd->tag) - 2);
+ cbd->tag[sizeof(cbd->tag) - 2] = hexdigests[GPOINTER_TO_INT(cbd) & 0xf];
+ cbd->tag[sizeof(cbd->tag) - 1] = 0;
+ }
+ else {
+ h = rspamd_random_uint64_fast();
+ rspamd_snprintf(cbd->tag, sizeof(cbd->tag), "%uxL", h);
+ }
+
+ if (resolver == NULL) {
+ if (task) {
+ resolver = task->resolver;
+ }
+ else {
+ resolver = lua_tcp_global_resolver(ev_base, cfg);
+ }
+ }
+
+ cbd->task = task;
+ cbd->cfg = cfg;
+ cbd->thread = lua_thread_pool_get_running_entry(cfg->lua_thread_pool);
+
+
+ cbd->handlers = g_queue_new();
+
+ cbd->event_loop = ev_base;
+ cbd->flags |= LUA_TCP_FLAG_SYNC;
+ cbd->fd = -1;
+ cbd->port = (guint16) port;
+
+ cbd->in = g_byte_array_new();
+
+ cbd->connect_cb = -1;
+
+ REF_INIT_RETAIN(cbd, lua_tcp_maybe_free);
+
+ if (task) {
+ rspamd_mempool_add_destructor(task->task_pool, lua_tcp_sync_session_dtor, cbd);
+ }
+
+ struct lua_tcp_handler *wh;
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_CONNECT;
+
+ g_queue_push_tail(cbd->handlers, wh);
+
+ if (session) {
+ cbd->session = session;
+
+ if (rspamd_session_blocked(session)) {
+ TCP_RELEASE(cbd);
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Session is being destroyed, requests are not allowed");
+
+ return 2;
+ }
+ }
+
+ if (rspamd_parse_inet_address(&cbd->addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ rspamd_inet_address_set_port(cbd->addr, (guint16) port);
+ /* Host is numeric IP, no need to resolve */
+ if (!lua_tcp_make_connection(cbd)) {
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate connection");
+
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ }
+ else {
+ if (task == NULL) {
+ if (!rspamd_dns_resolver_request(resolver, session, NULL, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate dns request");
+
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ else {
+ cbd->item = rspamd_symcache_get_cur_item(task);
+
+ if (!rspamd_dns_resolver_request_task(task, lua_tcp_dns_handler, cbd,
+ RDNS_REQUEST_A, host)) {
+ cbd->item = NULL; /* We have not registered watcher */
+ lua_pushboolean(L, FALSE);
+ lua_pushliteral(L, "Failed to initiate dns request");
+ TCP_RELEASE(cbd);
+
+ return 2;
+ }
+ else {
+ lua_tcp_register_watcher(cbd);
+ }
+ }
+ }
+
+ return lua_thread_yield(cbd->thread, 0);
+}
+
+static gint
+lua_tcp_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->ssl_conn) {
+ /* TODO: postpone close in case ssl is used ! */
+ rspamd_ssl_connection_free(cbd->ssl_conn);
+ cbd->ssl_conn = NULL;
+ }
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ cbd->addr = NULL;
+ }
+
+ if (cbd->up) {
+ rspamd_upstream_unref(cbd->up);
+ cbd->up = NULL;
+ }
+ /* Do not release refcount as it will be handled elsewhere */
+ /* TCP_RELEASE (cbd); */
+
+ return 0;
+}
+
+static gint
+lua_tcp_add_read(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ struct lua_tcp_handler *rh;
+ gchar *stop_pattern = NULL;
+ const gchar *p;
+ gsize plen = 0;
+ gint cbref = -1;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ p = lua_tolstring(L, 3, &plen);
+
+ if (p && plen > 0) {
+ stop_pattern = g_malloc(plen);
+ memcpy(stop_pattern, p, plen);
+ }
+ }
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = cbref;
+ rh->h.r.stop_pattern = stop_pattern;
+ rh->h.r.plen = plen;
+ msg_debug_tcp("added read event, cbref: %d", cbref);
+
+ g_queue_push_tail(cbd->handlers, rh);
+
+ return 0;
+}
+
+static gint
+lua_tcp_add_write(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ struct lua_tcp_handler *wh;
+ gint cbref = -1, tp;
+ struct iovec *iov = NULL;
+ guint niov = 0, total_out = 0;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TFUNCTION) {
+ lua_pushvalue(L, 2);
+ cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+
+ tp = lua_type(L, 3);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, 3, cbd, iov)) {
+ msg_err("tcp request has bad data argument");
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+
+ return 1;
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushvalue(L, 3);
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ lua_pop(L, 2);
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ lua_pushboolean(L, FALSE);
+ g_free(iov);
+ g_free(cbd);
+
+ return 1;
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ /* Cannot set write handler here */
+ wh->h.w.cbref = cbref;
+ msg_debug_tcp("added write event, cbref: %d", cbref);
+
+ g_queue_push_tail(cbd->handlers, wh);
+ lua_pushboolean(L, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_tcp_shift_callback(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_tcp_shift_handler(cbd);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ return 0;
+}
+
+static struct lua_tcp_cbdata *
+lua_check_sync_tcp(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{tcp_sync}");
+ luaL_argcheck(L, ud != NULL, pos, "'tcp' expected");
+ return ud ? *((struct lua_tcp_cbdata **) ud) : NULL;
+}
+
+static int
+lua_tcp_sync_close(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->fd != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ return 0;
+}
+
+static void
+lua_tcp_sync_session_dtor(gpointer ud)
+{
+ struct lua_tcp_cbdata *cbd = ud;
+ cbd->flags |= LUA_TCP_FLAG_FINISHED;
+
+ if (cbd->fd != -1) {
+ msg_debug("closing sync TCP connection");
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->fd);
+ cbd->fd = -1;
+ }
+
+ /* Task is gone, we should not try use it anymore */
+ cbd->task = NULL;
+
+ /* All events are removed when task is done, we should not refer them */
+ cbd->async_ev = NULL;
+}
+
+static int
+lua_tcp_sync_read_once(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ struct lua_tcp_handler *rh;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ struct thread_entry *thread = lua_thread_pool_get_running_entry(cbd->cfg->lua_thread_pool);
+
+ rh = g_malloc0(sizeof(*rh));
+ rh->type = LUA_WANT_READ;
+ rh->h.r.cbref = -1;
+
+ msg_debug_tcp("added read sync event, thread: %p", thread);
+
+ g_queue_push_tail(cbd->handlers, rh);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ TCP_RETAIN(cbd);
+
+ return lua_thread_yield(thread, 0);
+}
+
+static int
+lua_tcp_sync_write(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ struct lua_tcp_handler *wh;
+ gint tp;
+ struct iovec *iov = NULL;
+ guint niov = 0;
+ gsize total_out = 0;
+
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ struct thread_entry *thread = lua_thread_pool_get_running_entry(cbd->cfg->lua_thread_pool);
+
+ tp = lua_type(L, 2);
+ if (tp == LUA_TSTRING || tp == LUA_TUSERDATA) {
+ iov = g_malloc(sizeof(*iov));
+ niov = 1;
+
+ if (!lua_tcp_arg_toiovec(L, 2, cbd, iov)) {
+ msg_err("tcp request has bad data argument");
+ g_free(iov);
+ g_free(cbd);
+
+ return luaL_error(L, "invalid arguments second parameter (data) is expected to be either string or rspamd{text}");
+ }
+
+ total_out = iov[0].iov_len;
+ }
+ else if (tp == LUA_TTABLE) {
+ /* Count parts */
+ lua_pushvalue(L, 3);
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ niov++;
+ lua_pop(L, 1);
+ }
+
+ iov = g_malloc(sizeof(*iov) * niov);
+ lua_pushnil(L);
+ niov = 0;
+
+ while (lua_next(L, -2) != 0) {
+ if (!lua_tcp_arg_toiovec(L, -1, cbd, &iov[niov])) {
+ msg_err("tcp request has bad data argument at pos %d", niov);
+ g_free(iov);
+ g_free(cbd);
+
+ return luaL_error(L, "invalid arguments second parameter (data) is expected to be either string or rspamd{text}");
+ }
+
+ total_out += iov[niov].iov_len;
+ niov++;
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ wh = g_malloc0(sizeof(*wh));
+ wh->type = LUA_WANT_WRITE;
+ wh->h.w.iov = iov;
+ wh->h.w.iovlen = niov;
+ wh->h.w.total_bytes = total_out;
+ wh->h.w.pos = 0;
+ wh->h.w.cbref = -1;
+ msg_debug_tcp("added sync write event, thread: %p", thread);
+
+ g_queue_push_tail(cbd->handlers, wh);
+ lua_tcp_plan_handler_event(cbd, TRUE, TRUE);
+
+ TCP_RETAIN(cbd);
+
+ return lua_thread_yield(thread, 0);
+}
+
+static gint
+lua_tcp_sync_eof(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ lua_pushboolean(L, cbd->eof);
+
+ return 1;
+}
+
+static gint
+lua_tcp_sync_shutdown(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (cbd == NULL) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ shutdown(cbd->fd, SHUT_WR);
+
+ return 0;
+}
+
+static gint
+lua_tcp_starttls(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct lua_tcp_cbdata *cbd = lua_check_tcp(L, 1);
+ gpointer ssl_ctx;
+ gboolean verify_peer;
+
+ if (cbd == NULL || cbd->ssl_conn != NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (cbd->flags & LUA_TCP_FLAG_SSL_NOVERIFY) {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx_noverify;
+ verify_peer = FALSE;
+ }
+ else {
+ ssl_ctx = cbd->cfg->libs_ctx->ssl_ctx;
+ verify_peer = TRUE;
+ }
+
+ cbd->ssl_conn = rspamd_ssl_connection_new(ssl_ctx,
+ cbd->event_loop,
+ verify_peer,
+ cbd->tag);
+
+ if (!rspamd_ssl_connect_fd(cbd->ssl_conn, cbd->fd, cbd->hostname, &cbd->ev,
+ cbd->ev.timeout, lua_tcp_handler, lua_tcp_ssl_on_error, cbd)) {
+ lua_tcp_push_error(cbd, TRUE, "ssl connection failed: %s",
+ strerror(errno));
+ }
+
+ return 0;
+}
+
+static gint
+lua_tcp_sync_gc(lua_State *L)
+{
+ struct lua_tcp_cbdata *cbd = lua_check_sync_tcp(L, 1);
+ if (!cbd) {
+ return luaL_error(L, "invalid arguments [self is not rspamd{tcp_sync}]");
+ }
+
+ lua_tcp_maybe_free(cbd);
+ lua_tcp_fin(cbd);
+
+ return 0;
+}
+
+static gint
+lua_load_tcp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, tcp_libf);
+
+ return 1;
+}
+
+void luaopen_tcp(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_tcp", lua_load_tcp);
+ rspamd_lua_new_class(L, "rspamd{tcp}", tcp_libm);
+ rspamd_lua_new_class(L, "rspamd{tcp_sync}", tcp_sync_libm);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_tensor.c b/src/lua/lua_tensor.c
new file mode 100644
index 0000000..75e6139
--- /dev/null
+++ b/src/lua/lua_tensor.c
@@ -0,0 +1,817 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "lua_tensor.h"
+#include "contrib/kann/kautodiff.h"
+#include "blas-config.h"
+
+/***
+ * @module rspamd_tensor
+ * `rspamd_tensor` is a simple Lua library to abstract matrices and vectors
+ * Internally, they are represented as arrays of float variables
+ * So far, merely 1D and 2D tensors are supported
+ */
+
+LUA_FUNCTION_DEF(tensor, load);
+LUA_FUNCTION_DEF(tensor, save);
+LUA_FUNCTION_DEF(tensor, new);
+LUA_FUNCTION_DEF(tensor, fromtable);
+LUA_FUNCTION_DEF(tensor, destroy);
+LUA_FUNCTION_DEF(tensor, mul);
+LUA_FUNCTION_DEF(tensor, tostring);
+LUA_FUNCTION_DEF(tensor, index);
+LUA_FUNCTION_DEF(tensor, newindex);
+LUA_FUNCTION_DEF(tensor, len);
+LUA_FUNCTION_DEF(tensor, eigen);
+LUA_FUNCTION_DEF(tensor, mean);
+LUA_FUNCTION_DEF(tensor, transpose);
+LUA_FUNCTION_DEF(tensor, has_blas);
+LUA_FUNCTION_DEF(tensor, scatter_matrix);
+
+static luaL_reg rspamd_tensor_f[] = {
+ LUA_INTERFACE_DEF(tensor, load),
+ LUA_INTERFACE_DEF(tensor, new),
+ LUA_INTERFACE_DEF(tensor, fromtable),
+ LUA_INTERFACE_DEF(tensor, has_blas),
+ LUA_INTERFACE_DEF(tensor, scatter_matrix),
+ {NULL, NULL},
+};
+
+static luaL_reg rspamd_tensor_m[] = {
+ LUA_INTERFACE_DEF(tensor, save),
+ {"__gc", lua_tensor_destroy},
+ {"__mul", lua_tensor_mul},
+ {"mul", lua_tensor_mul},
+ {"tostring", lua_tensor_tostring},
+ {"__tostring", lua_tensor_tostring},
+ {"__index", lua_tensor_index},
+ {"__newindex", lua_tensor_newindex},
+ {"__len", lua_tensor_len},
+ LUA_INTERFACE_DEF(tensor, eigen),
+ LUA_INTERFACE_DEF(tensor, mean),
+ LUA_INTERFACE_DEF(tensor, transpose),
+ {NULL, NULL},
+};
+
+struct rspamd_lua_tensor *
+lua_newtensor(lua_State *L, int ndims, const int *dim, bool zero_fill, bool own)
+{
+ struct rspamd_lua_tensor *res;
+
+ res = lua_newuserdata(L, sizeof(struct rspamd_lua_tensor));
+ memset(res, 0, sizeof(*res));
+
+ res->ndims = ndims;
+ res->size = 1;
+
+ for (guint i = 0; i < ndims; i++) {
+ res->size *= dim[i];
+ res->dim[i] = dim[i];
+ }
+
+ /* To avoid allocating large stuff in Lua */
+ if (own) {
+ res->data = g_malloc(sizeof(rspamd_tensor_num_t) * res->size);
+
+ if (zero_fill) {
+ memset(res->data, 0, sizeof(rspamd_tensor_num_t) * res->size);
+ }
+ }
+ else {
+ /* Mark size negative to distinguish */
+ res->size = -(res->size);
+ }
+
+ rspamd_lua_setclass(L, TENSOR_CLASS, -1);
+
+ return res;
+}
+
+/***
+ * @function tensor.new(ndims, [dim1, ... dimN])
+ * Creates a new zero filled tensor with the specific number of dimensions
+ * @return
+ */
+static gint
+lua_tensor_new(lua_State *L)
+{
+ gint ndims = luaL_checkinteger(L, 1);
+
+ if (ndims > 0 && ndims <= 2) {
+ gint *dims = g_alloca(sizeof(gint) * ndims);
+
+ for (guint i = 0; i < ndims; i++) {
+ dims[i] = lua_tointeger(L, i + 2);
+ }
+
+ (void) lua_newtensor(L, ndims, dims, true, true);
+ }
+ else {
+ return luaL_error(L, "incorrect dimensions number: %d", ndims);
+ }
+
+ return 1;
+}
+
+/***
+ * @function tensor.fromtable(tbl)
+ * Creates a new zero filled tensor with the specific number of dimensions
+ * @return
+ */
+static gint
+lua_tensor_fromtable(lua_State *L)
+{
+ if (lua_istable(L, 1)) {
+ lua_rawgeti(L, 1, 1);
+
+ if (lua_isnumber(L, -1)) {
+ lua_pop(L, 1);
+ /* Input vector */
+ gint dims[2];
+ dims[0] = 1;
+ dims[1] = rspamd_lua_table_size(L, 1);
+
+ struct rspamd_lua_tensor *res = lua_newtensor(L, 2,
+ dims, false, true);
+
+ for (guint i = 0; i < dims[1]; i++) {
+ lua_rawgeti(L, 1, i + 1);
+ res->data[i] = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_istable(L, -1)) {
+ /* Input matrix */
+ lua_pop(L, 1);
+
+ /* Calculate the overall size */
+ gint nrows = rspamd_lua_table_size(L, 1), ncols = 0;
+ gint err;
+
+ for (gint i = 0; i < nrows; i++) {
+ lua_rawgeti(L, 1, i + 1);
+
+ if (ncols == 0) {
+ ncols = rspamd_lua_table_size(L, -1);
+
+ if (ncols == 0) {
+ lua_pop(L, 1);
+ err = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d",
+ i,
+ (int) ncols);
+
+ return err;
+ }
+ }
+ else {
+ if (ncols != rspamd_lua_table_size(L, -1)) {
+ gint t = rspamd_lua_table_size(L, -1);
+
+ lua_pop(L, 1);
+ err = luaL_error(L, "invalid params at pos %d: "
+ "bad input dimension %d; %d expected",
+ i,
+ t,
+ ncols);
+
+ return err;
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+
+ gint dims[2];
+ dims[0] = nrows;
+ dims[1] = ncols;
+
+ struct rspamd_lua_tensor *res = lua_newtensor(L, 2,
+ dims, false, true);
+
+ for (gint i = 0; i < nrows; i++) {
+ lua_rawgeti(L, 1, i + 1);
+
+ for (gint j = 0; j < ncols; j++) {
+ lua_rawgeti(L, -1, j + 1);
+
+ res->data[i * ncols + j] = lua_tonumber(L, -1);
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "incorrect table");
+ }
+ }
+ else {
+ return luaL_error(L, "incorrect input");
+ }
+
+ return 1;
+}
+
+
+/***
+ * @method tensor:destroy()
+ * Tensor destructor
+ * @return
+ */
+static gint
+lua_tensor_destroy(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ if (t->size > 0) {
+ g_free(t->data);
+ }
+ }
+
+ return 0;
+}
+
+/***
+ * @method tensor:save()
+ * Tensor serialisation function
+ * @return
+ */
+static gint
+lua_tensor_save(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint size;
+
+ if (t) {
+ if (t->size > 0) {
+ size = t->size;
+ }
+ else {
+ size = -(t->size);
+ }
+
+ gsize sz = sizeof(gint) * 4 + size * sizeof(rspamd_tensor_num_t);
+ guchar *data;
+
+ struct rspamd_lua_text *out = lua_new_text(L, NULL, 0, TRUE);
+
+ data = g_malloc(sz);
+ memcpy(data, &t->ndims, sizeof(int));
+ memcpy(data + sizeof(int), &size, sizeof(int));
+ memcpy(data + 2 * sizeof(int), t->dim, sizeof(int) * 2);
+ memcpy(data + 4 * sizeof(int), t->data,
+ size * sizeof(rspamd_tensor_num_t));
+
+ out->start = (const gchar *) data;
+ out->len = sz;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_tostring(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ GString *out = g_string_sized_new(128);
+
+ if (t->ndims == 1) {
+ /* Print as a vector */
+ for (gint i = 0; i < t->dim[0]; i++) {
+ rspamd_printf_gstring(out, "%.4f ", t->data[i]);
+ }
+ /* Trim last space */
+ out->len--;
+ }
+ else {
+ for (gint i = 0; i < t->dim[0]; i++) {
+ for (gint j = 0; j < t->dim[1]; j++) {
+ rspamd_printf_gstring(out, "%.4f ",
+ t->data[i * t->dim[1] + j]);
+ }
+ /* Trim last space */
+ out->len--;
+ rspamd_printf_gstring(out, "\n");
+ }
+ /* Trim last ; */
+ out->len--;
+ }
+
+ lua_pushlstring(L, out->str, out->len);
+
+ g_string_free(out, TRUE);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_index(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint idx;
+
+ if (t) {
+ if (lua_isnumber(L, 2)) {
+ idx = lua_tointeger(L, 2);
+
+ if (t->ndims == 1) {
+ /* Individual element */
+ if (idx <= t->dim[0]) {
+ lua_pushnumber(L, t->data[idx - 1]);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ /* Push row */
+ gint dim = t->dim[1];
+
+
+ if (idx <= t->dim[0]) {
+ /* Non-owning tensor */
+ struct rspamd_lua_tensor *res =
+ lua_newtensor(L, 1, &dim, false, false);
+ res->data = &t->data[(idx - 1) * t->dim[1]];
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ }
+ else if (lua_isstring(L, 2)) {
+ /* Access to methods */
+ lua_getmetatable(L, 1);
+ lua_pushvalue(L, 2);
+ lua_rawget(L, -2);
+ }
+ }
+
+ return 1;
+}
+static gint
+lua_tensor_newindex(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint idx;
+
+ if (t) {
+ if (lua_isnumber(L, 2)) {
+ idx = lua_tointeger(L, 2);
+
+ if (t->ndims == 1) {
+ /* Individual element */
+ if (idx <= t->dim[0] && idx > 0) {
+ rspamd_tensor_num_t value = lua_tonumber(L, 3), old;
+
+ old = t->data[idx - 1];
+ t->data[idx - 1] = value;
+ lua_pushnumber(L, old);
+ }
+ else {
+ return luaL_error(L, "invalid index: %d", idx);
+ }
+ }
+ else {
+ if (lua_isnumber(L, 3)) {
+ return luaL_error(L, "cannot assign number to a row");
+ }
+ else if (lua_isuserdata(L, 3)) {
+ /* Tensor assignment */
+ struct rspamd_lua_tensor *row = lua_check_tensor(L, 3);
+
+ if (row) {
+ if (row->ndims == 1) {
+ if (row->dim[0] == t->dim[1]) {
+ if (idx > 0 && idx <= t->dim[0]) {
+ idx--; /* Zero based index */
+ memcpy(&t->data[idx * t->dim[1]],
+ row->data,
+ t->dim[1] * sizeof(rspamd_tensor_num_t));
+
+ return 0;
+ }
+ else {
+ return luaL_error(L, "invalid index: %d", idx);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "cannot assign matrix to row");
+ }
+ }
+ else {
+ return luaL_error(L, "cannot assign row, invalid tensor");
+ }
+ }
+ else {
+ /* TODO: add table assignment */
+ return luaL_error(L, "cannot assign row, not a tensor");
+ }
+ }
+ }
+ else {
+ /* Access to methods? NYI */
+ return luaL_error(L, "cannot assign method of a tensor");
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @method tensor:mul(other, [transA, [transB]])
+ * Multiply two tensors (optionally transposed) and return a new tensor
+ * @return
+ */
+static gint
+lua_tensor_mul(lua_State *L)
+{
+ struct rspamd_lua_tensor *t1 = lua_check_tensor(L, 1),
+ *t2 = lua_check_tensor(L, 2), *res;
+ int transA = 0, transB = 0;
+
+ if (lua_isboolean(L, 3)) {
+ transA = lua_toboolean(L, 3);
+ }
+
+ if (lua_isboolean(L, 4)) {
+ transB = lua_toboolean(L, 4);
+ }
+
+ if (t1 && t2) {
+ gint dims[2], shadow_dims[2];
+ dims[0] = abs(transA ? t1->dim[1] : t1->dim[0]);
+ shadow_dims[0] = abs(transB ? t2->dim[1] : t2->dim[0]);
+ dims[1] = abs(transB ? t2->dim[0] : t2->dim[1]);
+ shadow_dims[1] = abs(transA ? t1->dim[0] : t1->dim[1]);
+
+ if (shadow_dims[0] != shadow_dims[1]) {
+ return luaL_error(L, "incompatible dimensions %d x %d * %d x %d",
+ dims[0], shadow_dims[1], shadow_dims[0], dims[1]);
+ }
+ else if (shadow_dims[0] == 0) {
+ /* Row * Column -> matrix */
+ shadow_dims[0] = 1;
+ shadow_dims[1] = 1;
+ }
+
+ if (dims[0] == 0) {
+ /* Column */
+ dims[0] = 1;
+
+ if (dims[1] == 0) {
+ /* Column * row -> number */
+ dims[1] = 1;
+ }
+ res = lua_newtensor(L, 2, dims, true, true);
+ }
+ else if (dims[1] == 0) {
+ /* Row */
+ res = lua_newtensor(L, 1, dims, true, true);
+ dims[1] = 1;
+ }
+ else {
+ res = lua_newtensor(L, 2, dims, true, true);
+ }
+
+ kad_sgemm_simple(transA, transB, dims[0], dims[1], shadow_dims[0],
+ t1->data, t2->data, res->data);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @function tensor.load(rspamd_text)
+ * Deserialize tensor
+ * @return
+ */
+static gint
+lua_tensor_load(lua_State *L)
+{
+ const guchar *data;
+ gsize sz;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid argument");
+ }
+
+ data = (const guchar *) t->start;
+ sz = t->len;
+ }
+ else {
+ data = (const guchar *) lua_tolstring(L, 1, &sz);
+ }
+
+ if (sz >= sizeof(gint) * 4) {
+ int ndims, nelts, dims[2];
+
+ memcpy(&ndims, data, sizeof(int));
+ memcpy(&nelts, data + sizeof(int), sizeof(int));
+ memcpy(dims, data + sizeof(int) * 2, sizeof(int) * 2);
+
+ if (sz == nelts * sizeof(rspamd_tensor_num_t) + sizeof(int) * 4) {
+ if (ndims == 1) {
+ if (nelts == dims[0]) {
+ struct rspamd_lua_tensor *t = lua_newtensor(L, ndims, dims, false, true);
+ memcpy(t->data, data + sizeof(int) * 4, nelts * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad dims: %d x %d != %d",
+ dims[0], 1, nelts);
+ }
+ }
+ else if (ndims == 2) {
+ if (nelts == dims[0] * dims[1]) {
+ struct rspamd_lua_tensor *t = lua_newtensor(L, ndims, dims, false, true);
+ memcpy(t->data, data + sizeof(int) * 4, nelts * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad dims: %d x %d != %d",
+ dims[0], dims[1], nelts);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid argument: bad ndims: %d", ndims);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid size: %d, %d required, %d elts", (int) sz,
+ (int) (nelts * sizeof(rspamd_tensor_num_t) + sizeof(int) * 4),
+ nelts);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments; sz = %d", (int) sz);
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_len(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+ gint nret = 1;
+
+ if (t) {
+ /* Return the main dimension first */
+ if (t->ndims == 1) {
+ lua_pushinteger(L, t->dim[0]);
+ }
+ else {
+ lua_pushinteger(L, t->dim[0]);
+ lua_pushinteger(L, t->dim[1]);
+ nret = 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return nret;
+}
+
+static gint
+lua_tensor_eigen(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *eigen;
+
+ if (t) {
+ if (t->ndims != 2 || t->dim[0] != t->dim[1]) {
+ return luaL_error(L, "expected square matrix NxN but got %dx%d",
+ t->dim[0], t->dim[1]);
+ }
+
+ eigen = lua_newtensor(L, 1, &t->dim[0], true, true);
+
+ if (!kad_ssyev_simple(t->dim[0], t->data, eigen->data)) {
+ lua_pop(L, 1);
+ return luaL_error(L, "kad_ssyev_simple failed (no blas?)");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline rspamd_tensor_num_t
+mean_vec(rspamd_tensor_num_t *x, gsize n)
+{
+ float sum = rspamd_sum_floats(x, &n);
+ return sum / (rspamd_tensor_num_t) n;
+}
+
+static gint
+lua_tensor_mean(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1);
+
+ if (t) {
+ if (t->ndims == 1) {
+ /* Mean of all elements in a vector */
+ lua_pushnumber(L, mean_vec(t->data, t->dim[0]));
+ }
+ else {
+ /* Row-wise mean vector output */
+ struct rspamd_lua_tensor *res;
+
+ res = lua_newtensor(L, 1, &t->dim[0], false, true);
+
+ for (int i = 0; i < t->dim[0]; i++) {
+ res->data[i] = mean_vec(&t->data[i * t->dim[1]], t->dim[1]);
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_transpose(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *res;
+ int dims[2];
+
+ if (t) {
+ if (t->ndims == 1) {
+ /* Row to column */
+ dims[0] = 1;
+ dims[1] = t->dim[0];
+ res = lua_newtensor(L, 2, dims, false, true);
+ memcpy(res->data, t->data, t->dim[0] * sizeof(rspamd_tensor_num_t));
+ }
+ else {
+ /* Cache friendly algorithm */
+ struct rspamd_lua_tensor *res;
+
+ dims[0] = t->dim[1];
+ dims[1] = t->dim[0];
+ res = lua_newtensor(L, 2, dims, false, true);
+
+ static const int block = 32;
+
+ for (int i = 0; i < t->dim[0]; i += block) {
+ for (int j = 0; j < t->dim[1]; ++j) {
+ for (int boff = 0; boff < block && i + boff < t->dim[0]; ++boff) {
+ res->data[j * t->dim[0] + i + boff] =
+ t->data[(i + boff) * t->dim[1] + j];
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_tensor_has_blas(lua_State *L)
+{
+#ifdef HAVE_CBLAS
+ lua_pushboolean(L, true);
+#else
+ lua_pushboolean(L, false);
+#endif
+
+ return 1;
+}
+
+static gint
+lua_tensor_scatter_matrix(lua_State *L)
+{
+ struct rspamd_lua_tensor *t = lua_check_tensor(L, 1), *res;
+ int dims[2];
+
+ if (t) {
+ if (t->ndims != 2) {
+ return luaL_error(L, "matrix required");
+ }
+
+ /* X * X square matrix */
+ dims[0] = t->dim[1];
+ dims[1] = t->dim[1];
+ res = lua_newtensor(L, 2, dims, true, true);
+
+ /* Auxiliary vars */
+ rspamd_tensor_num_t *means, /* means vector */
+ *tmp_row, /* temp row for Kahan's algorithm */
+ *tmp_square /* temp matrix for multiplications */;
+ means = g_malloc0(sizeof(rspamd_tensor_num_t) * t->dim[1]);
+ tmp_row = g_malloc0(sizeof(rspamd_tensor_num_t) * t->dim[1]);
+ tmp_square = g_malloc(sizeof(rspamd_tensor_num_t) * t->dim[1] * t->dim[1]);
+
+ /*
+ * Column based means
+ * means will have s, tmp_row will have c
+ */
+ for (int i = 0; i < t->dim[0]; i++) {
+ /* Cycle by rows */
+ for (int j = 0; j < t->dim[1]; j++) {
+ rspamd_tensor_num_t v = t->data[i * t->dim[1] + j];
+ rspamd_tensor_num_t y = v - tmp_row[j];
+ rspamd_tensor_num_t st = means[j] + y;
+ tmp_row[j] = (st - means[j]) - y;
+ means[j] = st;
+ }
+ }
+
+ for (int j = 0; j < t->dim[1]; j++) {
+ means[j] /= t->dim[0];
+ }
+
+ for (int i = 0; i < t->dim[0]; i++) {
+ /* Update for each sample */
+ for (int j = 0; j < t->dim[1]; j++) {
+ tmp_row[j] = t->data[i * t->dim[1] + j] - means[j];
+ }
+
+ memset(tmp_square, 0, t->dim[1] * t->dim[1] * sizeof(rspamd_tensor_num_t));
+ kad_sgemm_simple(1, 0, t->dim[1], t->dim[1], 1,
+ tmp_row, tmp_row, tmp_square);
+
+ for (int j = 0; j < t->dim[1]; j++) {
+ kad_saxpy(t->dim[1], 1.0, &tmp_square[j * t->dim[1]],
+ &res->data[j * t->dim[1]]);
+ }
+ }
+
+ g_free(tmp_row);
+ g_free(means);
+ g_free(tmp_square);
+ }
+ else {
+ return luaL_error(L, "tensor required");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_tensor(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, rspamd_tensor_f);
+
+ return 1;
+}
+
+
+void luaopen_tensor(lua_State *L)
+{
+ /* Metatables */
+ rspamd_lua_new_class(L, TENSOR_CLASS, rspamd_tensor_m);
+ lua_pop(L, 1); /* No need in metatable... */
+ rspamd_lua_add_preload(L, "rspamd_tensor", lua_load_tensor);
+ lua_settop(L, 0);
+}
+
+struct rspamd_lua_tensor *
+lua_check_tensor(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, TENSOR_CLASS);
+ luaL_argcheck(L, ud != NULL, pos, "'tensor' expected");
+ return ud ? ((struct rspamd_lua_tensor *) ud) : NULL;
+}
diff --git a/src/lua/lua_tensor.h b/src/lua/lua_tensor.h
new file mode 100644
index 0000000..2103868
--- /dev/null
+++ b/src/lua/lua_tensor.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LUA_TENSOR_H
+#define RSPAMD_LUA_TENSOR_H
+
+#define TENSOR_CLASS "rspamd{tensor}"
+
+typedef float rspamd_tensor_num_t;
+
+struct rspamd_lua_tensor {
+ int ndims;
+ int size; /* overall size (product of dims) */
+ int dim[2];
+ rspamd_tensor_num_t *data;
+};
+
+struct rspamd_lua_tensor *lua_check_tensor(lua_State *L, int pos);
+struct rspamd_lua_tensor *lua_newtensor(lua_State *L, int ndims,
+ const int *dim, bool zero_fill, bool own);
+
+#endif
diff --git a/src/lua/lua_text.c b/src/lua/lua_text.c
new file mode 100644
index 0000000..26a5c08
--- /dev/null
+++ b/src/lua/lua_text.c
@@ -0,0 +1,1789 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "libcryptobox/cryptobox.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "unix-std.h"
+
+/***
+ * @module rspamd_text
+ * This module provides access to opaque text structures used widely to prevent
+ * copying between Lua and C for various concerns: performance, security etc...
+ *
+ * You can convert rspamd_text into string but it will copy data.
+ */
+
+/***
+ * @function rspamd_text.fromstring(str)
+ * Creates rspamd_text from Lua string (copied to the text)
+ * @param {string} str string to use
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, fromstring);
+
+/***
+ * @function rspamd_text.null()
+ * Creates rspamd_text with NULL pointer for testing purposes
+ * @param {string} str string to use
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, null);
+/***
+ * @function rspamd_text.randombytes(nbytes)
+ * Creates rspamd_text with random bytes inside (raw bytes)
+ * @param {number} nbytes number of random bytes generated
+ * @return {rspamd_text} random bytes text
+ */
+LUA_FUNCTION_DEF(text, randombytes);
+
+/***
+ * @function rspamd_text.fromtable(tbl[, delim])
+ * Same as `table.concat` but generates rspamd_text instead of the Lua string
+ * @param {table} tbl table to use
+ * @param {string} delim optional delimiter
+ * @return {rspamd_text} resulting text
+ */
+LUA_FUNCTION_DEF(text, fromtable);
+/***
+ * @method rspamd_text:byte(pos[, pos2])
+ * Returns a byte at the position `pos` or bytes from `pos` to `pos2` if specified
+ * @param {integer} pos index
+ * @param {integer} pos2 index
+ * @return {integer} byte at the position `pos` or varargs of bytes
+ */
+LUA_FUNCTION_DEF(text, byte);
+/***
+ * @method rspamd_text:len()
+ * Returns length of a string
+ * @return {number} length of string in **bytes**
+ */
+LUA_FUNCTION_DEF(text, len);
+/***
+ * @method rspamd_text:str()
+ * Converts text to string by copying its content
+ * @return {string} copy of text as Lua string
+ */
+LUA_FUNCTION_DEF(text, str);
+/***
+ * @method rspamd_text:ptr()
+ * Converts text to lightuserdata
+ * @return {lightuserdata} pointer value of rspamd_text
+ */
+LUA_FUNCTION_DEF(text, ptr);
+/***
+ * @method rspamd_text:save_in_file(fname[, mode])
+ * Saves text in file
+ * @return {boolean} true if save has been completed
+ */
+LUA_FUNCTION_DEF(text, save_in_file);
+/***
+ * @method rspamd_text:span(start[, len])
+ * Returns a span for lua_text starting at pos [start] (1 indexed) and with
+ * length `len` (or to the end of the text)
+ * @param {integer} start start index
+ * @param {integer} len length of span
+ * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...)
+ */
+LUA_FUNCTION_DEF(text, span);
+/***
+ * @method rspamd_text:sub(start[, len])
+ * Returns a substring for lua_text similar to string.sub from Lua
+ * @return {rspamd_text} new rspamd_text with span (must be careful when using with owned texts...)
+ */
+LUA_FUNCTION_DEF(text, sub);
+/***
+ * @method rspamd_text:lines([stringify])
+ * Returns an iter over all lines as rspamd_text objects or as strings if `stringify` is true
+ * @param {boolean} stringify stringify lines
+ * @return {iterator} iterator triplet
+ */
+LUA_FUNCTION_DEF(text, lines);
+/***
+ * @method rspamd_text:split(regexp, [stringify])
+ * Returns an iter over all encounters of the specific regexp as rspamd_text objects or as strings if `stringify` is true
+ * @param {rspamd_regexp} regexp regexp (pcre syntax) used for splitting
+ * @param {boolean} stringify stringify lines
+ * @return {iterator} iterator triplet
+ */
+LUA_FUNCTION_DEF(text, split);
+/***
+ * @method rspamd_text:at(pos)
+ * Returns a byte at the position `pos`
+ * @param {integer} pos index
+ * @return {integer} byte at the position `pos` or nil if pos out of bound
+ */
+LUA_FUNCTION_DEF(text, at);
+/***
+ * @method rspamd_text:memchr(chr, [reverse])
+ * Returns the first or the last position of the character `chr` in the text or
+ * -1 in case if a character has not been found. Indexes start from `1`
+ * @param {string/number} chr character or a character code to find
+ * @param {boolean} reverse last character if `true`
+ * @return {integer} position of the character or `-1`
+ */
+LUA_FUNCTION_DEF(text, memchr);
+/***
+ * @method rspamd_text:bytes()
+ * Converts text to an array of bytes
+ * @return {table|integer} bytes in the array (as unsigned char)
+ */
+LUA_FUNCTION_DEF(text, bytes);
+/***
+ * @method rspamd_text:lower([is_utf, [inplace]])
+ * Return a new text with lowercased characters, if is_utf is true then Rspamd applies utf8 lowercase
+ * @param {boolean} is_utf apply utf8 lowercase
+ * @param {boolean} inplace lowercase the original text
+ * @return {rspamd_text} new rspamd_text (or the original text if inplace) with lowercased letters
+ */
+LUA_FUNCTION_DEF(text, lower);
+LUA_FUNCTION_DEF(text, take_ownership);
+/***
+ * @method rspamd_text:exclude_chars(set_to_exclude, [always_copy])
+ * Returns a text (if owned, then the original text is modified, if not, then it is copied and owned)
+ * where all chars from `set_to_exclude` are removed
+ * Patterns supported:
+ *
+ * - %s - all space characters
+ * - %n - all newline characters
+ * - %c - all control characters (it includes 8bit characters and spaces)
+ * - %8 - all 8 bit characters
+ * - %% - just a percent character
+ *
+ * @param {string} set_to_exclude characters to exclude
+ * @param {boolean} always_copy always copy the source text
+ * @return {rspamd_text} modified or copied text
+ */
+LUA_FUNCTION_DEF(text, exclude_chars);
+/***
+ * @method rspamd_text:oneline([always_copy])
+ * Returns a text (if owned, then the original text is modified, if not, then it is copied and owned)
+ * where the following transformations are made:
+ * - All spaces sequences are replaced with a single space
+ * - All newlines sequences are replaced with a single space
+ * - Trailing and leading spaces are removed
+ * - Control characters are excluded
+ * - UTF8 sequences are normalised
+ *
+ * @param {boolean} always_copy always copy the source text
+ * @return {rspamd_text} modified or copied text
+ */
+LUA_FUNCTION_DEF(text, oneline);
+/***
+ * @method rspamd_text:base32([b32type])
+ * Returns a text encoded in base32 (new rspamd_text is allocated)
+ *
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} new text encoded in base32
+ */
+LUA_FUNCTION_DEF(text, base32);
+/***
+ * @method rspamd_text:base64([line_length, [nline, [fold]]])
+ * Returns a text encoded in base64 (new rspamd_text is allocated)
+ *
+ * @param {number} line_length return text split with newlines up to this attribute
+ * @param {string} nline newline type: `cr`, `lf`, `crlf`
+ * @param {boolean} fold use folding when splitting into lines (false by default)
+ * @return {rspamd_text} new text encoded in base64
+ */
+LUA_FUNCTION_DEF(text, base64);
+/***
+ * @method rspamd_text:hex()
+ * Returns a text encoded in hex (new rspamd_text is allocated)
+ *
+ * @return {rspamd_text} new text encoded in hex
+ */
+LUA_FUNCTION_DEF(text, hex);
+/***
+ * @method rspamd_text:find(pattern [, init])
+ * Looks for the first match of pattern in the string s.
+ * If it finds a match, then find returns the indices of s where this occurrence
+ * starts and ends; otherwise, it returns nil. A third,
+ * optional numerical argument init specifies where to start the search;
+ * its default value is 1 and can be negative.
+ * This method currently supports merely a plain search, no patterns.
+ *
+ * @param {string} pattern pattern to find
+ * @param {number} init specifies where to start the search (1 default)
+ * @return {number,number/nil} If it finds a match, then find returns the indices of s where this occurrence starts and ends; otherwise, it returns nil
+ */
+LUA_FUNCTION_DEF(text, find);
+LUA_FUNCTION_DEF(text, gc);
+LUA_FUNCTION_DEF(text, eq);
+LUA_FUNCTION_DEF(text, lt);
+LUA_FUNCTION_DEF(text, concat);
+LUA_FUNCTION_DEF(text, strtoul);
+
+static const struct luaL_reg textlib_f[] = {
+ LUA_INTERFACE_DEF(text, fromstring),
+ {"from_string", lua_text_fromstring},
+ LUA_INTERFACE_DEF(text, fromtable),
+ {"from_table", lua_text_fromtable},
+ LUA_INTERFACE_DEF(text, null),
+ LUA_INTERFACE_DEF(text, randombytes),
+ {NULL, NULL}};
+
+static const struct luaL_reg textlib_m[] = {
+ LUA_INTERFACE_DEF(text, len),
+ LUA_INTERFACE_DEF(text, str),
+ LUA_INTERFACE_DEF(text, ptr),
+ LUA_INTERFACE_DEF(text, take_ownership),
+ LUA_INTERFACE_DEF(text, save_in_file),
+ LUA_INTERFACE_DEF(text, span),
+ LUA_INTERFACE_DEF(text, sub),
+ LUA_INTERFACE_DEF(text, lines),
+ LUA_INTERFACE_DEF(text, split),
+ LUA_INTERFACE_DEF(text, at),
+ LUA_INTERFACE_DEF(text, memchr),
+ LUA_INTERFACE_DEF(text, byte),
+ LUA_INTERFACE_DEF(text, bytes),
+ LUA_INTERFACE_DEF(text, lower),
+ LUA_INTERFACE_DEF(text, exclude_chars),
+ LUA_INTERFACE_DEF(text, oneline),
+ LUA_INTERFACE_DEF(text, base32),
+ LUA_INTERFACE_DEF(text, base64),
+ LUA_INTERFACE_DEF(text, hex),
+ LUA_INTERFACE_DEF(text, find),
+ LUA_INTERFACE_DEF(text, strtoul),
+ {"write", lua_text_save_in_file},
+ {"__len", lua_text_len},
+ {"__tostring", lua_text_str},
+ {"__gc", lua_text_gc},
+ {"__eq", lua_text_eq},
+ {"__lt", lua_text_lt},
+ {"__concat", lua_text_concat},
+ {NULL, NULL}};
+
+struct rspamd_lua_text *
+lua_check_text(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{text}");
+ luaL_argcheck(L, ud != NULL, pos, "'text' expected");
+ return ud ? (struct rspamd_lua_text *) ud : NULL;
+}
+
+struct rspamd_lua_text *
+lua_check_text_or_string(lua_State *L, gint pos)
+{
+ gint pos_type = lua_type(L, pos);
+
+ if (pos_type == LUA_TUSERDATA) {
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{text}");
+ luaL_argcheck(L, ud != NULL, pos, "'text' expected");
+ return ud ? (struct rspamd_lua_text *) ud : NULL;
+ }
+ else if (pos_type == LUA_TSTRING) {
+ /*
+ * Fake static lua_text, we allow to use this function multiple times
+ * by having a small array of static structures.
+ */
+ static unsigned cur_txt_idx = 0;
+ static struct rspamd_lua_text fake_text[4];
+ gsize len;
+ int sel_idx;
+
+ sel_idx = cur_txt_idx++ % G_N_ELEMENTS(fake_text);
+ fake_text[sel_idx].start = lua_tolstring(L, pos, &len);
+
+ if (len >= G_MAXUINT) {
+ return NULL;
+ }
+
+ fake_text[sel_idx].len = len;
+ fake_text[sel_idx].flags = RSPAMD_TEXT_FLAG_FAKE;
+
+ return &fake_text[sel_idx];
+ }
+
+ return NULL;
+}
+
+struct rspamd_lua_text *
+lua_new_text(lua_State *L, const gchar *start, gsize len, gboolean own)
+{
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ t->flags = 0;
+
+ if (own) {
+ gchar *storage;
+
+ if (len > 0) {
+ storage = g_malloc(len);
+
+ if (start != NULL) {
+ memcpy(storage, start, len);
+ }
+
+ t->start = storage;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ t->start = "";
+ }
+ }
+ else {
+ t->start = start;
+ }
+
+ t->len = len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ return t;
+}
+
+struct rspamd_lua_text *
+lua_new_text_task(lua_State *L, struct rspamd_task *task,
+ const gchar *start, gsize len, gboolean own)
+{
+ struct rspamd_lua_text *t;
+
+ t = lua_newuserdata(L, sizeof(*t));
+ t->flags = 0;
+
+ if (own) {
+ gchar *storage;
+
+ if (len > 0) {
+ storage = rspamd_mempool_alloc(task->task_pool, len);
+
+ if (start != NULL) {
+ memcpy(storage, start, len);
+ }
+
+ t->start = storage;
+ }
+ else {
+ t->start = "";
+ }
+ }
+ else {
+ t->start = start;
+ }
+
+ t->len = len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ return t;
+}
+
+bool lua_is_text_binary(struct rspamd_lua_text *t)
+{
+ if (t == NULL || t->len == 0) {
+ return false;
+ }
+
+ if (rspamd_str_has_8bit(t->start, t->len)) {
+ if (rspamd_fast_utf8_validate(t->start, t->len) == 0) {
+ return false;
+ }
+ return true;
+ }
+
+ return false;
+}
+
+
+static gint
+lua_text_fromstring(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *str;
+ gsize l = 0;
+ gboolean transparent = FALSE;
+
+ str = luaL_checklstring(L, 1, &l);
+
+ if (str) {
+ if (lua_isboolean(L, 2)) {
+ transparent = lua_toboolean(L, 2);
+ }
+
+ lua_new_text(L, str, l, !transparent);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ return 1;
+}
+
+static gint
+lua_text_null(lua_State *L)
+{
+ LUA_TRACE_POINT;
+
+ lua_new_text(L, NULL, 0, false);
+
+ return 1;
+}
+
+static gint
+lua_text_randombytes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint nbytes = luaL_checkinteger(L, 1);
+ struct rspamd_lua_text *out;
+
+ out = lua_new_text(L, NULL, nbytes, TRUE);
+ randombytes_buf((char *) out->start, nbytes);
+ out->len = nbytes;
+
+ return 1;
+}
+
+#define MAX_REC 10
+
+static void
+lua_text_tbl_length(lua_State *L, gsize dlen, gsize *dest, guint rec)
+{
+ gsize tblen, stlen;
+ struct rspamd_lua_text *elt;
+
+ if (rec > MAX_REC) {
+ luaL_error(L, "lua_text_tbl_length: recursion limit exceeded");
+
+ return;
+ }
+
+ tblen = rspamd_lua_table_size(L, -1);
+
+ for (gsize i = 0; i < tblen; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+#if LUA_VERSION_NUM >= 502
+ stlen = lua_rawlen(L, -1);
+#else
+ stlen = lua_objlen(L, -1);
+#endif
+ (*dest) += stlen;
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ elt = (struct rspamd_lua_text *) lua_touserdata(L, -1);
+
+ if (elt) {
+ (*dest) += elt->len;
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_text_tbl_length(L, dlen, dest, rec + 1);
+ }
+
+ if (i != tblen - 1) {
+ (*dest) += dlen;
+ }
+
+ lua_pop(L, 1);
+ }
+}
+
+static void
+lua_text_tbl_append(lua_State *L,
+ const gchar *delim,
+ gsize dlen,
+ gchar **dest,
+ guint rec)
+{
+ const gchar *st;
+ gsize tblen, stlen;
+ struct rspamd_lua_text *elt;
+
+ if (rec > MAX_REC) {
+ luaL_error(L, "lua_text_tbl_length: recursion limit exceeded");
+
+ return;
+ }
+
+ tblen = rspamd_lua_table_size(L, -1);
+
+ for (guint i = 0; i < tblen; i++) {
+ lua_rawgeti(L, -1, i + 1);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ st = lua_tolstring(L, -1, &stlen);
+ memcpy((*dest), st, stlen);
+ (*dest) += stlen;
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ elt = (struct rspamd_lua_text *) lua_touserdata(L, -1);
+
+ if (elt) {
+ memcpy((*dest), elt->start, elt->len);
+ (*dest) += elt->len;
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TTABLE) {
+ lua_text_tbl_append(L, delim, dlen, dest, rec + 1);
+ }
+
+ if (dlen && i != tblen - 1) {
+ memcpy((*dest), delim, dlen);
+ (*dest) += dlen;
+ }
+
+ lua_pop(L, 1);
+ }
+}
+
+static gint
+lua_text_fromtable(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *delim = "";
+ struct rspamd_lua_text *t;
+ gsize textlen = 0, dlen, oldtop = lua_gettop(L);
+ gchar *dest;
+
+ if (!lua_istable(L, 1)) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ delim = lua_tolstring(L, 2, &dlen);
+ }
+ else {
+ dlen = 0;
+ }
+
+ /* Calculate length needed */
+ lua_pushvalue(L, 1);
+ lua_text_tbl_length(L, dlen, &textlen, 0);
+ lua_pop(L, 1);
+
+ /* Allocate new text */
+ t = lua_newuserdata(L, sizeof(*t));
+ dest = g_malloc(textlen);
+ t->start = dest;
+ t->len = textlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+
+ lua_pushvalue(L, 1);
+ lua_text_tbl_append(L, delim, dlen, &dest, 0);
+ lua_pop(L, 1); /* Table arg */
+
+ gint newtop = lua_gettop(L);
+ g_assert(newtop == oldtop + 1);
+
+ return 1;
+}
+
+static gint
+lua_text_len(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gsize l = 0;
+
+ if (t != NULL) {
+ l = t->len;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, l);
+
+ return 1;
+}
+
+static gint
+lua_text_str(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ lua_pushlstring(L, t->start, t->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_ptr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ lua_pushlightuserdata(L, (gpointer) t->start);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_take_ownership(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gchar *dest;
+
+ if (t != NULL) {
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ /* We already own it */
+ lua_pushboolean(L, true);
+ }
+ else {
+ dest = g_malloc(t->len);
+ memcpy(dest, t->start, t->len);
+ t->start = dest;
+ t->flags |= RSPAMD_TEXT_FLAG_OWN;
+ lua_pushboolean(L, true);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_span(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gint64 start = lua_tointeger(L, 2), len = -1;
+
+ if (t && start >= 1 && start <= t->len) {
+ if (lua_isnumber(L, 3)) {
+ len = lua_tonumber(L, 3);
+ }
+
+ if (len == -1) {
+ len = t->len - (start - 1);
+ }
+
+ if (len < 0 || (len > (t->len - (start - 1)))) {
+ return luaL_error(L, "invalid length");
+ }
+
+ lua_new_text(L, t->start + (start - 1), len, FALSE);
+ }
+ else {
+ if (!t) {
+ return luaL_error(L, "invalid arguments, text required");
+ }
+ else {
+ return luaL_error(L, "invalid arguments: start offset %d "
+ "is larger than text len %d",
+ (int) start, (int) t->len);
+ }
+ }
+
+ return 1;
+}
+
+/* Helpers to behave exactly as Lua does */
+static inline gsize
+relative_pos_start(gint pos, gsize len)
+{
+ if (pos > 0) {
+ return pos;
+ }
+ else if (pos == 0) {
+ return 1;
+ }
+ else if (pos < -((gint) len)) {
+ return 1;
+ }
+
+ /* Negative pos inside str */
+ return len + ((gsize) pos) + 1;
+}
+
+static inline gsize
+relative_pos_end(gint pos, gsize len)
+{
+ if (pos > (gint) len) {
+ return len;
+ }
+ else if (pos >= 0) {
+ return (size_t) pos;
+ }
+ else if (pos < -((gint) len)) {
+ return 0;
+ }
+
+ return len + ((gsize) pos) + 1;
+}
+
+static gint
+lua_text_sub(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ size_t start = relative_pos_start(luaL_checkinteger(L, 2),
+ t->len);
+ size_t end = relative_pos_end(luaL_optinteger(L, 3, -1),
+ t->len);
+
+
+ if (start <= end) {
+ lua_new_text(L, t->start + (start - 1),
+ (end - start) + 1, FALSE);
+ }
+ else {
+ lua_new_text(L, "", 0, TRUE);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint64
+rspamd_lua_text_push_line(lua_State *L,
+ struct rspamd_lua_text *t,
+ gint64 start_offset,
+ const gchar *sep_pos,
+ gboolean stringify)
+{
+ const gchar *start;
+ gsize len;
+ gint64 ret;
+
+ start = t->start + start_offset;
+ len = sep_pos ? (sep_pos - start) : (t->len - start_offset);
+ ret = start_offset + len;
+
+ /* Trim line */
+ while (len > 0) {
+ if (start[len - 1] == '\r' || start[len - 1] == '\n') {
+ len--;
+ }
+ else {
+ break;
+ }
+ }
+
+ if (stringify) {
+ lua_pushlstring(L, start, len);
+ }
+ else {
+ struct rspamd_lua_text *ntext;
+
+ ntext = lua_newuserdata(L, sizeof(*ntext));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ ntext->start = start;
+ ntext->len = len;
+ ntext->flags = 0; /* Not own as it must be owned by a top object */
+ }
+
+ return ret;
+}
+
+static gint
+rspamd_lua_text_readline(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_touserdata(L, lua_upvalueindex(1));
+ gboolean stringify = lua_toboolean(L, lua_upvalueindex(2));
+ gint64 pos = lua_tointeger(L, lua_upvalueindex(3));
+
+ if (pos < 0) {
+ return luaL_error(L, "invalid pos: %d", (gint) pos);
+ }
+
+ if (pos >= t->len) {
+ /* We are done */
+ return 0;
+ }
+
+ const gchar *sep_pos;
+
+ /* We look just for `\n` ignoring `\r` as it is very rare nowadays */
+ sep_pos = memchr(t->start + pos, '\n', t->len - pos);
+
+ if (sep_pos == NULL) {
+ /* Either last `\n` or `\r` separated text */
+ sep_pos = memchr(t->start + pos, '\r', t->len - pos);
+ }
+
+ pos = rspamd_lua_text_push_line(L, t, pos, sep_pos, stringify);
+
+ /* Skip separators */
+ while (pos < t->len) {
+ if (t->start[pos] == '\n' || t->start[pos] == '\r') {
+ pos++;
+ }
+ else {
+ break;
+ }
+ }
+
+ /* Update pos */
+ lua_pushinteger(L, pos);
+ lua_replace(L, lua_upvalueindex(3));
+
+ return 1;
+}
+
+static gint
+lua_text_lines(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gboolean stringify = FALSE;
+
+ if (t) {
+ if (lua_isboolean(L, 2)) {
+ stringify = lua_toboolean(L, 2);
+ }
+
+ lua_pushvalue(L, 1);
+ lua_pushboolean(L, stringify);
+ lua_pushinteger(L, 0); /* Current pos */
+ lua_pushcclosure(L, rspamd_lua_text_readline, 3);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+rspamd_lua_text_regexp_split(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_touserdata(L, lua_upvalueindex(1)),
+ *new_t;
+ struct rspamd_lua_regexp *re = *(struct rspamd_lua_regexp **)
+ lua_touserdata(L, lua_upvalueindex(2));
+ gboolean stringify = lua_toboolean(L, lua_upvalueindex(3));
+ gint64 pos = lua_tointeger(L, lua_upvalueindex(4));
+ gboolean matched;
+
+ if (pos < 0) {
+ return luaL_error(L, "invalid pos: %d", (gint) pos);
+ }
+
+ if (pos >= t->len) {
+ /* We are done */
+ return 0;
+ }
+
+ const gchar *start, *end, *old_start;
+
+ end = t->start + pos;
+
+ for (;;) {
+ old_start = end;
+
+ matched = rspamd_regexp_search(re->re, t->start, t->len, &start, &end, FALSE,
+ NULL);
+
+ if (matched) {
+ if (start - old_start > 0) {
+ if (stringify) {
+ lua_pushlstring(L, old_start, start - old_start);
+ }
+ else {
+ new_t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ new_t->start = old_start;
+ new_t->len = start - old_start;
+ new_t->flags = 0;
+ }
+
+ break;
+ }
+ else {
+ if (start == end) {
+ matched = FALSE;
+ break;
+ }
+ /*
+ * All match separators (e.g. starting separator,
+ * we need to skip it). Continue iterations.
+ */
+ }
+ }
+ else {
+ /* No match, stop */
+ break;
+ }
+ }
+
+ if (!matched && (t->len > 0 && (end == NULL || end < t->start + t->len))) {
+ /* No more matches, but we might need to push the last element */
+ if (end == NULL) {
+ end = t->start;
+ }
+ /* No separators, need to push the whole remaining part */
+ if (stringify) {
+ lua_pushlstring(L, end, (t->start + t->len) - end);
+ }
+ else {
+ new_t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ new_t->start = end;
+ new_t->len = (t->start + t->len) - end;
+ new_t->flags = 0;
+ }
+
+ pos = t->len;
+ }
+ else {
+
+ pos = end - t->start;
+ }
+
+ /* Update pos */
+ lua_pushinteger(L, pos);
+ lua_replace(L, lua_upvalueindex(4));
+
+ return 1;
+}
+
+static gint
+lua_text_split(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ struct rspamd_lua_regexp *re;
+ gboolean stringify = FALSE, own_re = FALSE;
+
+ if (t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ re = lua_check_regexp(L, 2);
+ }
+ else {
+ rspamd_regexp_t *c_re;
+ GError *err = NULL;
+
+ c_re = rspamd_regexp_new(lua_tostring(L, 2), NULL, &err);
+ if (c_re == NULL) {
+
+ gint ret = luaL_error(L, "cannot parse regexp: %s, error: %s",
+ lua_tostring(L, 2),
+ err == NULL ? "undefined" : err->message);
+ if (err) {
+ g_error_free(err);
+ }
+
+ return ret;
+ }
+
+ re = g_malloc0(sizeof(struct rspamd_lua_regexp));
+ re->re = c_re;
+ re->re_pattern = g_strdup(lua_tostring(L, 2));
+ re->module = rspamd_lua_get_module_name(L);
+ own_re = TRUE;
+ }
+
+ if (re) {
+ if (lua_isboolean(L, 3)) {
+ stringify = lua_toboolean(L, 3);
+ }
+
+ /* Upvalues */
+ lua_pushvalue(L, 1); /* text */
+
+ if (own_re) {
+ struct rspamd_lua_regexp **pre;
+ pre = lua_newuserdata(L, sizeof(struct rspamd_lua_regexp *));
+ rspamd_lua_setclass(L, "rspamd{regexp}", -1);
+ *pre = re;
+ }
+ else {
+ lua_pushvalue(L, 2); /* regexp */
+ }
+
+ lua_pushboolean(L, stringify);
+ lua_pushinteger(L, 0); /* Current pos */
+ lua_pushcclosure(L, rspamd_lua_text_regexp_split, 4);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_text_at(lua_State *L)
+{
+ return lua_text_byte(L);
+}
+
+static gint
+lua_text_byte(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ gsize start = relative_pos_start(luaL_optinteger(L, 2, 1), t->len);
+ gsize end = relative_pos_end(luaL_optinteger(L, 3, start), t->len);
+ start--;
+
+ if (start >= end) {
+ return 0;
+ }
+
+ for (gsize i = start; i < end; i++) {
+ lua_pushinteger(L, t->start[i]);
+ }
+ return end - start;
+}
+
+static gint
+lua_text_memchr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ int c;
+ bool reverse = false;
+
+ if (lua_isnumber(L, 2)) {
+ c = lua_tonumber(L, 2);
+ }
+ else {
+ gsize l;
+ const gchar *str = lua_tolstring(L, 2, &l);
+
+ if (str) {
+ c = str[0];
+
+ if (l != 1) {
+ return luaL_error(L, "need exactly one character to search");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+
+ if (t) {
+ void *f;
+
+ if (lua_isboolean(L, 3)) {
+ reverse = lua_toboolean(L, 3);
+ }
+
+ if (reverse) {
+ f = rspamd_memrchr(t->start, c, t->len);
+ }
+ else {
+ f = memchr(t->start, c, t->len);
+ }
+
+ if (f) {
+ lua_pushinteger(L, ((const char *) f) - t->start + 1);
+ }
+ else {
+ lua_pushinteger(L, -1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_bytes(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ lua_createtable(L, t->len, 0);
+
+ for (gsize i = 0; i < t->len; i++) {
+ lua_pushinteger(L, (guchar) t->start[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_save_in_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ const gchar *fname = NULL;
+ guint mode = 00644;
+ gint fd = -1;
+ gboolean need_close = FALSE;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ fname = luaL_checkstring(L, 2);
+
+ if (lua_type(L, 3) == LUA_TNUMBER) {
+ mode = lua_tointeger(L, 3);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TNUMBER) {
+ /* Created fd */
+ fd = lua_tointeger(L, 2);
+ }
+
+ if (fd == -1) {
+ if (fname) {
+ fd = rspamd_file_xopen(fname, O_CREAT | O_WRONLY | O_EXCL, mode, 0);
+
+ if (fd == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+ need_close = TRUE;
+ }
+ else {
+ fd = STDOUT_FILENO;
+ }
+ }
+
+ if (write(fd, t->start, t->len) == -1) {
+ if (fd != STDOUT_FILENO) {
+ close(fd);
+ }
+
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ if (need_close) {
+ close(fd);
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_gc(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ g_assert(!(t->flags & RSPAMD_TEXT_FLAG_FAKE));
+
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ if (t->flags & RSPAMD_TEXT_FLAG_WIPE) {
+ rspamd_explicit_memzero((guchar *) t->start, t->len);
+ }
+
+ if (t->flags & RSPAMD_TEXT_FLAG_MMAPED) {
+ munmap((gpointer) t->start, t->len);
+ }
+ else {
+ if (t->flags & RSPAMD_TEXT_FLAG_SYSMALLOC) {
+ free((gpointer) t->start);
+ }
+ else {
+ g_free((gpointer) t->start);
+ }
+ }
+ }
+ }
+
+ return 0;
+}
+
+static gint
+lua_text_eq(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1->len == t2->len) {
+ lua_pushboolean(L, memcmp(t1->start, t2->start, t1->len) == 0);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_lt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ if (t1->len == t2->len) {
+ lua_pushboolean(L, memcmp(t1->start, t2->start, t1->len) < 0);
+ }
+ else {
+ lua_pushboolean(L, t1->len < t2->len);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_concat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1 = lua_check_text_or_string(L, 1),
+ *t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ struct rspamd_lua_text *final;
+
+ final = lua_new_text(L, NULL, t1->len + t2->len, TRUE);
+ memcpy((void *) final->start, t1->start, t1->len);
+ memcpy((void *) (final->start + t1->len), t2->start, t2->len);
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_wipe(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ rspamd_explicit_memzero((guchar *) t->start, t->len);
+ }
+ else {
+ return luaL_error(L, "cannot wipe not owned text");
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_text_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ out = lua_new_text(L, NULL, t->len * 8 / 5 + 2, TRUE);
+ out->len = rspamd_encode_base32_buf(t->start, t->len, (gchar *) out->start,
+ out->len, btype);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+ gsize line_len = 0;
+ gboolean fold = FALSE;
+
+ if (t != NULL) {
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ line_len = lua_tointeger(L, 2);
+
+ if (line_len <= 8) {
+ return luaL_error(L, "too small line length (at least 8 is required)");
+ }
+ }
+
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ if (lua_type(L, 4) == LUA_TBOOLEAN) {
+ fold = lua_toboolean(L, 4);
+ }
+
+ gsize sz_len;
+
+ out = lua_newuserdata(L, sizeof(*t));
+ out->flags = RSPAMD_TEXT_FLAG_OWN;
+ out->start = rspamd_encode_base64_common(t->start, t->len,
+ line_len, &sz_len, fold, how);
+ out->len = sz_len;
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *out;
+
+ if (t != NULL) {
+
+ out = lua_new_text(L, NULL, t->len * 2, TRUE);
+ out->len = rspamd_encode_hex_buf(t->start, t->len, (gchar *) out->start,
+ out->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_find(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gsize patlen, init = 1;
+ const gchar *pat = luaL_checklstring(L, 2, &patlen);
+
+ if (t != NULL && pat != NULL) {
+
+ if (lua_isnumber(L, 3)) {
+ init = relative_pos_start(lua_tointeger(L, 3), t->len);
+ }
+
+ init--;
+
+ if (init > t->len) {
+ return luaL_error(L, "invalid arguments to find: init too large");
+ }
+
+ goffset pos = rspamd_substring_search(t->start + init,
+ t->len - init,
+ pat, patlen);
+
+ if (pos == -1) {
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ lua_pushinteger(L, pos + 1);
+ lua_pushinteger(L, pos + patlen);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+#define BITOP(a, b, op) \
+ ((a)[(guint64) (b) / (8u * sizeof *(a))] op(guint64) 1 << ((guint64) (b) % (8u * sizeof *(a))))
+
+static gint
+lua_text_exclude_chars(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ gssize patlen;
+ const gchar *pat = lua_tolstring(L, 2, &patlen), *p, *end;
+ gchar *dest, *d;
+ guint64 byteset[32 / sizeof(guint64)]; /* Bitset for ascii */
+ gboolean copy = TRUE;
+ guint *plen;
+
+ if (t != NULL && pat && patlen > 0) {
+ if (lua_isboolean(L, 3)) {
+ copy = lua_toboolean(L, 3);
+ }
+ else if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ copy = FALSE;
+ }
+
+ if (!copy) {
+ dest = (gchar *) t->start;
+ plen = &t->len;
+ lua_pushvalue(L, 1); /* Push text as a result */
+ }
+ else {
+ /* We need to copy read only text */
+ struct rspamd_lua_text *nt;
+
+ dest = g_malloc(t->len);
+ nt = lua_newuserdata(L, sizeof(*nt));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ nt->len = t->len;
+ nt->flags = RSPAMD_TEXT_FLAG_OWN;
+ memcpy(dest, t->start, t->len);
+ nt->start = dest;
+ plen = &nt->len;
+ }
+
+ /* Fill pattern bitset */
+ memset(byteset, 0, sizeof byteset);
+
+ while (patlen > 0) {
+ if (*pat == '%') {
+ pat++;
+ patlen--;
+
+ if (patlen > 0) {
+ /*
+ * This stuff assumes little endian, but GUINT64_FROM_LE should
+ * deal with proper conversion
+ */
+ switch (*pat) {
+ case '%':
+ BITOP(byteset, *(guchar *) pat, |=);
+ break;
+ case 's':
+ /* "\r\n\t\f " */
+ byteset[0] |= GUINT64_FROM_LE(0x100003600LLU);
+ break;
+ case 'n':
+ /* newlines: "\r\n" */
+ byteset[0] |= GUINT64_FROM_LE(0x2400LLU);
+ break;
+ case '8':
+ /* 8 bit characters */
+ byteset[2] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ byteset[3] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ break;
+ case 'c':
+ /* Non printable (control) characters */
+ byteset[0] |= GUINT64_FROM_LE(0xffffffffLLU);
+ /* Del character */
+ byteset[1] |= GUINT64_FROM_LE(0x8000000000000000LLU);
+ break;
+ }
+ }
+ else {
+ /* Last '%' */
+ BITOP(byteset, (guchar) '%', |=);
+ }
+ }
+ else {
+ BITOP(byteset, *(guchar *) pat, |=);
+ }
+
+ pat++;
+ patlen--;
+ }
+ for (; patlen > 0 && BITOP(byteset, *(guchar *) pat, |=); pat++, patlen--)
+ ;
+
+ p = t->start;
+ end = t->start + t->len;
+ d = dest;
+
+ while (p < end) {
+ if (!BITOP(byteset, *(guchar *) p, &)) {
+ *d++ = *p;
+ }
+
+ p++;
+ }
+
+ *(plen) = d - dest;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_oneline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+ const gchar *p, *end;
+ gchar *dest, *d;
+ guint64 byteset[32 / sizeof(guint64)]; /* Bitset for ascii */
+ gboolean copy = TRUE, seen_8bit = FALSE;
+ guint *plen;
+
+ if (t != NULL) {
+ if (lua_isboolean(L, 2)) {
+ copy = lua_toboolean(L, 2);
+ }
+ else if (t->flags & RSPAMD_TEXT_FLAG_OWN) {
+ copy = FALSE;
+ }
+
+ if (!copy) {
+ dest = (gchar *) t->start;
+ plen = &t->len;
+ lua_pushvalue(L, 1); /* Push text as a result */
+ }
+ else {
+ /* We need to copy read only text */
+ struct rspamd_lua_text *nt;
+
+ dest = g_malloc(t->len);
+ nt = lua_newuserdata(L, sizeof(*nt));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ nt->len = t->len;
+ nt->flags = RSPAMD_TEXT_FLAG_OWN;
+ memcpy(dest, t->start, t->len);
+ nt->start = dest;
+ plen = &nt->len;
+ }
+
+ /* Fill pattern bitset */
+ memset(byteset, 0, sizeof byteset);
+ /* All spaces */
+ byteset[0] |= GUINT64_FROM_LE(0x100003600LLU);
+ /* Control characters */
+ byteset[0] |= GUINT64_FROM_LE(0xffffffffLLU);
+ /* Del character */
+ byteset[1] |= GUINT64_FROM_LE(0x8000000000000000LLU);
+ /* 8 bit characters */
+ byteset[2] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+ byteset[3] |= GUINT64_FROM_LE(0xffffffffffffffffLLU);
+
+ p = t->start;
+ end = t->start + t->len;
+ d = dest;
+
+ while (p < end) {
+ if (!BITOP(byteset, *(guchar *) p, &)) {
+ *d++ = *p;
+ }
+ else {
+ if ((*(guchar *) p) & 0x80) {
+ seen_8bit = TRUE;
+ *d++ = *p;
+ }
+ else {
+ if (*p == ' ') {
+ if (d != dest) {
+ *d++ = *p++;
+ }
+
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ continue; /* To avoid p++ */
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (d != dest) {
+ *d++ = ' ';
+ p++;
+ }
+
+ while (p < end && g_ascii_isspace(*p)) {
+ p++;
+ }
+
+ continue; /* To avoid p++ */
+ }
+ }
+ }
+
+ p++;
+ }
+
+ while (d > dest && g_ascii_isspace(*(d - 1))) {
+ d--;
+ }
+
+ if (seen_8bit) {
+ if (rspamd_fast_utf8_validate(dest, d - dest) != 0) {
+ /* Need to make it valid :( */
+ UChar32 uc;
+ goffset err_offset;
+ gsize remain = d - dest;
+ gchar *nd = dest;
+
+ while (remain > 0 && (err_offset = rspamd_fast_utf8_validate(nd, remain)) > 0) {
+ gint i = 0;
+
+ err_offset--; /* As it returns it 1 indexed */
+ nd += err_offset;
+ remain -= err_offset;
+
+ /* Each invalid character of input requires 3 bytes of output (+2 bytes) */
+ while (i < remain) {
+ gint old_pos = i;
+ U8_NEXT(nd, i, remain, uc);
+
+ if (uc < 0) {
+ nd[old_pos] = '?';
+ }
+ else {
+ break;
+ }
+ }
+
+ nd += i;
+ remain -= i;
+ }
+ }
+ }
+
+ *(plen) = d - dest;
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_lower(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1), *nt;
+ gboolean is_utf8 = FALSE, is_inplace = FALSE;
+
+ if (t != NULL) {
+ if (lua_isboolean(L, 2)) {
+ is_utf8 = lua_toboolean(L, 2);
+ }
+ if (lua_isboolean(L, 3)) {
+ is_inplace = lua_toboolean(L, 3);
+ }
+
+ if (is_inplace) {
+ nt = t;
+ lua_pushvalue(L, 1);
+ }
+ else {
+ nt = lua_new_text(L, t->start, t->len, TRUE);
+ }
+
+ if (!is_utf8) {
+ rspamd_str_lc((gchar *) nt->start, nt->len);
+ }
+ else {
+ rspamd_str_lc_utf8((gchar *) nt->start, nt->len);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_text_strtoul(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text(L, 1);
+
+ if (t) {
+ unsigned long ll;
+
+ if (rspamd_strtoul(t->start, t->len, &ll)) {
+ lua_pushinteger(L, ll);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/* Used to distinguish lua text metatable */
+static const guint rspamd_lua_text_cookie = 0x2b21ef6fU;
+
+static gint
+lua_load_text(lua_State *L)
+{
+ lua_newtable(L);
+ lua_pushstring(L, "cookie");
+ lua_pushnumber(L, rspamd_lua_text_cookie);
+ lua_settable(L, -3);
+ luaL_register(L, NULL, textlib_f);
+
+ return 1;
+}
+
+void luaopen_text(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{text}", textlib_m);
+ lua_pushstring(L, "cookie");
+ lua_pushnumber(L, rspamd_lua_text_cookie);
+ lua_settable(L, -3);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_text", lua_load_text);
+}
diff --git a/src/lua/lua_thread_pool.cxx b/src/lua/lua_thread_pool.cxx
new file mode 100644
index 0000000..295f33d
--- /dev/null
+++ b/src/lua/lua_thread_pool.cxx
@@ -0,0 +1,369 @@
+/*-
+ * Copyright 2018 Mikhail Galanin
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+
+#include <vector>
+
+#define msg_debug_lua_threads(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_lua_threads_log_id, "lua_threads", NULL, \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_threads)
+
+static struct thread_entry *thread_entry_new(lua_State *L);
+static void thread_entry_free(lua_State *L, struct thread_entry *ent);
+
+#define CFG_POOL_GET(cfg) (reinterpret_cast<lua_thread_pool *>((cfg)->lua_thread_pool))
+
+struct lua_thread_pool {
+ std::vector<struct thread_entry *> available_items;
+ lua_State *L;
+ gint max_items;
+ struct thread_entry *running_entry;
+ static const int default_max_items = 100;
+
+ lua_thread_pool(lua_State *L, gint max_items = default_max_items)
+ : L(L), max_items(max_items)
+ {
+ running_entry = nullptr;
+ available_items.reserve(max_items);
+
+ for (auto i = 0; i < MAX(2, max_items / 10); i++) {
+ auto *ent = thread_entry_new(L);
+ available_items.push_back(ent);
+ }
+ }
+
+ ~lua_thread_pool()
+ {
+ for (auto *ent: available_items) {
+ thread_entry_free(L, ent);
+ }
+ }
+
+ auto get_thread() -> struct thread_entry *
+ {
+ struct thread_entry *ent;
+
+ if (!available_items.empty()) {
+ ent = available_items.back();
+ available_items.pop_back();
+ }
+ else {
+ ent = thread_entry_new(L);
+ }
+
+ running_entry = ent;
+
+ return ent;
+ }
+
+ auto return_thread(struct thread_entry *thread_entry, const gchar *loc) -> void
+ {
+ /* we can't return a running/yielded thread into the pool */
+ g_assert(lua_status(thread_entry->lua_state) == 0);
+
+ if (running_entry == thread_entry) {
+ running_entry = NULL;
+ }
+
+ if (available_items.size() <= max_items) {
+ thread_entry->cd = NULL;
+ thread_entry->finish_callback = NULL;
+ thread_entry->error_callback = NULL;
+ thread_entry->task = NULL;
+ thread_entry->cfg = NULL;
+
+ msg_debug_lua_threads("%s: returned thread to the threads pool %ud items",
+ loc,
+ available_items.size());
+
+ available_items.push_back(thread_entry);
+ }
+ else {
+ msg_debug_lua_threads("%s: removed thread as thread pool has %ud items",
+ loc,
+ available_items.size());
+ thread_entry_free(L, thread_entry);
+ }
+ }
+
+ auto terminate_thread(struct thread_entry *thread_entry,
+ const gchar *loc,
+ bool enforce) -> void
+ {
+ struct thread_entry *ent = NULL;
+
+ if (!enforce) {
+ /* we should only terminate failed threads */
+ g_assert(lua_status(thread_entry->lua_state) != 0 &&
+ lua_status(thread_entry->lua_state) != LUA_YIELD);
+ }
+
+ if (running_entry == thread_entry) {
+ running_entry = NULL;
+ }
+
+ msg_debug_lua_threads("%s: terminated thread entry", loc);
+ thread_entry_free(L, thread_entry);
+
+ if (available_items.size() <= max_items) {
+ ent = thread_entry_new(L);
+ available_items.push_back(ent);
+ }
+ }
+
+ auto get_running_entry(void) -> struct thread_entry *
+ {
+ return running_entry;
+ };
+
+ auto set_running_entry(struct thread_entry *ent) -> struct thread_entry *
+ {
+ auto *old_entry = running_entry;
+ running_entry = ent;
+ return old_entry;
+ };
+};
+
+static struct thread_entry *
+thread_entry_new(lua_State *L)
+{
+ struct thread_entry *ent;
+ ent = g_new0(struct thread_entry, 1);
+ ent->lua_state = lua_newthread(L);
+ ent->thread_index = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return ent;
+}
+
+static void
+thread_entry_free(lua_State *L, struct thread_entry *ent)
+{
+ luaL_unref(L, LUA_REGISTRYINDEX, ent->thread_index);
+ g_free(ent);
+}
+
+struct lua_thread_pool *
+lua_thread_pool_new(lua_State *L)
+{
+ auto *pool = new lua_thread_pool(L);
+ return pool;
+}
+
+void lua_thread_pool_free(struct lua_thread_pool *pool)
+{
+ delete pool;
+}
+
+
+struct thread_entry *
+lua_thread_pool_get_for_task(struct rspamd_task *task)
+{
+ struct thread_entry *ent = CFG_POOL_GET(task->cfg)->get_thread();
+
+ ent->task = task;
+
+ return ent;
+}
+
+struct thread_entry *
+lua_thread_pool_get_for_config(struct rspamd_config *cfg)
+{
+ struct thread_entry *ent = CFG_POOL_GET(cfg)->get_thread();
+
+ ent->cfg = cfg;
+
+ return ent;
+}
+
+void lua_thread_pool_return_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry, const gchar *loc)
+{
+ pool->return_thread(thread_entry, loc);
+}
+
+void lua_thread_pool_terminate_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry, const gchar *loc,
+ bool enforce)
+{
+ pool->terminate_thread(thread_entry, loc, enforce);
+}
+
+struct thread_entry *
+lua_thread_pool_get_running_entry_full(struct lua_thread_pool *pool,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_get_running_entry_full", loc);
+ return pool->get_running_entry();
+}
+
+void lua_thread_pool_set_running_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_set_running_entry_full", loc);
+ pool->set_running_entry(thread_entry);
+}
+
+static void
+lua_thread_pool_set_running_entry_for_thread(struct thread_entry *thread_entry,
+ const gchar *loc)
+{
+ struct lua_thread_pool *pool;
+
+ if (thread_entry->task) {
+ pool = CFG_POOL_GET(thread_entry->task->cfg);
+ }
+ else {
+ pool = CFG_POOL_GET(thread_entry->cfg);
+ }
+
+ lua_thread_pool_set_running_entry_full(pool, thread_entry, loc);
+}
+
+void lua_thread_pool_prepare_callback_full(struct lua_thread_pool *pool,
+ struct lua_callback_state *cbs,
+ const gchar *loc)
+{
+ msg_debug_lua_threads("%s: lua_thread_pool_prepare_callback_full", loc);
+ cbs->thread_pool = pool;
+ cbs->previous_thread = lua_thread_pool_get_running_entry_full(pool, loc);
+ cbs->my_thread = pool->get_thread();
+ cbs->L = cbs->my_thread->lua_state;
+}
+
+void lua_thread_pool_restore_callback_full(struct lua_callback_state *cbs,
+ const gchar *loc)
+{
+ lua_thread_pool_return_full(cbs->thread_pool, cbs->my_thread, loc);
+ lua_thread_pool_set_running_entry_full(cbs->thread_pool,
+ cbs->previous_thread, loc);
+}
+
+static gint
+lua_do_resume_full(lua_State *L, gint narg, const gchar *loc)
+{
+#if LUA_VERSION_NUM >= 504
+ int nres;
+#endif
+ msg_debug_lua_threads("%s: lua_do_resume_full", loc);
+#if LUA_VERSION_NUM < 502
+ return lua_resume(L, narg);
+#else
+#if LUA_VERSION_NUM >= 504
+ return lua_resume(L, NULL, narg, &nres);
+#else
+ return lua_resume(L, NULL, narg);
+#endif
+#endif
+}
+
+static void
+lua_resume_thread_internal_full(struct thread_entry *thread_entry,
+ gint narg, const gchar *loc)
+{
+ gint ret;
+ struct lua_thread_pool *pool;
+ struct rspamd_task *task;
+
+ msg_debug_lua_threads("%s: lua_resume_thread_internal_full", loc);
+ ret = lua_do_resume_full(thread_entry->lua_state, narg, loc);
+
+ if (ret != LUA_YIELD) {
+ /*
+ LUA_YIELD state should not be handled here.
+ It should only happen when the thread initiated a asynchronous event and it will be restored as soon
+ the event is finished
+ */
+
+ if (thread_entry->task) {
+ pool = CFG_POOL_GET(thread_entry->task->cfg);
+ }
+ else {
+ pool = CFG_POOL_GET(thread_entry->cfg);
+ }
+
+ if (ret == 0) {
+ if (thread_entry->finish_callback) {
+ thread_entry->finish_callback(thread_entry, ret);
+ }
+
+ pool->return_thread(thread_entry, loc);
+ }
+ else {
+ rspamd_lua_traceback(thread_entry->lua_state);
+ if (thread_entry->error_callback) {
+ thread_entry->error_callback(thread_entry, ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+ else if (thread_entry->task) {
+ task = thread_entry->task;
+ msg_err_task("lua call failed (%d): %s", ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+ else {
+ msg_err("lua call failed (%d): %s", ret,
+ lua_tostring(thread_entry->lua_state, -1));
+ }
+
+ /*
+ * Maybe there is a way to recover here.
+ * For now, just remove faulty thread
+ */
+ pool->terminate_thread(thread_entry, loc, false);
+ }
+ }
+}
+
+void lua_thread_resume_full(struct thread_entry *thread_entry, gint narg,
+ const gchar *loc)
+{
+ /*
+ * The only state where we can resume from is LUA_YIELD
+ * Another acceptable status is OK (0) but in that case we should push function on stack
+ * to start the thread from, which is happening in lua_thread_call(), not in this function.
+ */
+ g_assert(lua_status(thread_entry->lua_state) == LUA_YIELD);
+ msg_debug_lua_threads("%s: lua_thread_resume_full", loc);
+ lua_thread_pool_set_running_entry_for_thread(thread_entry, loc);
+ lua_resume_thread_internal_full(thread_entry, narg, loc);
+}
+
+void lua_thread_call_full(struct thread_entry *thread_entry,
+ int narg, const gchar *loc)
+{
+ g_assert(lua_status(thread_entry->lua_state) == 0); /* we can't call running/yielded thread */
+ g_assert(thread_entry->task != NULL || thread_entry->cfg != NULL); /* we can't call without pool */
+
+ lua_resume_thread_internal_full(thread_entry, narg, loc);
+}
+
+gint lua_thread_yield_full(struct thread_entry *thread_entry,
+ gint nresults,
+ const gchar *loc)
+{
+ g_assert(lua_status(thread_entry->lua_state) == 0);
+
+ msg_debug_lua_threads("%s: lua_thread_yield_full", loc);
+ return lua_yield(thread_entry->lua_state, nresults);
+}
diff --git a/src/lua/lua_thread_pool.h b/src/lua/lua_thread_pool.h
new file mode 100644
index 0000000..b612ac3
--- /dev/null
+++ b/src/lua/lua_thread_pool.h
@@ -0,0 +1,194 @@
+#ifndef LUA_THREAD_POOL_H_
+#define LUA_THREAD_POOL_H_
+
+#include <lua.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct thread_entry;
+struct lua_thread_pool;
+
+typedef void (*lua_thread_finish_t)(struct thread_entry *thread, int ret);
+
+typedef void (*lua_thread_error_t)(struct thread_entry *thread, int ret, const char *msg);
+
+struct thread_entry {
+ lua_State *lua_state;
+ gint thread_index;
+ gpointer cd;
+
+ /* function to handle result of called method, can be NULL */
+ lua_thread_finish_t finish_callback;
+
+ /* function to log result, i.e. if you want to modify error logging message or somehow process this state, can be NUL */
+ lua_thread_error_t error_callback;
+ struct rspamd_task *task;
+ struct rspamd_config *cfg;
+};
+
+struct lua_callback_state {
+ lua_State *L;
+ struct thread_entry *my_thread;
+ struct thread_entry *previous_thread;
+ struct lua_thread_pool *thread_pool;
+};
+
+/**
+ * Allocates new thread pool on state L. Pre-creates number of lua-threads to use later on
+ *
+ * @param L
+ * @return
+ */
+struct lua_thread_pool *
+lua_thread_pool_new(lua_State *L);
+
+/**
+ * Destroys the pool
+ * @param pool
+ */
+void lua_thread_pool_free(struct lua_thread_pool *pool);
+
+/**
+ * Extracts a thread from the list of available ones.
+ * It immediately becomes the running one and should be used to run a Lua script/function straight away.
+ * as soon as the code is finished, it should be either returned into list of available threads by
+ * calling lua_thread_pool_return() or terminated by calling lua_thread_pool_terminate_entry()
+ * if the code finished with error.
+ *
+ * If the code performed YIELD, the thread is still running and it's live should be controlled by the callee
+ *
+ * @param task
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_for_task(struct rspamd_task *task);
+
+/**
+ * The same, but used when task is not available
+ *
+ * @param cfg
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_for_config(struct rspamd_config *cfg);
+
+/**
+ * Return thread into the list of available ones. It can't be done with yielded or dead threads.
+ *
+ * @param pool
+ * @param thread_entry
+ */
+void lua_thread_pool_return_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc);
+
+#define lua_thread_pool_return(pool, thread_entry) \
+ lua_thread_pool_return_full(pool, thread_entry, G_STRLOC)
+
+/**
+ * Currently running thread. Typically needed in yielding point - to fill-up continuation.
+ *
+ * @param pool
+ * @return
+ */
+struct thread_entry *
+lua_thread_pool_get_running_entry_full(struct lua_thread_pool *pool,
+ const gchar *loc);
+
+#define lua_thread_pool_get_running_entry(pool) \
+ lua_thread_pool_get_running_entry_full(pool, G_STRLOC)
+
+/**
+ * Updates currently running thread
+ *
+ * @param pool
+ * @param thread_entry
+ */
+void lua_thread_pool_set_running_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc);
+
+#define lua_thread_pool_set_running_entry(pool, thread_entry) \
+ lua_thread_pool_set_running_entry_full(pool, thread_entry, G_STRLOC)
+
+/**
+ * Prevents yielded thread to be used for callback execution. lua_thread_pool_restore_callback() should be called afterwards.
+ *
+ * @param pool
+ * @param cbs
+ */
+void lua_thread_pool_prepare_callback_full(struct lua_thread_pool *pool,
+ struct lua_callback_state *cbs, const gchar *loc);
+
+#define lua_thread_pool_prepare_callback(pool, cbs) \
+ lua_thread_pool_prepare_callback_full(pool, cbs, G_STRLOC)
+
+/**
+ * Restores state after lua_thread_pool_prepare_callback () usage
+ *
+ * @param cbs
+ */
+void lua_thread_pool_restore_callback_full(struct lua_callback_state *cbs,
+ const gchar *loc);
+
+#define lua_thread_pool_restore_callback(cbs) \
+ lua_thread_pool_restore_callback_full(cbs, G_STRLOC)
+
+/**
+ * Acts like lua_call but the tread is able to suspend execution.
+ * As soon as the call is over, call either thread_entry::finish_callback or thread_entry::error_callback.
+ *
+ * @param thread_entry
+ * @param narg
+ */
+void lua_thread_call_full(struct thread_entry *thread_entry,
+ int narg,
+ const gchar *loc);
+
+#define lua_thread_call(thread_entry, narg) \
+ lua_thread_call_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Yields thread. should be only called in return statement
+ * @param thread_entry
+ * @param nresults
+ * @return
+ */
+int lua_thread_yield_full(struct thread_entry *thread_entry, int nresults,
+ const gchar *loc);
+
+#define lua_thread_yield(thread_entry, narg) \
+ lua_thread_yield_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Resumes suspended by lua_yield_thread () thread
+ * @param task
+ * @param thread_entry
+ * @param narg
+ */
+void lua_thread_resume_full(struct thread_entry *thread_entry,
+ int narg,
+ const gchar *loc);
+
+#define lua_thread_resume(thread_entry, narg) \
+ lua_thread_resume_full(thread_entry, narg, G_STRLOC)
+
+/**
+ * Terminates thread pool entry and fill the pool with another thread entry if needed
+ * @param pool
+ * @param thread_entry
+ * @param loc
+ */
+void lua_thread_pool_terminate_entry_full(struct lua_thread_pool *pool,
+ struct thread_entry *thread_entry,
+ const gchar *loc, bool enforce);
+#define lua_thread_pool_terminate_entry(pool, thread_entry) \
+ lua_thread_pool_terminate_entry_full(pool, thread_entry, G_STRLOC, false)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LUA_THREAD_POOL_H_ */
diff --git a/src/lua/lua_trie.c b/src/lua/lua_trie.c
new file mode 100644
index 0000000..3b0b55e
--- /dev/null
+++ b/src/lua/lua_trie.c
@@ -0,0 +1,500 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "message.h"
+#include "libutil/multipattern.h"
+
+/***
+ * @module rspamd_trie
+ * Rspamd trie module provides the data structure suitable for searching of many
+ * patterns in arbitrary texts (or binary chunks). The algorithmic complexity of
+ * this algorithm is at most O(n + m + z), where `n` is the length of text, `m` is a length of pattern and `z` is a number of patterns in the text.
+ *
+ * Here is a typical example of trie usage:
+ * @example
+local rspamd_trie = require "rspamd_trie"
+local patterns = {'aab', 'ab', 'bcd\0ef'}
+
+local trie = rspamd_trie.create(patterns)
+
+local function trie_callback(number, pos)
+ print('Matched pattern number ' .. tostring(number) .. ' at pos: ' .. tostring(pos))
+end
+
+trie:match('some big text', trie_callback)
+ */
+
+/* Suffix trie */
+LUA_FUNCTION_DEF(trie, create);
+LUA_FUNCTION_DEF(trie, has_hyperscan);
+LUA_FUNCTION_DEF(trie, match);
+LUA_FUNCTION_DEF(trie, search_mime);
+LUA_FUNCTION_DEF(trie, search_rawmsg);
+LUA_FUNCTION_DEF(trie, search_rawbody);
+LUA_FUNCTION_DEF(trie, destroy);
+
+static const struct luaL_reg trielib_m[] = {
+ LUA_INTERFACE_DEF(trie, match),
+ LUA_INTERFACE_DEF(trie, search_mime),
+ LUA_INTERFACE_DEF(trie, search_rawmsg),
+ LUA_INTERFACE_DEF(trie, search_rawbody),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_trie_destroy},
+ {NULL, NULL}};
+static const struct luaL_reg trielib_f[] = {
+ LUA_INTERFACE_DEF(trie, create),
+ LUA_INTERFACE_DEF(trie, has_hyperscan),
+ {NULL, NULL}};
+
+static struct rspamd_multipattern *
+lua_check_trie(lua_State *L, gint idx)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{trie}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'trie' expected");
+ return ud ? *((struct rspamd_multipattern **) ud) : NULL;
+}
+
+static gint
+lua_trie_destroy(lua_State *L)
+{
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+
+ if (trie) {
+ rspamd_multipattern_destroy(trie);
+ }
+
+ return 0;
+}
+
+/***
+ * function trie.has_hyperscan()
+ * Checks for hyperscan support
+ *
+ * @return {bool} true if hyperscan is supported
+ */
+static gint
+lua_trie_has_hyperscan(lua_State *L)
+{
+ lua_pushboolean(L, rspamd_multipattern_has_hyperscan());
+ return 1;
+}
+
+/***
+ * function trie.create(patterns, [flags])
+ * Creates new trie data structure
+ * @param {table} array of string patterns
+ * @return {trie} new trie object
+ */
+static gint
+lua_trie_create(lua_State *L)
+{
+ struct rspamd_multipattern *trie, **ptrie;
+ gint npat = 0, flags = RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_GLOB;
+ GError *err = NULL;
+
+ if (lua_isnumber(L, 2)) {
+ flags = lua_tointeger(L, 2);
+ }
+
+ if (!lua_istable(L, 1)) {
+ return luaL_error(L, "lua trie expects array of patterns for now");
+ }
+ else {
+ lua_pushvalue(L, 1);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ npat++;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ trie = rspamd_multipattern_create_sized(npat, flags);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ const gchar *pat;
+ gsize patlen;
+
+ pat = lua_tolstring(L, -1, &patlen);
+ rspamd_multipattern_add_pattern_len(trie, pat, patlen, flags);
+ }
+
+ lua_pop(L, 1);
+ }
+
+ lua_pop(L, 1); /* table */
+
+ if (!rspamd_multipattern_compile(trie, &err)) {
+ msg_err("cannot compile multipattern: %e", err);
+ g_error_free(err);
+ rspamd_multipattern_destroy(trie);
+ lua_pushnil(L);
+ }
+ else {
+ ptrie = lua_newuserdata(L, sizeof(void *));
+ rspamd_lua_setclass(L, "rspamd{trie}", -1);
+ *ptrie = trie;
+ }
+ }
+
+ return 1;
+}
+
+#define PUSH_TRIE_MATCH(L, start, end, report_start) \
+ do { \
+ if (report_start) { \
+ lua_createtable(L, 2, 0); \
+ lua_pushinteger(L, (start)); \
+ lua_rawseti(L, -2, 1); \
+ lua_pushinteger(L, (end)); \
+ lua_rawseti(L, -2, 2); \
+ } \
+ else { \
+ lua_pushinteger(L, (end)); \
+ } \
+ } while (0)
+
+/* Normal callback type */
+static gint
+lua_trie_lua_cb_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint textpos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ lua_State *L = context;
+ gint ret;
+
+ gboolean report_start = lua_toboolean(L, -1);
+
+ /* Function */
+ lua_pushvalue(L, 3);
+ lua_pushinteger(L, strnum + 1);
+
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+
+ if (lua_pcall(L, 2, 1, 0) != 0) {
+ msg_info("call to trie callback has failed: %s",
+ lua_tostring(L, -1));
+ lua_pop(L, 1);
+
+ return 1;
+ }
+
+ ret = lua_tonumber(L, -1);
+ lua_pop(L, 1);
+
+ return ret;
+}
+
+/* Table like callback, expect result table on top of the stack */
+static gint
+lua_trie_table_callback(struct rspamd_multipattern *mp,
+ guint strnum,
+ gint match_start,
+ gint textpos,
+ const gchar *text,
+ gsize len,
+ void *context)
+{
+ lua_State *L = context;
+
+ gint report_start = lua_toboolean(L, -2);
+ /* Set table, indexed by pattern number */
+ lua_rawgeti(L, -1, strnum + 1);
+
+ if (lua_istable(L, -1)) {
+ /* Already have table, add offset */
+ gsize last = rspamd_lua_table_size(L, -1);
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+ lua_rawseti(L, -2, last + 1);
+ /* Remove table from the stack */
+ lua_pop(L, 1);
+ }
+ else {
+ /* Pop none */
+ lua_pop(L, 1);
+ /* New table */
+ lua_newtable(L);
+ PUSH_TRIE_MATCH(L, match_start, textpos, report_start);
+ lua_rawseti(L, -2, 1);
+ lua_rawseti(L, -2, strnum + 1);
+ }
+
+ return 0;
+}
+
+/*
+ * We assume that callback argument is at pos 3 and icase is in position 4
+ */
+static gint
+lua_trie_search_str(lua_State *L, struct rspamd_multipattern *trie,
+ const gchar *str, gsize len, rspamd_multipattern_cb_t cb)
+{
+ gint ret;
+ guint nfound = 0;
+
+ if ((ret = rspamd_multipattern_lookup(trie, str, len,
+ cb, L, &nfound)) == 0) {
+ return nfound;
+ }
+
+ return ret;
+}
+
+/***
+ * @method trie:match(input, [cb][, report_start])
+ * Search for patterns in `input` invoking `cb` optionally ignoring case
+ * @param {table or string} input one or several (if `input` is an array) strings of input text
+ * @param {function} cb callback called on each pattern match in form `function (idx, pos)` where `idx` is a numeric index of pattern (starting from 1) and `pos` is a numeric offset where the pattern ends
+ * @param {boolean} report_start report both start and end offset when matching patterns
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however). If `cb` is not defined then it returns a table of match positions indexed by pattern number
+ */
+static gint
+lua_trie_match(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE, report_start = FALSE;
+ struct rspamd_lua_text *t;
+ rspamd_multipattern_cb_t cb = lua_trie_lua_cb_callback;
+
+ gint old_top = lua_gettop(L);
+
+ if (trie) {
+ if (lua_type(L, 3) != LUA_TFUNCTION) {
+ if (lua_isboolean(L, 3)) {
+ report_start = lua_toboolean(L, 3);
+ }
+
+ lua_pushboolean(L, report_start);
+ /* Table like match */
+ lua_newtable(L);
+ cb = lua_trie_table_callback;
+ }
+ else {
+ if (lua_isboolean(L, 4)) {
+ report_start = lua_toboolean(L, 4);
+ }
+ lua_pushboolean(L, report_start);
+ }
+
+ if (lua_type(L, 2) == LUA_TTABLE) {
+ lua_pushvalue(L, 2);
+ lua_pushnil(L);
+
+ while (lua_next(L, -2) != 0) {
+ if (lua_isstring(L, -1)) {
+ text = lua_tolstring(L, -1, &len);
+
+ if (lua_trie_search_str(L, trie, text, len, cb)) {
+ found = TRUE;
+ }
+ }
+ else if (lua_isuserdata(L, -1)) {
+ t = lua_check_text(L, -1);
+
+ if (t) {
+ if (lua_trie_search_str(L, trie, t->start, t->len, cb)) {
+ found = TRUE;
+ }
+ }
+ }
+ lua_pop(L, 1);
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TSTRING) {
+ text = lua_tolstring(L, 2, &len);
+
+ if (lua_trie_search_str(L, trie, text, len, cb)) {
+ found = TRUE;
+ }
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 2);
+
+ if (t && lua_trie_search_str(L, trie, t->start, t->len, cb)) {
+ found = TRUE;
+ }
+ }
+ }
+
+ if (lua_type(L, 3) == LUA_TFUNCTION) {
+ lua_settop(L, old_top);
+ lua_pushboolean(L, found);
+ }
+ else {
+ lua_remove(L, -2);
+ }
+
+ return 1;
+}
+
+/***
+ * @method trie:search_mime(task, cb)
+ * This is a helper mehthod to search pattern within text parts of a message in rspamd task
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_mime(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ struct rspamd_mime_text_part *part;
+ const gchar *text;
+ gsize len, i;
+ gboolean found = FALSE;
+ rspamd_multipattern_cb_t cb = lua_trie_lua_cb_callback;
+
+ if (trie && task) {
+ PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+ {
+ if (!IS_TEXT_PART_EMPTY(part) && part->utf_content.len > 0) {
+ text = part->utf_content.begin;
+ len = part->utf_content.len;
+
+ if (lua_trie_search_str(L, trie, text, len, cb) != 0) {
+ found = TRUE;
+ }
+ }
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+/***
+ * @method trie:search_rawmsg(task, cb[, caseless])
+ * This is a helper mehthod to search pattern within the whole undecoded content of rspamd task
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_rawmsg(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE;
+
+ if (trie && task) {
+ text = task->msg.begin;
+ len = task->msg.len;
+
+ if (lua_trie_search_str(L, trie, text, len, lua_trie_lua_cb_callback) != 0) {
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+/***
+ * @method trie:search_rawbody(task, cb[, caseless])
+ * This is a helper mehthod to search pattern within the whole undecoded content of task's body (not including headers)
+ * @param {task} task object
+ * @param {function} cb callback called on each pattern match @see trie:match
+ * @param {boolean} caseless if `true` then match ignores symbols case (ASCII only)
+ * @return {boolean} `true` if any pattern has been found (`cb` might be called multiple times however)
+ */
+static gint
+lua_trie_search_rawbody(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_multipattern *trie = lua_check_trie(L, 1);
+ struct rspamd_task *task = lua_check_task(L, 2);
+ const gchar *text;
+ gsize len;
+ gboolean found = FALSE;
+
+ if (trie && task) {
+ if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+ text = task->msg.begin + MESSAGE_FIELD(task, raw_headers_content).len;
+ len = task->msg.len - MESSAGE_FIELD(task, raw_headers_content).len;
+ }
+ else {
+ /* Treat as raw message */
+ text = task->msg.begin;
+ len = task->msg.len;
+ }
+
+ if (lua_trie_search_str(L, trie, text, len, lua_trie_lua_cb_callback) != 0) {
+ found = TRUE;
+ }
+ }
+
+ lua_pushboolean(L, found);
+ return 1;
+}
+
+static gint
+lua_load_trie(lua_State *L)
+{
+ lua_newtable(L);
+
+ /* Flags */
+ lua_pushstring(L, "flags");
+ lua_newtable(L);
+
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_GLOB);
+ lua_setfield(L, -2, "glob");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_RE);
+ lua_setfield(L, -2, "re");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_ICASE);
+ lua_setfield(L, -2, "icase");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_UTF8);
+ lua_setfield(L, -2, "utf8");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_TLD);
+ lua_setfield(L, -2, "tld");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_DOTALL);
+ lua_setfield(L, -2, "dot_all");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_SINGLEMATCH);
+ lua_setfield(L, -2, "single_match");
+ lua_pushinteger(L, RSPAMD_MULTIPATTERN_NO_START);
+ lua_setfield(L, -2, "no_start");
+ lua_settable(L, -3);
+
+ /* Main content */
+ luaL_register(L, NULL, trielib_f);
+
+ return 1;
+}
+
+void luaopen_trie(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{trie}", trielib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_trie", lua_load_trie);
+}
diff --git a/src/lua/lua_udp.c b/src/lua/lua_udp.c
new file mode 100644
index 0000000..c79e35a
--- /dev/null
+++ b/src/lua/lua_udp.c
@@ -0,0 +1,594 @@
+/*-
+ * Copyright 2019 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_thread_pool.h"
+#include "utlist.h"
+#include "unix-std.h"
+#include <math.h>
+#include <src/libutil/libev_helper.h>
+
+static const gchar *M = "rspamd lua udp";
+
+/***
+ * @module rspamd_udp
+ * Rspamd UDP module is available from the version 1.9.0 and represents a generic
+ * UDP asynchronous client available from the LUA code.
+ * This module is quite simple: it can either send requests to some address or
+ * it can send requests and wait for replies, potentially handling retransmits.
+ * @example
+local logger = require "rspamd_logger"
+local udp = require "rspamd_udp"
+
+rspamd_config.SYM = function(task)
+ udp.sento{
+ host = addr, -- must be ip address object (e.g. received by upstream module)
+ port = 500,
+ data = {'str1', 'str2'}, -- can be table, string or rspamd_text
+ timeout = 0.5, -- default = 1s
+ task = task, -- if has task
+ session = session, -- optional
+ ev_base = ev_base, -- if no task available
+ -- You can include callback and then Rspamd will try to read replies
+ callback = function(success, data)
+ -- success is bool, data is either data or an error (string)
+ end,
+ retransmits = 0, -- Or more if retransmitting is necessary
+ }
+end
+ */
+
+static const double default_udp_timeout = 1.0;
+
+LUA_FUNCTION_DEF(udp, sendto);
+
+static const struct luaL_reg udp_libf[] = {
+ LUA_INTERFACE_DEF(udp, sendto),
+ {NULL, NULL}};
+
+struct lua_udp_cbdata {
+ struct ev_loop *event_loop;
+ struct rspamd_io_ev ev;
+ struct rspamd_async_event *async_ev;
+ struct rspamd_task *task;
+ rspamd_mempool_t *pool;
+ rspamd_inet_addr_t *addr;
+ struct rspamd_symcache_dynamic_item *item;
+ struct rspamd_async_session *s;
+ struct iovec *iov;
+ lua_State *L;
+ guint retransmits;
+ guint iovlen;
+ gint sock;
+ gint cbref;
+ gboolean sent;
+};
+
+#define msg_debug_udp(...) rspamd_conditional_debug_fast(NULL, cbd->addr, \
+ rspamd_lua_udp_log_id, "lua_udp", cbd->pool->tag.uid, \
+ G_STRFUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(lua_udp)
+
+static inline void
+lua_fill_iov(lua_State *L, rspamd_mempool_t *pool,
+ struct iovec *iov, gint pos)
+{
+ if (lua_type(L, pos) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, pos);
+
+ if (t) {
+ iov->iov_base = rspamd_mempool_alloc(pool, t->len);
+ iov->iov_len = t->len;
+ memcpy(iov->iov_base, t->start, t->len);
+ }
+ }
+ else {
+ const gchar *s;
+ gsize len;
+
+ s = lua_tolstring(L, pos, &len);
+
+ iov->iov_base = rspamd_mempool_alloc(pool, len);
+ iov->iov_len = len;
+ memcpy(iov->iov_base, s, len);
+ }
+}
+
+static void
+lua_udp_cbd_fin(gpointer p)
+{
+ struct lua_udp_cbdata *cbd = (struct lua_udp_cbdata *) p;
+
+ if (cbd->sock != -1) {
+ rspamd_ev_watcher_stop(cbd->event_loop, &cbd->ev);
+ close(cbd->sock);
+ }
+
+ if (cbd->addr) {
+ rspamd_inet_address_free(cbd->addr);
+ }
+
+ if (cbd->cbref) {
+ luaL_unref(cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
+ }
+}
+
+static void
+lua_udp_maybe_free(struct lua_udp_cbdata *cbd)
+{
+ if (cbd->item) {
+ rspamd_symcache_item_async_dec_check(cbd->task, cbd->item, M);
+ cbd->item = NULL;
+ }
+
+ if (cbd->async_ev) {
+ rspamd_session_remove_event(cbd->s, lua_udp_cbd_fin, cbd);
+ }
+ else {
+ lua_udp_cbd_fin(cbd);
+ }
+}
+
+
+enum rspamd_udp_send_result {
+ RSPAMD_SENT_OK,
+ RSPAMD_SENT_RETRY,
+ RSPAMD_SENT_FAILURE
+};
+
+static enum rspamd_udp_send_result
+lua_try_send_request(struct lua_udp_cbdata *cbd)
+{
+ struct msghdr msg;
+ gint r;
+
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_iov = cbd->iov;
+ msg.msg_iovlen = cbd->iovlen;
+ msg.msg_name = rspamd_inet_address_get_sa(cbd->addr, &msg.msg_namelen);
+
+ r = sendmsg(cbd->sock, &msg, 0);
+
+ if (r != -1) {
+ return RSPAMD_SENT_OK;
+ }
+
+ if (errno == EAGAIN || errno == EINTR) {
+ return RSPAMD_SENT_RETRY;
+ }
+
+ return RSPAMD_SENT_FAILURE;
+}
+
+static void
+lua_udp_maybe_push_error(struct lua_udp_cbdata *cbd, const gchar *err)
+{
+ if (cbd->cbref != -1) {
+ gint top;
+ lua_State *L = cbd->L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ /* Error message */
+ lua_pushboolean(L, false);
+ lua_pushstring(L, err);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 2, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ }
+
+ lua_udp_maybe_free(cbd);
+}
+
+static void
+lua_udp_push_data(struct lua_udp_cbdata *cbd, const gchar *data,
+ gssize len)
+{
+ if (cbd->cbref != -1) {
+ gint top;
+ lua_State *L = cbd->L;
+
+ top = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+
+ /* Error message */
+ lua_pushboolean(L, true);
+ lua_pushlstring(L, data, len);
+
+ if (cbd->item) {
+ rspamd_symcache_set_cur_item(cbd->task, cbd->item);
+ }
+
+ if (lua_pcall(L, 2, 0, 0) != 0) {
+ msg_info("callback call failed: %s", lua_tostring(L, -1));
+ }
+
+ lua_settop(L, top);
+ }
+
+ lua_udp_maybe_free(cbd);
+}
+
+static gboolean
+lua_udp_maybe_register_event(struct lua_udp_cbdata *cbd)
+{
+ if (cbd->s && !cbd->async_ev) {
+ if (cbd->item) {
+ cbd->async_ev = rspamd_session_add_event_full(cbd->s, lua_udp_cbd_fin,
+ cbd, M,
+ rspamd_symcache_dyn_item_name(cbd->task, cbd->item));
+ }
+ else {
+ cbd->async_ev = rspamd_session_add_event(cbd->s, lua_udp_cbd_fin,
+ cbd, M);
+ }
+
+ if (!cbd->async_ev) {
+ return FALSE;
+ }
+ }
+
+ if (cbd->task && !cbd->item) {
+ cbd->item = rspamd_symcache_get_cur_item(cbd->task);
+ rspamd_symcache_item_async_inc(cbd->task, cbd->item, M);
+ }
+
+ return TRUE;
+}
+
+static void
+lua_udp_io_handler(gint fd, short what, gpointer p)
+{
+ struct lua_udp_cbdata *cbd = (struct lua_udp_cbdata *) p;
+ gssize r;
+
+ if (what == EV_TIMEOUT) {
+ if (cbd->sent && cbd->retransmits > 0) {
+ r = lua_try_send_request(cbd);
+
+ if (r == RSPAMD_SENT_OK) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+ lua_udp_maybe_register_event(cbd);
+ cbd->retransmits--;
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_udp_maybe_push_error(cbd, "write error");
+ }
+ else {
+ cbd->retransmits--;
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_WRITE);
+ }
+ }
+ else {
+ if (!cbd->sent) {
+ lua_udp_maybe_push_error(cbd, "sent timeout");
+ }
+ else {
+ lua_udp_maybe_push_error(cbd, "read timeout");
+ }
+ }
+ }
+ else if (what == EV_WRITE) {
+ r = lua_try_send_request(cbd);
+
+ if (r == RSPAMD_SENT_OK) {
+ if (cbd->cbref != -1) {
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_READ);
+ cbd->sent = TRUE;
+ }
+ else {
+ lua_udp_maybe_free(cbd);
+ }
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_udp_maybe_push_error(cbd, "write error");
+ }
+ else {
+ cbd->retransmits--;
+ rspamd_ev_watcher_reschedule(cbd->event_loop, &cbd->ev, EV_WRITE);
+ }
+ }
+ else if (what == EV_READ) {
+ guchar udpbuf[4096];
+ socklen_t slen;
+ struct sockaddr *sa;
+
+ sa = rspamd_inet_address_get_sa(cbd->addr, &slen);
+
+ r = recvfrom(cbd->sock, udpbuf, sizeof(udpbuf), 0, sa, &slen);
+
+ if (r == -1) {
+ lua_udp_maybe_push_error(cbd, strerror(errno));
+ }
+ else {
+ lua_udp_push_data(cbd, udpbuf, r);
+ }
+ }
+}
+
+/***
+ * @function rspamd_udp.sendto({params})
+ * This function simply sends data to an external UDP service
+ *
+ * - `task`: rspamd task objects (implies `pool`, `session` and `ev_base` arguments)
+ * - `ev_base`: event base (if no task specified)
+ * - `session`: events session (no task, optional)
+ * - `pool`: memory pool (if no task specified)
+ * - `host`: IP or name of the peer (required)
+ * - `port`: remote port to use (if `host` has no port part this is required)
+ * - `data`: a table of strings or `rspamd_text` objects that contains data pieces
+ * - `retransmits`: number of retransmits if needed
+ * - `callback`: optional callback if reply should be read
+ * @return {boolean} true if request has been sent (additional string if it has not)
+ */
+static gint
+lua_udp_sendto(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ guint port;
+ struct ev_loop *ev_base = NULL;
+ struct lua_udp_cbdata *cbd;
+ struct rspamd_async_session *session = NULL;
+ struct rspamd_task *task = NULL;
+ rspamd_inet_addr_t *addr;
+ rspamd_mempool_t *pool = NULL;
+ gdouble timeout = default_udp_timeout;
+
+ if (lua_type(L, 1) == LUA_TTABLE) {
+ lua_pushstring(L, "port");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ port = lua_tointeger(L, -1);
+ }
+ else {
+ /* We assume that it is a unix socket */
+ port = 0;
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "host");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ host = luaL_checkstring(L, -1);
+
+ if (rspamd_parse_inet_address(&addr,
+ host, strlen(host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
+ if (port != 0) {
+ rspamd_inet_address_set_port(addr, port);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host: %s", host);
+ }
+ }
+ else if (lua_type(L, -1) == LUA_TUSERDATA) {
+ struct rspamd_lua_ip *lip;
+
+ lip = lua_check_ip(L, -1);
+
+ if (lip == NULL || lip->addr == NULL) {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host class");
+ }
+
+ addr = rspamd_inet_address_copy(lip->addr, NULL);
+
+ if (port != 0) {
+ rspamd_inet_address_set_port(addr, port);
+ }
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid host");
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "task");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TUSERDATA) {
+ task = lua_check_task(L, -1);
+ ev_base = task->event_loop;
+ session = task->s;
+ pool = task->task_pool;
+ }
+ lua_pop(L, 1);
+
+ if (task == NULL) {
+ lua_pushstring(L, "ev_base");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{ev_base}")) {
+ ev_base = *(struct ev_loop **) lua_touserdata(L, -1);
+ }
+ else {
+ ev_base = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "session");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{session}")) {
+ session = *(struct rspamd_async_session **) lua_touserdata(L, -1);
+ }
+ else {
+ session = NULL;
+ }
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "pool");
+ lua_gettable(L, -2);
+ if (rspamd_lua_check_udata_maybe(L, -1, "rspamd{mempool}")) {
+ pool = *(rspamd_mempool_t **) lua_touserdata(L, -1);
+ }
+ else {
+ pool = NULL;
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "timeout");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ timeout = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ if (!ev_base || !pool) {
+ rspamd_inet_address_free(addr);
+
+ return luaL_error(L, "invalid arguments");
+ }
+
+
+ cbd = rspamd_mempool_alloc0(pool, sizeof(*cbd));
+ cbd->event_loop = ev_base;
+ cbd->pool = pool;
+ cbd->s = session;
+ cbd->addr = addr;
+ cbd->sock = rspamd_socket_create(rspamd_inet_address_get_af(addr),
+ SOCK_DGRAM, 0, TRUE);
+ cbd->cbref = -1;
+ cbd->ev.timeout = timeout;
+
+ if (cbd->sock == -1) {
+ rspamd_inet_address_free(addr);
+
+ return luaL_error(L, "cannot open socket: %s", strerror(errno));
+ }
+
+ cbd->L = L;
+
+ gsize data_len;
+
+ lua_pushstring(L, "data");
+ lua_gettable(L, -2);
+
+ if (lua_type(L, -1) == LUA_TTABLE) {
+ data_len = rspamd_lua_table_size(L, -1);
+ cbd->iov = rspamd_mempool_alloc(pool,
+ sizeof(*cbd->iov) * data_len);
+
+ for (int i = 0; i < data_len; i++) {
+ lua_rawgeti(L, -1, i + 1);
+ lua_fill_iov(L, pool, &cbd->iov[i], -1);
+ lua_pop(L, 1);
+ }
+
+ cbd->iovlen = data_len;
+ }
+ else {
+ cbd->iov = rspamd_mempool_alloc(pool, sizeof(*cbd->iov));
+ cbd->iovlen = 1;
+ lua_fill_iov(L, pool, cbd->iov, -1);
+ }
+
+ lua_pop(L, 1);
+
+ lua_pushstring(L, "callback");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TFUNCTION) {
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ }
+ else {
+ lua_pop(L, 1);
+ }
+
+ lua_pushstring(L, "retransmits");
+ lua_gettable(L, -2);
+ if (lua_type(L, -1) == LUA_TNUMBER) {
+ cbd->retransmits = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ enum rspamd_udp_send_result r;
+
+ r = lua_try_send_request(cbd);
+ if (r == RSPAMD_SENT_OK) {
+ if (cbd->cbref == -1) {
+ lua_udp_maybe_free(cbd);
+ }
+ else {
+ if (!lua_udp_maybe_register_event(cbd)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "session error");
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+
+ rspamd_ev_watcher_init(&cbd->ev, cbd->sock, EV_READ,
+ lua_udp_io_handler, cbd);
+ rspamd_ev_watcher_start(cbd->event_loop, &cbd->ev, timeout);
+ cbd->sent = TRUE;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else if (r == RSPAMD_SENT_FAILURE) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+ else {
+ rspamd_ev_watcher_init(&cbd->ev, cbd->sock, EV_WRITE,
+ lua_udp_io_handler, cbd);
+ rspamd_ev_watcher_start(cbd->event_loop, &cbd->ev, timeout);
+
+ if (!lua_udp_maybe_register_event(cbd)) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, "session error");
+ lua_udp_maybe_free(cbd);
+
+ return 2;
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_udp(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, udp_libf);
+
+ return 1;
+}
+
+void luaopen_udp(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_udp", lua_load_udp);
+}
diff --git a/src/lua/lua_upstream.c b/src/lua/lua_upstream.c
new file mode 100644
index 0000000..583ee6a
--- /dev/null
+++ b/src/lua/lua_upstream.c
@@ -0,0 +1,672 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "lua_common.h"
+
+
+/***
+ * @module rspamd_upstream_list
+ * This module implements upstreams manipulation from LUA API. This functionality
+ * can be used for load balancing using different strategies including:
+ *
+ * - round-robin: balance upstreams one by one selecting accordingly to their weight
+ * - hash: use stable hashing algorithm to distribute values according to some static strings
+ * - master-slave: always prefer upstream with higher priority unless it is not available
+ *
+ * Here is an example of upstreams manipulations:
+ * @example
+local rspamd_logger = require "rspamd_logger"
+local rspamd_redis = require "rspamd_redis"
+local upstream_list = require "rspamd_upstream_list"
+local upstreams = upstream_list.create('127.0.0.1,10.0.0.1,10.0.0.2', 6379)
+
+local function sym_callback(task)
+ local upstream = upstreams:get_upstream_by_hash(task:get_from()[1]['domain'])
+
+ local function cb(task, err, data)
+ if err then
+ upstream:fail()
+ else
+ upstream:ok()
+ end
+ end
+
+ local addr = upstream:get_addr()
+ rspamd_redis.make_request(task, addr, cb,
+ 'PUSH', {'key', 'value'})
+end
+ */
+/* Upstream list functions */
+LUA_FUNCTION_DEF(upstream_list, create);
+LUA_FUNCTION_DEF(upstream_list, destroy);
+LUA_FUNCTION_DEF(upstream_list, all_upstreams);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_by_hash);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_round_robin);
+LUA_FUNCTION_DEF(upstream_list, get_upstream_master_slave);
+LUA_FUNCTION_DEF(upstream_list, add_watcher);
+
+static const struct luaL_reg upstream_list_m[] = {
+
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_by_hash),
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_round_robin),
+ LUA_INTERFACE_DEF(upstream_list, get_upstream_master_slave),
+ LUA_INTERFACE_DEF(upstream_list, all_upstreams),
+ LUA_INTERFACE_DEF(upstream_list, add_watcher),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_upstream_list_destroy},
+ {NULL, NULL}};
+static const struct luaL_reg upstream_list_f[] = {
+ LUA_INTERFACE_DEF(upstream_list, create),
+ {NULL, NULL}};
+
+/* Upstream functions */
+LUA_FUNCTION_DEF(upstream, ok);
+LUA_FUNCTION_DEF(upstream, fail);
+LUA_FUNCTION_DEF(upstream, get_addr);
+LUA_FUNCTION_DEF(upstream, get_name);
+LUA_FUNCTION_DEF(upstream, get_port);
+LUA_FUNCTION_DEF(upstream, destroy);
+
+static const struct luaL_reg upstream_m[] = {
+ LUA_INTERFACE_DEF(upstream, ok),
+ LUA_INTERFACE_DEF(upstream, fail),
+ LUA_INTERFACE_DEF(upstream, get_addr),
+ LUA_INTERFACE_DEF(upstream, get_port),
+ LUA_INTERFACE_DEF(upstream, get_name),
+ {"__tostring", rspamd_lua_class_tostring},
+ {"__gc", lua_upstream_destroy},
+ {NULL, NULL}};
+
+/* Upstream class */
+
+struct rspamd_lua_upstream *
+lua_check_upstream(lua_State *L, int pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{upstream}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'upstream' expected");
+ return ud ? (struct rspamd_lua_upstream *) ud : NULL;
+}
+
+/***
+ * @method upstream:get_addr()
+ * Get ip of upstream
+ * @return {ip} ip address object
+ */
+static gint
+lua_upstream_get_addr(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ rspamd_lua_ip_push(L, rspamd_upstream_addr_next(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:get_name()
+ * Get name of upstream
+ * @return {string} name of the upstream
+ */
+static gint
+lua_upstream_get_name(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ lua_pushstring(L, rspamd_upstream_name(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:get_port()
+ * Get port of upstream
+ * @return {int} port of the upstream
+ */
+static gint
+lua_upstream_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ lua_pushinteger(L, rspamd_upstream_port(up->up));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream:fail()
+ * Indicate upstream failure. After certain amount of failures during specified time frame, an upstream is marked as down and does not participate in rotations.
+ */
+static gint
+lua_upstream_fail(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+ gboolean fail_addr = FALSE;
+ const gchar *reason = "unknown";
+
+ if (up) {
+
+ if (lua_isboolean(L, 2)) {
+ fail_addr = lua_toboolean(L, 2);
+
+ if (lua_isstring(L, 3)) {
+ reason = lua_tostring(L, 3);
+ }
+ }
+ else if (lua_isstring(L, 2)) {
+ reason = lua_tostring(L, 2);
+ }
+
+ rspamd_upstream_fail(up->up, fail_addr, reason);
+ }
+
+ return 0;
+}
+
+/***
+ * @method upstream:ok()
+ * Indicates upstream success. Resets errors count for an upstream.
+ */
+static gint
+lua_upstream_ok(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ rspamd_upstream_ok(up->up);
+ }
+
+ return 0;
+}
+
+static gint
+lua_upstream_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_upstream *up = lua_check_upstream(L, 1);
+
+ if (up) {
+ /* Remove reference to the parent */
+ luaL_unref(L, LUA_REGISTRYINDEX, up->upref);
+ /* Upstream belongs to the upstream list, so no free here */
+ }
+
+ return 0;
+}
+
+/* Upstream list class */
+
+static struct upstream_list *
+lua_check_upstream_list(lua_State *L)
+{
+ void *ud = rspamd_lua_check_udata(L, 1, "rspamd{upstream_list}");
+
+ luaL_argcheck(L, ud != NULL, 1, "'upstream_list' expected");
+ return ud ? *((struct upstream_list **) ud) : NULL;
+}
+
+static struct rspamd_lua_upstream *
+lua_push_upstream(lua_State *L, gint up_idx, struct upstream *up)
+{
+ struct rspamd_lua_upstream *lua_ups;
+
+ if (up_idx < 0) {
+ up_idx = lua_gettop(L) + up_idx + 1;
+ }
+
+ lua_ups = lua_newuserdata(L, sizeof(*lua_ups));
+ lua_ups->up = up;
+ rspamd_lua_setclass(L, "rspamd{upstream}", -1);
+ /* Store parent in the upstream to prevent gc */
+ lua_pushvalue(L, up_idx);
+ lua_ups->upref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ return lua_ups;
+}
+
+/***
+ * @function upstream_list.create(cfg, def, [default_port])
+ * Create new upstream list from its string definition in form `<upstream>,<upstream>;<upstream>`
+ * @param {rspamd_config} cfg configuration reference
+ * @param {string} def upstream list definition
+ * @param {number} default_port default port for upstreams
+ * @return {upstream_list} upstream list structure
+ */
+static gint
+lua_upstream_list_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *new = NULL, **pnew;
+ struct rspamd_config *cfg = NULL;
+ const gchar *def;
+ guint default_port = 0;
+ gint top;
+
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ cfg = lua_check_config(L, 1);
+ top = 2;
+ }
+ else {
+ top = 1;
+ }
+
+ if (lua_gettop(L) >= top + 1) {
+ default_port = luaL_checknumber(L, top + 1);
+ }
+
+ if (lua_type(L, top) == LUA_TSTRING) {
+ def = luaL_checkstring(L, top);
+
+ new = rspamd_upstreams_create(cfg ? cfg->ups_ctx : NULL);
+
+ if (rspamd_upstreams_parse_line(new, def, default_port, NULL)) {
+ pnew = lua_newuserdata(L, sizeof(struct upstream_list *));
+ rspamd_lua_setclass(L, "rspamd{upstream_list}", -1);
+ *pnew = new;
+ }
+ else {
+ rspamd_upstreams_destroy(new);
+ lua_pushnil(L);
+ }
+ }
+ else if (lua_type(L, top) == LUA_TTABLE) {
+ new = rspamd_upstreams_create(cfg ? cfg->ups_ctx : NULL);
+ pnew = lua_newuserdata(L, sizeof(struct upstream_list *));
+ rspamd_lua_setclass(L, "rspamd{upstream_list}", -1);
+ *pnew = new;
+
+ lua_pushvalue(L, top);
+
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ def = lua_tostring(L, -1);
+
+ if (!def || !rspamd_upstreams_parse_line(new, def, default_port, NULL)) {
+ msg_warn("cannot parse upstream %s", def);
+ }
+ }
+
+ lua_pop(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/**
+ * Destroy a single upstream list object
+ * @param L
+ * @return
+ */
+static gint
+lua_upstream_list_destroy(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl = lua_check_upstream_list(L);
+
+ rspamd_upstreams_destroy(upl);
+
+ return 0;
+}
+
+/***
+ * @method upstream_list:get_upstream_by_hash(key)
+ * Get upstream by hash from key
+ * @param {string} key a string used as input for stable hash algorithm
+ * @return {upstream} upstream from a list corresponding to the given key
+ */
+static gint
+lua_upstream_list_get_upstream_by_hash(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+ const gchar *key;
+ gsize keyl;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+ key = luaL_checklstring(L, 2, &keyl);
+ if (key) {
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_HASHED, key,
+ (guint) keyl);
+
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream_list:get_upstream_round_robin()
+ * Get upstream round robin (by current weight)
+ * @return {upstream} upstream from a list in round-robin matter
+ */
+static gint
+lua_upstream_list_get_upstream_round_robin(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_ROUND_ROBIN, NULL, 0);
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+/***
+ * @method upstream_list:get_upstream_master_slave()
+ * Get upstream master slave order (by static priority)
+ * @return {upstream} upstream from a list in master-slave order
+ */
+static gint
+lua_upstream_list_get_upstream_master_slave(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream *selected;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+
+ selected = rspamd_upstream_get(upl, RSPAMD_UPSTREAM_MASTER_SLAVE,
+ NULL,
+ 0);
+ if (selected) {
+ lua_push_upstream(L, 1, selected);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct upstream_foreach_cbdata {
+ lua_State *L;
+ gint ups_pos;
+};
+
+static void lua_upstream_inserter(struct upstream *up, guint idx, void *ud)
+{
+ struct upstream_foreach_cbdata *cbd = (struct upstream_foreach_cbdata *) ud;
+
+ lua_push_upstream(cbd->L, cbd->ups_pos, up);
+ lua_rawseti(cbd->L, -2, idx + 1);
+}
+/***
+ * @method upstream_list:all_upstreams()
+ * Returns all upstreams for this list
+ * @return {table|upstream} all upstreams defined
+ */
+static gint
+lua_upstream_list_all_upstreams(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+ struct upstream_foreach_cbdata cbd;
+
+ upl = lua_check_upstream_list(L);
+ if (upl) {
+ cbd.L = L;
+ cbd.ups_pos = 1;
+
+ lua_createtable(L, rspamd_upstreams_count(upl), 0);
+ rspamd_upstreams_foreach(upl, lua_upstream_inserter, &cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static inline enum rspamd_upstreams_watch_event
+lua_str_to_upstream_flag(const gchar *str)
+{
+ enum rspamd_upstreams_watch_event fl = 0;
+
+ if (strcmp(str, "success") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_SUCCESS;
+ }
+ else if (strcmp(str, "failure") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_FAILURE;
+ }
+ else if (strcmp(str, "online") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_ONLINE;
+ }
+ else if (strcmp(str, "offline") == 0) {
+ fl = RSPAMD_UPSTREAM_WATCH_OFFLINE;
+ }
+ else {
+ msg_err("invalid flag: %s", str);
+ }
+
+ return fl;
+}
+
+static inline const gchar *
+lua_upstream_flag_to_str(enum rspamd_upstreams_watch_event fl)
+{
+ const gchar *res = "unknown";
+
+ /* Works with single flags, not combinations */
+ if (fl & RSPAMD_UPSTREAM_WATCH_SUCCESS) {
+ res = "success";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_FAILURE) {
+ res = "failure";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_ONLINE) {
+ res = "online";
+ }
+ else if (fl & RSPAMD_UPSTREAM_WATCH_OFFLINE) {
+ res = "offline";
+ }
+ else {
+ msg_err("invalid flag: %d", fl);
+ }
+
+ return res;
+}
+
+struct rspamd_lua_upstream_watcher_cbdata {
+ lua_State *L;
+ gint cbref;
+ gint parent_cbref; /* Reference to the upstream list */
+ struct upstream_list *upl;
+};
+
+static void
+lua_upstream_watch_func(struct upstream *up,
+ enum rspamd_upstreams_watch_event event,
+ guint cur_errors,
+ void *ud)
+{
+ struct rspamd_lua_upstream_watcher_cbdata *cdata =
+ (struct rspamd_lua_upstream_watcher_cbdata *) ud;
+ lua_State *L;
+ const gchar *what;
+ gint err_idx;
+
+ L = cdata->L;
+ what = lua_upstream_flag_to_str(event);
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cdata->cbref);
+ lua_pushstring(L, what);
+
+ struct rspamd_lua_upstream *lua_ups = lua_newuserdata(L, sizeof(*lua_ups));
+ lua_ups->up = up;
+ rspamd_lua_setclass(L, "rspamd{upstream}", -1);
+ /* Store parent in the upstream to prevent gc */
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cdata->parent_cbref);
+ lua_ups->upref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ lua_pushinteger(L, cur_errors);
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err("cannot call watch function for upstream: %s", lua_tostring(L, -1));
+ lua_settop(L, 0);
+
+ return;
+ }
+
+ lua_settop(L, 0);
+}
+
+static void
+lua_upstream_watch_dtor(gpointer ud)
+{
+ struct rspamd_lua_upstream_watcher_cbdata *cdata =
+ (struct rspamd_lua_upstream_watcher_cbdata *) ud;
+
+ luaL_unref(cdata->L, LUA_REGISTRYINDEX, cdata->cbref);
+ luaL_unref(cdata->L, LUA_REGISTRYINDEX, cdata->parent_cbref);
+ g_free(cdata);
+}
+
+/***
+ * @method upstream_list:add_watcher(what, cb)
+ * Add new watcher to the upstream lists events (table or a string):
+ * - `success` - called whenever upstream successfully used
+ * - `failure` - called on upstream error
+ * - `online` - called when upstream is being taken online from offline
+ * - `offline` - called when upstream is being taken offline from online
+ * Callback is a function: function(what, upstream, cur_errors) ... end
+ * @example
+ups:add_watcher('success', function(what, up, cur_errors) ... end)
+ups:add_watcher({'online', 'offline'}, function(what, up, cur_errors) ... end)
+ * @return nothing
+ */
+static gint
+lua_upstream_list_add_watcher(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct upstream_list *upl;
+
+ upl = lua_check_upstream_list(L);
+ if (upl &&
+ (lua_type(L, 2) == LUA_TTABLE || lua_type(L, 2) == LUA_TSTRING) &&
+ lua_type(L, 3) == LUA_TFUNCTION) {
+
+ enum rspamd_upstreams_watch_event flags = 0;
+ struct rspamd_lua_upstream_watcher_cbdata *cdata;
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ flags = lua_str_to_upstream_flag(lua_tostring(L, 2));
+ }
+ else {
+ for (lua_pushnil(L); lua_next(L, -2); lua_pop(L, 1)) {
+ if (lua_isstring(L, -1)) {
+ flags |= lua_str_to_upstream_flag(lua_tostring(L, -1));
+ }
+ else {
+ lua_pop(L, 1);
+
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ }
+
+ cdata = g_malloc0(sizeof(*cdata));
+ lua_pushvalue(L, 3); /* callback */
+ cdata->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+ cdata->L = L;
+ cdata->upl = upl;
+ lua_pushvalue(L, 1); /* upstream list itself */
+ cdata->parent_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_upstreams_add_watch_callback(upl, flags,
+ lua_upstream_watch_func, lua_upstream_watch_dtor, cdata);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 0;
+}
+
+static gint
+lua_load_upstream_list(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, upstream_list_f);
+
+ return 1;
+}
+
+void luaopen_upstream(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{upstream_list}", upstream_list_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_upstream_list", lua_load_upstream_list);
+
+ rspamd_lua_new_class(L, "rspamd{upstream}", upstream_m);
+ lua_pop(L, 1);
+}
diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c
new file mode 100644
index 0000000..913469f
--- /dev/null
+++ b/src/lua/lua_url.c
@@ -0,0 +1,1481 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "lua_url.h"
+
+
+/***
+ * @module rspamd_url
+ * This module provides routines to handle URL's and extract URL's from the text.
+ * Objects of this class are returned, for example, by `task:get_urls()` or `task:get_emails()`.
+ * You can also create `rspamd_url` from any text.
+ * @example
+local url = require "rspamd_url"
+local mpool = require "rspamd_mempool"
+
+url.init("/usr/share/rspamd/effective_tld_names.dat")
+local pool = mpool.create()
+local res = url.create(pool, 'Look at: http://user@test.example.com/test?query")
+local t = res:to_table()
+-- Content of t:
+-- url = ['http://test.example.com/test?query']
+-- host = ['test.example.com']
+-- user = ['user']
+-- path = ['test']
+-- tld = ['example.com']
+
+pool:destroy() -- res is destroyed here, so you should not use it afterwards
+
+local mistake = res:to_table() -- INVALID! as pool is destroyed
+ */
+
+/* URL methods */
+LUA_FUNCTION_DEF(url, get_length);
+LUA_FUNCTION_DEF(url, get_host);
+LUA_FUNCTION_DEF(url, get_port);
+LUA_FUNCTION_DEF(url, get_user);
+LUA_FUNCTION_DEF(url, get_path);
+LUA_FUNCTION_DEF(url, get_query);
+LUA_FUNCTION_DEF(url, get_fragment);
+LUA_FUNCTION_DEF(url, get_text);
+LUA_FUNCTION_DEF(url, tostring);
+LUA_FUNCTION_DEF(url, get_raw);
+LUA_FUNCTION_DEF(url, get_tld);
+LUA_FUNCTION_DEF(url, get_flags);
+LUA_FUNCTION_DEF(url, get_flags_num);
+LUA_FUNCTION_DEF(url, get_protocol);
+LUA_FUNCTION_DEF(url, to_table);
+LUA_FUNCTION_DEF(url, is_phished);
+LUA_FUNCTION_DEF(url, is_redirected);
+LUA_FUNCTION_DEF(url, is_obscured);
+LUA_FUNCTION_DEF(url, is_html_displayed);
+LUA_FUNCTION_DEF(url, is_subject);
+LUA_FUNCTION_DEF(url, get_phished);
+LUA_FUNCTION_DEF(url, set_redirected);
+LUA_FUNCTION_DEF(url, get_count);
+LUA_FUNCTION_DEF(url, get_visible);
+LUA_FUNCTION_DEF(url, create);
+LUA_FUNCTION_DEF(url, init);
+LUA_FUNCTION_DEF(url, all);
+LUA_FUNCTION_DEF(url, lt);
+LUA_FUNCTION_DEF(url, eq);
+LUA_FUNCTION_DEF(url, get_order);
+LUA_FUNCTION_DEF(url, get_part_order);
+
+static const struct luaL_reg urllib_m[] = {
+ LUA_INTERFACE_DEF(url, get_length),
+ LUA_INTERFACE_DEF(url, get_host),
+ LUA_INTERFACE_DEF(url, get_port),
+ LUA_INTERFACE_DEF(url, get_user),
+ LUA_INTERFACE_DEF(url, get_path),
+ LUA_INTERFACE_DEF(url, get_query),
+ LUA_INTERFACE_DEF(url, get_fragment),
+ LUA_INTERFACE_DEF(url, get_text),
+ LUA_INTERFACE_DEF(url, get_tld),
+ LUA_INTERFACE_DEF(url, get_raw),
+ LUA_INTERFACE_DEF(url, get_protocol),
+ LUA_INTERFACE_DEF(url, to_table),
+ LUA_INTERFACE_DEF(url, is_phished),
+ LUA_INTERFACE_DEF(url, is_redirected),
+ LUA_INTERFACE_DEF(url, is_obscured),
+ LUA_INTERFACE_DEF(url, is_html_displayed),
+ LUA_INTERFACE_DEF(url, is_subject),
+ LUA_INTERFACE_DEF(url, get_phished),
+
+ LUA_INTERFACE_DEF(url, get_visible),
+ LUA_INTERFACE_DEF(url, get_count),
+ LUA_INTERFACE_DEF(url, get_flags),
+ LUA_INTERFACE_DEF(url, get_flags_num),
+ LUA_INTERFACE_DEF(url, get_order),
+ LUA_INTERFACE_DEF(url, get_part_order),
+ {"get_redirected", lua_url_get_phished},
+ LUA_INTERFACE_DEF(url, set_redirected),
+ {"__tostring", lua_url_tostring},
+ {"__eq", lua_url_eq},
+ {"__lt", lua_url_lt},
+ {NULL, NULL}};
+
+static const struct luaL_reg urllib_f[] = {
+ LUA_INTERFACE_DEF(url, init),
+ LUA_INTERFACE_DEF(url, create),
+ LUA_INTERFACE_DEF(url, all),
+ {NULL, NULL}};
+
+struct rspamd_lua_url *
+lua_check_url(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{url}");
+ luaL_argcheck(L, ud != NULL, pos, "'url' expected");
+ return ud ? ((struct rspamd_lua_url *) ud) : NULL;
+}
+
+static gboolean
+lua_url_single_inserter(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ lua_State *L = ud;
+ struct rspamd_lua_url *lua_url;
+
+ lua_url = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_url->url = url;
+
+ return TRUE;
+}
+
+/***
+ * @method url:get_length()
+ * Get length of the url
+ * @return {number} length of url in bytes
+ */
+static gint
+lua_url_get_length(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushinteger(L, url->url->urllen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_host()
+ * Get domain part of the url
+ * @return {string} domain part of URL
+ */
+static gint
+lua_url_get_host(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url && url->url->hostlen > 0) {
+ lua_pushlstring(L, rspamd_url_host(url->url), url->url->hostlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_port()
+ * Get port of the url
+ * @return {number} url port
+ */
+static gint
+lua_url_get_port(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ if (rspamd_url_get_port_if_special(url->url) == 0) {
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushinteger(L, rspamd_url_get_port_if_special(url->url));
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+/***
+ * @method url:get_user()
+ * Get user part of the url (e.g. username in email)
+ * @return {string} user part of URL
+ */
+static gint
+lua_url_get_user(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && rspamd_url_user(url->url) != NULL) {
+ lua_pushlstring(L, rspamd_url_user(url->url), url->url->userlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_path()
+ * Get path of the url
+ * @return {string} path part of URL
+ */
+static gint
+lua_url_get_path(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->datalen > 0) {
+ lua_pushlstring(L, rspamd_url_data_unsafe(url->url), url->url->datalen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_query()
+ * Get query of the url
+ * @return {string} query part of URL
+ */
+static gint
+lua_url_get_query(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->querylen > 0) {
+ lua_pushlstring(L, rspamd_url_query_unsafe(url->url), url->url->querylen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_fragment()
+ * Get fragment of the url
+ * @return {string} fragment part of URL
+ */
+static gint
+lua_url_get_fragment(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->fragmentlen > 0) {
+ lua_pushlstring(L, rspamd_url_fragment_unsafe(url->url), url->url->fragmentlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_text()
+ * Get full content of the url
+ * @return {string} url string
+ */
+static gint
+lua_url_get_text(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushlstring(L, url->url->string, url->url->urllen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:tostring()
+ * Get full content of the url or user@domain in case of email
+ * @return {string} url as a string
+ */
+static gint
+lua_url_tostring(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url != NULL) {
+ if (url->url->protocol == PROTOCOL_MAILTO) {
+ gchar *tmp = g_malloc(url->url->userlen + 1 +
+ url->url->hostlen);
+ if (url->url->userlen) {
+ memcpy(tmp, url->url->string + url->url->usershift, url->url->userlen);
+ }
+
+ tmp[url->url->userlen] = '@';
+ memcpy(tmp + url->url->userlen + 1, rspamd_url_host_unsafe(url->url),
+ url->url->hostlen);
+
+ lua_pushlstring(L, tmp, url->url->userlen + 1 + url->url->hostlen);
+ g_free(tmp);
+ }
+ else {
+ lua_pushlstring(L, url->url->string, url->url->urllen);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_raw()
+ * Get full content of the url as it was parsed (e.g. with urldecode)
+ * @return {string} url string
+ */
+static gint
+lua_url_get_raw(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushlstring(L, url->url->raw, url->url->rawlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_phished()
+ * Check whether URL is treated as phished
+ * @return {boolean} `true` if URL is phished
+ */
+static gint
+lua_url_is_phished(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_PHISHED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_redirected()
+ * Check whether URL was redirected
+ * @return {boolean} `true` if URL is redirected
+ */
+static gint
+lua_url_is_redirected(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_REDIRECTED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_obscured()
+ * Check whether URL is treated as obscured or obfuscated (e.g. numbers in IP address or other hacks)
+ * @return {boolean} `true` if URL is obscured
+ */
+static gint
+lua_url_is_obscured(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_OBSCURED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+/***
+ * @method url:is_html_displayed()
+ * Check whether URL is just displayed in HTML (e.g. NOT a real href)
+ * @return {boolean} `true` if URL is displayed only
+ */
+static gint
+lua_url_is_html_displayed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_HTML_DISPLAYED);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:is_subject()
+ * Check whether URL is found in subject
+ * @return {boolean} `true` if URL is found in subject
+ */
+static gint
+lua_url_is_subject(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL) {
+ lua_pushboolean(L, url->url->flags & RSPAMD_URL_FLAG_SUBJECT);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_phished()
+ * Get another URL that pretends to be this URL (e.g. used in phishing)
+ * @return {url} phished URL
+ */
+static gint
+lua_url_get_phished(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *purl, *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->ext && url->url->ext->linked_url != NULL) {
+ /* XXX: in fact, this is the only possible combination of flags, so this check is redundant */
+ if (url->url->flags &
+ (RSPAMD_URL_FLAG_PHISHED | RSPAMD_URL_FLAG_REDIRECTED)) {
+ purl = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ purl->url = url->url->ext->linked_url;
+
+ return 1;
+ }
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+/***
+ * @method url:set_redirected(url, pool)
+ * Set url as redirected to another url
+ * @param {string|url} url new url that is redirecting an old one
+ * @param {pool} pool memory pool to allocate memory if needed
+ * @return {url} parsed redirected url (if needed)
+ */
+static gint
+lua_url_set_redirected(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1), *redir;
+ rspamd_mempool_t *pool = NULL;
+
+ if (url == NULL) {
+ return luaL_error(L, "url is required as the first argument");
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ /* Parse url */
+ if (lua_type(L, 3) != LUA_TUSERDATA) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ if (pool == NULL) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ gsize len;
+ const gchar *urlstr = lua_tolstring(L, 2, &len);
+
+ rspamd_url_find_single(pool, urlstr, len, RSPAMD_URL_FIND_ALL,
+ lua_url_single_inserter, L);
+
+ if (lua_type(L, -1) != LUA_TUSERDATA) {
+ /* URL is actually not found */
+ lua_pushnil(L);
+ }
+ else {
+ redir = lua_check_url(L, -1);
+
+ url->url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
+
+ if (url->url->ext == NULL) {
+ url->url->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+ }
+ url->url->ext->linked_url = redir->url;
+ }
+ }
+ else {
+ redir = lua_check_url(L, 2);
+
+ if (redir == NULL) {
+ return luaL_error(L, "url is required as the second argument");
+ }
+
+ pool = rspamd_lua_check_mempool(L, 3);
+
+ if (pool == NULL) {
+ return luaL_error(L, "mempool is required as the third argument");
+ }
+
+ url->url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
+ if (url->url->ext == NULL) {
+ url->url->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+ }
+ url->url->ext->linked_url = redir->url;
+
+ /* Push back on stack */
+ lua_pushvalue(L, 2);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_tld()
+ * Get effective second level domain part (eSLD) of the url host
+ * @return {string} effective second level domain part (eSLD) of the url host
+ */
+static gint
+lua_url_get_tld(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->tldlen > 0) {
+ lua_pushlstring(L, rspamd_url_tld_unsafe(url->url), url->url->tldlen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_protocol()
+ * Get protocol name
+ * @return {string} protocol as a string
+ */
+static gint
+lua_url_get_protocol(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->protocol != PROTOCOL_UNKNOWN) {
+ lua_pushstring(L, rspamd_url_protocol_name(url->url->protocol));
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_count()
+ * Return number of occurrences for this particular URL
+ * @return {number} number of occurrences
+ */
+static gint
+lua_url_get_count(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url != NULL) {
+ lua_pushinteger(L, url->url->count);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+* @method url:get_visible()
+* Get visible part of the url with html tags stripped
+* @return {string} url string
+*/
+static gint
+lua_url_get_visible(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url != NULL && url->url->ext && url->url->ext->visible_part) {
+ lua_pushstring(L, url->url->ext->visible_part);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:to_table()
+ * Return url as a table with the following fields:
+ *
+ * - `url`: full content
+ * - `host`: hostname part
+ * - `user`: user part
+ * - `path`: path part
+ * - `tld`: top level domain
+ * - `protocol`: url protocol
+ * @return {table} URL as a table
+ */
+static gint
+lua_url_to_table(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+ struct rspamd_url *u;
+
+ if (url != NULL) {
+ u = url->url;
+ lua_createtable(L, 0, 12);
+ lua_pushstring(L, "url");
+ lua_pushlstring(L, u->string, u->urllen);
+ lua_settable(L, -3);
+
+ if (u->hostlen > 0) {
+ lua_pushstring(L, "host");
+ lua_pushlstring(L, rspamd_url_host_unsafe(u), u->hostlen);
+ lua_settable(L, -3);
+ }
+
+ if (rspamd_url_get_port_if_special(u) != 0) {
+ lua_pushstring(L, "port");
+ lua_pushinteger(L, rspamd_url_get_port_if_special(u));
+ lua_settable(L, -3);
+ }
+
+ if (u->tldlen > 0) {
+ lua_pushstring(L, "tld");
+ lua_pushlstring(L, rspamd_url_tld_unsafe(u), u->tldlen);
+ lua_settable(L, -3);
+ }
+
+ if (u->userlen > 0) {
+ lua_pushstring(L, "user");
+ lua_pushlstring(L, rspamd_url_user(u), u->userlen);
+ lua_settable(L, -3);
+ }
+
+ if (u->datalen > 0) {
+ lua_pushstring(L, "path");
+ lua_pushlstring(L, rspamd_url_data_unsafe(u), u->datalen);
+ lua_settable(L, -3);
+ }
+
+ if (u->querylen > 0) {
+ lua_pushstring(L, "query");
+ lua_pushlstring(L, rspamd_url_query_unsafe(u), u->querylen);
+ lua_settable(L, -3);
+ }
+
+ if (u->fragmentlen > 0) {
+ lua_pushstring(L, "fragment");
+ lua_pushlstring(L, rspamd_url_fragment_unsafe(u), u->fragmentlen);
+ lua_settable(L, -3);
+ }
+
+
+ lua_pushstring(L, "protocol");
+ lua_pushstring(L, rspamd_url_protocol_name(u->protocol));
+ lua_settable(L, -3);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static rspamd_mempool_t *static_lua_url_pool;
+
+RSPAMD_CONSTRUCTOR(rspamd_urls_static_pool_ctor)
+{
+ static_lua_url_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(),
+ "static_lua_url", 0);
+}
+
+RSPAMD_DESTRUCTOR(rspamd_urls_static_pool_dtor)
+{
+ rspamd_mempool_delete(static_lua_url_pool);
+}
+
+/***
+ * @function url.create([mempool,] str, [{flags_table}])
+ * @param {rspamd_mempool} memory pool for URL, e.g. `task:get_mempool()`
+ * @param {string} text that contains URL (can also contain other stuff)
+ * @return {url} new url object that exists as long as the corresponding mempool exists
+ */
+static gint
+lua_url_create(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool;
+ struct rspamd_lua_text *t;
+ struct rspamd_lua_url *u;
+
+ if (lua_type(L, 1) == LUA_TUSERDATA) {
+ pool = rspamd_lua_check_mempool(L, 1);
+ t = lua_check_text_or_string(L, 2);
+ }
+ else {
+ pool = static_lua_url_pool;
+ t = lua_check_text_or_string(L, 2);
+ }
+
+ if (pool == NULL || t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ rspamd_url_find_single(pool, t->start, t->len, RSPAMD_URL_FIND_ALL,
+ lua_url_single_inserter, L);
+
+ if (lua_type(L, -1) != LUA_TUSERDATA) {
+ /* URL is actually not found */
+ lua_pushnil(L);
+
+ return 1;
+ }
+
+ u = (struct rspamd_lua_url *) lua_touserdata(L, -1);
+
+ if (lua_type(L, 3) == LUA_TTABLE) {
+ /* Add flags */
+ for (lua_pushnil(L); lua_next(L, 3); lua_pop(L, 1)) {
+ int nmask = 0;
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ u->url->flags |= nmask;
+ }
+ else {
+ lua_pop(L, 1);
+ return luaL_error(L, "invalid flag: %s", fname);
+ }
+ }
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @function url.init(tld_file)
+ * Initialize url library if not initialized yet by Rspamd
+ * @param {string} tld_file path to effective_tld_names.dat file (public suffix list)
+ * @return nothing
+ */
+static gint
+lua_url_init(lua_State *L)
+{
+ const gchar *tld_path;
+
+ tld_path = luaL_checkstring(L, 1);
+
+ rspamd_url_init(tld_path);
+
+ return 0;
+}
+
+static gboolean
+lua_url_table_inserter(struct rspamd_url *url, gsize start_offset,
+ gsize end_offset, gpointer ud)
+{
+ lua_State *L = ud;
+ struct rspamd_lua_url *lua_url;
+ gint n;
+
+ n = rspamd_lua_table_size(L, -1);
+ lua_url = lua_newuserdata(L, sizeof(struct rspamd_lua_url));
+ rspamd_lua_setclass(L, "rspamd{url}", -1);
+ lua_url->url = url;
+ lua_rawseti(L, -2, n + 1);
+
+ return TRUE;
+}
+
+
+static gint
+lua_url_all(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ rspamd_mempool_t *pool = rspamd_lua_check_mempool(L, 1);
+ const gchar *text;
+ size_t length;
+
+ if (pool == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ text = luaL_checklstring(L, 2, &length);
+
+ if (text != NULL) {
+ lua_newtable(L);
+ rspamd_url_find_multiple(pool, text, length,
+ RSPAMD_URL_FIND_ALL, NULL,
+ lua_url_table_inserter, L);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+/***
+ * @method url:get_flags()
+ * Return flags for a specified URL as map 'flag'->true for all flags set,
+ * possible flags are:
+ *
+ * - `phished`: URL is likely phished
+ * - `numeric`: URL is numeric (e.g. IP address)
+ * - `obscured`: URL was obscured
+ * - `redirected`: URL comes from redirector
+ * - `html_displayed`: URL is used just for displaying purposes
+ * - `text`: URL comes from the text
+ * - `subject`: URL comes from the subject
+ * - `host_encoded`: URL host part is encoded
+ * - `schema_encoded`: URL schema part is encoded
+ * - `query_encoded`: URL query part is encoded
+ * - `missing_slashes`: URL has some slashes missing
+ * - `idn`: URL has international characters
+ * - `has_port`: URL has port
+ * - `has_user`: URL has user part
+ * - `schemaless`: URL has no schema
+ * - `unnormalised`: URL has some unicode unnormalities
+ * - `zw_spaces`: URL has some zero width spaces
+ * - `url_displayed`: URL has some other url-like string in visible part
+ * - `image`: URL is from src attribute of img HTML tag
+ * @return {table} URL flags
+ */
+#define PUSH_FLAG(fl) \
+ do { \
+ if (flags & (fl)) { \
+ lua_pushstring(L, rspamd_url_flag_to_string(fl)); \
+ lua_pushboolean(L, true); \
+ lua_settable(L, -3); \
+ } \
+ } while (0)
+
+static gint
+lua_url_get_flags(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+ enum rspamd_url_flags flags;
+
+ if (url != NULL) {
+ flags = url->url->flags;
+
+ lua_createtable(L, 0, 4);
+
+ for (gint i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
+ PUSH_FLAG(1u << i);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+#undef PUSH_FLAG
+
+static gint
+lua_url_get_flags_num(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ lua_pushinteger(L, url->url->flags);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_get_order(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->order != (uint16_t) -1) {
+ lua_pushinteger(L, url->url->order);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_get_part_order(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *url = lua_check_url(L, 1);
+
+ if (url) {
+ if (url->url->part_order != (uint16_t) -1) {
+ lua_pushinteger(L, url->url->part_order);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+void lua_tree_url_callback(gpointer key, gpointer value, gpointer ud)
+{
+ struct rspamd_lua_url *lua_url;
+ struct rspamd_url *url = (struct rspamd_url *) value;
+ struct lua_tree_cb_data *cb = ud;
+
+ if ((url->protocol & cb->protocols_mask) == url->protocol) {
+
+ /* Handle different flags application logic */
+ switch (cb->flags_mode) {
+ case url_flags_mode_include_any:
+ if (url->flags != (url->flags & cb->flags_mask)) {
+ return;
+ }
+ break;
+ case url_flags_mode_include_explicit:
+ if ((url->flags & cb->flags_mask) != cb->flags_mask) {
+ return;
+ }
+ break;
+ case url_flags_mode_exclude_include:
+ if ((url->flags & cb->flags_exclude_mask) != 0) {
+ return;
+ }
+ if ((url->flags & cb->flags_mask) == 0) {
+ return;
+ }
+ break;
+ }
+
+ if (cb->skip_prob > 0) {
+ gdouble coin = rspamd_random_double_fast_seed(&cb->random_seed);
+
+ if (coin < cb->skip_prob) {
+ return;
+ }
+ }
+
+ lua_url = lua_newuserdata(cb->L, sizeof(struct rspamd_lua_url));
+ lua_pushvalue(cb->L, cb->metatable_pos);
+ lua_setmetatable(cb->L, -2);
+ lua_url->url = url;
+ lua_rawseti(cb->L, -2, cb->i++);
+ }
+}
+
+gboolean
+lua_url_cbdata_fill(lua_State *L,
+ gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ guint default_flags,
+ gsize max_urls)
+{
+ gint protocols_mask = 0;
+
+ gint pos_arg_type = lua_type(L, pos);
+ guint flags_mask = default_flags;
+ gboolean seen_flags = FALSE, seen_protocols = FALSE;
+
+ memset(cbd, 0, sizeof(*cbd));
+ cbd->flags_mode = url_flags_mode_include_any;
+
+ if (pos_arg_type == LUA_TBOOLEAN) {
+ protocols_mask = default_protocols;
+ if (lua_toboolean(L, 2)) {
+ protocols_mask |= PROTOCOL_MAILTO;
+ }
+ }
+ else if (pos_arg_type == LUA_TTABLE) {
+ if (rspamd_lua_geti(L, 1, pos) == LUA_TNIL) {
+ /* New method: indexed table */
+
+ lua_getfield(L, pos, "flags");
+ if (lua_istable(L, -1)) {
+ gint top = lua_gettop(L);
+
+ lua_getfield(L, pos, "flags_mode");
+ if (lua_isstring(L, -1)) {
+ const gchar *mode_str = lua_tostring(L, -1);
+
+ if (strcmp(mode_str, "explicit") == 0) {
+ cbd->flags_mode = url_flags_mode_include_explicit;
+ /*
+ * Ignore default flags in this mode and include
+ * merely flags specified by a caller
+ */
+ flags_mask = 0;
+ }
+ }
+ lua_pop(L, 1);
+
+ for (lua_pushnil(L); lua_next(L, top); lua_pop(L, 1)) {
+ int nmask = 0;
+
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+
+ seen_flags = TRUE;
+ }
+ else {
+ flags_mask |= default_flags;
+ }
+ lua_pop(L, 1);
+
+ lua_getfield(L, pos, "protocols");
+ if (lua_istable(L, -1)) {
+ gint top = lua_gettop(L);
+
+ for (lua_pushnil(L); lua_next(L, top); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ seen_protocols = TRUE;
+ }
+ else {
+ protocols_mask = default_protocols;
+ }
+ lua_pop(L, 1);
+
+ if (!seen_protocols) {
+ lua_getfield(L, pos, "emails");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ protocols_mask |= PROTOCOL_MAILTO;
+ }
+ }
+ lua_pop(L, 1);
+ }
+
+ if (!seen_flags) {
+ lua_getfield(L, pos, "images");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ flags_mask |= RSPAMD_URL_FLAG_IMAGE;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ lua_pop(L, 1);
+ }
+
+ if (!seen_flags) {
+ lua_getfield(L, pos, "content");
+ if (lua_isboolean(L, -1)) {
+ if (lua_toboolean(L, -1)) {
+ flags_mask |= RSPAMD_URL_FLAG_CONTENT;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_CONTENT;
+ }
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_CONTENT;
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_getfield(L, pos, "max_urls");
+ if (lua_isnumber(L, -1)) {
+ max_urls = lua_tonumber(L, -1);
+ }
+ lua_pop(L, 1);
+
+ lua_getfield(L, pos, "sort");
+ if (lua_isboolean(L, -1)) {
+ cbd->sort = TRUE;
+ }
+ lua_pop(L, 1);
+ }
+ else {
+ /* Plain table of the protocols */
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ }
+
+ lua_pop(L, 1); /* After rspamd_lua_geti */
+ }
+ else if (pos_arg_type == LUA_TSTRING) {
+ const gchar *plist = lua_tostring(L, pos);
+ gchar **strvec;
+ gchar *const *cvec;
+
+ strvec = g_strsplit_set(plist, ",;", -1);
+ cvec = strvec;
+
+ while (*cvec) {
+ int nmask;
+
+ nmask = rspamd_url_protocol_from_string(*cvec);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", *cvec);
+ g_strfreev(strvec);
+
+ return FALSE;
+ }
+
+ cvec++;
+ }
+
+ g_strfreev(strvec);
+ }
+ else if (pos_arg_type == LUA_TNONE || pos_arg_type == LUA_TNIL) {
+ protocols_mask = default_protocols;
+ flags_mask = default_flags;
+ }
+ else {
+ return FALSE;
+ }
+
+ if (lua_type(L, pos + 1) == LUA_TBOOLEAN) {
+ if (lua_toboolean(L, pos + 1)) {
+ flags_mask |= RSPAMD_URL_FLAG_IMAGE;
+ }
+ else {
+ flags_mask &= ~RSPAMD_URL_FLAG_IMAGE;
+ }
+ }
+
+ cbd->i = 1;
+ cbd->L = L;
+ cbd->max_urls = max_urls;
+ cbd->protocols_mask = protocols_mask;
+ cbd->flags_mask = flags_mask;
+
+ /* This needs to be removed from the stack */
+ rspamd_lua_class_metatable(L, "rspamd{url}");
+ cbd->metatable_pos = lua_gettop(L);
+ (void) lua_checkstack(L, cbd->metatable_pos + 4);
+
+ return TRUE;
+}
+
+gboolean
+lua_url_cbdata_fill_exclude_include(lua_State *L,
+ gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ gsize max_urls)
+{
+ guint protocols_mask = default_protocols;
+ guint include_flags_mask, exclude_flags_mask;
+
+ gint pos_arg_type = lua_type(L, pos);
+
+ memset(cbd, 0, sizeof(*cbd));
+ cbd->flags_mode = url_flags_mode_exclude_include;
+
+ /* Include flags */
+ if (pos_arg_type == LUA_TTABLE) {
+ include_flags_mask = 0; /* Reset to no flags */
+
+ for (lua_pushnil(L); lua_next(L, pos); lua_pop(L, 1)) {
+ int nmask = 0;
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ include_flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url include flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ include_flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+ }
+ else if (pos_arg_type == LUA_TNIL || pos_arg_type == LUA_TNONE) {
+ /* Include all flags */
+ include_flags_mask = ~0U;
+ }
+ else {
+ msg_info("bad arguments: wrong include mask");
+ return FALSE;
+ }
+
+ /* Exclude flags */
+ pos_arg_type = lua_type(L, pos + 1);
+ if (pos_arg_type == LUA_TTABLE) {
+ exclude_flags_mask = 0; /* Reset to no flags */
+
+ for (lua_pushnil(L); lua_next(L, pos + 1); lua_pop(L, 1)) {
+ int nmask = 0;
+
+ if (lua_type(L, -1) == LUA_TSTRING) {
+ const gchar *fname = lua_tostring(L, -1);
+
+ if (rspamd_url_flag_from_string(fname, &nmask)) {
+ exclude_flags_mask |= nmask;
+ }
+ else {
+ msg_info("bad url exclude flag: %s", fname);
+ return FALSE;
+ }
+ }
+ else {
+ exclude_flags_mask |= lua_tointeger(L, -1);
+ }
+ }
+ }
+ else if (pos_arg_type == LUA_TNIL || pos_arg_type == LUA_TNONE) {
+ /* Empty all exclude flags */
+ exclude_flags_mask = 0U;
+ }
+ else {
+ msg_info("bad arguments: wrong exclude mask");
+ return FALSE;
+ }
+
+ if (lua_type(L, pos + 2) == LUA_TTABLE) {
+ protocols_mask = 0U; /* Reset all protocols */
+
+ for (lua_pushnil(L); lua_next(L, pos + 2); lua_pop(L, 1)) {
+ int nmask;
+ const gchar *pname = lua_tostring(L, -1);
+
+ nmask = rspamd_url_protocol_from_string(pname);
+
+ if (nmask != PROTOCOL_UNKNOWN) {
+ protocols_mask |= nmask;
+ }
+ else {
+ msg_info("bad url protocol: %s", pname);
+ return FALSE;
+ }
+ }
+ }
+ else {
+ protocols_mask = default_protocols;
+ }
+
+ cbd->i = 1;
+ cbd->L = L;
+ cbd->max_urls = max_urls;
+ cbd->protocols_mask = protocols_mask;
+ cbd->flags_mask = include_flags_mask;
+ cbd->flags_exclude_mask = exclude_flags_mask;
+
+ /* This needs to be removed from the stack */
+ rspamd_lua_class_metatable(L, "rspamd{url}");
+ cbd->metatable_pos = lua_gettop(L);
+ (void) lua_checkstack(L, cbd->metatable_pos + 4);
+
+ return TRUE;
+}
+
+
+void lua_url_cbdata_dtor(struct lua_tree_cb_data *cbd)
+{
+ if (cbd->metatable_pos != -1) {
+ lua_remove(cbd->L, cbd->metatable_pos);
+ }
+}
+
+gsize lua_url_adjust_skip_prob(float timestamp,
+ guchar digest[16],
+ struct lua_tree_cb_data *cb,
+ gsize sz)
+{
+ if (cb->max_urls > 0 && sz > cb->max_urls) {
+ cb->skip_prob = 1.0 - ((gdouble) cb->max_urls) / (gdouble) sz;
+ /*
+ * Use task dependent probabilistic seed to ensure that
+ * consequent task:get_urls return the same list of urls
+ * We use both digest and timestamp here to avoid attack surface
+ * based just on digest.
+ */
+ memcpy(&cb->random_seed, digest, 4);
+ memcpy(((unsigned char *) &cb->random_seed) + 4, &timestamp, 4);
+ sz = cb->max_urls;
+ }
+
+ return sz;
+}
+
+static gint
+lua_url_eq(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *u1 = lua_check_url(L, 1),
+ *u2 = lua_check_url(L, 2);
+
+ if (u1 && u2) {
+ lua_pushboolean(L, (rspamd_url_cmp(u1->url, u2->url) == 0));
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static gint
+lua_url_lt(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_url *u1 = lua_check_url(L, 1),
+ *u2 = lua_check_url(L, 2);
+
+ if (u1 && u2) {
+ lua_pushinteger(L, rspamd_url_cmp(u1->url, u2->url));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_url(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, urllib_f);
+
+ /* Push flags */
+ lua_createtable(L, 0, RSPAMD_URL_MAX_FLAG_SHIFT);
+ for (int i = 0; i < RSPAMD_URL_MAX_FLAG_SHIFT; i++) {
+ guint flag = 1u << i;
+
+ lua_pushinteger(L, flag);
+ lua_setfield(L, -2, rspamd_url_flag_to_string(flag));
+ }
+
+ lua_setfield(L, -2, "flags");
+
+ return 1;
+}
+
+void luaopen_url(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{url}", urllib_m);
+ lua_pop(L, 1);
+
+ rspamd_lua_add_preload(L, "rspamd_url", lua_load_url);
+}
diff --git a/src/lua/lua_url.h b/src/lua/lua_url.h
new file mode 100644
index 0000000..a78dffa
--- /dev/null
+++ b/src/lua/lua_url.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright 2020 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LUA_URL_H
+#define RSPAMD_LUA_URL_H
+
+#include "lua_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct lua_tree_cb_data {
+ lua_State *L;
+ int i;
+ int metatable_pos;
+ guint flags_mask;
+ guint flags_exclude_mask;
+ guint protocols_mask;
+ enum {
+ url_flags_mode_include_any,
+ url_flags_mode_include_explicit,
+ url_flags_mode_exclude_include,
+ } flags_mode;
+ gboolean sort;
+ gsize max_urls;
+ gdouble skip_prob;
+ guint64 random_seed;
+};
+
+void lua_tree_url_callback(gpointer key, gpointer value, gpointer ud);
+
+/**
+ * Fills a cbdata table based on the parameter at position pos
+ * @param L
+ * @param pos
+ * @param cbd
+ * @return
+ */
+gboolean lua_url_cbdata_fill(lua_State *L, gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ guint default_flags,
+ gsize max_urls);
+
+gboolean lua_url_cbdata_fill_exclude_include(lua_State *L, gint pos,
+ struct lua_tree_cb_data *cbd,
+ guint default_protocols,
+ gsize max_urls);
+
+/**
+ * Cleanup url cbdata
+ * @param cbd
+ */
+void lua_url_cbdata_dtor(struct lua_tree_cb_data *cbd);
+
+/**
+ * Adjust probabilistic skip of the urls
+ * @param timestamp
+ * @param digest
+ * @param cb
+ * @param sz
+ * @param max_urls
+ * @return
+ */
+gsize lua_url_adjust_skip_prob(float timestamp,
+ guchar digest[16],
+ struct lua_tree_cb_data *cb,
+ gsize sz);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
new file mode 100644
index 0000000..152c02d
--- /dev/null
+++ b/src/lua/lua_util.c
@@ -0,0 +1,3585 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+#include "unix-std.h"
+#include "lua_compress.h"
+#include "libmime/email_addr.h"
+#include "libmime/content_type.h"
+#include "libmime/mime_headers.h"
+#include "libutil/hash.h"
+
+#include "lua_parsers.h"
+
+#ifdef WITH_LUA_REPL
+#include "replxx.h"
+#endif
+
+#include <math.h>
+#include <glob.h>
+
+#include "unicode/uspoof.h"
+#include "unicode/uscript.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+/***
+ * @module rspamd_util
+ * This module contains some generic purpose utilities that could be useful for
+ * testing and production rules.
+ */
+
+/***
+ * @function util.create_event_base()
+ * Creates new event base for processing asynchronous events
+ * @return {ev_base} new event processing base
+ */
+LUA_FUNCTION_DEF(util, create_event_base);
+/***
+ * @function util.load_rspamd_config(filename)
+ * Load rspamd config from the specified file
+ * @return {confg} new configuration object suitable for access
+ */
+LUA_FUNCTION_DEF(util, load_rspamd_config);
+/***
+ * @function util.config_from_ucl(any, string)
+ * Load rspamd config from ucl represented by any lua table
+ * @return {confg} new configuration object suitable for access
+ */
+LUA_FUNCTION_DEF(util, config_from_ucl);
+/***
+ * @function util.encode_base64(input[, str_len, [newlines_type]])
+ * Encodes data in base64 breaking lines if needed
+ * @param {text or string} input input data
+ * @param {number} str_len optional size of lines or 0 if split is not needed
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_base64);
+/***
+ * @function util.encode_qp(input[, str_len, [newlines_type]])
+ * Encodes data in quoted printable breaking lines if needed
+ * @param {text or string} input input data
+ * @param {number} str_len optional size of lines or 0 if split is not needed
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_qp);
+
+/***
+ * @function util.decode_qp(input)
+ * Decodes data from quoted printable
+ * @param {text or string} input input data
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_qp);
+
+/***
+ * @function util.decode_base64(input)
+ * Decodes data from base64 ignoring whitespace characters
+ * @param {text or string} input data to decode; if `rspamd{text}` is used then the string is modified **in-place**
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_base64);
+
+/***
+ * @function util.encode_base32(input, [b32type = 'default'])
+ * Encodes data in base32 breaking lines if needed
+ * @param {text or string} input input data
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} encoded data chunk
+ */
+LUA_FUNCTION_DEF(util, encode_base32);
+/***
+ * @function util.decode_base32(input, [b32type = 'default'])
+ * Decodes data from base32 ignoring whitespace characters
+ * @param {text or string} input data to decode
+ * @param {string} b32type base32 type (default, bleach, rfc)
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_base32);
+
+/***
+ * @function util.decode_url(input)
+ * Decodes data from url encoding
+ * @param {text or string} input data to decode
+ * @return {rspamd_text} decoded data chunk
+ */
+LUA_FUNCTION_DEF(util, decode_url);
+
+/***
+ * @function util.tokenize_text(input[, exceptions])
+ * Create tokens from a text using optional exceptions list
+ * @param {text/string} input input data
+ * @param {table} exceptions, a table of pairs containing <start_pos,length> of exceptions in the input
+ * @return {table/strings} list of strings representing words in the text
+ */
+LUA_FUNCTION_DEF(util, tokenize_text);
+LUA_FUNCTION_DEF(util, process_message);
+/***
+ * @function util.tanh(num)
+ * Calculates hyperbolic tangent of the specified floating point value
+ * @param {number} num input number
+ * @return {number} hyperbolic tangent of the variable
+ */
+LUA_FUNCTION_DEF(util, tanh);
+
+/***
+ * @function util.parse_html(input)
+ * Parses HTML and returns the according text
+ * @param {string|text} in input HTML
+ * @return {rspamd_text} processed text with no HTML tags
+ */
+LUA_FUNCTION_DEF(util, parse_html);
+
+/***
+ * @function util.levenshtein_distance(s1, s2)
+ * Returns levenstein distance between two strings
+ * @param {string} s1 the first string
+ * @param {string} s2 the second string
+ * @return {number} number of differences in two strings
+ */
+LUA_FUNCTION_DEF(util, levenshtein_distance);
+
+/***
+ * @function util.fold_header(name, value, [how, [stop_chars]])
+ * Fold rfc822 header according to the folding rules
+ *
+ * @param {string} name name of the header
+ * @param {string} value value of the header
+ * @param {string} how "cr" for \r, "lf" for \n and "crlf" for \r\n (default)
+ * @param {string} stop_chars also fold header when the
+ * @return {string} Folded value of the header
+ */
+LUA_FUNCTION_DEF(util, fold_header);
+
+/***
+ * @function util.is_uppercase(str)
+ * Returns true if a string is all uppercase
+ *
+ * @param {string} str input string
+ * @return {bool} true if a string is all uppercase
+ */
+LUA_FUNCTION_DEF(util, is_uppercase);
+
+/***
+ * @function util.humanize_number(num)
+ * Returns humanized representation of given number (like 1k instead of 1000)
+ *
+ * @param {number} num number to humanize
+ * @return {string} humanized representation of a number
+ */
+LUA_FUNCTION_DEF(util, humanize_number);
+
+/***
+ * @function util.get_tld(host)
+ * Returns effective second level domain part (eSLD) for the specified host
+ *
+ * @param {string} host hostname
+ * @return {string} eSLD part of the hostname or the full hostname if eSLD was not found
+ */
+LUA_FUNCTION_DEF(util, get_tld);
+
+/***
+ * @function util.glob(pattern)
+ * Returns results for the glob match for the specified pattern
+ *
+ * @param {string} pattern glob pattern to match ('?' and '*' are supported)
+ * @return {table/string} list of matched files
+ */
+LUA_FUNCTION_DEF(util, glob);
+
+/***
+ * @function util.parse_mail_address(str, [pool])
+ * Parses email address and returns a table of tables in the following format:
+ *
+ * - `raw` - the original value without any processing
+ * - `name` - name of internet address in UTF8, e.g. for `Vsevolod Stakhov <blah@foo.com>` it returns `Vsevolod Stakhov`
+ * - `addr` - address part of the address
+ * - `user` - user part (if present) of the address, e.g. `blah`
+ * - `domain` - domain part (if present), e.g. `foo.com`
+ * - `flags` - table with following keys set to true if given condition fulfilled:
+ * - [valid] - valid SMTP address in conformity with https://tools.ietf.org/html/rfc5321#section-4.1.
+ * - [ip] - domain is IPv4/IPv6 address
+ * - [braced] - angled `<blah@foo.com>` address
+ * - [quoted] - quoted user part
+ * - [empty] - empty address
+ * - [backslash] - user part contains backslash
+ * - [8bit] - contains 8bit characters
+ *
+ * @param {string} str input string
+ * @param {rspamd_mempool} pool memory pool to use
+ * @return {table/tables} parsed list of mail addresses
+ */
+LUA_FUNCTION_DEF(util, parse_mail_address);
+
+/***
+ * @function util.strlen_utf8(str)
+ * Returns length of string encoded in utf-8 in characters.
+ * If invalid characters are found, then this function returns number of bytes.
+ * @param {string} str utf8 encoded string
+ * @return {number} number of characters in string
+ */
+LUA_FUNCTION_DEF(util, strlen_utf8);
+
+/***
+ * @function util.lower_utf8(str)
+ * Converts utf8 string to lower case
+ * @param {string} str utf8 encoded string
+ * @return {string} lowercased utf8 string
+ */
+LUA_FUNCTION_DEF(util, lower_utf8);
+
+/***
+ * @function util.normalize_utf8(str)
+ * Gets a string in UTF8 and normalises it to NFKC_Casefold form
+ * @param {string} str utf8 encoded string
+ * @return {string,integer} lowercased utf8 string + result of the normalisation (use bit.band to check):
+ * RSPAMD_UNICODE_NORM_NORMAL = 0,
+ * RSPAMD_UNICODE_NORM_UNNORMAL = (1 << 0),
+ * RSPAMD_UNICODE_NORM_ZERO_SPACES = (1 << 1),
+ * RSPAMD_UNICODE_NORM_ERROR = (1 << 2),
+ * RSPAMD_UNICODE_NORM_OVERFLOW = (1 << 3)
+ */
+LUA_FUNCTION_DEF(util, normalize_utf8);
+
+
+/***
+ * @function util.transliterate(str)
+ * Converts utf8 encoded string to latin transliteration
+ * @param {string/text} str utf8 encoded string
+ * @return {text} transliterated string
+ */
+LUA_FUNCTION_DEF(util, transliterate);
+
+/***
+ * @function util.strequal_caseless(str1, str2)
+ * Compares two strings regardless of their case using ascii comparison.
+ * Returns `true` if `str1` is equal to `str2`
+ * @param {string} str1 utf8 encoded string
+ * @param {string} str2 utf8 encoded string
+ * @return {bool} result of comparison
+ */
+LUA_FUNCTION_DEF(util, strequal_caseless);
+
+
+/***
+ * @function util.strequal_caseless_utf8(str1, str2)
+ * Compares two utf8 strings regardless of their case using utf8 collation rules.
+ * Returns `true` if `str1` is equal to `str2`
+ * @param {string} str1 utf8 encoded string
+ * @param {string} str2 utf8 encoded string
+ * @return {bool} result of comparison
+ */
+LUA_FUNCTION_DEF(util, strequal_caseless_utf8);
+
+
+/***
+ * @function util.get_ticks()
+ * Returns current number of ticks as floating point number
+ * @return {number} number of current clock ticks (monotonically increasing)
+ */
+LUA_FUNCTION_DEF(util, get_ticks);
+
+/***
+ * @function util.get_time()
+ * Returns current time as unix time in floating point representation
+ * @return {number} number of seconds since 01.01.1970
+ */
+LUA_FUNCTION_DEF(util, get_time);
+
+/***
+ * @function util.time_to_string(seconds)
+ * Converts time from Unix time to HTTP date format
+ * @param {number} seconds unix timestamp
+ * @return {string} date as HTTP date
+ */
+LUA_FUNCTION_DEF(util, time_to_string);
+
+/***
+ * @function util.stat(fname)
+ * Performs stat(2) on a specified filepath and returns table of values
+ *
+ * - `size`: size of file in bytes
+ * - `type`: type of filepath: `regular`, `directory`, `special`
+ * - `mtime`: modification time as unix time
+ *
+ * @return {string,table} string is returned when error is occurred
+ * @example
+ *
+ * local err,st = util.stat('/etc/password')
+ *
+ * if err then
+ * -- handle error
+ * else
+ * print(st['size'])
+ * end
+ */
+LUA_FUNCTION_DEF(util, stat);
+
+/***
+ * @function util.unlink(fname)
+ * Removes the specified file from the filesystem
+ *
+ * @param {string} fname filename to remove
+ * @return {boolean,[string]} true if file has been deleted or false,'error string'
+ */
+LUA_FUNCTION_DEF(util, unlink);
+
+/***
+ * @function util.lock_file(fname, [fd])
+ * Lock the specified file. This function returns {number} which must be passed to `util.unlock_file` after usage
+ * or you'll have a resource leak
+ *
+ * @param {string} fname filename to lock
+ * @param {number} fd use the specified fd instead of opening one
+ * @return {number|nil,string} number if locking was successful or nil + error otherwise
+ */
+LUA_FUNCTION_DEF(util, lock_file);
+
+/***
+ * @function util.unlock_file(fd, [close_fd])
+ * Unlock the specified file closing the file descriptor associated.
+ *
+ * @param {number} fd descriptor to unlock
+ * @param {boolean} close_fd close descriptor on unlocking (default: TRUE)
+ * @return {boolean[,string]} true if a file was unlocked
+ */
+LUA_FUNCTION_DEF(util, unlock_file);
+
+/***
+ * @function util.create_file(fname, [mode])
+ * Creates the specified file with the default mode 0644
+ *
+ * @param {string} fname filename to create
+ * @param {number} mode open mode (you should use octal number here)
+ * @return {number|nil,string} file descriptor or pair nil + error string
+ */
+LUA_FUNCTION_DEF(util, create_file);
+
+/***
+ * @function util.close_file(fd)
+ * Closes descriptor fd
+ *
+ * @param {number} fd descriptor to close
+ * @return {boolean[,string]} true if a file was closed
+ */
+LUA_FUNCTION_DEF(util, close_file);
+
+/***
+ * @function util.random_hex(size)
+ * Returns random hex string of the specified size
+ *
+ * @param {number} len length of desired string in bytes
+ * @return {string} string with random hex digests
+ */
+LUA_FUNCTION_DEF(util, random_hex);
+
+/***
+ * @function util.zstd_compress(data, [level=1])
+ * Compresses input using zstd compression
+ *
+ * @param {string/rspamd_text} data input data
+ * @return {rspamd_text} compressed data
+ */
+LUA_FUNCTION_DEF(util, zstd_compress);
+
+/***
+ * @function util.zstd_decompress(data)
+ * Decompresses input using zstd algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @return {error,rspamd_text} pair of error + decompressed text
+ */
+LUA_FUNCTION_DEF(util, zstd_decompress);
+
+/***
+ * @function util.gzip_decompress(data, [size_limit])
+ * Decompresses input using gzip algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @param {integer} size_limit optional size limit
+ * @return {rspamd_text} decompressed text
+ */
+LUA_FUNCTION_DEF(util, gzip_decompress);
+
+/***
+ * @function util.inflate(data, [size_limit])
+ * Decompresses input using inflate algorithm
+ *
+ * @param {string/rspamd_text} data compressed data
+ * @param {integer} size_limit optional size limit
+ * @return {rspamd_text} decompressed text
+ */
+LUA_FUNCTION_DEF(util, inflate);
+
+/***
+ * @function util.gzip_compress(data, [level=1])
+ * Compresses input using gzip compression
+ *
+ * @param {string/rspamd_text} data input data
+ * @return {rspamd_text} compressed data
+ */
+LUA_FUNCTION_DEF(util, gzip_compress);
+
+/***
+ * @function util.normalize_prob(prob, [bias = 0.5])
+ * Normalize probabilities using polynom
+ *
+ * @param {number} prob probability param
+ * @param {number} bias number to subtract for making the final solution
+ * @return {number} normalized number
+ */
+LUA_FUNCTION_DEF(util, normalize_prob);
+/***
+ * @function util.is_utf_spoofed(str, [str2])
+ * Returns true if a string is spoofed (possibly with another string `str2`)
+ * @return {boolean} true if a string is spoofed
+ */
+LUA_FUNCTION_DEF(util, is_utf_spoofed);
+
+/**
+* @function util.is_utf_mixed_script(str)
+* Returns true if a string contains mixed unicode scripts
+* @param {string} String to check
+* @return {boolean} true if a string contains chars with mixed unicode script
+*/
+LUA_FUNCTION_DEF(util, is_utf_mixed_script);
+
+/**
+* @function util.is_utf_outside_range(str, range_start, range_end)
+* Returns true if a string contains chars outside range
+* @param {string} String to check
+* @param {number} start of character range similar to uset_addRange
+* @param {number} end of character range similar to uset_addRange
+* @return {boolean} true if a string contains chars outside selected utf range
+*/
+LUA_FUNCTION_DEF(util, is_utf_outside_range);
+
+/***
+* @function util.get_string_stats(str)
+* Returns table with number of letters and digits in string
+* @return {table} with string stats keys are "digits" and "letters"
+*/
+LUA_FUNCTION_DEF(util, get_string_stats);
+
+/***
+ * @function util.is_valid_utf8(str)
+ * Returns true if a string is valid UTF8 string
+ * @return {boolean} true if a string is spoofed
+ */
+LUA_FUNCTION_DEF(util, is_valid_utf8);
+
+/***
+ * @function util.has_obscured_unicode(str)
+ * Returns true if a string has obscure UTF symbols (zero width spaces, order marks), ignores invalid utf characters
+ * @return {boolean} true if a has obscured unicode characters (+ character and offset if found)
+ */
+LUA_FUNCTION_DEF(util, has_obscured_unicode);
+
+/***
+ * @function util.readline([prompt])
+ * Returns string read from stdin with history and editing support
+ * @return {string} string read from the input (with line endings stripped)
+ */
+LUA_FUNCTION_DEF(util, readline);
+
+/***
+ * @function util.readpassphrase([prompt])
+ * Returns string read from stdin disabling echo
+ * @return {string} string read from the input (with line endings stripped)
+ */
+LUA_FUNCTION_DEF(util, readpassphrase);
+
+/***
+ * @function util.file_exists(file)
+ * Checks if a specified file exists and is available for reading
+ * @return {boolean,string} true if file exists + string error if not
+ */
+LUA_FUNCTION_DEF(util, file_exists);
+
+/***
+ * @function util.mkdir(dir[, recursive])
+ * Creates a specified directory
+ * @return {boolean[,error]} true if directory has been created
+ */
+LUA_FUNCTION_DEF(util, mkdir);
+
+/***
+ * @function util.umask(mask)
+ * Sets new umask. Accepts either numeric octal string, e.g. '022' or a plain
+ * number, e.g. 0x12 (since Lua does not support octal integrals)
+ * @return {number} old umask
+ */
+LUA_FUNCTION_DEF(util, umask);
+
+/***
+ * @function util.isatty()
+ * Returns if stdout is a tty
+ * @return {boolean} true in case of output being tty
+ */
+LUA_FUNCTION_DEF(util, isatty);
+
+/***
+ * @function util.pack(fmt, ...)
+ *
+ * Backport of Lua 5.3 `string.pack` function:
+ * Returns a binary string containing the values v1, v2, etc. packed (that is,
+ * serialized in binary form) according to the format string `fmt`
+ * A format string is a sequence of conversion options. The conversion
+ * options are as follows:
+ *
+ * * <: sets little endian
+ * * >: sets big endian
+ * * =: sets native endian
+ * * ![n]: sets maximum alignment to n (default is native alignment)
+ * * b: a signed byte (char)
+ * * B: an unsigned byte (char)
+ * * h: a signed short (native size)
+ * * H: an unsigned short (native size)
+ * * l: a signed long (native size)
+ * * L: an unsigned long (native size)
+ * * j: a lua_Integer
+ * * J: a lua_Unsigned
+ * * T: a size_t (native size)
+ * * i[n]: a signed int with n bytes (default is native size)
+ * * I[n]: an unsigned int with n bytes (default is native size)
+ * * f: a float (native size)
+ * * d: a double (native size)
+ * * n: a lua_Number
+ * * cn: a fixed-sized string with n bytes
+ * * z: a zero-terminated string
+ * * s[n]: a string preceded by its length coded as an unsigned integer with
+ * * n bytes (default is a size_t)
+ * * x: one byte of padding
+ * * Xop: an empty item that aligns according to option op (which is otherwise ignored)
+ * * ' ': (empty space) ignored
+ *
+ * (A "[n]" means an optional integral numeral.) Except for padding, spaces,
+ * and configurations (options "xX <=>!"), each option corresponds to an
+ * argument (in string.pack) or a result (in string.unpack).
+ *
+ * For options "!n", "sn", "in", and "In", n can be any integer between 1 and
+ * All integral options check overflows; string.pack checks whether the given
+ * value fits in the given size; string.unpack checks whether the read value
+ * fits in a Lua integer.
+ *
+ * Any format string starts as if prefixed by "!1=", that is, with maximum
+ * alignment of 1 (no alignment) and native endianness.
+ *
+ * Alignment works as follows: For each option, the format gets extra padding
+ * until the data starts at an offset that is a multiple of the minimum
+ * between the option size and the maximum alignment; this minimum must be a
+ * power of 2. Options "c" and "z" are not aligned; option "s" follows the
+ * alignment of its starting integer.
+ *
+ * All padding is filled with zeros by string.pack (and ignored by unpack).
+ */
+LUA_FUNCTION_DEF(util, pack);
+
+/***
+ * @function util.packsize(fmt)
+ *
+ * Returns size of the packed binary string returned for the same `fmt` argument
+ * by @see util.pack
+ */
+LUA_FUNCTION_DEF(util, packsize);
+
+/***
+ * @function util.unpack(fmt, s [, pos])
+ * Unpacks string `s` according to the format string `fmt` as described in
+ * @see util.pack
+ *
+ * @returns {multiple} list of unpacked values according to `fmt`
+ */
+LUA_FUNCTION_DEF(util, unpack);
+
+/***
+ * @function util.caseless_hash(str[, seed])
+ * Calculates caseless non-crypto hash from a string or rspamd text
+ * @param str string or lua_text
+ * @param seed mandatory seed (0xdeadbabe by default)
+ * @return {int64} boxed int64_t
+ */
+LUA_FUNCTION_DEF(util, caseless_hash);
+
+/***
+ * @function util.caseless_hash_fast(str[, seed])
+ * Calculates caseless non-crypto hash from a string or rspamd text
+ * @param str string or lua_text
+ * @param seed mandatory seed (0xdeadbabe by default)
+ * @return {number} number from int64_t
+ */
+LUA_FUNCTION_DEF(util, caseless_hash_fast);
+
+/***
+ * @function util.get_hostname()
+ * Returns hostname for this machine
+ * @return {string} hostname
+ */
+LUA_FUNCTION_DEF(util, get_hostname);
+
+/***
+ * @function util.parse_content_type(ct_string, mempool)
+ * Parses content-type string to a table:
+ * - `type`
+ * - `subtype`
+ * - `charset`
+ * - `boundary`
+ * - other attributes
+ *
+ * @param {string} ct_string content type as string
+ * @param {rspamd_mempool} mempool needed to store temporary data (e.g. task pool)
+ * @return table or nil if cannot parse content type
+ */
+LUA_FUNCTION_DEF(util, parse_content_type);
+
+/***
+ * @function util.mime_header_encode(hdr)
+ * Encodes header if needed
+ * @param {string} hdr input header
+ * @return encoded header
+ */
+LUA_FUNCTION_DEF(util, mime_header_encode);
+
+/***
+ * @function util.btc_polymod(input_values)
+ * Performs bitcoin polymod function
+ * @param {table|numbers} input_values
+ * @return {boolean} true if polymod has been successful
+ */
+LUA_FUNCTION_DEF(util, btc_polymod);
+
+/***
+ * @function util.parse_smtp_date(str[, local_tz])
+ * Converts an SMTP date string to unix timestamp
+ * @param {string} str input string
+ * @param {boolean} local_tz convert to local tz if `true`
+ * @return {number} time as unix timestamp (converted to float)
+ */
+LUA_FUNCTION_DEF(util, parse_smtp_date);
+
+
+static const struct luaL_reg utillib_f[] = {
+ LUA_INTERFACE_DEF(util, create_event_base),
+ LUA_INTERFACE_DEF(util, load_rspamd_config),
+ LUA_INTERFACE_DEF(util, config_from_ucl),
+ LUA_INTERFACE_DEF(util, process_message),
+ LUA_INTERFACE_DEF(util, encode_base64),
+ LUA_INTERFACE_DEF(util, encode_qp),
+ LUA_INTERFACE_DEF(util, decode_qp),
+ LUA_INTERFACE_DEF(util, decode_base64),
+ LUA_INTERFACE_DEF(util, encode_base32),
+ LUA_INTERFACE_DEF(util, decode_base32),
+ LUA_INTERFACE_DEF(util, decode_url),
+ LUA_INTERFACE_DEF(util, tokenize_text),
+ LUA_INTERFACE_DEF(util, tanh),
+ LUA_INTERFACE_DEF(util, parse_html),
+ LUA_INTERFACE_DEF(util, levenshtein_distance),
+ LUA_INTERFACE_DEF(util, fold_header),
+ LUA_INTERFACE_DEF(util, is_uppercase),
+ LUA_INTERFACE_DEF(util, humanize_number),
+ LUA_INTERFACE_DEF(util, get_tld),
+ LUA_INTERFACE_DEF(util, glob),
+ {"parse_addr", lua_util_parse_mail_address},
+ LUA_INTERFACE_DEF(util, parse_mail_address),
+ LUA_INTERFACE_DEF(util, strlen_utf8),
+ LUA_INTERFACE_DEF(util, lower_utf8),
+ LUA_INTERFACE_DEF(util, normalize_utf8),
+ LUA_INTERFACE_DEF(util, transliterate),
+ LUA_INTERFACE_DEF(util, strequal_caseless),
+ LUA_INTERFACE_DEF(util, strequal_caseless_utf8),
+ LUA_INTERFACE_DEF(util, get_ticks),
+ LUA_INTERFACE_DEF(util, get_time),
+ LUA_INTERFACE_DEF(util, time_to_string),
+ LUA_INTERFACE_DEF(util, stat),
+ LUA_INTERFACE_DEF(util, unlink),
+ LUA_INTERFACE_DEF(util, lock_file),
+ LUA_INTERFACE_DEF(util, unlock_file),
+ LUA_INTERFACE_DEF(util, create_file),
+ LUA_INTERFACE_DEF(util, close_file),
+ LUA_INTERFACE_DEF(util, random_hex),
+ LUA_INTERFACE_DEF(util, zstd_compress),
+ LUA_INTERFACE_DEF(util, zstd_decompress),
+ LUA_INTERFACE_DEF(util, gzip_compress),
+ LUA_INTERFACE_DEF(util, gzip_decompress),
+ LUA_INTERFACE_DEF(util, inflate),
+ LUA_INTERFACE_DEF(util, normalize_prob),
+ LUA_INTERFACE_DEF(util, caseless_hash),
+ LUA_INTERFACE_DEF(util, caseless_hash_fast),
+ LUA_INTERFACE_DEF(util, is_utf_spoofed),
+ LUA_INTERFACE_DEF(util, is_utf_mixed_script),
+ LUA_INTERFACE_DEF(util, is_utf_outside_range),
+ LUA_INTERFACE_DEF(util, get_string_stats),
+ LUA_INTERFACE_DEF(util, is_valid_utf8),
+ LUA_INTERFACE_DEF(util, has_obscured_unicode),
+ LUA_INTERFACE_DEF(util, readline),
+ LUA_INTERFACE_DEF(util, readpassphrase),
+ LUA_INTERFACE_DEF(util, file_exists),
+ LUA_INTERFACE_DEF(util, mkdir),
+ LUA_INTERFACE_DEF(util, umask),
+ LUA_INTERFACE_DEF(util, isatty),
+ LUA_INTERFACE_DEF(util, get_hostname),
+ LUA_INTERFACE_DEF(util, parse_content_type),
+ LUA_INTERFACE_DEF(util, mime_header_encode),
+ LUA_INTERFACE_DEF(util, pack),
+ LUA_INTERFACE_DEF(util, unpack),
+ LUA_INTERFACE_DEF(util, packsize),
+ LUA_INTERFACE_DEF(util, btc_polymod),
+ LUA_INTERFACE_DEF(util, parse_smtp_date),
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(int64, tostring);
+LUA_FUNCTION_DEF(int64, fromstring);
+LUA_FUNCTION_DEF(int64, tonumber);
+LUA_FUNCTION_DEF(int64, hex);
+
+static const struct luaL_reg int64lib_f[] = {
+ LUA_INTERFACE_DEF(int64, fromstring),
+ {NULL, NULL}};
+static const struct luaL_reg int64lib_m[] = {
+ LUA_INTERFACE_DEF(int64, tostring),
+ LUA_INTERFACE_DEF(int64, tonumber),
+ LUA_INTERFACE_DEF(int64, hex),
+ {"__tostring", lua_int64_tostring},
+ {NULL, NULL}};
+
+LUA_FUNCTION_DEF(ev_base, loop);
+
+static const struct luaL_reg ev_baselib_m[] = {
+ LUA_INTERFACE_DEF(ev_base, loop),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static gint64
+lua_check_int64(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{int64}");
+ luaL_argcheck(L, ud != NULL, pos, "'int64' expected");
+ return ud ? *((gint64 *) ud) : 0LL;
+}
+
+
+static gint
+lua_util_create_event_base(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct ev_loop **pev_base;
+
+ pev_base = lua_newuserdata(L, sizeof(struct ev_loop *));
+ rspamd_lua_setclass(L, "rspamd{ev_base}", -1);
+ *pev_base = ev_loop_new(EVFLAG_SIGNALFD | EVBACKEND_ALL);
+
+ return 1;
+}
+
+static gint
+lua_util_load_rspamd_config(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg, **pcfg;
+ const gchar *cfg_name;
+
+ cfg_name = luaL_checkstring(L, 1);
+
+ if (cfg_name) {
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
+
+ if (rspamd_config_read(cfg, cfg_name, NULL, NULL, NULL, FALSE, NULL)) {
+ msg_err_config("cannot load config from %s", cfg_name);
+ lua_pushnil(L);
+ }
+ else {
+ rspamd_config_post_load(cfg, 0);
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ }
+ }
+
+ return 1;
+}
+
+static gint
+parse_config_options(const char *str_options)
+{
+ gint ret = 0;
+ gchar **vec;
+ const gchar *str;
+ guint i, l;
+
+ vec = g_strsplit_set(str_options, ",;", -1);
+ if (vec) {
+ l = g_strv_length(vec);
+ for (i = 0; i < l; i++) {
+ str = vec[i];
+
+ if (g_ascii_strcasecmp(str, "INIT_URL") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_URL;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_LIBS") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_LIBS;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_SYMCACHE") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_SYMCACHE;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_VALIDATE") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_VALIDATE;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_NO_TLD") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_NO_TLD;
+ }
+ else if (g_ascii_strcasecmp(str, "INIT_PRELOAD_MAPS") == 0) {
+ ret |= RSPAMD_CONFIG_INIT_PRELOAD_MAPS;
+ }
+ else {
+ msg_warn("bad type: %s", str);
+ }
+ }
+
+ g_strfreev(vec);
+ }
+
+ return ret;
+}
+
+static gint
+lua_util_config_from_ucl(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = NULL, **pcfg;
+ struct rspamd_rcl_sections_map *top;
+ GError *err = NULL;
+ ucl_object_t *obj;
+ const char *str_options = NULL;
+ gint int_options = 0;
+
+
+ obj = ucl_object_lua_import(L, 1);
+ if (lua_gettop(L) == 2) {
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ str_options = lua_tostring(L, 2);
+ int_options = parse_config_options(str_options);
+ }
+ else {
+ msg_err("config_from_ucl: second parameter is expected to be string");
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ }
+
+ if (obj) {
+ cfg = rspamd_config_new(RSPAMD_CONFIG_INIT_SKIP_LUA);
+ cfg->lua_state = L;
+
+ cfg->cfg_ucl_obj = obj;
+ top = rspamd_rcl_config_init(cfg, NULL);
+
+ if (!rspamd_rcl_parse(top, cfg, cfg, cfg->cfg_pool, cfg->cfg_ucl_obj, &err)) {
+ msg_err("rcl parse error: %s", err->message);
+ ucl_object_unref(obj);
+ lua_pushnil(L);
+ }
+ else {
+
+ if (int_options & RSPAMD_CONFIG_INIT_LIBS) {
+ cfg->libs_ctx = rspamd_init_libs();
+ }
+
+ rspamd_config_post_load(cfg, int_options);
+ pcfg = lua_newuserdata(L, sizeof(struct rspamd_config *));
+ rspamd_lua_setclass(L, "rspamd{config}", -1);
+ *pcfg = cfg;
+ }
+
+ rspamd_rcl_sections_free(top);
+ }
+
+ return 1;
+}
+
+static gboolean
+lua_util_task_fin(struct rspamd_task *task, void *ud)
+{
+ ucl_object_t **target = ud;
+
+ *target = rspamd_protocol_write_ucl(task, RSPAMD_PROTOCOL_DEFAULT);
+ rdns_resolver_release(task->resolver->r);
+
+ return TRUE;
+}
+
+static gint
+lua_util_process_message(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_config *cfg = lua_check_config(L, 1);
+ const gchar *message;
+ gsize mlen;
+ struct rspamd_task *task;
+ struct ev_loop *base;
+ ucl_object_t *res = NULL;
+
+ message = luaL_checklstring(L, 2, &mlen);
+
+ if (cfg != NULL && message != NULL) {
+ base = ev_loop_new(EVFLAG_SIGNALFD | EVBACKEND_ALL);
+ rspamd_init_filters(cfg, false, false);
+ task = rspamd_task_new(NULL, cfg, NULL, NULL, base, FALSE);
+ task->msg.begin = rspamd_mempool_alloc(task->task_pool, mlen);
+ rspamd_strlcpy((gpointer) task->msg.begin, message, mlen);
+ task->msg.len = mlen;
+ task->fin_callback = lua_util_task_fin;
+ task->fin_arg = &res;
+ task->resolver = rspamd_dns_resolver_init(NULL, base, cfg);
+ task->s = rspamd_session_create(task->task_pool, rspamd_task_fin,
+ NULL, (event_finalizer_t) rspamd_task_free, task);
+
+ if (!rspamd_task_load_message(task, NULL, message, mlen)) {
+ lua_pushnil(L);
+ }
+ else {
+ if (rspamd_task_process(task, RSPAMD_TASK_PROCESS_ALL)) {
+ ev_loop(base, 0);
+
+ if (res != NULL) {
+ ucl_object_push_lua(L, res, true);
+
+ ucl_object_unref(res);
+ }
+ else {
+ ucl_object_push_lua(L,
+ rspamd_protocol_write_ucl(task, RSPAMD_PROTOCOL_DEFAULT),
+ true);
+ rdns_resolver_release(task->resolver->r);
+ rspamd_session_destroy(task->s);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ ev_loop_destroy(base);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ gchar *out;
+ gsize outlen;
+ long str_lim = 0;
+ gboolean fold = FALSE;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (lua_gettop(L) > 1) {
+ str_lim = luaL_checkinteger(L, 2);
+ fold = str_lim > 0;
+ }
+
+ if (t == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+
+ if (fold) {
+ out = rspamd_encode_base64(t->start, t->len, str_lim, &outlen);
+ }
+ else {
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ out = rspamd_encode_base64_fold(t->start, t->len, str_lim, &outlen, how);
+ }
+
+ if (out != NULL) {
+ lua_new_text(L, out, outlen, TRUE);
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_qp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gchar *out;
+ gsize inlen, outlen;
+ guint str_lim = 0;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_gettop(L) > 1) {
+ str_lim = luaL_checknumber(L, 2);
+ }
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ enum rspamd_newlines_type how = RSPAMD_TASK_NEWLINES_CRLF;
+
+ if (lua_type(L, 3) == LUA_TSTRING) {
+ const gchar *how_str = lua_tostring(L, 3);
+
+ if (g_ascii_strcasecmp(how_str, "cr") == 0) {
+ how = RSPAMD_TASK_NEWLINES_CR;
+ }
+ else if (g_ascii_strcasecmp(how_str, "lf") == 0) {
+ how = RSPAMD_TASK_NEWLINES_LF;
+ }
+ else if (g_ascii_strcasecmp(how_str, "crlf") != 0) {
+ return luaL_error(L, "invalid newline style: %s", how_str);
+ }
+ }
+
+ out = rspamd_encode_qp_fold(s, inlen, str_lim, &outlen, how);
+
+ if (out != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = out;
+ t->len = outlen;
+ /* Need destruction */
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_qp(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t, *out;
+ const gchar *s = NULL;
+ gsize inlen = 0;
+ gssize outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (s == NULL) {
+ lua_pushnil(L);
+ }
+ else {
+ out = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ out->start = g_malloc(inlen + 1);
+ out->flags = RSPAMD_TEXT_FLAG_OWN;
+ outlen = rspamd_decode_qp_buf(s, inlen, (gchar *) out->start, inlen + 1);
+
+ if (outlen > 0) {
+ out->len = outlen;
+ }
+ else {
+ /*
+ * It removes out and frees memory on gc due to RSPAMD_TEXT_FLAG_OWN
+ */
+ lua_pop(L, 1);
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_base64(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gsize inlen = 0, outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (s != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->len = (inlen / 4) * 3 + 3;
+ t->start = g_malloc(t->len);
+
+ rspamd_cryptobox_base64_decode(s, inlen, (guchar *) t->start,
+ &outlen);
+ t->len = outlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_encode_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gchar *out;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+ gsize inlen, outlen;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (s == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+ else {
+ out = rspamd_encode_base32(s, inlen, btype);
+
+ if (out != NULL) {
+ t = lua_newuserdata(L, sizeof(*t));
+ outlen = strlen(out);
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = out;
+ t->len = outlen;
+ /* Need destruction */
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_base32(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ const gchar *s = NULL;
+ gsize inlen, outlen;
+ enum rspamd_base32_type btype = RSPAMD_BASE32_DEFAULT;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ s = luaL_checklstring(L, 1, &inlen);
+ }
+ else if (lua_type(L, 1) == LUA_TUSERDATA) {
+ t = lua_check_text(L, 1);
+
+ if (t != NULL) {
+ s = t->start;
+ inlen = t->len;
+ }
+ }
+
+ if (lua_type(L, 2) == LUA_TSTRING) {
+ btype = rspamd_base32_decode_type_from_str(lua_tostring(L, 2));
+
+ if (btype == RSPAMD_BASE32_INVALID) {
+ return luaL_error(L, "invalid b32 type: %s", lua_tostring(L, 2));
+ }
+ }
+
+ if (s != NULL) {
+ guchar *decoded;
+
+ decoded = rspamd_decode_base32(s, inlen, &outlen, btype);
+
+ if (decoded) {
+ t = lua_newuserdata(L, sizeof(*t));
+ rspamd_lua_setclass(L, "rspamd{text}", -1);
+ t->start = (const gchar *) decoded;
+ t->len = outlen;
+ t->flags = RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_decode_url(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t != NULL) {
+ struct rspamd_lua_text *out = lua_new_text(L, NULL, t->len, TRUE);
+
+ out->len = rspamd_url_decode((char *) out->start, t->start, t->len);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_tokenize_text(lua_State *L)
+{
+ return lua_parsers_tokenize_text(L);
+}
+
+static gint
+lua_util_tanh(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble in = luaL_checknumber(L, 1);
+
+ lua_pushnumber(L, tanh(in));
+
+ return 1;
+}
+
+static gint
+lua_util_parse_html(lua_State *L)
+{
+ return lua_parsers_parse_html(L);
+}
+
+static gint
+lua_util_levenshtein_distance(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint dist = 0;
+ guint replace_cost = 1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+ if (lua_isnumber(L, 3)) {
+ replace_cost = lua_tointeger(L, 3);
+ }
+
+ if (t1 && t2) {
+ dist = rspamd_strings_levenshtein_distance(t1->start, t1->len, t2->start, t2->len,
+ replace_cost);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushinteger(L, dist);
+
+ return 1;
+}
+
+static gint
+lua_util_fold_header(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *how, *stop_chars = NULL;
+ struct rspamd_lua_text *name, *value;
+ GString *folded;
+
+ name = lua_check_text_or_string(L, 1);
+ value = lua_check_text_or_string(L, 2);
+
+ if (name && value) {
+
+ if (lua_isstring(L, 3)) {
+
+ how = lua_tostring(L, 3);
+
+ if (lua_isstring(L, 4)) {
+ stop_chars = lua_tostring(L, 4);
+ }
+
+ if (strcmp(how, "cr") == 0) {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len,
+ 0,
+ RSPAMD_TASK_NEWLINES_CR, stop_chars);
+ }
+ else if (strcmp(how, "lf") == 0) {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_LF, stop_chars);
+ }
+ else {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_CRLF, stop_chars);
+ }
+ }
+ else {
+ folded = rspamd_header_value_fold(name->start, name->len,
+ value->start, value->len, 0,
+ RSPAMD_TASK_NEWLINES_CRLF, stop_chars);
+ }
+
+ if (folded) {
+ lua_pushlstring(L, folded->str, folded->len);
+ g_string_free(folded, TRUE);
+
+ return 1;
+ }
+ }
+
+ lua_pushnil(L);
+ return 1;
+}
+
+static gint
+lua_util_is_uppercase(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint32 i = 0;
+ UChar32 uc;
+ guint nlc = 0, nuc = 0;
+
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+ if (t) {
+ while (i >= 0 && i < t->len) {
+ U8_NEXT(t->start, i, t->len, uc);
+
+ if (uc < 0) {
+ break;
+ }
+
+ if (u_isupper(uc)) {
+ nuc++;
+ }
+ else if (u_islower(uc)) {
+ nlc++;
+ }
+ }
+ }
+
+ if (nuc > 0 && nlc == 0) {
+ lua_pushboolean(L, TRUE);
+ }
+ else {
+ lua_pushboolean(L, FALSE);
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_humanize_number(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint64 number = luaL_checkinteger(L, 1);
+ gchar numbuf[32];
+
+
+ rspamd_snprintf(numbuf, sizeof(numbuf), "%hL", number);
+ lua_pushstring(L, numbuf);
+
+ return 1;
+}
+
+static gint
+lua_util_get_tld(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *host;
+ gsize hostlen;
+ rspamd_ftok_t tld;
+
+ host = luaL_checklstring(L, 1, &hostlen);
+
+ if (host) {
+ if (!rspamd_url_find_tld(host, hostlen, &tld)) {
+ lua_pushlstring(L, host, hostlen);
+ }
+ else {
+ lua_pushlstring(L, tld.begin, tld.len);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_glob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *pattern;
+ glob_t gl;
+ gint top, i, flags = 0;
+
+ top = lua_gettop(L);
+ memset(&gl, 0, sizeof(gl));
+
+ for (i = 1; i <= top; i++, flags |= GLOB_APPEND) {
+ pattern = luaL_checkstring(L, i);
+
+ if (pattern) {
+ if (glob(pattern, flags, NULL, &gl) != 0) {
+ /* There is no way to return error here, so just create an table */
+ lua_createtable(L, 0, 0);
+ globfree(&gl);
+
+ return 1;
+ }
+ }
+ }
+
+ lua_createtable(L, gl.gl_pathc, 0);
+ /* Push results */
+ for (i = 0; i < (gint) gl.gl_pathc; i++) {
+ lua_pushstring(L, gl.gl_pathv[i]);
+ lua_rawseti(L, -2, i + 1);
+ }
+
+ globfree(&gl);
+
+ return 1;
+}
+
+static gint
+lua_util_parse_mail_address(lua_State *L)
+{
+ return lua_parsers_parse_mail_address(L);
+}
+
+static gint
+lua_util_strlen_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ gint32 i = 0, nchars = 0;
+ UChar32 uc;
+
+ while (i < t->len) {
+ U8_NEXT((guint8 *) t->start, i, t->len, uc);
+ nchars++;
+ }
+
+ lua_pushinteger(L, nchars);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_lower_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+
+ gchar *dst;
+ UChar32 uc;
+ UBool err = 0;
+ gint32 i = 0, j = 0;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ dst = g_malloc(t->len);
+
+ while (i < t->len && err == 0) {
+ U8_NEXT((guint8 *) t->start, i, t->len, uc);
+ uc = u_tolower(uc);
+ U8_APPEND(dst, j, t->len, uc, err);
+ }
+
+ if (lua_isstring(L, 1)) {
+ lua_pushlstring(L, dst, j);
+ g_free(dst);
+ }
+ else {
+ t = lua_new_text(L, dst, j, FALSE);
+ /* We have actually allocated text data before */
+ t->flags |= RSPAMD_TEXT_FLAG_OWN;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_normalize_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ bool is_text = lua_type(L, 1) == LUA_TUSERDATA;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ char *cpy = g_malloc(t->len + 1);
+ memcpy(cpy, t->start, t->len);
+ cpy[t->len] = '\0';
+ gsize len = t->len;
+ enum rspamd_utf8_normalise_result res = rspamd_normalise_unicode_inplace(cpy, &len);
+
+ if (is_text) {
+ struct rspamd_lua_text *out = lua_new_text(L, cpy, len, FALSE);
+ out->flags |= RSPAMD_TEXT_FLAG_OWN;
+ }
+ else {
+ lua_pushlstring(L, cpy, len);
+ g_free(cpy);
+ }
+
+ lua_pushinteger(L, res);
+
+ return 2;
+}
+
+static gint
+lua_util_transliterate(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t;
+ t = lua_check_text_or_string(L, 1);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ gsize outlen;
+ char *transliterated = rspamd_utf8_transliterate(t->start, t->len, &outlen);
+ lua_new_text(L, transliterated, outlen, TRUE);
+
+ return 1;
+}
+
+static gint
+lua_util_strequal_caseless(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint ret = -1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+
+ if (t1->len == t2->len) {
+ ret = rspamd_lc_cmp(t1->start, t2->start, t1->len);
+ }
+ else {
+ ret = t1->len - t2->len;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, (ret == 0) ? true : false);
+ return 1;
+}
+
+static gint
+lua_util_strequal_caseless_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t1, *t2;
+ gint ret = -1;
+
+ t1 = lua_check_text_or_string(L, 1);
+ t2 = lua_check_text_or_string(L, 2);
+
+ if (t1 && t2) {
+ ret = rspamd_utf8_strcmp_sizes(t1->start, t1->len, t2->start, t2->len);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, (ret == 0) ? true : false);
+
+ return 1;
+}
+
+static gint
+lua_util_get_ticks(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble ticks;
+ gboolean rdtsc = FALSE;
+
+ if (lua_isboolean(L, 1)) {
+ rdtsc = lua_toboolean(L, 1);
+ }
+
+ ticks = rspamd_get_ticks(rdtsc);
+ lua_pushnumber(L, ticks);
+
+ return 1;
+}
+
+static gint
+lua_util_get_time(lua_State *L)
+{
+ LUA_TRACE_POINT;
+
+ lua_pushnumber(L, ev_time());
+
+ return 1;
+}
+
+static gint
+lua_util_time_to_string(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble seconds;
+ char timebuf[128];
+
+ if (lua_isnumber(L, 1)) {
+ seconds = lua_tonumber(L, 1);
+ }
+ else {
+ seconds = ev_time();
+ }
+
+ rspamd_http_date_format(timebuf, sizeof(timebuf), seconds);
+ lua_pushstring(L, timebuf);
+
+ return 1;
+}
+
+static gint
+lua_util_stat(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ struct stat st;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (stat(fpath, &st) == -1) {
+ lua_pushstring(L, strerror(errno));
+ lua_pushnil(L);
+ }
+ else {
+ lua_pushnil(L);
+ lua_createtable(L, 0, 3);
+
+ lua_pushstring(L, "size");
+ lua_pushinteger(L, st.st_size);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "mtime");
+ lua_pushinteger(L, st.st_mtime);
+ lua_settable(L, -3);
+
+ lua_pushstring(L, "type");
+ if (S_ISREG(st.st_mode)) {
+ lua_pushstring(L, "regular");
+ }
+ else if (S_ISDIR(st.st_mode)) {
+ lua_pushstring(L, "directory");
+ }
+ else {
+ lua_pushstring(L, "special");
+ }
+ lua_settable(L, -3);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_util_unlink(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ gint ret;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ ret = unlink(fpath);
+
+ if (ret == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_lock_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fpath;
+ gint fd = -1;
+ gboolean own = FALSE;
+
+#if !HAVE_FLOCK
+ struct flock fl = {
+ .l_type = F_WRLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+#endif
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (lua_isnumber(L, 2)) {
+ fd = lua_tointeger(L, 2);
+ }
+ else {
+ fd = open(fpath, O_RDONLY);
+ own = TRUE;
+ }
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+#if HAVE_FLOCK
+ if (flock(fd, LOCK_EX) == -1) {
+#else
+ if (fcntl(fd, F_SETLKW, &fl) == -1) {
+#endif
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ if (own) {
+ close(fd);
+ }
+
+ return 2;
+ }
+
+ lua_pushinteger(L, fd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_unlock_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd = -1, ret, serrno;
+ gboolean do_close = TRUE;
+
+#if !HAVE_FLOCK
+ struct flock fl = {
+ .l_type = F_UNLCK,
+ .l_whence = SEEK_SET,
+ .l_start = 0,
+ .l_len = 0};
+#endif
+
+ if (lua_isnumber(L, 1)) {
+ fd = lua_tointeger(L, 1);
+
+ if (lua_isboolean(L, 2)) {
+ do_close = lua_toboolean(L, 2);
+ }
+
+#if HAVE_FLOCK
+ ret = flock(fd, LOCK_UN);
+#else
+ ret = fcntl(fd, F_SETLKW, &fl);
+#endif
+
+ if (do_close) {
+ serrno = errno;
+ close(fd);
+ errno = serrno;
+ }
+
+ if (ret == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_create_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd, mode = 00644;
+ const gchar *fpath;
+
+ fpath = luaL_checkstring(L, 1);
+
+ if (fpath) {
+ if (lua_isnumber(L, 2)) {
+ mode = lua_tointeger(L, 2);
+ }
+
+ fd = rspamd_file_xopen(fpath, O_RDWR | O_CREAT | O_TRUNC, mode, 0);
+
+ if (fd == -1) {
+ lua_pushnil(L);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushinteger(L, fd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_close_file(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint fd = -1;
+
+ if (lua_isnumber(L, 1)) {
+ fd = lua_tointeger(L, 1);
+
+ if (close(fd) == -1) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_random_hex(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar *buf;
+ gint buflen;
+
+ buflen = lua_tointeger(L, 1);
+
+ if (buflen <= 0) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ buf = g_malloc(buflen);
+ rspamd_random_hex(buf, buflen);
+ lua_pushlstring(L, buf, buflen);
+ g_free(buf);
+
+ return 1;
+}
+
+static gint
+lua_util_zstd_compress(lua_State *L)
+{
+ return lua_compress_zstd_compress(L);
+}
+
+static gint
+lua_util_zstd_decompress(lua_State *L)
+{
+ return lua_compress_zstd_decompress(L);
+}
+
+static gint
+lua_util_gzip_compress(lua_State *L)
+{
+ return lua_compress_zlib_compress(L);
+}
+
+static gint
+lua_util_gzip_decompress(lua_State *L)
+{
+ return lua_compress_zlib_decompress(L, true);
+}
+
+static gint
+lua_util_inflate(lua_State *L)
+{
+ return lua_compress_zlib_decompress(L, false);
+}
+
+static gint
+lua_util_normalize_prob(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gdouble x, bias = 0.5;
+
+ x = lua_tonumber(L, 1);
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ bias = lua_tonumber(L, 2);
+ }
+
+ lua_pushnumber(L, rspamd_normalize_probability(x, bias));
+
+ return 1;
+}
+
+static gint
+lua_util_caseless_hash(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint64 seed = 0xdeadbabe, h;
+ struct rspamd_lua_text *t = NULL;
+ gint64 *r;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ seed = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ seed = lua_check_int64(L, 2);
+ }
+
+ h = rspamd_icase_hash(t->start, t->len, seed);
+ r = lua_newuserdata(L, sizeof(*r));
+ *r = h;
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+
+ return 1;
+}
+
+static gint
+lua_util_caseless_hash_fast(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ guint64 seed = 0xdeadbabe, h;
+ struct rspamd_lua_text *t = NULL;
+ union {
+ guint64 i;
+ double d;
+ } u;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t == NULL || t->start == NULL) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ if (lua_type(L, 2) == LUA_TNUMBER) {
+ seed = lua_tointeger(L, 2);
+ }
+ else if (lua_type(L, 2) == LUA_TUSERDATA) {
+ seed = lua_check_int64(L, 2);
+ }
+
+ /*
+ * Here, we loose entropy from 64 bits to 52 bits roughly, however,
+ * it is still fine for practical applications
+ */
+
+ h = rspamd_icase_hash(t->start, t->len, seed);
+ u.i = G_GUINT64_CONSTANT(0x3FF) << 52 | h >> 12;
+ lua_pushnumber(L, u.d - 1.0);
+
+ return 1;
+}
+
+static gint
+lua_util_is_utf_spoofed(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize l1, l2;
+ gint ret, nres = 2;
+ const gchar *s1 = lua_tolstring(L, 1, &l1),
+ *s2 = lua_tolstring(L, 2, &l2);
+ static USpoofChecker *spc, *spc_sgl;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (s1 && s2) {
+ if (spc == NULL) {
+ spc = uspoof_open(&uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+ }
+
+ ret = uspoof_areConfusableUTF8(spc, s1, l1, s2, l2, &uc_err);
+ }
+ else if (s1) {
+ /* We have just s1, not s2 */
+ if (spc_sgl == NULL) {
+ spc_sgl = uspoof_open(&uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+
+ uspoof_setChecks(spc_sgl,
+ USPOOF_INVISIBLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_ANY_CASE,
+ &uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("Cannot set proper checks for uspoof: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(spc);
+ return 1;
+ }
+ }
+
+ ret = uspoof_checkUTF8(spc_sgl, s1, l1, NULL, &uc_err);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, !!(ret != 0));
+
+ switch (ret) {
+ case 0:
+ nres = 1;
+ break;
+ case USPOOF_SINGLE_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "single");
+ break;
+ case USPOOF_MIXED_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "multiple");
+ break;
+ case USPOOF_WHOLE_SCRIPT_CONFUSABLE:
+ lua_pushstring(L, "whole");
+ break;
+ default:
+ lua_pushstring(L, "unknown");
+ break;
+ }
+
+ return nres;
+}
+
+static gint
+lua_util_is_utf_mixed_script(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len_of_string;
+ const guchar *string_to_check = lua_tolstring(L, 1, &len_of_string);
+ UScriptCode last_script_code = USCRIPT_INVALID_CODE;
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (string_to_check) {
+ uint index = 0;
+ UChar32 char_to_check = 0;
+
+ while (index < len_of_string) {
+ U8_NEXT(string_to_check, index, len_of_string, char_to_check);
+
+ if (char_to_check < 0) {
+ return luaL_error(L, "passed string is not valid utf");
+ }
+
+ UScriptCode current_script_code = uscript_getScript(char_to_check, &uc_err);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot get unicode script for character, error: %s",
+ u_errorName(uc_err));
+ lua_pushboolean(L, false);
+
+ return 1;
+ }
+
+ if (current_script_code != USCRIPT_COMMON &&
+ current_script_code != USCRIPT_INHERITED) {
+
+ if (last_script_code == USCRIPT_INVALID_CODE) {
+ last_script_code = current_script_code;
+ }
+ else {
+ if (last_script_code != current_script_code) {
+ lua_pushboolean(L, true);
+
+ return 1;
+ }
+ }
+ }
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_util_get_string_stats(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint num_of_digits = 0, num_of_letters = 0;
+ struct rspamd_lua_text *t;
+
+ t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ const gchar *p = t->start, *end = t->start + t->len;
+ while (p < end) {
+ if (g_ascii_isdigit(*p)) {
+ num_of_digits++;
+ }
+ else if (g_ascii_isalpha(*p)) {
+ num_of_letters++;
+ }
+ p++;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "digits");
+ lua_pushinteger(L, num_of_digits);
+ lua_settable(L, -3);
+ lua_pushstring(L, "letters");
+ lua_pushinteger(L, num_of_letters);
+ lua_settable(L, -3);
+
+ return 1;
+}
+
+
+static gint
+lua_util_is_utf_outside_range(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint ret;
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+ guint32 range_start = lua_tointeger(L, 2);
+ guint32 range_end = lua_tointeger(L, 3);
+
+ static rspamd_lru_hash_t *validators;
+
+ if (validators == NULL) {
+ validators = rspamd_lru_hash_new_full(16, g_free, (GDestroyNotify) uspoof_close, g_int64_hash, g_int64_equal);
+ }
+
+ if (t) {
+ guint64 hash_key = (guint64) range_end << 32 || range_start;
+
+ USpoofChecker *validator = rspamd_lru_hash_lookup(validators, &hash_key, 0);
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (validator == NULL) {
+ USet *allowed_chars;
+ guint64 *creation_hash_key = g_malloc(sizeof(guint64));
+ *creation_hash_key = hash_key;
+
+ validator = uspoof_open(&uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("cannot init spoof checker: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ allowed_chars = uset_openEmpty();
+ uset_addRange(allowed_chars, range_start, range_end);
+ uspoof_setAllowedChars(validator, allowed_chars, &uc_err);
+
+ uspoof_setChecks(validator,
+ USPOOF_CHAR_LIMIT | USPOOF_ANY_CASE, &uc_err);
+
+ uset_close(allowed_chars);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err("Cannot configure uspoof: %s", u_errorName(uc_err));
+ lua_pushboolean(L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ rspamd_lru_hash_insert(validators, creation_hash_key, validator,
+ 0, 0);
+ }
+
+ gint32 pos = 0;
+ ret = uspoof_checkUTF8(validator, t->start, t->len, &pos,
+ &uc_err);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ lua_pushboolean(L, !!(ret != 0));
+
+ return 1;
+}
+
+
+static gint
+lua_util_get_hostname(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar *hostbuf;
+ gsize hostlen;
+
+ hostlen = sysconf(_SC_HOST_NAME_MAX);
+
+ if (hostlen <= 0) {
+ hostlen = 256;
+ }
+ else {
+ hostlen++;
+ }
+
+ hostbuf = g_alloca(hostlen);
+ memset(hostbuf, 0, hostlen);
+ gethostname(hostbuf, hostlen - 1);
+
+ lua_pushstring(L, hostbuf);
+
+ return 1;
+}
+
+static gint
+lua_util_parse_content_type(lua_State *L)
+{
+ return lua_parsers_parse_content_type(L);
+}
+
+
+static gint
+lua_util_mime_header_encode(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len;
+ const gchar *hdr = luaL_checklstring(L, 1, &len);
+ gchar *encoded;
+
+ if (!hdr) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ encoded = rspamd_mime_header_encode(hdr, len);
+ lua_pushstring(L, encoded);
+ g_free(encoded);
+
+ return 1;
+}
+
+static gint
+lua_util_is_valid_utf8(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ if (t) {
+ goffset error_offset = rspamd_fast_utf8_validate(t->start, t->len);
+
+ if (error_offset == 0) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ lua_pushinteger(L, error_offset);
+
+ return 2;
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_util_has_obscured_unicode(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gint32 i = 0, prev_i;
+ UChar32 uc;
+
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ while (i < t->len) {
+ prev_i = i;
+ U8_NEXT(t->start, i, t->len, uc);
+
+ if (uc > 0) {
+ if (IS_OBSCURED_CHAR(uc)) {
+ lua_pushboolean(L, true);
+ lua_pushinteger(L, uc); /* Character */
+ lua_pushinteger(L, prev_i); /* Offset */
+
+ return 3;
+ }
+ }
+ }
+
+ lua_pushboolean(L, false);
+
+ return 1;
+}
+
+static gint
+lua_util_readline(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *prompt = "";
+ gchar *input;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ prompt = lua_tostring(L, 1);
+ }
+#ifdef WITH_LUA_REPL
+ static Replxx *rx_instance = NULL;
+
+ if (rx_instance == NULL) {
+ rx_instance = replxx_init();
+ /* See https://github.com/AmokHuginnsson/replxx/issues/137 */
+ replxx_history_add(rx_instance, "");
+ }
+
+ input = (gchar *) replxx_input(rx_instance, prompt);
+
+ if (input) {
+ lua_pushstring(L, input);
+ }
+ else {
+ lua_pushnil(L);
+ }
+#else
+ size_t linecap = 0;
+ ssize_t linelen;
+
+ fprintf(stdout, "%s ", prompt);
+
+ linelen = getline(&input, &linecap, stdin);
+
+ if (linelen > 0) {
+ if (input[linelen - 1] == '\n') {
+ linelen--;
+ }
+
+ lua_pushlstring(L, input, linelen);
+ free(input);
+ }
+ else {
+ lua_pushnil(L);
+ }
+#endif
+
+ return 1;
+}
+
+static gint
+lua_util_readpassphrase(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar test_password[8192];
+ gsize r;
+
+ r = rspamd_read_passphrase(test_password, sizeof(test_password), 0, NULL);
+
+ if (r > 0) {
+ lua_pushlstring(L, test_password, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ /* In fact, we still pass it to Lua which is not very safe */
+ rspamd_explicit_memzero(test_password, sizeof(test_password));
+
+ return 1;
+}
+
+static gint
+lua_util_file_exists(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *fname = luaL_checkstring(L, 1);
+ gint serrno;
+
+ if (fname) {
+ if (access(fname, R_OK) == -1) {
+ serrno = errno;
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(serrno));
+ }
+ else {
+ lua_pushboolean(L, true);
+ lua_pushnil(L);
+ }
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 2;
+}
+
+static gint
+lua_util_mkdir(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *dname = luaL_checkstring(L, 1);
+ gboolean recursive = FALSE;
+ gint r = -1;
+
+ if (dname) {
+ if (lua_isboolean(L, 2)) {
+ recursive = lua_toboolean(L, 2);
+ }
+
+ if (recursive) {
+ char path[PATH_MAX];
+ gsize len, i;
+
+ len = rspamd_strlcpy(path, dname, sizeof(path));
+
+ /* Strip last / */
+ if (path[len - 1] == '/') {
+ path[len - 1] = '\0';
+ len--;
+ }
+
+ for (i = 1; i < len; i++) {
+ if (path[i] == '/') {
+ path[i] = '\0';
+
+ errno = 0;
+ r = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+
+ if (r == -1 && errno != EEXIST) {
+ break;
+ }
+
+ path[i] = '/';
+ }
+ }
+
+ /* Final path component */
+ r = mkdir(path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+ }
+ else {
+ r = mkdir(dname, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
+ }
+
+ if (r == -1 && errno != EEXIST) {
+ lua_pushboolean(L, false);
+ lua_pushstring(L, strerror(errno));
+
+ return 2;
+ }
+
+ lua_pushboolean(L, true);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_util_umask(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ mode_t mask = 0, old;
+
+ if (lua_type(L, 1) == LUA_TSTRING) {
+ const gchar *str = lua_tostring(L, 1);
+
+ if (str[0] == '0') {
+ /* e.g. '022' */
+ mask = strtol(str, NULL, 8);
+ }
+ else {
+ /* XXX: implement modestring parsing at some point */
+ return luaL_error(L, "invalid arguments");
+ }
+ }
+ else if (lua_type(L, 1) == LUA_TNUMBER) {
+ mask = lua_tointeger(L, 1);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ old = umask(mask);
+
+ lua_pushinteger(L, old);
+
+ return 1;
+}
+
+static gint
+lua_util_isatty(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ if (isatty(STDOUT_FILENO)) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+/* Backport from Lua 5.3 */
+
+/******************************************************************************
+* Copyright (C) 1994-2016 Lua.org, PUC-Rio.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+/*
+** {======================================================
+** PACK/UNPACK
+** =======================================================
+*/
+
+
+/* value used for padding */
+#if !defined(LUA_PACKPADBYTE)
+#define LUA_PACKPADBYTE 0x00
+#endif
+
+/* maximum size for the binary representation of an integer */
+#define MAXINTSIZE 16
+
+/* number of bits in a character */
+#define NB CHAR_BIT
+
+/* mask for one character (NB 1's) */
+#define MC ((1 << NB) - 1)
+
+/* size of a lua_Integer */
+#define SZINT ((int) sizeof(lua_Integer))
+
+#define MAX_SIZET ((size_t) (~(size_t) 0))
+
+#define MAXSIZE \
+ (sizeof(size_t) < sizeof(int) ? MAX_SIZET : (size_t) (INT_MAX))
+
+
+/* dummy union to get native endianness */
+static const union {
+ int dummy;
+ char little; /* true if machine is little endian */
+} nativeendian = {1};
+
+
+/* dummy structure to get native alignment requirements */
+struct cD {
+ char c;
+ union {
+ double d;
+ void *p;
+ lua_Integer i;
+ lua_Number n;
+ } u;
+};
+
+#define MAXALIGN (offsetof(struct cD, u))
+
+/*
+** Union for serializing floats
+*/
+typedef union Ftypes {
+ float f;
+ double d;
+ lua_Number n;
+ char buff[5 * sizeof(lua_Number)]; /* enough for any float type */
+} Ftypes;
+
+
+/*
+** information to pack/unpack stuff
+*/
+typedef struct Header {
+ lua_State *L;
+ int islittle;
+ int maxalign;
+} Header;
+
+/*
+** options for pack/unpack
+*/
+typedef enum KOption {
+ Kint, /* signed integers */
+ Kuint, /* unsigned integers */
+ Kfloat, /* floating-point numbers */
+ Kchar, /* fixed-length strings */
+ Kstring, /* strings with prefixed length */
+ Kzstr, /* zero-terminated strings */
+ Kpadding, /* padding */
+ Kpaddalign, /* padding for alignment */
+ Knop /* no-op (configuration or spaces) */
+} KOption;
+
+#if LUA_VERSION_NUM <= 502
+#define lua_Unsigned size_t
+#endif
+
+#if LUA_VERSION_NUM < 502
+
+#define lua_Unsigned size_t
+
+typedef struct luaL_Buffer_53 {
+ luaL_Buffer b; /* make incorrect code crash! */
+ char *ptr;
+ size_t nelems;
+ size_t capacity;
+ lua_State *L2;
+} luaL_Buffer_53;
+
+#define luaL_Buffer luaL_Buffer_53
+#define COMPAT53_PREFIX lua
+#undef COMPAT53_API
+
+#if defined(__GNUC__) || defined(__clang__)
+#define COMPAT53_API __attribute__((__unused__)) static
+#else
+#define COMPAT53_API static
+#endif
+
+#define COMPAT53_CONCAT_HELPER(a, b) a##b
+#define COMPAT53_CONCAT(a, b) COMPAT53_CONCAT_HELPER(a, b)
+
+#define luaL_buffinit COMPAT53_CONCAT(COMPAT53_PREFIX, _buffinit_53)
+COMPAT53_API void luaL_buffinit(lua_State *L, luaL_Buffer_53 *B);
+#define luaL_prepbuffsize COMPAT53_CONCAT(COMPAT53_PREFIX, _prepbufsize_53)
+COMPAT53_API char *luaL_prepbuffsize(luaL_Buffer_53 *B, size_t s);
+#define luaL_addlstring COMPAT53_CONCAT(COMPAT53_PREFIX, _addlstring_53)
+COMPAT53_API void luaL_addlstring(luaL_Buffer_53 *B, const char *s, size_t l);
+#define luaL_addvalue COMPAT53_CONCAT(COMPAT53_PREFIX, _addvalue_53)
+COMPAT53_API void luaL_addvalue(luaL_Buffer_53 *B);
+#define luaL_pushresult COMPAT53_CONCAT(COMPAT53_PREFIX, _pushresult_53)
+COMPAT53_API void luaL_pushresult(luaL_Buffer_53 *B);
+#undef luaL_buffinitsize
+#define luaL_buffinitsize(L, B, s) \
+ (luaL_buffinit(L, B), luaL_prepbuffsize(B, s))
+
+#undef luaL_prepbuffer
+#define luaL_prepbuffer(B) \
+ luaL_prepbuffsize(B, LUAL_BUFFERSIZE)
+
+#undef luaL_addchar
+#define luaL_addchar(B, c) \
+ ((void) ((B)->nelems < (B)->capacity || luaL_prepbuffsize(B, 1)), \
+ ((B)->ptr[(B)->nelems++] = (c)))
+
+#undef luaL_addsize
+#define luaL_addsize(B, s) \
+ ((B)->nelems += (s))
+
+#undef luaL_addstring
+#define luaL_addstring(B, s) \
+ luaL_addlstring(B, s, strlen(s))
+
+#undef luaL_pushresultsize
+#define luaL_pushresultsize(B, s) \
+ (luaL_addsize(B, s), luaL_pushresult(B))
+
+COMPAT53_API void
+luaL_buffinit(lua_State *L, luaL_Buffer_53 *B)
+{
+ /* make it crash if used via pointer to a 5.1-style luaL_Buffer */
+ B->b.p = NULL;
+ B->b.L = NULL;
+ B->b.lvl = 0;
+ /* reuse the buffer from the 5.1-style luaL_Buffer though! */
+ B->ptr = B->b.buffer;
+ B->nelems = 0;
+ B->capacity = LUAL_BUFFERSIZE;
+ B->L2 = L;
+}
+
+
+COMPAT53_API char *
+luaL_prepbuffsize(luaL_Buffer_53 *B, size_t s)
+{
+ if (B->capacity - B->nelems < s) { /* needs to grow */
+ char *newptr = NULL;
+ size_t newcap = B->capacity * 2;
+ if (newcap - B->nelems < s)
+ newcap = B->nelems + s;
+ if (newcap < B->capacity) /* overflow */
+ luaL_error(B->L2, "buffer too large");
+ newptr = (char *) lua_newuserdata(B->L2, newcap);
+ memcpy(newptr, B->ptr, B->nelems);
+ if (B->ptr != B->b.buffer) {
+ lua_replace(B->L2, -2); /* remove old buffer */
+ }
+ B->ptr = newptr;
+ B->capacity = newcap;
+ }
+ return B->ptr + B->nelems;
+}
+
+
+COMPAT53_API void
+luaL_addlstring(luaL_Buffer_53 *B, const char *s, size_t l)
+{
+ memcpy(luaL_prepbuffsize(B, l), s, l);
+ luaL_addsize(B, l);
+}
+
+
+COMPAT53_API void
+luaL_addvalue(luaL_Buffer_53 *B)
+{
+ size_t len = 0;
+ const char *s = lua_tolstring(B->L2, -1, &len);
+ if (!s)
+ luaL_error(B->L2, "cannot convert value to string");
+ if (B->ptr != B->b.buffer) {
+ lua_insert(B->L2, -2); /* userdata buffer must be at stack top */
+ }
+ luaL_addlstring(B, s, len);
+ lua_remove(B->L2, B->ptr != B->b.buffer ? -2 : -1);
+}
+
+
+COMPAT53_API void
+luaL_pushresult(luaL_Buffer_53 *B)
+{
+ lua_pushlstring(B->L2, B->ptr, B->nelems);
+ if (B->ptr != B->b.buffer) {
+ lua_replace(B->L2, -2); /* remove userdata buffer */
+ }
+}
+
+#endif
+
+/*
+** Read an integer numeral from string 'fmt' or return 'df' if
+** there is no numeral
+*/
+static int
+digit(int c)
+{
+ return '0' <= c && c <= '9';
+}
+
+static int
+getnum(const char **fmt, int df)
+{
+ if (!digit(**fmt)) /* no number? */
+ return df; /* return default value */
+ else {
+ int a = 0;
+ do {
+ a = a * 10 + (*((*fmt)++) - '0');
+ } while (digit(**fmt) && a <= ((int) MAXSIZE - 9) / 10);
+ return a;
+ }
+}
+
+
+/*
+** Read an integer numeral and raises an error if it is larger
+** than the maximum size for integers.
+*/
+static int
+getnumlimit(Header *h, const char **fmt, int df)
+{
+ int sz = getnum(fmt, df);
+ if (sz > MAXINTSIZE || sz <= 0)
+ luaL_error(h->L, "integral size (%d) out of limits [1,%d]",
+ sz, MAXINTSIZE);
+ return sz;
+}
+
+
+/*
+** Initialize Header
+*/
+static void
+initheader(lua_State *L, Header *h)
+{
+ h->L = L;
+ h->islittle = nativeendian.little;
+ h->maxalign = 1;
+}
+
+
+/*
+** Read and classify next option. 'size' is filled with option's size.
+*/
+static KOption
+getoption(Header *h, const char **fmt, int *size)
+{
+ int opt = *((*fmt)++);
+ *size = 0; /* default */
+ switch (opt) {
+ case 'b':
+ *size = sizeof(char);
+ return Kint;
+ case 'B':
+ *size = sizeof(char);
+ return Kuint;
+ case 'h':
+ *size = sizeof(short);
+ return Kint;
+ case 'H':
+ *size = sizeof(short);
+ return Kuint;
+ case 'l':
+ *size = sizeof(long);
+ return Kint;
+ case 'L':
+ *size = sizeof(long);
+ return Kuint;
+ case 'j':
+ *size = sizeof(lua_Integer);
+ return Kint;
+ case 'J':
+ *size = sizeof(lua_Integer);
+ return Kuint;
+ case 'T':
+ *size = sizeof(size_t);
+ return Kuint;
+ case 'f':
+ *size = sizeof(float);
+ return Kfloat;
+ case 'd':
+ *size = sizeof(double);
+ return Kfloat;
+ case 'n':
+ *size = sizeof(lua_Number);
+ return Kfloat;
+ case 'i':
+ *size = getnumlimit(h, fmt, sizeof(int));
+ return Kint;
+ case 'I':
+ *size = getnumlimit(h, fmt, sizeof(int));
+ return Kuint;
+ case 's':
+ *size = getnumlimit(h, fmt, sizeof(size_t));
+ return Kstring;
+ case 'c':
+ *size = getnum(fmt, -1);
+ if (*size == -1)
+ luaL_error(h->L, "missing size for format option 'c'");
+ return Kchar;
+ case 'z':
+ return Kzstr;
+ case 'x':
+ *size = 1;
+ return Kpadding;
+ case 'X':
+ return Kpaddalign;
+ case ' ':
+ break;
+ case '<':
+ h->islittle = 1;
+ break;
+ case '>':
+ h->islittle = 0;
+ break;
+ case '=':
+ h->islittle = nativeendian.little;
+ break;
+ case '!':
+ h->maxalign = getnumlimit(h, fmt, MAXALIGN);
+ break;
+ default:
+ luaL_error(h->L, "invalid format option '%c'", opt);
+ }
+ return Knop;
+}
+
+
+/*
+** Read, classify, and fill other details about the next option.
+** 'psize' is filled with option's size, 'notoalign' with its
+** alignment requirements.
+** Local variable 'size' gets the size to be aligned. (Kpadal option
+** always gets its full alignment, other options are limited by
+** the maximum alignment ('maxalign'). Kchar option needs no alignment
+** despite its size.
+*/
+static KOption
+getdetails(Header *h, size_t totalsize,
+ const char **fmt, int *psize, int *ntoalign)
+{
+ KOption opt = getoption(h, fmt, psize);
+ int align = *psize; /* usually, alignment follows size */
+ if (opt == Kpaddalign) { /* 'X' gets alignment from following option */
+ if (**fmt == '\0' || getoption(h, fmt, &align) == Kchar || align == 0)
+ luaL_argerror(h->L, 1, "invalid next option for option 'X'");
+ }
+ if (align <= 1 || opt == Kchar) /* need no alignment? */
+ *ntoalign = 0;
+ else {
+ if (align > h->maxalign) /* enforce maximum alignment */
+ align = h->maxalign;
+ if ((align & (align - 1)) != 0) /* is 'align' not a power of 2? */
+ luaL_argerror(h->L, 1, "format asks for alignment not power of 2");
+ *ntoalign = (align - (int) (totalsize & (align - 1))) & (align - 1);
+ }
+ return opt;
+}
+
+
+/*
+** Pack integer 'n' with 'size' bytes and 'islittle' endianness.
+** The final 'if' handles the case when 'size' is larger than
+** the size of a Lua integer, correcting the extra sign-extension
+** bytes if necessary (by default they would be zeros).
+*/
+static void
+packint(luaL_Buffer *b, lua_Unsigned n,
+ int islittle, int size, int neg)
+{
+ char *buff = luaL_prepbuffsize(b, size);
+ int i;
+ buff[islittle ? 0 : size - 1] = (char) (n & MC); /* first byte */
+ for (i = 1; i < size; i++) {
+ n >>= NB;
+ buff[islittle ? i : size - 1 - i] = (char) (n & MC);
+ }
+ if (neg && size > SZINT) { /* negative number need sign extension? */
+ for (i = SZINT; i < size; i++) /* correct extra bytes */
+ buff[islittle ? i : size - 1 - i] = (char) MC;
+ }
+ luaL_addsize(b, size); /* add result to buffer */
+}
+
+
+/*
+** Copy 'size' bytes from 'src' to 'dest', correcting endianness if
+** given 'islittle' is different from native endianness.
+*/
+static void
+copywithendian(volatile char *dest, volatile const char *src,
+ int size, int islittle)
+{
+ if (islittle == nativeendian.little) {
+ while (size-- != 0)
+ *(dest++) = *(src++);
+ }
+ else {
+ dest += size - 1;
+ while (size-- != 0)
+ *(dest--) = *(src++);
+ }
+}
+
+
+static int
+lua_util_pack(lua_State *L)
+{
+ luaL_Buffer b;
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1); /* format string */
+ int arg = 1; /* current argument to pack */
+ size_t totalsize = 0; /* accumulate total size of result */
+ initheader(L, &h);
+ lua_pushnil(L); /* mark to separate arguments from string buffer */
+ luaL_buffinit(L, &b);
+
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, totalsize, &fmt, &size, &ntoalign);
+ totalsize += ntoalign + size;
+ while (ntoalign-- > 0)
+ luaL_addchar(&b, LUA_PACKPADBYTE); /* fill alignment */
+ arg++;
+ switch (opt) {
+ case Kint: { /* signed integers */
+ lua_Integer n = luaL_checkinteger(L, arg);
+ if (size < SZINT) { /* need overflow check? */
+ lua_Integer lim = (lua_Integer) 1 << ((size * NB) - 1);
+ luaL_argcheck(L, -lim <= n && n < lim, arg, "integer overflow");
+ }
+ packint(&b, (lua_Unsigned) n, h.islittle, size, (n < 0));
+ break;
+ }
+ case Kuint: { /* unsigned integers */
+ lua_Integer n = luaL_checkinteger(L, arg);
+ if (size < SZINT) /* need overflow check? */
+ luaL_argcheck(L,
+ (lua_Unsigned) n < ((lua_Unsigned) 1 << (size * NB)),
+ arg,
+ "unsigned overflow");
+ packint(&b, (lua_Unsigned) n, h.islittle, size, 0);
+ break;
+ }
+ case Kfloat: { /* floating-point options */
+ volatile Ftypes u;
+ char *buff = luaL_prepbuffsize(&b, size);
+ lua_Number n = luaL_checknumber(L, arg); /* get argument */
+ if (size == sizeof(u.f))
+ u.f = (float) n; /* copy it into 'u' */
+ else if (size == sizeof(u.d))
+ u.d = (double) n;
+ else
+ u.n = n;
+ /* move 'u' to final result, correcting endianness if needed */
+ copywithendian(buff, u.buff, size, h.islittle);
+ luaL_addsize(&b, size);
+ break;
+ }
+ case Kchar: { /* fixed-size string */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ if ((size_t) size <=
+ len) /* string larger than (or equal to) needed? */
+ luaL_addlstring(&b,
+ s,
+ size); /* truncate string to asked size */
+ else { /* string smaller than needed */
+ luaL_addlstring(&b, s, len); /* add it all */
+ while (len++ < (size_t) size) /* pad extra space */
+ luaL_addchar(&b, LUA_PACKPADBYTE);
+ }
+ break;
+ }
+ case Kstring: { /* strings with length count */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ luaL_argcheck(L, size >= (int) sizeof(size_t) || len < ((size_t) 1 << (size * NB)),
+ arg, "string length does not fit in given size");
+ packint(&b,
+ (lua_Unsigned) len,
+ h.islittle,
+ size,
+ 0); /* pack length */
+ luaL_addlstring(&b, s, len);
+ totalsize += len;
+ break;
+ }
+ case Kzstr: { /* zero-terminated string */
+ size_t len;
+ const char *s = luaL_checklstring(L, arg, &len);
+ luaL_argcheck(L, strlen(s) == len, arg, "string contains zeros");
+ luaL_addlstring(&b, s, len);
+ luaL_addchar(&b, '\0'); /* add zero at the end */
+ totalsize += len + 1;
+ break;
+ }
+ case Kpadding:
+ luaL_addchar(&b, LUA_PACKPADBYTE); /* FALLTHROUGH */
+ case Kpaddalign:
+ case Knop:
+ arg--; /* undo increment */
+ break;
+ }
+ }
+ luaL_pushresult(&b);
+ return 1;
+}
+
+
+static int
+lua_util_packsize(lua_State *L)
+{
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1); /* format string */
+ size_t totalsize = 0; /* accumulate total size of result */
+ initheader(L, &h);
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, totalsize, &fmt, &size, &ntoalign);
+ size += ntoalign; /* total space used by option */
+ luaL_argcheck(L, totalsize <= MAXSIZE - size, 1,
+ "format result too large");
+ totalsize += size;
+ switch (opt) {
+ case Kstring: /* strings with length count */
+ case Kzstr: /* zero-terminated string */
+ luaL_argerror(L, 1, "variable-length format");
+ /* call never return, but to avoid warnings: */ /* FALLTHROUGH */
+ default:
+ break;
+ }
+ }
+ lua_pushinteger(L, (lua_Integer) totalsize);
+ return 1;
+}
+
+
+/*
+** Unpack an integer with 'size' bytes and 'islittle' endianness.
+** If size is smaller than the size of a Lua integer and integer
+** is signed, must do sign extension (propagating the sign to the
+** higher bits); if size is larger than the size of a Lua integer,
+** it must check the unread bytes to see whether they do not cause an
+** overflow.
+*/
+static lua_Integer
+unpackint(lua_State *L, const char *str,
+ int islittle, int size, int issigned)
+{
+ lua_Unsigned res = 0;
+ int i;
+ int limit = (size <= SZINT) ? size : SZINT;
+ for (i = limit - 1; i >= 0; i--) {
+ res <<= NB;
+ res |= (lua_Unsigned) (unsigned char) str[islittle ? i : size - 1 - i];
+ }
+ if (size < SZINT) { /* real size smaller than lua_Integer? */
+ if (issigned) { /* needs sign extension? */
+ lua_Unsigned mask = (lua_Unsigned) 1 << (size * NB - 1);
+ res = ((res ^ mask) - mask); /* do sign extension */
+ }
+ }
+ else if (size > SZINT) { /* must check unread bytes */
+ int mask = (!issigned || (lua_Integer) res >= 0) ? 0 : MC;
+ for (i = limit; i < size; i++) {
+ if ((unsigned char) str[islittle ? i : size - 1 - i] != mask)
+ luaL_error(L,
+ "%d-byte integer does not fit into Lua Integer",
+ size);
+ }
+ }
+ return (lua_Integer) res;
+}
+
+static lua_Integer
+posrelat(lua_Integer pos, size_t len)
+{
+ if (pos >= 0)
+ return pos;
+ else if (0u - (size_t) pos > len)
+ return 0;
+ else
+ return (lua_Integer) len + pos + 1;
+}
+
+static int
+lua_util_unpack(lua_State *L)
+{
+ Header h;
+ const char *fmt = luaL_checkstring(L, 1);
+ size_t ld;
+ const char *data;
+ int n = 0; /* number of results */
+
+ if (lua_type(L, 2) == LUA_TUSERDATA) {
+ struct rspamd_lua_text *t = lua_check_text(L, 2);
+
+ if (!t) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ data = t->start;
+ ld = t->len;
+ }
+ else {
+ data = luaL_checklstring(L, 2, &ld);
+ }
+
+ size_t pos = (size_t) posrelat(luaL_optinteger(L, 3, 1), ld) - 1;
+ luaL_argcheck(L, pos <= ld, 3, "initial position out of string");
+
+ initheader(L, &h);
+
+ while (*fmt != '\0') {
+ int size, ntoalign;
+ KOption opt = getdetails(&h, pos, &fmt, &size, &ntoalign);
+ if ((size_t) ntoalign + size > ~pos || pos + ntoalign + size > ld)
+ luaL_argerror(L, 2, "data string too short");
+ pos += ntoalign; /* skip alignment */
+ /* stack space for item + next position */
+ luaL_checkstack(L, 2, "too many results");
+ n++;
+ switch (opt) {
+ case Kint:
+ case Kuint: {
+ lua_Integer res = unpackint(L, data + pos, h.islittle, size,
+ (opt == Kint));
+ lua_pushinteger(L, res);
+ break;
+ }
+ case Kfloat: {
+ volatile Ftypes u;
+ lua_Number num;
+ copywithendian(u.buff, data + pos, size, h.islittle);
+ if (size == sizeof(u.f))
+ num = (lua_Number) u.f;
+ else if (size == sizeof(u.d))
+ num = (lua_Number) u.d;
+ else
+ num = u.n;
+ lua_pushnumber(L, num);
+ break;
+ }
+ case Kchar: {
+ lua_pushlstring(L, data + pos, size);
+ break;
+ }
+ case Kstring: {
+ size_t len = (size_t) unpackint(L,
+ data + pos,
+ h.islittle,
+ size,
+ 0);
+ luaL_argcheck(L,
+ pos + len + size <= ld,
+ 2,
+ "data string too short");
+ lua_pushlstring(L, data + pos + size, len);
+ pos += len; /* skip string */
+ break;
+ }
+ case Kzstr: {
+ size_t len = (int) strlen(data + pos);
+ lua_pushlstring(L, data + pos, len);
+ pos += len + 1; /* skip string plus final '\0' */
+ break;
+ }
+ case Kpaddalign:
+ case Kpadding:
+ case Knop:
+ n--; /* undo increment */
+ break;
+ }
+ pos += size;
+ }
+ lua_pushinteger(L, pos + 1); /* next position */
+ return n + 1;
+}
+
+static int
+lua_util_btc_polymod(lua_State *L)
+{
+ guint64 c = 1;
+
+ if (lua_type(L, 1) != LUA_TTABLE) {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ for (lua_pushnil(L); lua_next(L, 1); lua_pop(L, 1)) {
+ guint8 c0 = c >> 35;
+ guint64 d = lua_tointeger(L, -1);
+
+ c = ((c & 0x07ffffffff) << 5) ^ d;
+
+ if (c0 & 0x01) c ^= 0x98f2bc8e61;
+ if (c0 & 0x02) c ^= 0x79b76d99e2;
+ if (c0 & 0x04) c ^= 0xf33e5fb3c4;
+ if (c0 & 0x08) c ^= 0xae2eabe2a8;
+ if (c0 & 0x10) c ^= 0x1e4f43e470;
+ }
+
+ if ((c ^ 1) == 0) {
+ lua_pushboolean(L, true);
+ }
+ else {
+ lua_pushboolean(L, false);
+ }
+
+ return 1;
+}
+
+static int
+lua_util_parse_smtp_date(lua_State *L)
+{
+ return lua_parsers_parse_smtp_date(L);
+}
+
+
+static gint
+lua_load_util(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, utillib_f);
+
+ return 1;
+}
+
+static gint
+lua_load_int64(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, int64lib_f);
+
+ return 1;
+}
+
+
+void luaopen_util(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{ev_base}", ev_baselib_m);
+ lua_pop(L, 1);
+ rspamd_lua_new_class(L, "rspamd{int64}", int64lib_m);
+ lua_pop(L, 1);
+ rspamd_lua_add_preload(L, "rspamd_util", lua_load_util);
+ rspamd_lua_add_preload(L, "rspamd_int64", lua_load_int64);
+}
+
+static int
+lua_int64_tostring(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gchar buf[32];
+ bool is_signed = false;
+
+ if (lua_isboolean(L, 2)) {
+ is_signed = lua_toboolean(L, 2);
+ }
+
+ if (is_signed) {
+ rspamd_snprintf(buf, sizeof(buf), "%L", n);
+ }
+ else {
+ rspamd_snprintf(buf, sizeof(buf), "%uL", n);
+ }
+ lua_pushstring(L, buf);
+
+ return 1;
+}
+
+static int
+lua_int64_fromstring(lua_State *L)
+{
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, 1);
+
+ if (t && t->len > 0) {
+ guint64 u64;
+ const char *p = t->start;
+ gsize len = t->len;
+ bool neg = false;
+
+ /*
+ * We use complicated negation to allow both signed and unsinged values to
+ * fit into result.
+ * So we read int64 as unsigned and copy it to signed number.
+ * If we wanted u64 this allows to have the same memory representation of
+ * signed and unsigned.
+ * If we wanted signed i64 we still can use -1000500 and it will be parsed
+ * properly
+ */
+ if (*p == '-') {
+ neg = true;
+ p++;
+ len--;
+ }
+ if (!rspamd_strtou64(p, len, &u64)) {
+ lua_pushnil(L);
+ lua_pushstring(L, "invalid number");
+ return 2;
+ }
+
+ gint64 *i64_p = lua_newuserdata(L, sizeof(gint64));
+ rspamd_lua_setclass(L, "rspamd{int64}", -1);
+ memcpy(i64_p, &u64, sizeof(u64));
+
+ if (neg) {
+ *i64_p = -(*i64_p);
+ }
+ }
+ else {
+ }
+
+ return 1;
+}
+
+static int
+lua_int64_tonumber(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gdouble d;
+
+ d = n;
+ lua_pushinteger(L, d);
+
+ return 1;
+}
+
+static int
+lua_int64_hex(lua_State *L)
+{
+ gint64 n = lua_check_int64(L, 1);
+ gchar buf[32];
+
+ rspamd_snprintf(buf, sizeof(buf), "%XL", n);
+ lua_pushstring(L, buf);
+
+ return 1;
+}
+
+static int
+lua_ev_base_loop(lua_State *L)
+{
+ int flags = 0;
+ struct ev_loop *ev_base;
+
+ ev_base = lua_check_ev_base(L, 1);
+ if (lua_isnumber(L, 2)) {
+ flags = lua_tointeger(L, 2);
+ }
+
+ int ret = ev_run(ev_base, flags);
+ lua_pushinteger(L, ret);
+
+ return 1;
+}
diff --git a/src/lua/lua_worker.c b/src/lua/lua_worker.c
new file mode 100644
index 0000000..025b97b
--- /dev/null
+++ b/src/lua/lua_worker.c
@@ -0,0 +1,883 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lua_common.h"
+#include "unix-std.h"
+#include "worker_util.h"
+#include "rspamd_control.h"
+#include "ottery.h"
+
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#endif
+
+#include <sys/wait.h>
+#include <src/libserver/rspamd_control.h>
+
+/***
+ * @module rspamd_worker
+ * This module provides methods to access worker related functions in various
+ * places, such as periodic events or on_load events.
+ */
+
+
+LUA_FUNCTION_DEF(worker, get_name);
+LUA_FUNCTION_DEF(worker, get_stat);
+LUA_FUNCTION_DEF(worker, get_index);
+LUA_FUNCTION_DEF(worker, get_count);
+LUA_FUNCTION_DEF(worker, get_pid);
+LUA_FUNCTION_DEF(worker, is_scanner);
+LUA_FUNCTION_DEF(worker, is_primary_controller);
+LUA_FUNCTION_DEF(worker, spawn_process);
+LUA_FUNCTION_DEF(worker, get_mem_stats);
+LUA_FUNCTION_DEF(worker, add_control_handler);
+
+const luaL_reg worker_reg[] = {
+ LUA_INTERFACE_DEF(worker, get_name),
+ {"get_type", lua_worker_get_name},
+ LUA_INTERFACE_DEF(worker, get_stat),
+ LUA_INTERFACE_DEF(worker, get_index),
+ LUA_INTERFACE_DEF(worker, get_count),
+ LUA_INTERFACE_DEF(worker, get_pid),
+ LUA_INTERFACE_DEF(worker, spawn_process),
+ LUA_INTERFACE_DEF(worker, is_scanner),
+ LUA_INTERFACE_DEF(worker, is_primary_controller),
+ LUA_INTERFACE_DEF(worker, get_mem_stats),
+ LUA_INTERFACE_DEF(worker, add_control_handler),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+static struct rspamd_worker *
+lua_check_worker(lua_State *L, gint pos)
+{
+ void *ud = rspamd_lua_check_udata(L, pos, "rspamd{worker}");
+ luaL_argcheck(L, ud != NULL, pos, "'worker' expected");
+ return ud ? *((struct rspamd_worker **) ud) : NULL;
+}
+
+static gint
+lua_worker_get_stat(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ rspamd_mempool_stat_t mem_st;
+ struct rspamd_stat *stat, stat_copy;
+ ucl_object_t *top, *sub;
+ gint i;
+ guint64 spam = 0, ham = 0;
+
+ memset(&mem_st, 0, sizeof(mem_st));
+ rspamd_mempool_stat(&mem_st);
+ memcpy(&stat_copy, w->srv->stat, sizeof(stat_copy));
+ stat = &stat_copy;
+ top = ucl_object_typed_new(UCL_OBJECT);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_scanned), "scanned", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(stat->messages_learned), "learned", 0, false);
+ if (stat->messages_scanned > 0) {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ ucl_object_fromint(stat->actions_stat[i]),
+ rspamd_action_to_str(i), 0, false);
+ if (i < METRIC_ACTION_GREYLIST) {
+ spam += stat->actions_stat[i];
+ }
+ else {
+ ham += stat->actions_stat[i];
+ }
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+ else {
+ sub = ucl_object_typed_new(UCL_OBJECT);
+ for (i = METRIC_ACTION_REJECT; i <= METRIC_ACTION_NOACTION; i++) {
+ ucl_object_insert_key(sub,
+ 0,
+ rspamd_action_to_str(i), 0, false);
+ }
+ ucl_object_insert_key(top, sub, "actions", 0, false);
+ }
+ ucl_object_insert_key(top, ucl_object_fromint(spam), "spam_count", 0, false);
+ ucl_object_insert_key(top, ucl_object_fromint(ham), "ham_count", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->connections_count), "connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(stat->control_connections_count),
+ "control_connections", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_allocated), "pools_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.pools_freed), "pools_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.bytes_allocated), "bytes_allocated", 0,
+ false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.chunks_allocated),
+ "chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.shared_chunks_allocated),
+ "shared_chunks_allocated", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(mem_st.chunks_freed), "chunks_freed", 0, false);
+ ucl_object_insert_key(top,
+ ucl_object_fromint(
+ mem_st.oversized_chunks),
+ "chunks_oversized", 0, false);
+
+ ucl_object_push_lua(L, top, true);
+ ucl_object_unref(top);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_name(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushstring(L, g_quark_to_string(w->type));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_index(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->index);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_count(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->cf->count);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_get_pid(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushinteger(L, w->pid);
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+
+static gint
+lua_worker_is_scanner(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushboolean(L, rspamd_worker_is_scanner(w));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+static gint
+lua_worker_is_primary_controller(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+ lua_pushboolean(L, rspamd_worker_is_primary_controller(w));
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_control_cbdata {
+ lua_State *L;
+ rspamd_mempool_t *pool;
+ struct rspamd_worker *w;
+ struct rspamd_config *cfg;
+ struct ev_loop *event_loop;
+ struct rspamd_async_session *session;
+ enum rspamd_control_type cmd;
+ gint cbref;
+ gint fd;
+};
+
+static gboolean
+lua_worker_control_fin_session(void *ud)
+{
+ struct rspamd_control_reply rep;
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+ rspamd_mempool_t *pool;
+
+ pool = cbd->pool;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = cbd->cmd;
+
+ if (write(cbd->fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err_pool("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ return TRUE;
+}
+
+static void
+lua_worker_control_session_dtor(void *ud)
+{
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+
+ rspamd_mempool_delete(cbd->pool);
+}
+
+static gboolean
+lua_worker_control_handler(struct rspamd_main *rspamd_main,
+ struct rspamd_worker *worker,
+ gint fd,
+ gint attached_fd,
+ struct rspamd_control_command *cmd,
+ gpointer ud)
+{
+ struct rspamd_async_session *session, **psession;
+ struct rspamd_control_cbdata *cbd = (struct rspamd_control_cbdata *) ud;
+ rspamd_mempool_t *pool;
+ lua_State *L;
+ gint err_idx, status;
+
+ L = cbd->L;
+ pool = cbd->pool;
+ session = rspamd_session_create(cbd->pool,
+ lua_worker_control_fin_session,
+ NULL,
+ lua_worker_control_session_dtor,
+ cbd);
+ cbd->session = session;
+ cbd->fd = fd;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbd->cbref);
+ psession = lua_newuserdata(L, sizeof(*psession));
+ rspamd_lua_setclass(L, "rspamd{session}", -1);
+ *psession = session;
+
+ /* Command name */
+ lua_pushstring(L, rspamd_control_command_to_string(cmd->type));
+
+ /* Command's extras */
+ lua_newtable(L);
+
+ switch (cmd->type) {
+ case RSPAMD_CONTROL_CHILD_CHANGE:
+ lua_pushinteger(L, cmd->cmd.child_change.pid);
+ lua_setfield(L, -2, "pid");
+ switch (cmd->cmd.child_change.what) {
+ case rspamd_child_offline:
+ lua_pushstring(L, "offline");
+ lua_setfield(L, -2, "what");
+ break;
+ case rspamd_child_online:
+ lua_pushstring(L, "online");
+ lua_setfield(L, -2, "what");
+ break;
+ case rspamd_child_terminated:
+ lua_pushstring(L, "terminated");
+ lua_setfield(L, -2, "what");
+ status = cmd->cmd.child_change.additional;
+
+ if (WIFEXITED(status)) {
+ lua_pushinteger(L, WEXITSTATUS(status));
+ lua_setfield(L, -2, "exit_code");
+ }
+
+ if (WIFSIGNALED(status)) {
+ lua_pushinteger(L, WTERMSIG(status));
+ lua_setfield(L, -2, "signal");
+#ifdef WCOREDUMP
+ lua_pushboolean(L, WCOREDUMP(status));
+ lua_setfield(L, -2, "core");
+#endif
+ }
+ break;
+ }
+ break;
+ case RSPAMD_CONTROL_MONITORED_CHANGE:
+ lua_pushinteger(L, cmd->cmd.monitored_change.sender);
+ lua_setfield(L, -2, "sender");
+ lua_pushboolean(L, cmd->cmd.monitored_change.alive);
+ lua_setfield(L, -2, "alive");
+ lua_pushlstring(L, cmd->cmd.monitored_change.tag,
+ sizeof(cmd->cmd.monitored_change.tag));
+ lua_setfield(L, -2, "tag");
+ break;
+ case RSPAMD_CONTROL_HYPERSCAN_LOADED:
+ lua_pushstring(L, cmd->cmd.hs_loaded.cache_dir);
+ lua_setfield(L, -2, "cache_dir");
+ lua_pushboolean(L, cmd->cmd.hs_loaded.forced);
+ lua_setfield(L, -2, "forced");
+ break;
+ case RSPAMD_CONTROL_STAT:
+ case RSPAMD_CONTROL_RELOAD:
+ case RSPAMD_CONTROL_RERESOLVE:
+ case RSPAMD_CONTROL_RECOMPILE:
+ case RSPAMD_CONTROL_LOG_PIPE:
+ case RSPAMD_CONTROL_FUZZY_STAT:
+ case RSPAMD_CONTROL_FUZZY_SYNC:
+ default:
+ break;
+ }
+
+ if (lua_pcall(L, 3, 0, err_idx) != 0) {
+ msg_err_pool("cannot init lua parser script: %s", lua_tostring(L, -1));
+ lua_settop(L, err_idx - 1);
+
+ struct rspamd_control_reply rep;
+
+ memset(&rep, 0, sizeof(rep));
+ rep.type = cbd->cmd;
+ rep.reply.monitored_change.status = -1;
+
+ if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) {
+ msg_err_pool("cannot write reply to the control socket: %s",
+ strerror(errno));
+ }
+
+ rspamd_session_destroy(session);
+ }
+ else {
+ lua_settop(L, err_idx - 1);
+ rspamd_session_pending(session);
+ }
+
+ return TRUE;
+}
+
+static gint
+lua_worker_add_control_handler(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+ struct rspamd_config *cfg = lua_check_config(L, 2);
+ struct ev_loop *event_loop = lua_check_ev_base(L, 3);
+ const gchar *cmd_name = luaL_checkstring(L, 4);
+ enum rspamd_control_type cmd;
+ struct rspamd_control_cbdata *cbd;
+
+ if (w && cfg && event_loop && cmd_name && lua_isfunction(L, 5)) {
+ cmd = rspamd_control_command_from_string(cmd_name);
+
+ if (cmd == RSPAMD_CONTROL_MAX) {
+ return luaL_error(L, "invalid command type: %s", cmd_name);
+ }
+
+ rspamd_mempool_t *pool = rspamd_mempool_new(
+ rspamd_mempool_suggest_size(), "lua_control", 0);
+ cbd = rspamd_mempool_alloc0(pool, sizeof(*cbd));
+ cbd->pool = pool;
+ cbd->event_loop = event_loop;
+ cbd->w = w;
+ cbd->cfg = cfg;
+ cbd->cmd = cmd;
+ cbd->L = L;
+ /* Refcount callback */
+ lua_pushvalue(L, 5);
+ cbd->cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+ rspamd_control_worker_add_cmd_handler(w, cmd, lua_worker_control_handler,
+ cbd);
+ }
+ else {
+ return luaL_error(L, "invalid arguments, need worker, cfg, "
+ "ev_loop, cmd_name and callback function");
+ }
+
+ return 0;
+}
+
+#ifdef WITH_JEMALLOC
+static void
+lua_worker_jemalloc_stats_cb(void *ud, const char *msg)
+{
+ lua_State *L = (lua_State *) ud;
+
+ lua_pushstring(L, msg);
+}
+#endif
+
+static gint
+lua_worker_get_mem_stats(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+
+ if (w) {
+#ifdef WITH_JEMALLOC
+ malloc_stats_print(lua_worker_jemalloc_stats_cb, (void *) L, NULL);
+#else
+ lua_pushstring(L, "no stats, jemalloc support is required");
+#endif
+ }
+ else {
+ return luaL_error(L, "invalid arguments");
+ }
+
+ return 1;
+}
+
+struct rspamd_lua_process_cbdata {
+ gint sp[2];
+ gint func_cbref;
+ gint cb_cbref;
+ gboolean replied;
+ gboolean is_error;
+ pid_t cpid;
+ lua_State *L;
+ guint64 sz;
+ GString *io_buf;
+ GString *out_buf;
+ goffset out_pos;
+ struct rspamd_worker *wrk;
+ struct ev_loop *event_loop;
+ ev_io ev;
+};
+
+static void
+rspamd_lua_execute_lua_subprocess(lua_State *L,
+ struct rspamd_lua_process_cbdata *cbdata)
+{
+ gint err_idx, r;
+ guint64 wlen = 0;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+
+ if (lua_pcall(L, 0, 1, err_idx) != 0) {
+ const gchar *s = lua_tostring(L, -1);
+ gsize slen = strlen(s);
+
+ msg_err("call to subprocess failed: %s", s);
+ /* Indicate error */
+ wlen = (1ULL << 63u) + slen;
+
+ r = write(cbdata->sp[1], &wlen, sizeof(wlen));
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+
+ r = write(cbdata->sp[1], s, slen);
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+ }
+ else {
+ struct rspamd_lua_text *t = lua_check_text_or_string(L, -1);
+
+ if (t) {
+ wlen = t->len;
+ r = write(cbdata->sp[1], &wlen, sizeof(wlen));
+
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+
+ r = write(cbdata->sp[1], t->start, wlen);
+
+ if (r == -1) {
+ msg_err("write failed: %s", strerror(errno));
+ }
+ }
+ else {
+ msg_err("subprocess: invalid return value: %s",
+ lua_typename(L, lua_type(L, -1)));
+ }
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+static void
+rspamd_lua_call_on_complete(lua_State *L,
+ struct rspamd_lua_process_cbdata *cbdata,
+ const gchar *err_msg,
+ const gchar *data, gsize datalen)
+{
+ gint err_idx;
+
+ lua_pushcfunction(L, &rspamd_lua_traceback);
+ err_idx = lua_gettop(L);
+
+ lua_rawgeti(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+
+ if (err_msg) {
+ lua_pushstring(L, err_msg);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (data) {
+ lua_pushlstring(L, data, datalen);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ if (lua_pcall(L, 2, 0, err_idx) != 0) {
+ msg_err("call to on_complete script failed: %s",
+ lua_tostring(L, -1));
+ }
+
+ lua_settop(L, err_idx - 1);
+}
+
+static gboolean
+rspamd_lua_cld_handler(struct rspamd_worker_signal_handler *sigh, void *ud)
+{
+ struct rspamd_lua_process_cbdata *cbdata = ud;
+ struct rspamd_srv_command srv_cmd;
+ lua_State *L;
+ pid_t died;
+ gint res = 0;
+
+ /* Are we called by a correct children ? */
+ died = waitpid(cbdata->cpid, &res, WNOHANG);
+
+ if (died <= 0) {
+ /* Wait more */
+ return TRUE;
+ }
+
+ L = cbdata->L;
+ msg_info("handled SIGCHLD from %P", cbdata->cpid);
+
+ if (!cbdata->replied) {
+ /* We still need to call on_complete callback */
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Worker has died without reply", NULL, 0);
+ }
+
+ /* Free structures */
+ close(cbdata->sp[0]);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_string_free(cbdata->io_buf, TRUE);
+
+ if (cbdata->out_buf) {
+ g_string_free(cbdata->out_buf, TRUE);
+ }
+
+ /* Notify main */
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_ON_FORK;
+ srv_cmd.cmd.on_fork.state = child_dead;
+ srv_cmd.cmd.on_fork.cpid = cbdata->cpid;
+ srv_cmd.cmd.on_fork.ppid = getpid();
+ rspamd_srv_send_command(cbdata->wrk, cbdata->event_loop, &srv_cmd, -1,
+ NULL, NULL);
+ g_free(cbdata);
+
+ /* We are done with this SIGCHLD */
+ return FALSE;
+}
+
+static void
+rspamd_lua_subprocess_io(EV_P_ ev_io *w, int revents)
+{
+ struct rspamd_lua_process_cbdata *cbdata =
+ (struct rspamd_lua_process_cbdata *) w->data;
+ gssize r;
+
+ if (cbdata->sz == (guint64) -1) {
+ guint64 sz;
+
+ /* We read size of reply + flags first */
+ r = read(cbdata->sp[0], cbdata->io_buf->str + cbdata->io_buf->len,
+ sizeof(guint64) - cbdata->io_buf->len);
+
+ if (r == 0) {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Unexpected EOF", NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ else if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ return;
+ }
+ else {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ strerror(errno), NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ }
+
+ cbdata->io_buf->len += r;
+
+ if (cbdata->io_buf->len == sizeof(guint64)) {
+ memcpy((guchar *) &sz, cbdata->io_buf->str, sizeof(sz));
+
+ if (sz & (1ULL << 63)) {
+ cbdata->is_error = TRUE;
+ sz &= ~(1ULL << 63);
+ }
+
+ cbdata->io_buf->len = 0;
+ cbdata->sz = sz;
+ g_string_set_size(cbdata->io_buf, sz + 1);
+ cbdata->io_buf->len = 0;
+ }
+ }
+ else {
+ /* Read data */
+ r = read(cbdata->sp[0], cbdata->io_buf->str + cbdata->io_buf->len,
+ cbdata->sz - cbdata->io_buf->len);
+
+ if (r == 0) {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ "Unexpected EOF", NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ else if (r == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ return;
+ }
+ else {
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ strerror(errno), NULL, 0);
+ cbdata->replied = TRUE;
+ kill(cbdata->cpid, SIGTERM);
+
+ return;
+ }
+ }
+
+ cbdata->io_buf->len += r;
+
+ if (cbdata->io_buf->len == cbdata->sz) {
+ gchar rep[4];
+
+ ev_io_stop(cbdata->event_loop, &cbdata->ev);
+ /* Finished reading data */
+ if (cbdata->is_error) {
+ cbdata->io_buf->str[cbdata->io_buf->len] = '\0';
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ cbdata->io_buf->str, NULL, 0);
+ }
+ else {
+ rspamd_lua_call_on_complete(cbdata->L, cbdata,
+ NULL, cbdata->io_buf->str, cbdata->io_buf->len);
+ }
+
+ cbdata->replied = TRUE;
+
+ /* Write reply to the child */
+ rspamd_socket_blocking(cbdata->sp[0]);
+ memset(rep, 0, sizeof(rep));
+ (void) !write(cbdata->sp[0], rep, sizeof(rep));
+ }
+ }
+}
+
+static gint
+lua_worker_spawn_process(lua_State *L)
+{
+ struct rspamd_worker *w = lua_check_worker(L, 1);
+ struct rspamd_lua_process_cbdata *cbdata;
+ struct rspamd_abstract_worker_ctx *actx;
+ struct rspamd_srv_command srv_cmd;
+ const gchar *cmdline = NULL, *input = NULL, *proctitle = NULL;
+ gsize inputlen = 0;
+ pid_t pid;
+ GError *err = NULL;
+ gint func_cbref, cb_cbref;
+
+ if (!rspamd_lua_parse_table_arguments(L, 2, &err,
+ RSPAMD_LUA_PARSE_ARGUMENTS_DEFAULT,
+ "func=F;exec=S;stdin=V;*on_complete=F;proctitle=S", &func_cbref,
+ &cmdline, &inputlen, &input, &cb_cbref, &proctitle)) {
+ msg_err("cannot get parameters list: %e", err);
+
+ if (err) {
+ g_error_free(err);
+ }
+
+ return 0;
+ }
+
+ cbdata = g_malloc0(sizeof(*cbdata));
+ cbdata->cb_cbref = cb_cbref;
+ cbdata->func_cbref = func_cbref;
+
+ if (input) {
+ cbdata->out_buf = g_string_new_len(input, inputlen);
+ cbdata->out_pos = 0;
+ }
+
+ if (rspamd_socketpair(cbdata->sp, SOCK_STREAM) == -1) {
+ msg_err("cannot spawn socketpair: %s", strerror(errno));
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_free(cbdata);
+
+ return 0;
+ }
+
+ actx = w->ctx;
+ cbdata->wrk = w;
+ cbdata->L = L;
+ cbdata->event_loop = actx->event_loop;
+ cbdata->sz = (guint64) -1;
+
+ pid = fork();
+
+ if (pid == -1) {
+ msg_err("cannot spawn process: %s", strerror(errno));
+ close(cbdata->sp[0]);
+ close(cbdata->sp[1]);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->func_cbref);
+ luaL_unref(L, LUA_REGISTRYINDEX, cbdata->cb_cbref);
+ g_free(cbdata);
+
+ return 0;
+ }
+ else if (pid == 0) {
+ /* Child */
+ gint rc;
+ gchar inbuf[4];
+
+ rspamd_log_on_fork(w->cf->type, w->srv->cfg, w->srv->logger);
+ rc = ottery_init(w->srv->cfg->libs_ctx->ottery_cfg);
+
+ if (rc != OTTERY_ERR_NONE) {
+ msg_err("cannot initialize PRNG: %d", rc);
+ abort();
+ }
+ rspamd_random_seed_fast();
+#ifdef HAVE_EVUTIL_RNG_INIT
+ evutil_secure_rng_init();
+#endif
+
+ close(cbdata->sp[0]);
+ /* Here we assume that we can block on writing results */
+ rspamd_socket_blocking(cbdata->sp[1]);
+ g_hash_table_remove_all(w->signal_events);
+ ev_loop_destroy(cbdata->event_loop);
+
+ if (proctitle) {
+ rspamd_setproctitle("lua process: %s", proctitle);
+ }
+ else {
+ rspamd_setproctitle("lua process: unnamed");
+ }
+
+ cbdata->event_loop = ev_loop_new(EVFLAG_SIGNALFD);
+ rspamd_worker_unblock_signals();
+ rspamd_lua_execute_lua_subprocess(L, cbdata);
+
+ /* Wait for parent to reply and exit */
+ rc = read(cbdata->sp[1], inbuf, sizeof(inbuf));
+
+ if (rc >= sizeof(inbuf) &&
+ memcmp(inbuf, "\0\0\0\0", sizeof(inbuf)) == 0) {
+ exit(EXIT_SUCCESS);
+ }
+ else {
+ msg_err("got invalid reply from parent");
+
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ cbdata->cpid = pid;
+ cbdata->io_buf = g_string_sized_new(8);
+ /* Notify main */
+ memset(&srv_cmd, 0, sizeof(srv_cmd));
+ srv_cmd.type = RSPAMD_SRV_ON_FORK;
+ srv_cmd.cmd.on_fork.state = child_create;
+ srv_cmd.cmd.on_fork.cpid = pid;
+ srv_cmd.cmd.on_fork.ppid = getpid();
+ rspamd_srv_send_command(w, cbdata->event_loop, &srv_cmd, -1, NULL, NULL);
+
+ close(cbdata->sp[1]);
+ rspamd_socket_nonblocking(cbdata->sp[0]);
+ /* Parent */
+ rspamd_worker_set_signal_handler(SIGCHLD, w, cbdata->event_loop,
+ rspamd_lua_cld_handler,
+ cbdata);
+
+ /* Add result pipe waiting */
+ ev_io_init(&cbdata->ev, rspamd_lua_subprocess_io, cbdata->sp[0], EV_READ);
+ cbdata->ev.data = cbdata;
+ ev_io_start(cbdata->event_loop, &cbdata->ev);
+
+ return 0;
+}
+
+void luaopen_worker(lua_State *L)
+{
+ rspamd_lua_new_class(L, "rspamd{worker}", worker_reg);
+}
diff --git a/src/lua/lua_xmlrpc.c b/src/lua/lua_xmlrpc.c
new file mode 100644
index 0000000..efb2b22
--- /dev/null
+++ b/src/lua/lua_xmlrpc.c
@@ -0,0 +1,796 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "lua_common.h"
+
+
+LUA_FUNCTION_DEF(xmlrpc, parse_reply);
+LUA_FUNCTION_DEF(xmlrpc, make_request);
+
+static const struct luaL_reg xmlrpclib_m[] = {
+ LUA_INTERFACE_DEF(xmlrpc, parse_reply),
+ LUA_INTERFACE_DEF(xmlrpc, make_request),
+ {"__tostring", rspamd_lua_class_tostring},
+ {NULL, NULL}};
+
+#define msg_debug_xmlrpc(...) rspamd_conditional_debug_fast(NULL, NULL, \
+ rspamd_xmlrpc_log_id, "xmlrpc", "", \
+ RSPAMD_LOG_FUNC, \
+ __VA_ARGS__)
+
+INIT_LOG_MODULE(xmlrpc)
+
+enum lua_xmlrpc_state {
+ read_method_response = 0,
+ read_params = 1,
+ read_param = 2,
+ read_param_value = 3,
+ read_param_element = 4,
+ read_struct = 5,
+ read_struct_member_name = 6,
+ read_struct_member_value = 7,
+ read_struct_element = 8,
+ read_string = 9,
+ read_int = 10,
+ read_double = 11,
+ read_array = 12,
+ read_array_value = 13,
+ read_array_element = 14,
+ error_state = 99,
+ success_state = 100,
+};
+
+enum lua_xmlrpc_stack {
+ st_array = 1,
+ st_struct = 2,
+};
+
+struct lua_xmlrpc_ud {
+ enum lua_xmlrpc_state parser_state;
+ GQueue *st;
+ gint param_count;
+ gboolean got_text;
+ lua_State *L;
+};
+
+static void xmlrpc_start_element(GMarkupParseContext *context,
+ const gchar *name,
+ const gchar **attribute_names,
+ const gchar **attribute_values,
+ gpointer user_data,
+ GError **error);
+static void xmlrpc_end_element(GMarkupParseContext *context,
+ const gchar *element_name,
+ gpointer user_data,
+ GError **error);
+static void xmlrpc_error(GMarkupParseContext *context,
+ GError *error,
+ gpointer user_data);
+static void xmlrpc_text(GMarkupParseContext *context,
+ const gchar *text,
+ gsize text_len,
+ gpointer user_data,
+ GError **error);
+
+static GMarkupParser xmlrpc_parser = {
+ .start_element = xmlrpc_start_element,
+ .end_element = xmlrpc_end_element,
+ .passthrough = NULL,
+ .text = xmlrpc_text,
+ .error = xmlrpc_error,
+};
+
+static GQuark
+xmlrpc_error_quark(void)
+{
+ return g_quark_from_static_string("xmlrpc-error-quark");
+}
+
+static void
+xmlrpc_start_element(GMarkupParseContext *context,
+ const gchar *name,
+ const gchar **attribute_names,
+ const gchar **attribute_values,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ enum lua_xmlrpc_state last_state;
+
+ last_state = ud->parser_state;
+
+ msg_debug_xmlrpc("got start element %s on state %d", name, last_state);
+
+ switch (ud->parser_state) {
+ case read_method_response:
+ /* Expect tag methodResponse */
+ if (g_ascii_strcasecmp(name, "methodResponse") == 0) {
+ ud->parser_state = read_params;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_params:
+ /* Expect tag params */
+ if (g_ascii_strcasecmp(name, "params") == 0) {
+ ud->parser_state = read_param;
+ /* result -> table of params indexed by int */
+ lua_newtable(ud->L);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param:
+ /* Expect tag param */
+ if (g_ascii_strcasecmp(name, "param") == 0) {
+ ud->parser_state = read_param_value;
+ /* Create new param */
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_value:
+ /* Expect tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_element:
+ /* Expect tag struct */
+ if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct:
+ /* Parse structure */
+ /* Expect tag member */
+ if (g_ascii_strcasecmp(name, "member") == 0) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_name:
+ /* Expect tag name */
+ if (g_ascii_strcasecmp(name, "name") == 0) {
+ ud->parser_state = read_struct_member_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_value:
+ /* Accept value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_element:
+ /* Parse any values */
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ /* Structure */
+ else if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array:
+ /* Parse array */
+ /* Expect data */
+ if (g_ascii_strcasecmp(name, "data") == 0) {
+ ud->parser_state = read_array_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_value:
+ /* Accept array value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_array_element;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_element:
+ /* Parse any values */
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0) {
+ ud->parser_state = read_string;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "int") == 0) {
+ ud->parser_state = read_int;
+ ud->got_text = FALSE;
+ }
+ else if (g_ascii_strcasecmp(name, "double") == 0) {
+ ud->parser_state = read_double;
+ ud->got_text = FALSE;
+ }
+ /* Structure */
+ else if (g_ascii_strcasecmp(name, "struct") == 0) {
+ ud->parser_state = read_struct;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_struct));
+ msg_debug_xmlrpc("push struct");
+ }
+ else if (g_ascii_strcasecmp(name, "array") == 0) {
+ ud->parser_state = read_array;
+ /* Create new param of table type */
+ lua_newtable(ud->L);
+ g_queue_push_head(ud->st, GINT_TO_POINTER(st_array));
+ msg_debug_xmlrpc("push array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ default:
+ break;
+ }
+
+ msg_debug_xmlrpc("switched state on start tag %d->%d", last_state,
+ ud->parser_state);
+
+ if (ud->parser_state == error_state) {
+ g_set_error(error,
+ xmlrpc_error_quark(), 1, "xmlrpc parse error on state: %d, while parsing start tag: %s",
+ last_state, name);
+ }
+}
+
+static void
+xmlrpc_end_element(GMarkupParseContext *context,
+ const gchar *name,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ enum lua_xmlrpc_state last_state;
+ int last_queued;
+
+ last_state = ud->parser_state;
+
+ msg_debug_xmlrpc("got end element %s on state %d", name, last_state);
+
+ switch (ud->parser_state) {
+ case read_method_response:
+ ud->parser_state = error_state;
+ break;
+ case read_params:
+ /* Got methodResponse */
+ if (g_ascii_strcasecmp(name, "methodResponse") == 0) {
+ /* End processing */
+ ud->parser_state = success_state;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param:
+ /* Got tag params */
+ if (g_ascii_strcasecmp(name, "params") == 0) {
+ ud->parser_state = read_params;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_value:
+ /* Got tag param */
+ if (g_ascii_strcasecmp(name, "param") == 0) {
+ ud->parser_state = read_param;
+ lua_rawseti(ud->L, -2, ++ud->param_count);
+ msg_debug_xmlrpc("set param element idx: %d", ud->param_count);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_param_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_value;
+ }
+ else {
+ if (GPOINTER_TO_INT(g_queue_peek_head(ud->st)) == st_struct) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ ud->parser_state = read_array_value;
+ }
+ }
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct:
+ /* Got tag struct */
+ if (g_ascii_strcasecmp(name, "struct") == 0) {
+ g_assert(GPOINTER_TO_INT(g_queue_pop_head(ud->st)) == st_struct);
+
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ last_queued = GPOINTER_TO_INT(g_queue_peek_head(ud->st));
+ if (last_queued == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+
+ msg_debug_xmlrpc("pop struct");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_name:
+ /* Got tag member */
+ if (g_ascii_strcasecmp(name, "member") == 0) {
+ ud->parser_state = read_struct;
+ /* Set table */
+ msg_debug_xmlrpc("set struct element idx: %s",
+ lua_tostring(ud->L, -2));
+ lua_settable(ud->L, -3);
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_member_value:
+ /* Got tag name */
+ if (g_ascii_strcasecmp(name, "name") == 0) {
+ ud->parser_state = read_struct_member_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_struct_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ ud->parser_state = read_struct_member_name;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_string:
+ case read_int:
+ case read_double:
+ /* Parse any values */
+ /* Handle empty tags */
+ if (!ud->got_text) {
+ lua_pushnil(ud->L);
+ }
+ else {
+ ud->got_text = FALSE;
+ }
+ /* Primitives */
+ if (g_ascii_strcasecmp(name, "string") == 0 ||
+ g_ascii_strcasecmp(name, "int") == 0 ||
+ g_ascii_strcasecmp(name, "double") == 0) {
+ if (GPOINTER_TO_INT(g_queue_peek_head(ud->st)) == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array:
+ /* Got tag array */
+ if (g_ascii_strcasecmp(name, "array") == 0) {
+ g_assert(GPOINTER_TO_INT(g_queue_pop_head(ud->st)) == st_array);
+
+ if (g_queue_get_length(ud->st) == 0) {
+ ud->parser_state = read_param_element;
+ }
+ else {
+ last_queued = GPOINTER_TO_INT(g_queue_peek_head(ud->st));
+ if (last_queued == st_struct) {
+ ud->parser_state = read_struct_element;
+ }
+ else {
+ ud->parser_state = read_array_element;
+ }
+ }
+
+ msg_debug_xmlrpc("pop array");
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_value:
+ /* Got tag data */
+ if (g_ascii_strcasecmp(name, "data") == 0) {
+ ud->parser_state = read_array;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ case read_array_element:
+ /* Got tag value */
+ if (g_ascii_strcasecmp(name, "value") == 0) {
+ guint tbl_len = rspamd_lua_table_size(ud->L, -2);
+ lua_rawseti(ud->L, -2, tbl_len + 1);
+ msg_debug_xmlrpc("set array element idx: %d", tbl_len + 1);
+ ud->parser_state = read_array_value;
+ }
+ else {
+ /* Error state */
+ ud->parser_state = error_state;
+ }
+ break;
+ default:
+ break;
+ }
+
+ msg_debug_xmlrpc("switched state on end tag %d->%d",
+ last_state, ud->parser_state);
+
+ if (ud->parser_state == error_state) {
+ g_set_error(error,
+ xmlrpc_error_quark(), 1, "xmlrpc parse error on state: %d, while parsing end tag: %s",
+ last_state, name);
+ }
+}
+
+static void
+xmlrpc_text(GMarkupParseContext *context,
+ const gchar *text,
+ gsize text_len,
+ gpointer user_data,
+ GError **error)
+{
+ struct lua_xmlrpc_ud *ud = user_data;
+ gulong num;
+ gdouble dnum;
+
+ /* Strip line */
+ while (text_len > 0 && g_ascii_isspace(*text)) {
+ text++;
+ text_len--;
+ }
+ while (text_len > 0 && g_ascii_isspace(text[text_len - 1])) {
+ text_len--;
+ }
+
+ if (text_len > 0) {
+ msg_debug_xmlrpc("got data on state %d", ud->parser_state);
+ switch (ud->parser_state) {
+ case read_struct_member_value:
+ /* Push key */
+ lua_pushlstring(ud->L, text, text_len);
+ break;
+ case read_string:
+ /* Push string value */
+ lua_pushlstring(ud->L, text, text_len);
+ break;
+ case read_int:
+ /* Push integer value */
+ rspamd_strtoul(text, text_len, &num);
+ lua_pushinteger(ud->L, num);
+ break;
+ case read_double:
+ /* Push integer value */
+ dnum = strtod(text, NULL);
+ lua_pushnumber(ud->L, dnum);
+ break;
+ default:
+ break;
+ }
+ ud->got_text = TRUE;
+ }
+}
+
+static void
+xmlrpc_error(GMarkupParseContext *context, GError *error, gpointer user_data)
+{
+ msg_err("xmlrpc parser error: %s", error->message);
+}
+
+static gint
+lua_xmlrpc_parse_reply(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ const gchar *data;
+ GMarkupParseContext *ctx;
+ GError *err = NULL;
+ struct lua_xmlrpc_ud ud;
+ gsize s;
+ gboolean res;
+
+ data = luaL_checklstring(L, 1, &s);
+
+ if (data != NULL) {
+ ud.L = L;
+ ud.parser_state = read_method_response;
+ ud.param_count = 0;
+ ud.st = g_queue_new();
+
+ ctx = g_markup_parse_context_new(&xmlrpc_parser,
+ G_MARKUP_TREAT_CDATA_AS_TEXT, &ud, NULL);
+ res = g_markup_parse_context_parse(ctx, data, s, &err);
+
+ g_markup_parse_context_free(ctx);
+ if (!res) {
+ lua_pushnil(L);
+ }
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ /* Return table or nil */
+ return 1;
+}
+
+static gint
+lua_xmlrpc_parse_table(lua_State *L,
+ gint pos,
+ gchar *databuf,
+ gint pr,
+ gsize size)
+{
+ gint r = pr, num;
+ double dnum;
+
+ r += rspamd_snprintf(databuf + r, size - r, "<struct>");
+ lua_pushnil(L); /* first key */
+ while (lua_next(L, pos) != 0) {
+ /* uses 'key' (at index -2) and 'value' (at index -1) */
+ if (lua_type(L, -2) != LUA_TSTRING) {
+ /* Ignore non sting keys */
+ lua_pop(L, 1);
+ continue;
+ }
+ r += rspamd_snprintf(databuf + r,
+ size - r,
+ "<member><name>%s</name><value>",
+ lua_tostring(L, -2));
+ switch (lua_type(L, -1)) {
+ case LUA_TNUMBER:
+ num = lua_tointeger(L, -1);
+ dnum = lua_tonumber(L, -1);
+
+ /* Try to avoid conversion errors */
+ if (dnum != (double) num) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<double>%f</double>",
+ dnum);
+ }
+ else {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<int>%d</int>",
+ num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ r += rspamd_snprintf(databuf + r,
+ size - r,
+ "<boolean>%d</boolean>",
+ lua_toboolean(L, -1) ? 1 : 0);
+ break;
+ case LUA_TSTRING:
+ r += rspamd_snprintf(databuf + r, size - r, "<string>%s</string>",
+ lua_tostring(L, -1));
+ break;
+ case LUA_TTABLE:
+ /* Recursive call */
+ r += lua_xmlrpc_parse_table(L, -1, databuf + r, r, size);
+ break;
+ }
+ r += rspamd_snprintf(databuf + r, size - r, "</value></member>");
+ /* removes 'value'; keeps 'key' for next iteration */
+ lua_pop(L, 1);
+ }
+ r += rspamd_snprintf(databuf + r, size - r, "</struct>");
+
+ return r - pr;
+}
+
+/*
+ * Internal limitation: xmlrpc request must NOT be more than
+ * BUFSIZ * 2 (16384 bytes)
+ */
+static gint
+lua_xmlrpc_make_request(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gchar databuf[BUFSIZ * 2];
+ const gchar *func;
+ gint r, top, i, num;
+ double dnum;
+
+ func = luaL_checkstring(L, 1);
+
+ if (func) {
+ r = rspamd_snprintf(databuf, sizeof(databuf),
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<methodCall><methodName>%s</methodName><params>",
+ func);
+ /* Extract arguments */
+ top = lua_gettop(L);
+ /* Get additional options */
+ for (i = 2; i <= top; i++) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<param><value>");
+ switch (lua_type(L, i)) {
+ case LUA_TNUMBER:
+ num = lua_tointeger(L, i);
+ dnum = lua_tonumber(L, i);
+
+ /* Try to avoid conversion errors */
+ if (dnum != (double) num) {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<double>%f</double>",
+ dnum);
+ }
+ else {
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<int>%d</int>",
+ num);
+ }
+ break;
+ case LUA_TBOOLEAN:
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<boolean>%d</boolean>",
+ lua_toboolean(L, i) ? 1 : 0);
+ break;
+ case LUA_TSTRING:
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "<string>%s</string>",
+ lua_tostring(L, i));
+ break;
+ case LUA_TTABLE:
+ r +=
+ lua_xmlrpc_parse_table(L, i, databuf, r, sizeof(databuf));
+ break;
+ }
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "</value></param>");
+ }
+
+ r += rspamd_snprintf(databuf + r,
+ sizeof(databuf) - r,
+ "</params></methodCall>");
+ lua_pushlstring(L, databuf, r);
+ }
+ else {
+ lua_pushnil(L);
+ }
+
+ return 1;
+}
+
+static gint
+lua_load_xmlrpc(lua_State *L)
+{
+ lua_newtable(L);
+ luaL_register(L, NULL, xmlrpclib_m);
+
+ return 1;
+}
+
+void luaopen_xmlrpc(lua_State *L)
+{
+ rspamd_lua_add_preload(L, "rspamd_xmlrpc", lua_load_xmlrpc);
+}
diff --git a/src/lua/rspamd.luadoc b/src/lua/rspamd.luadoc
new file mode 100644
index 0000000..7f2c5cc
--- /dev/null
+++ b/src/lua/rspamd.luadoc
@@ -0,0 +1,124 @@
+--- Rspamd interaction package
+-- contains several subclasses:
+-- config - for parsing config files
+-- metric - for handling metrics callbacks
+-- task - for interaction with task object
+-- message - gate to GMime functions
+-- textpart - a single textual part of message
+module Rspamd
+
+--- Each lua module has global rspamd_config that can be used for getting config
+-- options and registering callbacks (via metric interface)
+
+------------------------------------- CONFIG -----------------------------------------
+--
+--- Get module option from config
+-- @param mname module name
+-- @param option option
+-- @return string with value
+function config:get_module_opt (mname, option)
+
+--- Get all module options as a table like ['param' => 'value']
+-- @param mname module name
+-- @return table with options
+function config:get_all_opt (mname)
+
+--- Get specified metric
+-- @param name metric name
+-- @return metric object
+function config:get_metric (name)
+
+------------------------------------- METRIC -----------------------------------------
+
+--- Register symbol in metric
+-- @param symbol name of symbol
+-- @param weight weight of symbol
+-- @param callback function that would be called as callback for symbol
+function metric:register_symbol (symbol, weight, callback)
+
+------------------------------------- TASK -------------------------------------------
+
+--- Get message object from task
+-- @return message object
+function task:get_message ()
+
+--- Insert result to specified metric with specified weight (obsoleted)
+-- @param metric metric name
+-- @param symbol symbol name
+-- @param weight weight of symbol
+function task:insert_result (metric, symbol, weight)
+
+--- Get all urls as array
+-- @return array of urls in textual form
+function task:get_urls ()
+
+--- Get all text parts
+-- @return array of textpart objects
+function task:get_text_parts ()
+
+--- Get raw headers
+-- @return string that contains raw headers
+function task:get_raw_headers ()
+
+--- Get array of received headers
+-- @return array of received headers that are tables itself
+function task:get_received_headers ()
+
+--- Resolve A record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_a (host, callback)
+
+--- Resolve PTR record using rspamd async resolver
+-- @param host host to resolve
+-- @param callback name of callback function
+function task:resolve_dns_ptr (host, callback)
+
+--- Callback function for dns resolving
+-- @param task task object
+-- @param to_resolve ptr or a record that was resolved
+-- @param results results of dns query (array or nil)
+-- @param err resolver error or nil
+function dns_cb(task, to_resolve, results, err)
+
+------------------------------------- TEXTPART ---------------------------------------
+
+--- Get part's content
+-- @return string that contains part's content
+function textpart:get_content ()
+
+--- Check if part is empty
+-- @return boolean value
+function textpart:is_empty ()
+
+--- Check if part is html
+-- @return boolean value
+function textpart:is_html ()
+
+--- Get part's fuzzy
+-- @return string that contains part's fuzzy
+function textpart:get_fuzzy ()
+
+------------------------------------- MESSAGE ----------------------------------------
+
+--- Get message subject
+-- @return message subject
+function message:get_subject ()
+
+--- Get message id
+-- @return message id
+function message:get_message_id ()
+
+--- Get sender of message
+-- @return sender's credits
+function message:get_sender ()
+
+--- Get reply-to field
+-- @return value of reply-to header
+function message:get_reply_to ()
+
+--- Get header
+-- @param header_name name of header
+-- @return array of headers with specified name
+function message:get_header (header_name)
+