Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 578 insertions, 0 deletions

diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h
new file mode 100644
index 0000000..2c67cba
--- /dev/null
+++ b/src/libserver/rspamd_symcache.h
@@ -0,0 +1,578 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_SYMBOLS_CACHE_H
+#define RSPAMD_SYMBOLS_CACHE_H
+
+#include "config.h"
+#include "ucl.h"
+#include "cfg_file.h"
+#include "contrib/libev/ev.h"
+
+#include <lua.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_config;
+struct rspamd_symcache;
+struct rspamd_worker;
+struct rspamd_symcache_dynamic_item;
+struct rspamd_symcache_item;
+struct rspamd_config_settings_elt;
+
+typedef void (*symbol_func_t)(struct rspamd_task *task,
+							  struct rspamd_symcache_dynamic_item *item,
+							  gpointer user_data);
+
+enum rspamd_symbol_type {
+	SYMBOL_TYPE_NORMAL = (1u << 0u),
+	SYMBOL_TYPE_VIRTUAL = (1u << 1u),
+	SYMBOL_TYPE_CALLBACK = (1u << 2u),
+	SYMBOL_TYPE_GHOST = (1u << 3u),
+	SYMBOL_TYPE_SKIPPED = (1u << 4u),
+	SYMBOL_TYPE_COMPOSITE = (1u << 5u),
+	SYMBOL_TYPE_CLASSIFIER = (1u << 6u),
+	SYMBOL_TYPE_FINE = (1u << 7u),
+	SYMBOL_TYPE_EMPTY = (1u << 8u),      /* Allow execution on empty tasks */
+	SYMBOL_TYPE_CONNFILTER = (1u << 9u), /* Connection stage filter */
+	SYMBOL_TYPE_PREFILTER = (1u << 10u),
+	SYMBOL_TYPE_POSTFILTER = (1u << 11u),
+	SYMBOL_TYPE_NOSTAT = (1u << 12u),             /* Skip as statistical symbol */
+	SYMBOL_TYPE_IDEMPOTENT = (1u << 13u),         /* Symbol cannot change metric */
+	SYMBOL_TYPE_TRIVIAL = (1u << 14u),            /* Symbol is trivial */
+	SYMBOL_TYPE_MIME_ONLY = (1u << 15u),          /* Symbol is mime only */
+	SYMBOL_TYPE_EXPLICIT_DISABLE = (1u << 16u),   /* Symbol should be disabled explicitly only */
+	SYMBOL_TYPE_IGNORE_PASSTHROUGH = (1u << 17u), /* Symbol ignores passthrough result */
+	SYMBOL_TYPE_EXPLICIT_ENABLE = (1u << 18u),    /* Symbol should be enabled explicitly only */
+	SYMBOL_TYPE_USE_CORO = (1u << 19u),           /* Symbol uses lua coroutines */
+};
+
+/**
+ * Abstract structure for saving callback data for symbols
+ */
+struct rspamd_abstract_callback_data {
+	guint64 magic;
+	char data[];
+};
+
+/**
+ * Shared memory block specific for each symbol
+ */
+struct rspamd_symcache_item_stat {
+	struct rspamd_counter_data time_counter;
+	gdouble avg_time;
+	gdouble weight;
+	guint hits;
+	guint64 total_hits;
+	struct rspamd_counter_data frequency_counter;
+	gdouble avg_frequency;
+	gdouble stddev_frequency;
+};
+
+/**
+ * Creates new cache structure
+ * @return
+ */
+struct rspamd_symcache *rspamd_symcache_new(struct rspamd_config *cfg);
+
+/**
+ * Remove the cache structure syncing data if needed
+ * @param cache
+ */
+void rspamd_symcache_destroy(struct rspamd_symcache *cache);
+
+/**
+ * Saves symbols cache to disk if possible
+ * @param cache
+ */
+void rspamd_symcache_save(struct rspamd_symcache *cache);
+
+/**
+ * Load symbols cache from file, must be called _after_ init_symbols_cache
+ */
+gboolean rspamd_symcache_init(struct rspamd_symcache *cache);
+
+/**
+ * Generic function to register a symbol
+ * @param cache
+ * @param name
+ * @param weight
+ * @param priority
+ * @param func
+ * @param user_data
+ * @param type
+ * @param parent
+ */
+gint rspamd_symcache_add_symbol(struct rspamd_symcache *cache,
+								const gchar *name,
+								gint priority,
+								symbol_func_t func,
+								gpointer user_data,
+								int type,
+								gint parent);
+
+/**
+ * Adds augmentation to the symbol
+ * @param cache
+ * @param sym_id
+ * @param augmentation
+ * @return
+ */
+bool rspamd_symcache_add_symbol_augmentation(struct rspamd_symcache *cache,
+											 int sym_id,
+											 const char *augmentation,
+											 const char *value);
+
+/**
+ * Add callback to be executed whenever symbol has peak value
+ * @param cache
+ * @param cbref
+ */
+void rspamd_symcache_set_peak_callback(struct rspamd_symcache *cache,
+									   gint cbref);
+
+/**
+ * Add delayed condition to the specific symbol in cache. So symbol can be absent
+ * to the moment of addition
+ * @param cache
+ * @param id id of symbol
+ * @param L lua state pointer
+ * @param cbref callback reference (returned by luaL_ref)
+ * @return TRUE if condition has been added
+ */
+gboolean rspamd_symcache_add_condition_delayed(struct rspamd_symcache *cache,
+											   const gchar *sym,
+											   lua_State *L, gint cbref);
+
+/**
+ * Find symbol in cache by id and returns its id resolving virtual symbols if
+ * applicable
+ * @param cache
+ * @param name
+ * @return id of symbol or (-1) if a symbol has not been found
+ */
+gint rspamd_symcache_find_symbol(struct rspamd_symcache *cache,
+								 const gchar *name);
+
+/**
+ * Get statistics for a specific symbol
+ * @param cache
+ * @param name
+ * @param frequency
+ * @param tm
+ * @return
+ */
+gboolean rspamd_symcache_stat_symbol(struct rspamd_symcache *cache,
+									 const gchar *name,
+									 gdouble *frequency,
+									 gdouble *freq_stddev,
+									 gdouble *tm,
+									 guint *nhits);
+
+/**
+ * Returns number of symbols registered in symbols cache
+ * @param cache
+ * @return number of symbols in the cache
+ */
+guint rspamd_symcache_stats_symbols_count(struct rspamd_symcache *cache);
+
+/**
+ * Validate cache items against theirs weights defined in metrics
+ * @param cache symbols cache
+ * @param cfg configuration
+ * @param strict do strict checks - symbols MUST be described in metrics
+ */
+gboolean rspamd_symcache_validate(struct rspamd_symcache *cache,
+								  struct rspamd_config *cfg,
+								  gboolean strict);
+
+/**
+ * Call function for cached symbol using saved callback
+ * @param task task object
+ * @param cache symbols cache
+ * @param saved_item pointer to currently saved item
+ */
+gboolean rspamd_symcache_process_symbols(struct rspamd_task *task,
+										 struct rspamd_symcache *cache,
+										 guint stage);
+
+/**
+ * Return statistics about the cache as ucl object (array of objects one per item)
+ * @param cache
+ * @return
+ */
+ucl_object_t *rspamd_symcache_counters(struct rspamd_symcache *cache);
+
+/**
+ * Start cache reloading
+ * @param cache
+ * @param ev_base
+ */
+void *rspamd_symcache_start_refresh(struct rspamd_symcache *cache,
+									struct ev_loop *ev_base,
+									struct rspamd_worker *w);
+
+/**
+ * Increases counter for a specific symbol
+ * @param cache
+ * @param symbol
+ */
+void rspamd_symcache_inc_frequency(struct rspamd_symcache *_cache,
+								   struct rspamd_symcache_item *item,
+								   const gchar *sym_name);
+
+/**
+ * Add delayed dependency that is resolved on cache post-load routine
+ * @param cache
+ * @param from
+ * @param to
+ */
+void rspamd_symcache_add_delayed_dependency(struct rspamd_symcache *cache,
+											const gchar *from, const gchar *to);
+
+/**
+ * Get abstract callback data for a symbol (or its parent symbol)
+ * @param cache cache object
+ * @param symbol symbol name
+ * @return abstract callback data or NULL if symbol is absent or has no data attached
+ */
+struct rspamd_abstract_callback_data *rspamd_symcache_get_cbdata(
+	struct rspamd_symcache *cache, const gchar *symbol);
+
+/**
+ * Returns symbol's parent name (or symbol name itself)
+ * @param cache
+ * @param symbol
+ * @return
+ */
+const gchar *rspamd_symcache_get_parent(struct rspamd_symcache *cache,
+										const gchar *symbol);
+
+guint rspamd_symcache_get_symbol_flags(struct rspamd_symcache *cache,
+									   const gchar *symbol);
+
+void rspamd_symcache_get_symbol_details(struct rspamd_symcache *cache,
+										const gchar *symbol,
+										ucl_object_t *this_sym_ucl);
+
+
+/**
+ * Process settings for task
+ * @param task
+ * @param cache
+ * @return
+ */
+gboolean rspamd_symcache_process_settings(struct rspamd_task *task,
+										  struct rspamd_symcache *cache);
+
+
+/**
+ * Checks if a symbol specified has been checked (or disabled)
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+gboolean rspamd_symcache_is_checked(struct rspamd_task *task,
+									struct rspamd_symcache *cache,
+									const gchar *symbol);
+
+/**
+ * Returns checksum for all cache items
+ * @param cache
+ * @return
+ */
+guint64 rspamd_symcache_get_cksum(struct rspamd_symcache *cache);
+
+/**
+ * Checks if a symbols is enabled (not checked and conditions return true if present)
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+gboolean rspamd_symcache_is_symbol_enabled(struct rspamd_task *task,
+										   struct rspamd_symcache *cache,
+										   const gchar *symbol);
+
+/**
+ * Enable this symbol for task
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return TRUE if a symbol has been enabled (not executed before)
+ */
+gboolean rspamd_symcache_enable_symbol(struct rspamd_task *task,
+									   struct rspamd_symcache *cache,
+									   const gchar *symbol);
+
+/**
+ * Enable this symbol for task
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return TRUE if a symbol has been disabled (not executed before)
+ */
+gboolean rspamd_symcache_disable_symbol(struct rspamd_task *task,
+										struct rspamd_symcache *cache,
+										const gchar *symbol);
+
+/**
+ * Disable execution of a symbol or a pattern (a string enclosed in `//`) permanently
+ * @param task
+ * @param cache
+ * @param symbol
+ * @return
+ */
+void rspamd_symcache_disable_symbol_static(struct rspamd_symcache *cache,
+										   const gchar *symbol);
+/**
+ * Add a symbol or a pattern to the list of explicitly and statically enabled symbols
+ * @param cache
+ * @param symbol
+ * @return
+ */
+void rspamd_symcache_enable_symbol_static(struct rspamd_symcache *cache,
+										  const gchar *symbol);
+
+/**
+ * Process specific function for each cache element (in order they are added)
+ * @param cache
+ * @param func
+ * @param ud
+ */
+void rspamd_symcache_foreach(struct rspamd_symcache *cache,
+							 void (*func)(struct rspamd_symcache_item *item, gpointer /* userdata */),
+							 gpointer ud);
+
+/**
+ * Returns the current item being processed (if any)
+ * @param task
+ * @return
+ */
+struct rspamd_symcache_dynamic_item *rspamd_symcache_get_cur_item(struct rspamd_task *task);
+
+/**
+ * Replaces the current item being processed.
+ * Returns the current item being processed (if any)
+ * @param task
+ * @param item
+ * @return
+ */
+struct rspamd_symcache_dynamic_item *rspamd_symcache_set_cur_item(struct rspamd_task *task,
+																  struct rspamd_symcache_dynamic_item *item);
+
+
+/**
+ * Finalize the current async element potentially calling its deps
+ */
+void rspamd_symcache_finalize_item(struct rspamd_task *task,
+								   struct rspamd_symcache_dynamic_item *item);
+
+/*
+ * Increase number of async events pending for an item
+ */
+guint rspamd_symcache_item_async_inc_full(struct rspamd_task *task,
+										  struct rspamd_symcache_dynamic_item *item,
+										  const gchar *subsystem,
+										  const gchar *loc);
+
+#define rspamd_symcache_item_async_inc(task, item, subsystem) \
+	rspamd_symcache_item_async_inc_full(task, item, subsystem, G_STRLOC)
+
+/*
+ * Decrease number of async events pending for an item, asserts if no events pending
+ */
+guint rspamd_symcache_item_async_dec_full(struct rspamd_task *task,
+										  struct rspamd_symcache_dynamic_item *item,
+										  const gchar *subsystem,
+										  const gchar *loc);
+
+#define rspamd_symcache_item_async_dec(task, item, subsystem) \
+	rspamd_symcache_item_async_dec_full(task, item, subsystem, G_STRLOC)
+
+/**
+ * Decrease number of async events pending for an item, asserts if no events pending
+ * If no events are left, this function calls `rspamd_symbols_cache_finalize_item` and returns TRUE
+ * @param task
+ * @param item
+ * @return
+ */
+gboolean rspamd_symcache_item_async_dec_check_full(struct rspamd_task *task,
+												   struct rspamd_symcache_dynamic_item *item,
+												   const gchar *subsystem,
+												   const gchar *loc);
+
+#define rspamd_symcache_item_async_dec_check(task, item, subsystem) \
+	rspamd_symcache_item_async_dec_check_full(task, item, subsystem, G_STRLOC)
+
+/**
+ * Disables execution of all symbols, excluding those specified in `skip_mask`
+ * @param task
+ * @param cache
+ * @param skip_mask
+ */
+void rspamd_symcache_disable_all_symbols(struct rspamd_task *task,
+										 struct rspamd_symcache *cache,
+										 guint skip_mask);
+
+/**
+ * Iterates over the list of the enabled composites calling specified function
+ * @param task
+ * @param cache
+ * @param func
+ * @param fd
+ */
+void rspamd_symcache_composites_foreach(struct rspamd_task *task,
+										struct rspamd_symcache *cache,
+										GHFunc func,
+										gpointer fd);
+
+/**
+ * Sets allowed settings ids for a symbol
+ * @param cache
+ * @param symbol
+ * @param ids
+ * @param nids
+ */
+bool rspamd_symcache_set_allowed_settings_ids(struct rspamd_symcache *cache,
+											  const gchar *symbol,
+											  const guint32 *ids,
+											  guint nids);
+/**
+ * Sets denied settings ids for a symbol
+ * @param cache
+ * @param symbol
+ * @param ids
+ * @param nids
+ */
+bool rspamd_symcache_set_forbidden_settings_ids(struct rspamd_symcache *cache,
+												const gchar *symbol,
+												const guint32 *ids,
+												guint nids);
+
+/**
+ * Returns allowed ids for a symbol as a constant array
+ * @param cache
+ * @param symbol
+ * @param nids
+ * @return
+ */
+const guint32 *rspamd_symcache_get_allowed_settings_ids(struct rspamd_symcache *cache,
+														const gchar *symbol,
+														guint *nids);
+
+/**
+ * Returns denied ids for a symbol as a constant array
+ * @param cache
+ * @param symbol
+ * @param nids
+ * @return
+ */
+const guint32 *rspamd_symcache_get_forbidden_settings_ids(struct rspamd_symcache *cache,
+														  const gchar *symbol,
+														  guint *nids);
+
+
+/**
+ * Processes settings_elt in cache and converts it to a set of
+ * adjustments for forbidden/allowed settings_ids for each symbol
+ * @param cache
+ * @param elt
+ */
+void rspamd_symcache_process_settings_elt(struct rspamd_symcache *cache,
+										  struct rspamd_config_settings_elt *elt);
+
+/**
+ * Check if a symbol is allowed for execution/insertion, this does not involve
+ * condition scripts to be checked (so it is intended to be fast).
+ * @param task
+ * @param item
+ * @param exec_only
+ * @return
+ */
+gboolean rspamd_symcache_is_item_allowed(struct rspamd_task *task,
+										 struct rspamd_symcache_item *item,
+										 gboolean exec_only);
+
+/**
+ * Returns symcache item flags
+ * @param item
+ * @return
+ */
+gint rspamd_symcache_dyn_item_flags(struct rspamd_task *task,
+									struct rspamd_symcache_dynamic_item *dyn_item);
+gint rspamd_symcache_item_flags(struct rspamd_symcache_item *item);
+
+/**
+ * Returns cache item name
+ * @param item
+ * @return
+ */
+const gchar *rspamd_symcache_dyn_item_name(struct rspamd_task *task,
+										   struct rspamd_symcache_dynamic_item *dyn_item);
+const gchar *rspamd_symcache_item_name(struct rspamd_symcache_item *item);
+
+/**
+ * Returns the current item stat
+ * @param item
+ * @return
+ */
+const struct rspamd_symcache_item_stat *
+rspamd_symcache_item_stat(struct rspamd_symcache_item *item);
+
+/**
+ * Enable profiling for task (e.g. when a slow rule has been found)
+ * @param task
+ */
+void rspamd_symcache_enable_profile(struct rspamd_task *task);
+
+struct rspamd_symcache_timeout_item {
+	double timeout;
+	const struct rspamd_symcache_item *item;
+};
+
+struct rspamd_symcache_timeout_result {
+	double max_timeout;
+	struct rspamd_symcache_timeout_item *items;
+	size_t nitems;
+};
+/**
+ * Gets maximum timeout announced by symbols cache
+ * @param cache
+ * @return new symcache timeout_result structure, that should be freed by call
+ * `rspamd_symcache_timeout_result_free`
+ */
+struct rspamd_symcache_timeout_result *rspamd_symcache_get_max_timeout(struct rspamd_symcache *cache);
+
+/**
+ * Frees results obtained from the previous function
+ * @param res
+ */
+void rspamd_symcache_timeout_result_free(struct rspamd_symcache_timeout_result *res);
+
+/**
+ * Destroy internal state of the symcache runtime
+ * @param task
+ */
+void rspamd_symcache_runtime_destroy(struct rspamd_task *task);
+#ifdef __cplusplus
+}
+#endif
+
+#endif