From 133a45c109da5310add55824db21af5239951f93 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 10 Apr 2024 23:30:40 +0200 Subject: Adding upstream version 3.8.1. Signed-off-by: Daniel Baumann --- src/libserver/maps/map.c | 3195 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 3195 insertions(+) create mode 100644 src/libserver/maps/map.c (limited to 'src/libserver/maps/map.c') diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c new file mode 100644 index 0000000..7f6a48f --- /dev/null +++ b/src/libserver/maps/map.c @@ -0,0 +1,3195 @@ +/* + * Copyright 2023 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Implementation of map files handling + */ + +#include "config.h" +#include "map.h" +#include "map_private.h" +#include "libserver/http/http_connection.h" +#include "libserver/http/http_private.h" +#include "rspamd.h" +#include "contrib/libev/ev.h" +#include "contrib/uthash/utlist.h" + +#ifdef SYS_ZSTD +#include "zstd.h" +#else +#include "contrib/zstd/zstd.h" +#endif + +#undef MAP_DEBUG_REFS +#ifdef MAP_DEBUG_REFS +#define MAP_RETAIN(x, t) \ + do { \ + msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": retain ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount + 1); \ + REF_RETAIN(x); \ + } while (0) + +#define MAP_RELEASE(x, t) \ + do { \ + msg_err(G_GNUC_PRETTY_FUNCTION ": " t ": release ref %p, refcount: %d -> %d", (x), (x)->ref.refcount, (x)->ref.refcount - 1); \ + REF_RELEASE(x); \ + } while (0) +#else +#define MAP_RETAIN(x, t) REF_RETAIN(x) +#define MAP_RELEASE(x, t) REF_RELEASE(x) +#endif + +enum rspamd_map_periodic_opts { + RSPAMD_MAP_SCHEDULE_NORMAL = 0, + RSPAMD_MAP_SCHEDULE_ERROR = (1u << 0u), + RSPAMD_MAP_SCHEDULE_LOCKED = (1u << 1u), + RSPAMD_MAP_SCHEDULE_INIT = (1u << 2u), +}; + +static void free_http_cbdata_common(struct http_callback_data *cbd, + gboolean plan_new); +static void free_http_cbdata_dtor(gpointer p); +static void free_http_cbdata(struct http_callback_data *cbd); +static void rspamd_map_process_periodic(struct map_periodic_cbdata *cbd); +static void rspamd_map_schedule_periodic(struct rspamd_map *map, int how); +static gboolean read_map_file_chunks(struct rspamd_map *map, + struct map_cb_data *cbdata, + const gchar *fname, + gsize len, + goffset off); +static gboolean rspamd_map_save_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct http_map_data *htdata, + const guchar *data, + gsize len); +static gboolean rspamd_map_update_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct http_map_data *htdata); + +guint rspamd_map_log_id = (guint) -1; +RSPAMD_CONSTRUCTOR(rspamd_map_log_init) +{ + rspamd_map_log_id = rspamd_logger_add_debug_module("map"); +} + +/** + * Write HTTP request + */ +static void +write_http_request(struct http_callback_data *cbd) +{ + gchar datebuf[128]; + struct rspamd_http_message *msg; + + msg = rspamd_http_new_message(HTTP_REQUEST); + if (cbd->check) { + msg->method = HTTP_HEAD; + } + + msg->url = rspamd_fstring_append(msg->url, + cbd->data->path, strlen(cbd->data->path)); + + if (cbd->check) { + if (cbd->data->last_modified != 0) { + rspamd_http_date_format(datebuf, sizeof(datebuf), + cbd->data->last_modified); + rspamd_http_message_add_header(msg, "If-Modified-Since", + datebuf); + } + if (cbd->data->etag) { + rspamd_http_message_add_header_len(msg, "If-None-Match", + cbd->data->etag->str, cbd->data->etag->len); + } + } + + msg->url = rspamd_fstring_append(msg->url, cbd->data->rest, + strlen(cbd->data->rest)); + + if (cbd->data->userinfo) { + rspamd_http_message_add_header(msg, "Authorization", + cbd->data->userinfo); + } + + MAP_RETAIN(cbd, "http_callback_data"); + rspamd_http_connection_write_message(cbd->conn, + msg, + cbd->data->host, + NULL, + cbd, + cbd->timeout); +} + +/** + * Callback for destroying HTTP callback data + */ +static void +free_http_cbdata_common(struct http_callback_data *cbd, gboolean plan_new) +{ + struct map_periodic_cbdata *periodic = cbd->periodic; + + if (cbd->shmem_data) { + rspamd_http_message_shmem_unref(cbd->shmem_data); + } + + if (cbd->pk) { + rspamd_pubkey_unref(cbd->pk); + } + + if (cbd->conn) { + rspamd_http_connection_unref(cbd->conn); + cbd->conn = NULL; + } + + if (cbd->addrs) { + rspamd_inet_addr_t *addr; + guint i; + + PTR_ARRAY_FOREACH(cbd->addrs, i, addr) + { + rspamd_inet_address_free(addr); + } + + g_ptr_array_free(cbd->addrs, TRUE); + } + + + MAP_RELEASE(cbd->bk, "rspamd_map_backend"); + + if (periodic) { + /* Detached in case of HTTP error */ + MAP_RELEASE(periodic, "periodic"); + } + + g_free(cbd); +} + +static void +free_http_cbdata(struct http_callback_data *cbd) +{ + cbd->map->tmp_dtor = NULL; + cbd->map->tmp_dtor_data = NULL; + + free_http_cbdata_common(cbd, TRUE); +} + +static void +free_http_cbdata_dtor(gpointer p) +{ + struct http_callback_data *cbd = p; + struct rspamd_map *map; + + map = cbd->map; + if (cbd->stage == http_map_http_conn) { + REF_RELEASE(cbd); + } + else { + /* We cannot terminate DNS requests sent */ + cbd->stage = http_map_terminated; + } + + msg_warn_map("%s: " + "connection with http server is terminated: worker is stopping", + map->name); +} + +/* + * HTTP callbacks + */ +static void +http_map_error(struct rspamd_http_connection *conn, + GError *err) +{ + struct http_callback_data *cbd = conn->ud; + struct rspamd_map *map; + + map = cbd->map; + + if (cbd->periodic) { + cbd->periodic->errored = TRUE; + msg_err_map("error reading %s(%s): " + "connection with http server terminated incorrectly: %e", + cbd->bk->uri, + cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "", + err); + + rspamd_map_process_periodic(cbd->periodic); + } + + MAP_RELEASE(cbd, "http_callback_data"); +} + +static void +rspamd_map_cache_cb(struct ev_loop *loop, ev_timer *w, int revents) +{ + struct rspamd_http_map_cached_cbdata *cache_cbd = (struct rspamd_http_map_cached_cbdata *) + w->data; + struct rspamd_map *map; + struct http_map_data *data; + + map = cache_cbd->map; + data = cache_cbd->data; + + if (cache_cbd->gen != cache_cbd->data->gen) { + /* We have another update, so this cache element is obviously expired */ + /* + * Important!: we do not set cache availability to zero here, as there + * might be fresh cache + */ + msg_info_map("cached data is now expired (gen mismatch %L != %L) for %s; shm name=%s; refcount=%d", + cache_cbd->gen, cache_cbd->data->gen, map->name, cache_cbd->shm->shm_name, + cache_cbd->shm->ref.refcount); + MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata"); + ev_timer_stop(loop, &cache_cbd->timeout); + g_free(cache_cbd); + } + else if (cache_cbd->data->last_checked >= cache_cbd->last_checked) { + /* + * We checked map but we have not found anything more recent, + * reschedule cache check + */ + if (cache_cbd->map->poll_timeout > + rspamd_get_calendar_ticks() - cache_cbd->data->last_checked) { + w->repeat = cache_cbd->map->poll_timeout - + (rspamd_get_calendar_ticks() - cache_cbd->data->last_checked); + } + else { + w->repeat = cache_cbd->map->poll_timeout; + } + + if (w->repeat < 0) { + msg_info_map("cached data for %s has skewed check time: %d last checked, " + "%d poll timeout, %.2f diff; shm name=%s; refcount=%d", + map->name, (int) cache_cbd->data->last_checked, + (int) cache_cbd->map->poll_timeout, + (rspamd_get_calendar_ticks() - cache_cbd->data->last_checked), + cache_cbd->shm->shm_name, + cache_cbd->shm->ref.refcount); + w->repeat = 0.0; + } + + cache_cbd->last_checked = cache_cbd->data->last_checked; + msg_debug_map("cached data is up to date for %s", map->name); + ev_timer_again(loop, &cache_cbd->timeout); + } + else { + data->cur_cache_cbd = NULL; + g_atomic_int_set(&data->cache->available, 0); + msg_info_map("cached data is now expired for %s; shm name=%s; refcount=%d", + map->name, + cache_cbd->shm->shm_name, + cache_cbd->shm->ref.refcount); + MAP_RELEASE(cache_cbd->shm, "rspamd_http_map_cached_cbdata"); + ev_timer_stop(loop, &cache_cbd->timeout); + g_free(cache_cbd); + } +} + +static int +http_map_finish(struct rspamd_http_connection *conn, + struct rspamd_http_message *msg) +{ + struct http_callback_data *cbd = conn->ud; + struct rspamd_map *map; + struct rspamd_map_backend *bk; + struct http_map_data *data; + struct rspamd_http_map_cached_cbdata *cache_cbd; + const rspamd_ftok_t *expires_hdr, *etag_hdr; + char next_check_date[128]; + guchar *in = NULL; + gsize dlen = 0; + + map = cbd->map; + bk = cbd->bk; + data = bk->data.hd; + + if (msg->code == 200) { + + if (cbd->check) { + msg_info_map("need to reread map from %s", cbd->bk->uri); + cbd->periodic->need_modify = TRUE; + /* Reset the whole chain */ + cbd->periodic->cur_backend = 0; + /* Reset cache, old cached data will be cleaned on timeout */ + g_atomic_int_set(&data->cache->available, 0); + data->cur_cache_cbd = NULL; + + rspamd_map_process_periodic(cbd->periodic); + MAP_RELEASE(cbd, "http_callback_data"); + + return 0; + } + + cbd->data->last_checked = msg->date; + + if (msg->last_modified) { + cbd->data->last_modified = msg->last_modified; + } + else { + cbd->data->last_modified = msg->date; + } + + + /* Unsigned version - just open file */ + cbd->shmem_data = rspamd_http_message_shmem_ref(msg); + cbd->data_len = msg->body_buf.len; + + if (cbd->data_len == 0) { + msg_err_map("cannot read empty map"); + goto err; + } + + g_assert(cbd->shmem_data != NULL); + + in = rspamd_shmem_xmap(cbd->shmem_data->shm_name, PROT_READ, &dlen); + + if (in == NULL) { + msg_err_map("cannot read tempfile %s: %s", + cbd->shmem_data->shm_name, + strerror(errno)); + goto err; + } + + /* Check for expires */ + double cached_timeout = map->poll_timeout * 2; + + expires_hdr = rspamd_http_message_find_header(msg, "Expires"); + + if (expires_hdr) { + time_t hdate; + + hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len); + + if (hdate != (time_t) -1 && hdate > msg->date) { + cached_timeout = map->next_check - msg->date + + map->poll_timeout * 2; + + map->next_check = hdate; + } + else { + msg_info_map("invalid expires header: %T, ignore it", expires_hdr); + map->next_check = 0; + } + } + + /* Check for etag */ + etag_hdr = rspamd_http_message_find_header(msg, "ETag"); + + if (etag_hdr) { + if (cbd->data->etag) { + /* Remove old etag */ + rspamd_fstring_free(cbd->data->etag); + } + + cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin, + etag_hdr->len); + } + else { + if (cbd->data->etag) { + /* Remove and clear old etag */ + rspamd_fstring_free(cbd->data->etag); + cbd->data->etag = NULL; + } + } + + MAP_RETAIN(cbd->shmem_data, "shmem_data"); + cbd->data->gen++; + /* + * We know that a map is in the locked state + */ + g_atomic_int_set(&data->cache->available, 1); + /* Store cached data */ + rspamd_strlcpy(data->cache->shmem_name, cbd->shmem_data->shm_name, + sizeof(data->cache->shmem_name)); + data->cache->len = cbd->data_len; + data->cache->last_modified = cbd->data->last_modified; + cache_cbd = g_malloc0(sizeof(*cache_cbd)); + cache_cbd->shm = cbd->shmem_data; + cache_cbd->event_loop = cbd->event_loop; + cache_cbd->map = map; + cache_cbd->data = cbd->data; + cache_cbd->last_checked = cbd->data->last_checked; + cache_cbd->gen = cbd->data->gen; + MAP_RETAIN(cache_cbd->shm, "shmem_data"); + msg_info_map("stored map data in a shared memory cache: %s", + cache_cbd->shm->shm_name); + + ev_timer_init(&cache_cbd->timeout, rspamd_map_cache_cb, cached_timeout, + 0.0); + ev_timer_start(cbd->event_loop, &cache_cbd->timeout); + cache_cbd->timeout.data = cache_cbd; + data->cur_cache_cbd = cache_cbd; + + if (map->next_check) { + rspamd_http_date_format(next_check_date, sizeof(next_check_date), + map->next_check); + } + else { + rspamd_http_date_format(next_check_date, sizeof(next_check_date), + rspamd_get_calendar_ticks() + map->poll_timeout); + } + + + if (cbd->bk->is_compressed) { + ZSTD_DStream *zstream; + ZSTD_inBuffer zin; + ZSTD_outBuffer zout; + guchar *out; + gsize outlen, r; + + zstream = ZSTD_createDStream(); + ZSTD_initDStream(zstream); + + zin.pos = 0; + zin.src = in; + zin.size = dlen; + + if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) { + outlen = ZSTD_DStreamOutSize(); + } + + out = g_malloc(outlen); + + zout.dst = out; + zout.pos = 0; + zout.size = outlen; + + while (zin.pos < zin.size) { + r = ZSTD_decompressStream(zstream, &zout, &zin); + + if (ZSTD_isError(r)) { + msg_err_map("%s(%s): cannot decompress data: %s", + cbd->bk->uri, + rspamd_inet_address_to_string_pretty(cbd->addr), + ZSTD_getErrorName(r)); + ZSTD_freeDStream(zstream); + g_free(out); + MAP_RELEASE(cbd->shmem_data, "shmem_data"); + goto err; + } + + if (zout.pos == zout.size) { + /* We need to extend output buffer */ + zout.size = zout.size * 2 + 1.0; + out = g_realloc(zout.dst, zout.size); + zout.dst = out; + } + } + + ZSTD_freeDStream(zstream); + msg_info_map("%s(%s): read map data %z bytes compressed, " + "%z uncompressed, next check at %s", + cbd->bk->uri, + rspamd_inet_address_to_string_pretty(cbd->addr), + dlen, zout.pos, next_check_date); + map->read_callback(out, zout.pos, &cbd->periodic->cbdata, TRUE); + rspamd_map_save_http_cached_file(map, bk, cbd->data, out, zout.pos); + g_free(out); + } + else { + msg_info_map("%s(%s): read map data %z bytes, next check at %s", + cbd->bk->uri, + rspamd_inet_address_to_string_pretty(cbd->addr), + dlen, next_check_date); + rspamd_map_save_http_cached_file(map, bk, cbd->data, in, cbd->data_len); + map->read_callback(in, cbd->data_len, &cbd->periodic->cbdata, TRUE); + } + + MAP_RELEASE(cbd->shmem_data, "shmem_data"); + + cbd->periodic->cur_backend++; + munmap(in, dlen); + rspamd_map_process_periodic(cbd->periodic); + } + else if (msg->code == 304 && cbd->check) { + cbd->data->last_checked = msg->date; + + if (msg->last_modified) { + cbd->data->last_modified = msg->last_modified; + } + else { + cbd->data->last_modified = msg->date; + } + + expires_hdr = rspamd_http_message_find_header(msg, "Expires"); + + if (expires_hdr) { + time_t hdate; + + hdate = rspamd_http_parse_date(expires_hdr->begin, expires_hdr->len); + if (hdate != (time_t) -1 && hdate > msg->date) { + map->next_check = hdate; + } + else { + msg_info_map("invalid expires header: %T, ignore it", expires_hdr); + map->next_check = 0; + } + } + + etag_hdr = rspamd_http_message_find_header(msg, "ETag"); + + if (etag_hdr) { + if (cbd->data->etag) { + /* Remove old etag */ + rspamd_fstring_free(cbd->data->etag); + cbd->data->etag = rspamd_fstring_new_init(etag_hdr->begin, + etag_hdr->len); + } + } + + if (map->next_check) { + rspamd_http_date_format(next_check_date, sizeof(next_check_date), + map->next_check); + msg_info_map("data is not modified for server %s, next check at %s " + "(http cache based: %T)", + cbd->data->host, next_check_date, expires_hdr); + } + else { + rspamd_http_date_format(next_check_date, sizeof(next_check_date), + rspamd_get_calendar_ticks() + map->poll_timeout); + msg_info_map("data is not modified for server %s, next check at %s " + "(timer based)", + cbd->data->host, next_check_date); + } + + rspamd_map_update_http_cached_file(map, bk, cbd->data); + cbd->periodic->cur_backend++; + rspamd_map_process_periodic(cbd->periodic); + } + else { + msg_info_map("cannot load map %s from %s: HTTP error %d", + bk->uri, cbd->data->host, msg->code); + goto err; + } + + MAP_RELEASE(cbd, "http_callback_data"); + return 0; + +err: + cbd->periodic->errored = 1; + rspamd_map_process_periodic(cbd->periodic); + MAP_RELEASE(cbd, "http_callback_data"); + + return 0; +} + +static gboolean +read_map_file_chunks(struct rspamd_map *map, struct map_cb_data *cbdata, + const gchar *fname, gsize len, goffset off) +{ + gint fd; + gssize r, avail; + gsize buflen = 1024 * 1024; + gchar *pos, *bytes; + + fd = rspamd_file_xopen(fname, O_RDONLY, 0, TRUE); + + if (fd == -1) { + msg_err_map("can't open map for buffered reading %s: %s", + fname, strerror(errno)); + return FALSE; + } + + if (lseek(fd, off, SEEK_SET) == -1) { + msg_err_map("can't seek in map to pos %d for buffered reading %s: %s", + (gint) off, fname, strerror(errno)); + close(fd); + + return FALSE; + } + + buflen = MIN(len, buflen); + bytes = g_malloc(buflen); + avail = buflen; + pos = bytes; + + while ((r = read(fd, pos, avail)) > 0) { + gchar *end = bytes + (pos - bytes) + r; + msg_debug_map("%s: read map chunk, %z bytes", fname, + r); + pos = map->read_callback(bytes, end - bytes, cbdata, r == len); + + if (pos && pos > bytes && pos < end) { + guint remain = end - pos; + + memmove(bytes, pos, remain); + pos = bytes + remain; + /* Need to preserve the remain */ + avail = ((gssize) buflen) - remain; + + if (avail <= 0) { + /* Try realloc, too large element */ + g_assert(buflen >= remain); + bytes = g_realloc(bytes, buflen * 2); + + pos = bytes + remain; /* Adjust */ + avail += buflen; + buflen *= 2; + } + } + else { + avail = buflen; + pos = bytes; + } + + len -= r; + } + + if (r == -1) { + msg_err_map("can't read from map %s: %s", fname, strerror(errno)); + close(fd); + g_free(bytes); + + return FALSE; + } + + close(fd); + g_free(bytes); + + return TRUE; +} + +static gboolean +rspamd_map_check_sig_pk_mem(const guchar *sig, + gsize siglen, + struct rspamd_map *map, + const guchar *input, + gsize inlen, + struct rspamd_cryptobox_pubkey *pk) +{ + GString *b32_key; + gboolean ret = TRUE; + + if (siglen != rspamd_cryptobox_signature_bytes(RSPAMD_CRYPTOBOX_MODE_25519)) { + msg_err_map("can't open signature for %s: invalid size: %z", map->name, siglen); + + ret = FALSE; + } + + if (ret && !rspamd_cryptobox_verify(sig, siglen, input, inlen, + rspamd_pubkey_get_pk(pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { + msg_err_map("can't verify signature for %s: incorrect signature", map->name); + + ret = FALSE; + } + + if (ret) { + b32_key = rspamd_pubkey_print(pk, + RSPAMD_KEYPAIR_BASE32 | RSPAMD_KEYPAIR_PUBKEY); + msg_info_map("verified signature for %s using trusted key %v", + map->name, b32_key); + g_string_free(b32_key, TRUE); + } + + return ret; +} + +static gboolean +rspamd_map_check_file_sig(const char *fname, + struct rspamd_map *map, + struct rspamd_map_backend *bk, + const guchar *input, + gsize inlen) +{ + guchar *data; + struct rspamd_cryptobox_pubkey *pk = NULL; + GString *b32_key; + gboolean ret = TRUE; + gsize len = 0; + gchar fpath[PATH_MAX]; + + if (bk->trusted_pubkey == NULL) { + /* Try to load and check pubkey */ + rspamd_snprintf(fpath, sizeof(fpath), "%s.pub", fname); + data = rspamd_file_xmap(fpath, PROT_READ, &len, TRUE); + + if (data == NULL) { + msg_err_map("can't open pubkey %s: %s", fpath, strerror(errno)); + return FALSE; + } + + pk = rspamd_pubkey_from_base32(data, len, RSPAMD_KEYPAIR_SIGN, + RSPAMD_CRYPTOBOX_MODE_25519); + munmap(data, len); + + if (pk == NULL) { + msg_err_map("can't load pubkey %s", fpath); + return FALSE; + } + + /* We just check pk against the trusted db of keys */ + b32_key = rspamd_pubkey_print(pk, + RSPAMD_KEYPAIR_BASE32 | RSPAMD_KEYPAIR_PUBKEY); + g_assert(b32_key != NULL); + + if (g_hash_table_lookup(map->cfg->trusted_keys, b32_key->str) == NULL) { + msg_err_map("pubkey loaded from %s is untrusted: %v", fpath, + b32_key); + g_string_free(b32_key, TRUE); + rspamd_pubkey_unref(pk); + + return FALSE; + } + + g_string_free(b32_key, TRUE); + } + else { + pk = rspamd_pubkey_ref(bk->trusted_pubkey); + } + + rspamd_snprintf(fpath, sizeof(fpath), "%s.sig", fname); + data = rspamd_shmem_xmap(fpath, PROT_READ, &len); + + if (data == NULL) { + msg_err_map("can't open signature %s: %s", fpath, strerror(errno)); + ret = FALSE; + } + + if (ret) { + ret = rspamd_map_check_sig_pk_mem(data, len, map, input, inlen, pk); + munmap(data, len); + } + + rspamd_pubkey_unref(pk); + + return ret; +} + +/** + * Callback for reading data from file + */ +static gboolean +read_map_file(struct rspamd_map *map, struct file_map_data *data, + struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic) +{ + gchar *bytes; + gsize len; + struct stat st; + + if (map->read_callback == NULL || map->fin_callback == NULL) { + msg_err_map("%s: bad callback for reading map file", + data->filename); + return FALSE; + } + + if (stat(data->filename, &st) == -1) { + /* File does not exist, skipping */ + if (errno != ENOENT) { + msg_err_map("%s: map file is unavailable for reading: %s", + data->filename, strerror(errno)); + + return FALSE; + } + else { + msg_info_map("%s: map file is not found; " + "it will be read automatically if created", + data->filename); + return TRUE; + } + } + + ev_stat_stat(map->event_loop, &data->st_ev); + len = st.st_size; + + if (bk->is_signed) { + bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE); + + if (bytes == NULL) { + msg_err_map("can't open map %s: %s", data->filename, strerror(errno)); + return FALSE; + } + + if (!rspamd_map_check_file_sig(data->filename, map, bk, bytes, len)) { + munmap(bytes, len); + + return FALSE; + } + + munmap(bytes, len); + } + + if (len > 0) { + if (map->no_file_read) { + /* We just call read callback with backend name */ + map->read_callback(data->filename, strlen(data->filename), + &periodic->cbdata, TRUE); + } + else { + if (bk->is_compressed) { + bytes = rspamd_file_xmap(data->filename, PROT_READ, &len, TRUE); + + if (bytes == NULL) { + msg_err_map("can't open map %s: %s", data->filename, strerror(errno)); + return FALSE; + } + + ZSTD_DStream *zstream; + ZSTD_inBuffer zin; + ZSTD_outBuffer zout; + guchar *out; + gsize outlen, r; + + zstream = ZSTD_createDStream(); + ZSTD_initDStream(zstream); + + zin.pos = 0; + zin.src = bytes; + zin.size = len; + + if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) { + outlen = ZSTD_DStreamOutSize(); + } + + out = g_malloc(outlen); + + zout.dst = out; + zout.pos = 0; + zout.size = outlen; + + while (zin.pos < zin.size) { + r = ZSTD_decompressStream(zstream, &zout, &zin); + + if (ZSTD_isError(r)) { + msg_err_map("%s: cannot decompress data: %s", + data->filename, + ZSTD_getErrorName(r)); + ZSTD_freeDStream(zstream); + g_free(out); + munmap(bytes, len); + return FALSE; + } + + if (zout.pos == zout.size) { + /* We need to extend output buffer */ + zout.size = zout.size * 2 + 1; + out = g_realloc(zout.dst, zout.size); + zout.dst = out; + } + } + + ZSTD_freeDStream(zstream); + msg_info_map("%s: read map data, %z bytes compressed, " + "%z uncompressed)", + data->filename, + len, zout.pos); + map->read_callback(out, zout.pos, &periodic->cbdata, TRUE); + g_free(out); + + munmap(bytes, len); + } + else { + /* Perform buffered read: fail-safe */ + if (!read_map_file_chunks(map, &periodic->cbdata, data->filename, + len, 0)) { + return FALSE; + } + } + } + } + else { + /* Empty map */ + map->read_callback(NULL, 0, &periodic->cbdata, TRUE); + } + + return TRUE; +} + +static gboolean +read_map_static(struct rspamd_map *map, struct static_map_data *data, + struct rspamd_map_backend *bk, struct map_periodic_cbdata *periodic) +{ + guchar *bytes; + gsize len; + + if (map->read_callback == NULL || map->fin_callback == NULL) { + msg_err_map("%s: bad callback for reading map file", map->name); + data->processed = TRUE; + return FALSE; + } + + bytes = data->data; + len = data->len; + + if (len > 0) { + if (bk->is_compressed) { + ZSTD_DStream *zstream; + ZSTD_inBuffer zin; + ZSTD_outBuffer zout; + guchar *out; + gsize outlen, r; + + zstream = ZSTD_createDStream(); + ZSTD_initDStream(zstream); + + zin.pos = 0; + zin.src = bytes; + zin.size = len; + + if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) { + outlen = ZSTD_DStreamOutSize(); + } + + out = g_malloc(outlen); + + zout.dst = out; + zout.pos = 0; + zout.size = outlen; + + while (zin.pos < zin.size) { + r = ZSTD_decompressStream(zstream, &zout, &zin); + + if (ZSTD_isError(r)) { + msg_err_map("%s: cannot decompress data: %s", + map->name, + ZSTD_getErrorName(r)); + ZSTD_freeDStream(zstream); + g_free(out); + + return FALSE; + } + + if (zout.pos == zout.size) { + /* We need to extend output buffer */ + zout.size = zout.size * 2 + 1; + out = g_realloc(zout.dst, zout.size); + zout.dst = out; + } + } + + ZSTD_freeDStream(zstream); + msg_info_map("%s: read map data, %z bytes compressed, " + "%z uncompressed)", + map->name, + len, zout.pos); + map->read_callback(out, zout.pos, &periodic->cbdata, TRUE); + g_free(out); + } + else { + msg_info_map("%s: read map data, %z bytes", + map->name, len); + map->read_callback(bytes, len, &periodic->cbdata, TRUE); + } + } + else { + map->read_callback(NULL, 0, &periodic->cbdata, TRUE); + } + + data->processed = TRUE; + + return TRUE; +} + +static void +rspamd_map_periodic_dtor(struct map_periodic_cbdata *periodic) +{ + struct rspamd_map *map; + + map = periodic->map; + msg_debug_map("periodic dtor %p", periodic); + + if (periodic->need_modify || periodic->cbdata.errored) { + /* Need to notify the real data structure */ + periodic->map->fin_callback(&periodic->cbdata, periodic->map->user_data); + + if (map->on_load_function) { + map->on_load_function(map, map->on_load_ud); + } + } + else { + /* Not modified */ + } + + if (periodic->locked) { + g_atomic_int_set(periodic->map->locked, 0); + msg_debug_map("unlocked map %s", periodic->map->name); + + if (periodic->map->wrk->state == rspamd_worker_state_running) { + rspamd_map_schedule_periodic(periodic->map, + RSPAMD_SYMBOL_RESULT_NORMAL); + } + else { + msg_debug_map("stop scheduling periodics for %s; terminating state", + periodic->map->name); + } + } + + g_free(periodic); +} + +/* Called on timer execution */ +static void +rspamd_map_periodic_callback(struct ev_loop *loop, ev_timer *w, int revents) +{ + struct map_periodic_cbdata *cbd = (struct map_periodic_cbdata *) w->data; + + MAP_RETAIN(cbd, "periodic"); + ev_timer_stop(loop, w); + rspamd_map_process_periodic(cbd); + MAP_RELEASE(cbd, "periodic"); +} + +static void +rspamd_map_schedule_periodic(struct rspamd_map *map, int how) +{ + const gdouble error_mult = 20.0, lock_mult = 0.1; + static const gdouble min_timer_interval = 2.0; + const gchar *reason = "unknown reason"; + gdouble jittered_sec; + gdouble timeout; + struct map_periodic_cbdata *cbd; + + if (map->scheduled_check || (map->wrk && + map->wrk->state != rspamd_worker_state_running)) { + /* + * Do not schedule check if some check is already scheduled or + * if worker is going to die + */ + return; + } + + if (!(how & RSPAMD_MAP_SCHEDULE_INIT) && map->static_only) { + /* No need to schedule anything for static maps */ + return; + } + + if (map->non_trivial && map->next_check != 0) { + timeout = map->next_check - rspamd_get_calendar_ticks(); + map->next_check = 0; + + if (timeout > 0 && timeout < map->poll_timeout) { + /* Early check case, jitter */ + gdouble poll_timeout = map->poll_timeout; + + if (how & RSPAMD_MAP_SCHEDULE_ERROR) { + poll_timeout = map->poll_timeout * error_mult; + reason = "early active non-trivial check (after error)"; + } + else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) { + poll_timeout = map->poll_timeout * lock_mult; + reason = "early active non-trivial check (after being locked)"; + } + else { + reason = "early active non-trivial check"; + } + + jittered_sec = MIN(timeout, poll_timeout); + } + else if (timeout <= 0) { + /* Data is already expired, need to check */ + if (how & RSPAMD_MAP_SCHEDULE_ERROR) { + /* In case of error we still need to increase delay */ + jittered_sec = map->poll_timeout * error_mult; + reason = "expired non-trivial data (after error)"; + } + else { + jittered_sec = 0.0; + reason = "expired non-trivial data"; + } + } + else { + /* No need to check now, wait till next_check */ + jittered_sec = timeout; + reason = "valid non-trivial data"; + } + } + else { + /* No valid information when to check a map, plan a timer based check */ + timeout = map->poll_timeout; + + if (how & RSPAMD_MAP_SCHEDULE_INIT) { + if (map->active_http) { + /* Spill maps load to get better chances to hit ssl cache */ + timeout = rspamd_time_jitter(0.0, 2.0); + } + else { + timeout = 0.0; + } + + reason = "init scheduled check"; + } + else { + if (how & RSPAMD_MAP_SCHEDULE_ERROR) { + timeout = map->poll_timeout * error_mult; + reason = "errored scheduled check"; + } + else if (how & RSPAMD_MAP_SCHEDULE_LOCKED) { + timeout = map->poll_timeout * lock_mult; + reason = "locked scheduled check"; + } + else { + reason = "normal scheduled check"; + } + } + + jittered_sec = rspamd_time_jitter(timeout, 0); + } + + /* Now, we do some sanity checks for jittered seconds */ + if (!(how & RSPAMD_MAP_SCHEDULE_INIT)) { + /* Never allow too low interval between timer checks, it is expensive */ + if (jittered_sec < min_timer_interval) { + jittered_sec = rspamd_time_jitter(min_timer_interval, 0); + } + + if (map->non_trivial) { + /* + * Even if we are reported that we need to reload cache often, we + * still want to be sane in terms of events... + */ + if (jittered_sec < min_timer_interval * 2.0) { + if (map->nelts > 0) { + jittered_sec = min_timer_interval * 3.0; + } + } + } + } + + cbd = g_malloc0(sizeof(*cbd)); + cbd->cbdata.prev_data = *map->user_data; + cbd->cbdata.cur_data = NULL; + cbd->cbdata.map = map; + cbd->map = map; + map->scheduled_check = cbd; + REF_INIT_RETAIN(cbd, rspamd_map_periodic_dtor); + + cbd->ev.data = cbd; + ev_timer_init(&cbd->ev, rspamd_map_periodic_callback, jittered_sec, 0.0); + ev_timer_start(map->event_loop, &cbd->ev); + + msg_debug_map("schedule new periodic event %p in %.3f seconds for %s; reason: %s", + cbd, jittered_sec, map->name, reason); +} + +static gint +rspamd_map_af_to_weight(const rspamd_inet_addr_t *addr) +{ + int ret; + + switch (rspamd_inet_address_get_af(addr)) { + case AF_UNIX: + ret = 2; + break; + case AF_INET: + ret = 1; + break; + default: + ret = 0; + break; + } + + return ret; +} + +static gint +rspamd_map_dns_address_sort_func(gconstpointer a, gconstpointer b) +{ + const rspamd_inet_addr_t *ip1 = *(const rspamd_inet_addr_t **) a, + *ip2 = *(const rspamd_inet_addr_t **) b; + gint w1, w2; + + w1 = rspamd_map_af_to_weight(ip1); + w2 = rspamd_map_af_to_weight(ip2); + + /* Inverse order */ + return w2 - w1; +} + +static void +rspamd_map_dns_callback(struct rdns_reply *reply, void *arg) +{ + struct http_callback_data *cbd = arg; + struct rdns_reply_entry *cur_rep; + struct rspamd_map *map; + guint flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED; + + map = cbd->map; + + msg_debug_map("got dns reply with code %s on stage %d", + rdns_strerror(reply->code), cbd->stage); + + if (cbd->stage == http_map_terminated) { + MAP_RELEASE(cbd, "http_callback_data"); + return; + } + + if (reply->code == RDNS_RC_NOERROR) { + DL_FOREACH(reply->entries, cur_rep) + { + rspamd_inet_addr_t *addr; + addr = rspamd_inet_address_from_rnds(cur_rep); + + if (addr != NULL) { + rspamd_inet_address_set_port(addr, cbd->data->port); + g_ptr_array_add(cbd->addrs, (void *) addr); + } + } + + if (cbd->stage == http_map_resolve_host2) { + /* We have still one request pending */ + cbd->stage = http_map_resolve_host1; + } + else if (cbd->stage == http_map_resolve_host1) { + cbd->stage = http_map_http_conn; + } + } + else if (cbd->stage < http_map_http_conn) { + if (cbd->stage == http_map_resolve_host2) { + /* We have still one request pending */ + cbd->stage = http_map_resolve_host1; + } + else if (cbd->addrs->len == 0) { + /* We could not resolve host, so cowardly fail here */ + msg_err_map("cannot resolve %s: %s", cbd->data->host, + rdns_strerror(reply->code)); + cbd->periodic->errored = 1; + rspamd_map_process_periodic(cbd->periodic); + } + else { + /* We have at least one address, so we can continue... */ + cbd->stage = http_map_http_conn; + } + } + + if (cbd->stage == http_map_http_conn && cbd->addrs->len > 0) { + rspamd_ptr_array_shuffle(cbd->addrs); + gint idx = 0; + /* + * For the existing addr we can just select any address as we have + * data available + */ + if (cbd->map->nelts > 0 && rspamd_random_double_fast() > 0.5) { + /* Already shuffled, use whatever is the first */ + cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx); + } + else { + /* Always prefer IPv4 as IPv6 is almost all the time broken */ + g_ptr_array_sort(cbd->addrs, rspamd_map_dns_address_sort_func); + cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx); + } + + retry: + msg_debug_map("try open http connection to %s", + rspamd_inet_address_to_string_pretty(cbd->addr)); + if (cbd->bk->protocol == MAP_PROTO_HTTPS) { + flags |= RSPAMD_HTTP_CLIENT_SSL; + } + cbd->conn = rspamd_http_connection_new_client(NULL, + NULL, + http_map_error, + http_map_finish, + flags, + cbd->addr); + + if (cbd->conn != NULL) { + write_http_request(cbd); + } + else { + if (idx < cbd->addrs->len - 1) { + /* We can retry */ + idx++; + rspamd_inet_addr_t *prev_addr = cbd->addr; + cbd->addr = (rspamd_inet_addr_t *) g_ptr_array_index(cbd->addrs, idx); + msg_info_map("cannot connect to %s to get data for %s: %s, retry with %s (%d of %d)", + rspamd_inet_address_to_string_pretty(prev_addr), + cbd->bk->uri, + strerror(errno), + rspamd_inet_address_to_string_pretty(cbd->addr), + idx + 1, cbd->addrs->len); + goto retry; + } + else { + /* Nothing else left */ + cbd->periodic->errored = TRUE; + msg_err_map("error reading %s(%s): " + "connection with http server terminated incorrectly: %s", + cbd->bk->uri, + cbd->addr ? rspamd_inet_address_to_string_pretty(cbd->addr) : "", + strerror(errno)); + + rspamd_map_process_periodic(cbd->periodic); + } + } + } + + MAP_RELEASE(cbd, "http_callback_data"); +} + +static gboolean +rspamd_map_read_cached(struct rspamd_map *map, struct rspamd_map_backend *bk, + struct map_periodic_cbdata *periodic, const gchar *host) +{ + gsize mmap_len, len; + gpointer in; + struct http_map_data *data; + + data = bk->data.hd; + + in = rspamd_shmem_xmap(data->cache->shmem_name, PROT_READ, &mmap_len); + + if (in == NULL) { + msg_err("cannot map cache from %s: %s", data->cache->shmem_name, + strerror(errno)); + return FALSE; + } + + if (mmap_len < data->cache->len) { + msg_err("cannot map cache from %s: truncated length %z, %z expected", + data->cache->shmem_name, + mmap_len, data->cache->len); + munmap(in, mmap_len); + + return FALSE; + } + + /* + * Len is taken from the shmem file size that can be larger than the + * actual data length, as we use shared memory as a growing buffer for the + * HTTP input. + * Hence, we need to use len from the saved cache data, counting that it is + * at least not more than the cached file length (this is checked above). + */ + len = data->cache->len; + + if (bk->is_compressed) { + ZSTD_DStream *zstream; + ZSTD_inBuffer zin; + ZSTD_outBuffer zout; + guchar *out; + gsize outlen, r; + + zstream = ZSTD_createDStream(); + ZSTD_initDStream(zstream); + + zin.pos = 0; + zin.src = in; + zin.size = len; + + if ((outlen = ZSTD_getDecompressedSize(zin.src, zin.size)) == 0) { + outlen = ZSTD_DStreamOutSize(); + } + + out = g_malloc(outlen); + + zout.dst = out; + zout.pos = 0; + zout.size = outlen; + + while (zin.pos < zin.size) { + r = ZSTD_decompressStream(zstream, &zout, &zin); + + if (ZSTD_isError(r)) { + msg_err_map("%s: cannot decompress data: %s", + bk->uri, + ZSTD_getErrorName(r)); + ZSTD_freeDStream(zstream); + g_free(out); + munmap(in, mmap_len); + return FALSE; + } + + if (zout.pos == zout.size) { + /* We need to extend output buffer */ + zout.size = zout.size * 2 + 1; + out = g_realloc(zout.dst, zout.size); + zout.dst = out; + } + } + + ZSTD_freeDStream(zstream); + msg_info_map("%s: read map data cached %z bytes compressed, " + "%z uncompressed", + bk->uri, + len, zout.pos); + map->read_callback(out, zout.pos, &periodic->cbdata, TRUE); + g_free(out); + } + else { + msg_info_map("%s: read map data cached %z bytes", bk->uri, len); + map->read_callback(in, len, &periodic->cbdata, TRUE); + } + + munmap(in, mmap_len); + + return TRUE; +} + +static gboolean +rspamd_map_has_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk) +{ + gchar path[PATH_MAX]; + guchar digest[rspamd_cryptobox_HASHBYTES]; + struct rspamd_config *cfg = map->cfg; + struct stat st; + + if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') { + return FALSE; + } + + rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0); + rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir, + G_DIR_SEPARATOR, 20, digest); + + if (stat(path, &st) != -1 && st.st_size > + sizeof(struct rspamd_http_file_data)) { + return TRUE; + } + + return FALSE; +} + +static gboolean +rspamd_map_save_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct http_map_data *htdata, + const guchar *data, + gsize len) +{ + gchar path[PATH_MAX]; + guchar digest[rspamd_cryptobox_HASHBYTES]; + struct rspamd_config *cfg = map->cfg; + gint fd; + struct rspamd_http_file_data header; + + if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') { + return FALSE; + } + + rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0); + rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir, + G_DIR_SEPARATOR, 20, digest); + + fd = rspamd_file_xopen(path, O_WRONLY | O_TRUNC | O_CREAT, + 00600, FALSE); + + if (fd == -1) { + return FALSE; + } + + if (!rspamd_file_lock(fd, FALSE)) { + msg_err_map("cannot lock file %s: %s", path, strerror(errno)); + close(fd); + + return FALSE; + } + + memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic)); + header.mtime = htdata->last_modified; + header.next_check = map->next_check; + header.data_off = sizeof(header); + + if (htdata->etag) { + header.data_off += RSPAMD_FSTRING_LEN(htdata->etag); + header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag); + } + else { + header.etag_len = 0; + } + + if (write(fd, &header, sizeof(header)) != sizeof(header)) { + msg_err_map("cannot write file %s (header stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + + if (header.etag_len > 0) { + if (write(fd, RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len) != + header.etag_len) { + msg_err_map("cannot write file %s (etag stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + } + + /* Now write the rest */ + if (write(fd, data, len) != len) { + msg_err_map("cannot write file %s (data stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + + rspamd_file_unlock(fd, FALSE); + close(fd); + + msg_info_map("saved data from %s in %s, %uz bytes", bk->uri, path, len + sizeof(header) + header.etag_len); + + return TRUE; +} + +static gboolean +rspamd_map_update_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct http_map_data *htdata) +{ + gchar path[PATH_MAX]; + guchar digest[rspamd_cryptobox_HASHBYTES]; + struct rspamd_config *cfg = map->cfg; + gint fd; + struct rspamd_http_file_data header; + + if (!rspamd_map_has_http_cached_file(map, bk)) { + return FALSE; + } + + rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0); + rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir, + G_DIR_SEPARATOR, 20, digest); + + fd = rspamd_file_xopen(path, O_WRONLY, + 00600, FALSE); + + if (fd == -1) { + return FALSE; + } + + if (!rspamd_file_lock(fd, FALSE)) { + msg_err_map("cannot lock file %s: %s", path, strerror(errno)); + close(fd); + + return FALSE; + } + + memcpy(header.magic, rspamd_http_file_magic, sizeof(rspamd_http_file_magic)); + header.mtime = htdata->last_modified; + header.next_check = map->next_check; + header.data_off = sizeof(header); + + if (htdata->etag) { + header.data_off += RSPAMD_FSTRING_LEN(htdata->etag); + header.etag_len = RSPAMD_FSTRING_LEN(htdata->etag); + } + else { + header.etag_len = 0; + } + + if (write(fd, &header, sizeof(header)) != sizeof(header)) { + msg_err_map("cannot update file %s (header stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + + if (header.etag_len > 0) { + if (write(fd, RSPAMD_FSTRING_DATA(htdata->etag), header.etag_len) != + header.etag_len) { + msg_err_map("cannot update file %s (etag stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + } + + rspamd_file_unlock(fd, FALSE); + close(fd); + + return TRUE; +} + + +static gboolean +rspamd_map_read_http_cached_file(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct http_map_data *htdata, + struct map_cb_data *cbdata) +{ + gchar path[PATH_MAX]; + guchar digest[rspamd_cryptobox_HASHBYTES]; + struct rspamd_config *cfg = map->cfg; + gint fd; + struct stat st; + struct rspamd_http_file_data header; + + if (cfg->maps_cache_dir == NULL || cfg->maps_cache_dir[0] == '\0') { + return FALSE; + } + + rspamd_cryptobox_hash(digest, bk->uri, strlen(bk->uri), NULL, 0); + rspamd_snprintf(path, sizeof(path), "%s%c%*xs.map", cfg->maps_cache_dir, + G_DIR_SEPARATOR, 20, digest); + + fd = rspamd_file_xopen(path, O_RDONLY, 00600, FALSE); + + if (fd == -1) { + return FALSE; + } + + if (!rspamd_file_lock(fd, FALSE)) { + msg_err_map("cannot lock file %s: %s", path, strerror(errno)); + close(fd); + + return FALSE; + } + + (void) fstat(fd, &st); + + if (read(fd, &header, sizeof(header)) != sizeof(header)) { + msg_err_map("cannot read file %s (header stage): %s", path, strerror(errno)); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + + if (memcmp(header.magic, rspamd_http_file_magic, + sizeof(rspamd_http_file_magic)) != 0) { + msg_warn_map("invalid or old version magic in file %s; ignore it", path); + rspamd_file_unlock(fd, FALSE); + close(fd); + + return FALSE; + } + + double now = rspamd_get_calendar_ticks(); + + if (header.next_check > now) { + map->next_check = header.next_check; + } + else { + map->next_check = now; + } + + htdata->last_modified = header.mtime; + + if (header.etag_len > 0) { + rspamd_fstring_t *etag = rspamd_fstring_sized_new(header.etag_len); + + if (read(fd, RSPAMD_FSTRING_DATA(etag), header.etag_len) != header.etag_len) { + msg_err_map("cannot read file %s (etag stage): %s", path, + strerror(errno)); + rspamd_file_unlock(fd, FALSE); + rspamd_fstring_free(etag); + close(fd); + + return FALSE; + } + + etag->len = header.etag_len; + + if (htdata->etag) { + /* FIXME: should be dealt somehow better */ + msg_warn_map("etag is already defined as %V; cached is %V; ignore cached", + htdata->etag, etag); + rspamd_fstring_free(etag); + } + else { + htdata->etag = etag; + } + } + + rspamd_file_unlock(fd, FALSE); + close(fd); + + /* Now read file data */ + /* Perform buffered read: fail-safe */ + if (!read_map_file_chunks(map, cbdata, path, + st.st_size - header.data_off, header.data_off)) { + return FALSE; + } + + struct tm tm; + gchar ncheck_buf[32], lm_buf[32]; + + rspamd_localtime(map->next_check, &tm); + strftime(ncheck_buf, sizeof(ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm); + rspamd_localtime(htdata->last_modified, &tm); + strftime(lm_buf, sizeof(lm_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm); + + msg_info_map("read cached data for %s from %s, %uz bytes; next check at: %s;" + " last modified on: %s; etag: %V", + bk->uri, + path, + (size_t) (st.st_size - header.data_off), + ncheck_buf, + lm_buf, + htdata->etag); + + return TRUE; +} + +/** + * Async HTTP callback + */ +static void +rspamd_map_common_http_callback(struct rspamd_map *map, + struct rspamd_map_backend *bk, + struct map_periodic_cbdata *periodic, + gboolean check) +{ + struct http_map_data *data; + struct http_callback_data *cbd; + guint flags = RSPAMD_HTTP_CLIENT_SIMPLE | RSPAMD_HTTP_CLIENT_SHARED; + + data = bk->data.hd; + + if (g_atomic_int_get(&data->cache->available) == 1) { + /* Read cached data */ + if (check) { + if (data->last_modified < data->cache->last_modified) { + msg_info_map("need to reread cached map triggered by %s " + "(%d our modify time, %d cached modify time)", + bk->uri, + (int) data->last_modified, + (int) data->cache->last_modified); + periodic->need_modify = TRUE; + /* Reset the whole chain */ + periodic->cur_backend = 0; + rspamd_map_process_periodic(periodic); + } + else { + if (map->active_http) { + /* Check even if there is a cached version */ + goto check; + } + else { + /* Switch to the next backend */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); + } + } + + return; + } + else { + if (map->active_http && + data->last_modified > data->cache->last_modified) { + goto check; + } + else if (rspamd_map_read_cached(map, bk, periodic, data->host)) { + /* Switch to the next backend */ + periodic->cur_backend++; + data->last_modified = data->cache->last_modified; + rspamd_map_process_periodic(periodic); + + return; + } + } + } + else if (!map->active_http) { + /* Switch to the next backend */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); + + return; + } + +check: + cbd = g_malloc0(sizeof(struct http_callback_data)); + + cbd->event_loop = map->event_loop; + cbd->addrs = g_ptr_array_sized_new(4); + cbd->map = map; + cbd->data = data; + cbd->check = check; + cbd->periodic = periodic; + MAP_RETAIN(periodic, "periodic"); + cbd->bk = bk; + MAP_RETAIN(bk, "rspamd_map_backend"); + cbd->stage = http_map_terminated; + REF_INIT_RETAIN(cbd, free_http_cbdata); + + msg_debug_map("%s map data from %s", check ? "checking" : "reading", + data->host); + + /* Try address */ + rspamd_inet_addr_t *addr = NULL; + + if (rspamd_parse_inet_address(&addr, data->host, + strlen(data->host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + rspamd_inet_address_set_port(addr, cbd->data->port); + g_ptr_array_add(cbd->addrs, (void *) addr); + + if (bk->protocol == MAP_PROTO_HTTPS) { + flags |= RSPAMD_HTTP_CLIENT_SSL; + } + + cbd->conn = rspamd_http_connection_new_client( + NULL, + NULL, + http_map_error, + http_map_finish, + flags, + addr); + + if (cbd->conn != NULL) { + cbd->stage = http_map_http_conn; + write_http_request(cbd); + cbd->addr = addr; + MAP_RELEASE(cbd, "http_callback_data"); + } + else { + msg_warn_map("cannot load map: cannot connect to %s: %s", + data->host, strerror(errno)); + MAP_RELEASE(cbd, "http_callback_data"); + } + + return; + } + else if (map->r->r) { + /* Send both A and AAAA requests */ + guint nreq = 0; + + if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd, + map->cfg->dns_timeout, map->cfg->dns_retransmits, 1, + data->host, RDNS_REQUEST_A)) { + MAP_RETAIN(cbd, "http_callback_data"); + nreq++; + } + if (rdns_make_request_full(map->r->r, rspamd_map_dns_callback, cbd, + map->cfg->dns_timeout, map->cfg->dns_retransmits, 1, + data->host, RDNS_REQUEST_AAAA)) { + MAP_RETAIN(cbd, "http_callback_data"); + nreq++; + } + + if (nreq == 2) { + cbd->stage = http_map_resolve_host2; + } + else if (nreq == 1) { + cbd->stage = http_map_resolve_host1; + } + + map->tmp_dtor = free_http_cbdata_dtor; + map->tmp_dtor_data = cbd; + } + else { + msg_warn_map("cannot load map: DNS resolver is not initialized"); + cbd->periodic->errored = TRUE; + } + + MAP_RELEASE(cbd, "http_callback_data"); +} + +static void +rspamd_map_http_check_callback(struct map_periodic_cbdata *cbd) +{ + struct rspamd_map *map; + struct rspamd_map_backend *bk; + + map = cbd->map; + bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend); + + rspamd_map_common_http_callback(map, bk, cbd, TRUE); +} + +static void +rspamd_map_http_read_callback(struct map_periodic_cbdata *cbd) +{ + struct rspamd_map *map; + struct rspamd_map_backend *bk; + + map = cbd->map; + bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend); + rspamd_map_common_http_callback(map, bk, cbd, FALSE); +} + +static void +rspamd_map_file_check_callback(struct map_periodic_cbdata *periodic) +{ + struct rspamd_map *map; + struct file_map_data *data; + struct rspamd_map_backend *bk; + + map = periodic->map; + bk = g_ptr_array_index(map->backends, periodic->cur_backend); + data = bk->data.fd; + + if (data->need_modify) { + periodic->need_modify = TRUE; + periodic->cur_backend = 0; + data->need_modify = FALSE; + + rspamd_map_process_periodic(periodic); + + return; + } + + map = periodic->map; + /* Switch to the next backend as the rest is handled by ev_stat */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); +} + +static void +rspamd_map_static_check_callback(struct map_periodic_cbdata *periodic) +{ + struct rspamd_map *map; + struct static_map_data *data; + struct rspamd_map_backend *bk; + + map = periodic->map; + bk = g_ptr_array_index(map->backends, periodic->cur_backend); + data = bk->data.sd; + + if (!data->processed) { + periodic->need_modify = TRUE; + periodic->cur_backend = 0; + + rspamd_map_process_periodic(periodic); + + return; + } + + /* Switch to the next backend */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); +} + +static void +rspamd_map_file_read_callback(struct map_periodic_cbdata *periodic) +{ + struct rspamd_map *map; + struct file_map_data *data; + struct rspamd_map_backend *bk; + + map = periodic->map; + + bk = g_ptr_array_index(map->backends, periodic->cur_backend); + data = bk->data.fd; + + msg_info_map("rereading map file %s", data->filename); + + if (!read_map_file(map, data, bk, periodic)) { + periodic->errored = TRUE; + } + + /* Switch to the next backend */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); +} + +static void +rspamd_map_static_read_callback(struct map_periodic_cbdata *periodic) +{ + struct rspamd_map *map; + struct static_map_data *data; + struct rspamd_map_backend *bk; + + map = periodic->map; + + bk = g_ptr_array_index(map->backends, periodic->cur_backend); + data = bk->data.sd; + + msg_info_map("rereading static map"); + + if (!read_map_static(map, data, bk, periodic)) { + periodic->errored = TRUE; + } + + /* Switch to the next backend */ + periodic->cur_backend++; + rspamd_map_process_periodic(periodic); +} + +static void +rspamd_map_process_periodic(struct map_periodic_cbdata *cbd) +{ + struct rspamd_map_backend *bk; + struct rspamd_map *map; + + map = cbd->map; + map->scheduled_check = NULL; + + if (!map->file_only && !cbd->locked) { + if (!g_atomic_int_compare_and_exchange(cbd->map->locked, + 0, 1)) { + msg_debug_map( + "don't try to reread map %s as it is locked by other process, " + "will reread it later", + cbd->map->name); + rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_LOCKED); + MAP_RELEASE(cbd, "periodic"); + + return; + } + else { + msg_debug_map("locked map %s", cbd->map->name); + cbd->locked = TRUE; + } + } + + if (cbd->errored) { + /* We should not check other backends if some backend has failed*/ + rspamd_map_schedule_periodic(cbd->map, RSPAMD_MAP_SCHEDULE_ERROR); + + if (cbd->locked) { + g_atomic_int_set(cbd->map->locked, 0); + cbd->locked = FALSE; + } + + /* Also set error flag for the map consumer */ + cbd->cbdata.errored = true; + + msg_debug_map("unlocked map %s, refcount=%d", cbd->map->name, + cbd->ref.refcount); + MAP_RELEASE(cbd, "periodic"); + + return; + } + + /* For each backend we need to check for modifications */ + if (cbd->cur_backend >= cbd->map->backends->len) { + /* Last backend */ + msg_debug_map("finished map: %d of %d", cbd->cur_backend, + cbd->map->backends->len); + MAP_RELEASE(cbd, "periodic"); + + return; + } + + if (cbd->map->wrk && cbd->map->wrk->state == rspamd_worker_state_running) { + bk = g_ptr_array_index(cbd->map->backends, cbd->cur_backend); + g_assert(bk != NULL); + + if (cbd->need_modify) { + /* Load data from the next backend */ + switch (bk->protocol) { + case MAP_PROTO_HTTP: + case MAP_PROTO_HTTPS: + rspamd_map_http_read_callback(cbd); + break; + case MAP_PROTO_FILE: + rspamd_map_file_read_callback(cbd); + break; + case MAP_PROTO_STATIC: + rspamd_map_static_read_callback(cbd); + break; + } + } + else { + /* Check the next backend */ + switch (bk->protocol) { + case MAP_PROTO_HTTP: + case MAP_PROTO_HTTPS: + rspamd_map_http_check_callback(cbd); + break; + case MAP_PROTO_FILE: + rspamd_map_file_check_callback(cbd); + break; + case MAP_PROTO_STATIC: + rspamd_map_static_check_callback(cbd); + break; + } + } + } +} + +static void +rspamd_map_on_stat(struct ev_loop *loop, ev_stat *w, int revents) +{ + struct rspamd_map *map = (struct rspamd_map *) w->data; + + if (w->attr.st_nlink > 0) { + msg_info_map("old mtime is %t (size = %Hz), " + "new mtime is %t (size = %Hz) for map file %s", + w->prev.st_mtime, (gsize) w->prev.st_size, + w->attr.st_mtime, (gsize) w->attr.st_size, + w->path); + + /* Fire need modify flag */ + struct rspamd_map_backend *bk; + guint i; + + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + if (bk->protocol == MAP_PROTO_FILE) { + bk->data.fd->need_modify = TRUE; + } + } + + map->next_check = 0; + + if (map->scheduled_check) { + ev_timer_stop(map->event_loop, &map->scheduled_check->ev); + MAP_RELEASE(map->scheduled_check, "rspamd_map_on_stat"); + map->scheduled_check = NULL; + } + + rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT); + } +} + +/* Start watching event for all maps */ +void rspamd_map_watch(struct rspamd_config *cfg, + struct ev_loop *event_loop, + struct rspamd_dns_resolver *resolver, + struct rspamd_worker *worker, + enum rspamd_map_watch_type how) +{ + GList *cur = cfg->maps; + struct rspamd_map *map; + struct rspamd_map_backend *bk; + guint i; + + g_assert(how > RSPAMD_MAP_WATCH_MIN && how < RSPAMD_MAP_WATCH_MAX); + + /* First of all do synced read of data */ + while (cur) { + map = cur->data; + map->event_loop = event_loop; + map->r = resolver; + + if (map->wrk == NULL && how != RSPAMD_MAP_WATCH_WORKER) { + /* Generic scanner map */ + map->wrk = worker; + + if (how == RSPAMD_MAP_WATCH_PRIMARY_CONTROLLER) { + map->active_http = TRUE; + } + else { + map->active_http = FALSE; + } + } + else if (map->wrk != NULL && map->wrk == worker) { + /* Map is bound to a specific worker */ + map->active_http = TRUE; + } + else { + /* Skip map for this worker as irrelevant */ + cur = g_list_next(cur); + continue; + } + + if (!map->active_http) { + /* Check cached version more frequently as it is cheap */ + + if (map->poll_timeout >= cfg->map_timeout && + cfg->map_file_watch_multiplier < 1.0) { + map->poll_timeout = + map->poll_timeout * cfg->map_file_watch_multiplier; + } + } + + map->file_only = TRUE; + map->static_only = TRUE; + + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + bk->event_loop = event_loop; + + if (bk->protocol == MAP_PROTO_FILE) { + struct file_map_data *data; + + data = bk->data.fd; + + if (map->user_data == NULL || *map->user_data == NULL) { + /* Map has not been read, init it's reading if possible */ + struct stat st; + + if (stat(data->filename, &st) != -1) { + data->need_modify = TRUE; + } + } + + ev_stat_init(&data->st_ev, rspamd_map_on_stat, + data->filename, map->poll_timeout * cfg->map_file_watch_multiplier); + data->st_ev.data = map; + ev_stat_start(event_loop, &data->st_ev); + map->static_only = FALSE; + } + else if ((bk->protocol == MAP_PROTO_HTTP || + bk->protocol == MAP_PROTO_HTTPS)) { + if (map->active_http) { + map->non_trivial = TRUE; + } + + map->static_only = FALSE; + map->file_only = FALSE; + } + } + + rspamd_map_schedule_periodic(map, RSPAMD_MAP_SCHEDULE_INIT); + + cur = g_list_next(cur); + } +} + +void rspamd_map_preload(struct rspamd_config *cfg) +{ + GList *cur = cfg->maps; + struct rspamd_map *map; + struct rspamd_map_backend *bk; + guint i; + gboolean map_ok; + + /* First of all do synced read of data */ + while (cur) { + map = cur->data; + map_ok = TRUE; + + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + if (!(bk->protocol == MAP_PROTO_FILE || + bk->protocol == MAP_PROTO_STATIC)) { + + if (bk->protocol == MAP_PROTO_HTTP || + bk->protocol == MAP_PROTO_HTTPS) { + if (!rspamd_map_has_http_cached_file(map, bk)) { + + if (!map->fallback_backend) { + map_ok = FALSE; + } + break; + } + else { + continue; /* We are yet fine */ + } + } + map_ok = FALSE; + break; + } + } + + if (map_ok) { + struct map_periodic_cbdata fake_cbd; + gboolean succeed = TRUE; + + memset(&fake_cbd, 0, sizeof(fake_cbd)); + fake_cbd.cbdata.state = 0; + fake_cbd.cbdata.prev_data = *map->user_data; + fake_cbd.cbdata.cur_data = NULL; + fake_cbd.cbdata.map = map; + fake_cbd.map = map; + + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + fake_cbd.cur_backend = i; + + if (bk->protocol == MAP_PROTO_FILE) { + if (!read_map_file(map, bk->data.fd, bk, &fake_cbd)) { + succeed = FALSE; + break; + } + } + else if (bk->protocol == MAP_PROTO_STATIC) { + if (!read_map_static(map, bk->data.sd, bk, &fake_cbd)) { + succeed = FALSE; + break; + } + } + else if (bk->protocol == MAP_PROTO_HTTP || + bk->protocol == MAP_PROTO_HTTPS) { + if (!rspamd_map_read_http_cached_file(map, bk, bk->data.hd, + &fake_cbd.cbdata)) { + + if (map->fallback_backend) { + /* Try fallback */ + g_assert(map->fallback_backend->protocol == + MAP_PROTO_FILE); + if (!read_map_file(map, + map->fallback_backend->data.fd, + map->fallback_backend, &fake_cbd)) { + succeed = FALSE; + break; + } + } + else { + succeed = FALSE; + break; + } + } + } + else { + g_assert_not_reached(); + } + } + + if (succeed) { + map->fin_callback(&fake_cbd.cbdata, map->user_data); + + if (map->on_load_function) { + map->on_load_function(map, map->on_load_ud); + } + } + else { + msg_info_map("preload of %s failed", map->name); + } + } + + cur = g_list_next(cur); + } +} + +void rspamd_map_remove_all(struct rspamd_config *cfg) +{ + struct rspamd_map *map; + GList *cur; + struct rspamd_map_backend *bk; + struct map_cb_data cbdata; + guint i; + + for (cur = cfg->maps; cur != NULL; cur = g_list_next(cur)) { + map = cur->data; + + if (map->tmp_dtor) { + map->tmp_dtor(map->tmp_dtor_data); + } + + if (map->dtor) { + cbdata.prev_data = NULL; + cbdata.map = map; + cbdata.cur_data = *map->user_data; + + map->dtor(&cbdata); + *map->user_data = NULL; + } + + if (map->on_load_ud_dtor) { + map->on_load_ud_dtor(map->on_load_ud); + } + + for (i = 0; i < map->backends->len; i++) { + bk = g_ptr_array_index(map->backends, i); + + MAP_RELEASE(bk, "rspamd_map_backend"); + } + + if (map->fallback_backend) { + MAP_RELEASE(map->fallback_backend, "rspamd_map_backend"); + } + } + + g_list_free(cfg->maps); + cfg->maps = NULL; +} + +static const gchar * +rspamd_map_check_proto(struct rspamd_config *cfg, + const gchar *map_line, struct rspamd_map_backend *bk) +{ + const gchar *pos = map_line, *end, *end_key; + + g_assert(bk != NULL); + g_assert(pos != NULL); + + end = pos + strlen(pos); + + /* Static check */ + if (g_ascii_strcasecmp(pos, "static") == 0) { + bk->protocol = MAP_PROTO_STATIC; + bk->uri = g_strdup(pos); + + return pos; + } + else if (g_ascii_strcasecmp(pos, "zst+static") == 0) { + bk->protocol = MAP_PROTO_STATIC; + bk->uri = g_strdup(pos + 4); + bk->is_compressed = TRUE; + + return pos + 4; + } + + for (;;) { + if (g_ascii_strncasecmp(pos, "sign+", sizeof("sign+") - 1) == 0) { + bk->is_signed = TRUE; + pos += sizeof("sign+") - 1; + } + else if (g_ascii_strncasecmp(pos, "fallback+", sizeof("fallback+") - 1) == 0) { + bk->is_fallback = TRUE; + pos += sizeof("fallback+") - 1; + } + else if (g_ascii_strncasecmp(pos, "key=", sizeof("key=") - 1) == 0) { + pos += sizeof("key=") - 1; + end_key = memchr(pos, '+', end - pos); + + if (end_key != NULL) { + bk->trusted_pubkey = rspamd_pubkey_from_base32(pos, end_key - pos, + RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); + + if (bk->trusted_pubkey == NULL) { + msg_err_config("cannot read pubkey from map: %s", + map_line); + return NULL; + } + pos = end_key + 1; + } + else if (end - pos > 64) { + /* Try hex encoding */ + bk->trusted_pubkey = rspamd_pubkey_from_hex(pos, 64, + RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); + + if (bk->trusted_pubkey == NULL) { + msg_err_config("cannot read pubkey from map: %s", + map_line); + return NULL; + } + pos += 64; + } + else { + msg_err_config("cannot read pubkey from map: %s", + map_line); + return NULL; + } + + if (*pos == '+' || *pos == ':') { + pos++; + } + } + else { + /* No known flags */ + break; + } + } + + bk->protocol = MAP_PROTO_FILE; + + if (g_ascii_strncasecmp(pos, "http://", sizeof("http://") - 1) == 0) { + bk->protocol = MAP_PROTO_HTTP; + /* Include http:// */ + bk->uri = g_strdup(pos); + pos += sizeof("http://") - 1; + } + else if (g_ascii_strncasecmp(pos, "https://", sizeof("https://") - 1) == 0) { + bk->protocol = MAP_PROTO_HTTPS; + /* Include https:// */ + bk->uri = g_strdup(pos); + pos += sizeof("https://") - 1; + } + else if (g_ascii_strncasecmp(pos, "file://", sizeof("file://") - 1) == 0) { + pos += sizeof("file://") - 1; + /* Exclude file:// */ + bk->uri = g_strdup(pos); + } + else if (*pos == '/') { + /* Trivial file case */ + bk->uri = g_strdup(pos); + } + else { + msg_err_config("invalid map fetching protocol: %s", map_line); + + return NULL; + } + + if (bk->protocol != MAP_PROTO_FILE && bk->is_signed) { + msg_err_config("signed maps are no longer supported for HTTP(s): %s", map_line); + } + + return pos; +} + +gboolean +rspamd_map_is_map(const gchar *map_line) +{ + gboolean ret = FALSE; + + g_assert(map_line != NULL); + + if (map_line[0] == '/') { + ret = TRUE; + } + else if (g_ascii_strncasecmp(map_line, "sign+", sizeof("sign+") - 1) == 0) { + ret = TRUE; + } + else if (g_ascii_strncasecmp(map_line, "fallback+", sizeof("fallback+") - 1) == 0) { + ret = TRUE; + } + else if (g_ascii_strncasecmp(map_line, "file://", sizeof("file://") - 1) == 0) { + ret = TRUE; + } + else if (g_ascii_strncasecmp(map_line, "http://", sizeof("http://") - 1) == 0) { + ret = TRUE; + } + else if (g_ascii_strncasecmp(map_line, "https://", sizeof("https://") - 1) == 0) { + ret = TRUE; + } + + return ret; +} + +static void +rspamd_map_backend_dtor(struct rspamd_map_backend *bk) +{ + switch (bk->protocol) { + case MAP_PROTO_FILE: + if (bk->data.fd) { + ev_stat_stop(bk->event_loop, &bk->data.fd->st_ev); + g_free(bk->data.fd->filename); + g_free(bk->data.fd); + } + break; + case MAP_PROTO_STATIC: + if (bk->data.sd) { + if (bk->data.sd->data) { + g_free(bk->data.sd->data); + } + + g_free(bk->data.sd); + } + break; + case MAP_PROTO_HTTP: + case MAP_PROTO_HTTPS: + if (bk->data.hd) { + struct http_map_data *data = bk->data.hd; + + g_free(data->host); + g_free(data->path); + g_free(data->rest); + + if (data->userinfo) { + g_free(data->userinfo); + } + + if (data->etag) { + rspamd_fstring_free(data->etag); + } + + /* + * Clear cached file, but check if a worker is an active http worker + * as cur_cache_cbd is meaningful merely for active worker, who actually + * owns the cache + */ + if (bk->map && bk->map->active_http) { + if (g_atomic_int_compare_and_exchange(&data->cache->available, 1, 0)) { + if (data->cur_cache_cbd) { + msg_info("clear shared memory cache for a map in %s as backend \"%s\" is closing", + data->cur_cache_cbd->shm->shm_name, + bk->uri); + MAP_RELEASE(data->cur_cache_cbd->shm, + "rspamd_http_map_cached_cbdata"); + ev_timer_stop(data->cur_cache_cbd->event_loop, + &data->cur_cache_cbd->timeout); + g_free(data->cur_cache_cbd); + data->cur_cache_cbd = NULL; + } + } + } + + g_free(bk->data.hd); + } + break; + } + + if (bk->trusted_pubkey) { + rspamd_pubkey_unref(bk->trusted_pubkey); + } + + g_free(bk->uri); + g_free(bk); +} + +static struct rspamd_map_backend * +rspamd_map_parse_backend(struct rspamd_config *cfg, const gchar *map_line) +{ + struct rspamd_map_backend *bk; + struct file_map_data *fdata = NULL; + struct http_map_data *hdata = NULL; + struct static_map_data *sdata = NULL; + struct http_parser_url up; + const gchar *end, *p; + rspamd_ftok_t tok; + + bk = g_malloc0(sizeof(*bk)); + REF_INIT_RETAIN(bk, rspamd_map_backend_dtor); + + if (!rspamd_map_check_proto(cfg, map_line, bk)) { + goto err; + } + + if (bk->is_fallback && bk->protocol != MAP_PROTO_FILE) { + msg_err_config("fallback backend must be file for %s", bk->uri); + + goto err; + } + + end = map_line + strlen(map_line); + if (end - map_line > 5) { + p = end - 5; + if (g_ascii_strcasecmp(p, ".zstd") == 0) { + bk->is_compressed = TRUE; + } + p = end - 4; + if (g_ascii_strcasecmp(p, ".zst") == 0) { + bk->is_compressed = TRUE; + } + } + + /* Now check for each proto separately */ + if (bk->protocol == MAP_PROTO_FILE) { + fdata = g_malloc0(sizeof(struct file_map_data)); + + if (access(bk->uri, R_OK) == -1) { + if (errno != ENOENT) { + msg_err_config("cannot open file '%s': %s", bk->uri, strerror(errno)); + goto err; + } + + msg_info_config( + "map '%s' is not found, but it can be loaded automatically later", + bk->uri); + } + + fdata->filename = g_strdup(bk->uri); + bk->data.fd = fdata; + } + else if (bk->protocol == MAP_PROTO_HTTP || bk->protocol == MAP_PROTO_HTTPS) { + hdata = g_malloc0(sizeof(struct http_map_data)); + + memset(&up, 0, sizeof(up)); + if (http_parser_parse_url(bk->uri, strlen(bk->uri), FALSE, + &up) != 0) { + msg_err_config("cannot parse HTTP url: %s", bk->uri); + goto err; + } + else { + if (!(up.field_set & 1u << UF_HOST)) { + msg_err_config("cannot parse HTTP url: %s: no host", bk->uri); + goto err; + } + + tok.begin = bk->uri + up.field_data[UF_HOST].off; + tok.len = up.field_data[UF_HOST].len; + hdata->host = rspamd_ftokdup(&tok); + + if (up.field_set & (1u << UF_PORT)) { + hdata->port = up.port; + } + else { + if (bk->protocol == MAP_PROTO_HTTP) { + hdata->port = 80; + } + else { + hdata->port = 443; + } + } + + if (up.field_set & (1u << UF_PATH)) { + tok.begin = bk->uri + up.field_data[UF_PATH].off; + tok.len = up.field_data[UF_PATH].len; + + hdata->path = rspamd_ftokdup(&tok); + + /* We also need to check query + fragment */ + if (up.field_set & ((1u << UF_QUERY) | (1u << UF_FRAGMENT))) { + tok.begin = bk->uri + up.field_data[UF_PATH].off + + up.field_data[UF_PATH].len; + tok.len = strlen(tok.begin); + hdata->rest = rspamd_ftokdup(&tok); + } + else { + hdata->rest = g_strdup(""); + } + } + + if (up.field_set & (1u << UF_USERINFO)) { + /* Create authorisation header for basic auth */ + guint len = sizeof("Basic ") + + up.field_data[UF_USERINFO].len * 8 / 5 + 4; + hdata->userinfo = g_malloc(len); + rspamd_snprintf(hdata->userinfo, len, "Basic %*Bs", + (int) up.field_data[UF_USERINFO].len, + bk->uri + up.field_data[UF_USERINFO].off); + + msg_debug("added userinfo for the map from the URL: %s", hdata->host); + } + else { + /* Try to obtain authentication data from options in the configuration */ + const ucl_object_t *auth_obj, *opts_obj; + + opts_obj = ucl_object_lookup(cfg->cfg_ucl_obj, "options"); + if (opts_obj != NULL) { + auth_obj = ucl_object_lookup(opts_obj, "http_auth"); + if (auth_obj != NULL && ucl_object_type(auth_obj) == UCL_OBJECT) { + const ucl_object_t *host_obj; + + /* + * Search first by the full URL and then by the host part + */ + host_obj = ucl_object_lookup(auth_obj, map_line); + + if (host_obj == NULL) { + host_obj = ucl_object_lookup(auth_obj, hdata->host); + } + + if (host_obj != NULL && ucl_object_type(host_obj) == UCL_OBJECT) { + const ucl_object_t *user_obj, *password_obj; + + user_obj = ucl_object_lookup(host_obj, "user"); + password_obj = ucl_object_lookup(host_obj, "password"); + + if (user_obj != NULL && password_obj != NULL && + ucl_object_type(user_obj) == UCL_STRING && + ucl_object_type(password_obj) == UCL_STRING) { + + gchar *tmpbuf; + unsigned tlen; + + /* User + password + ':' */ + tlen = strlen(ucl_object_tostring(user_obj)) + + strlen(ucl_object_tostring(password_obj)) + 1; + tmpbuf = g_malloc(tlen + 1); + rspamd_snprintf(tmpbuf, tlen + 1, "%s:%s", + ucl_object_tostring(user_obj), + ucl_object_tostring(password_obj)); + /* Base64 encoding is not so greedy, but we add some space for simplicity */ + tlen *= 2; + tlen += sizeof("Basic ") - 1; + hdata->userinfo = g_malloc(tlen + 1); + rspamd_snprintf(hdata->userinfo, tlen + 1, "Basic %Bs", tmpbuf); + g_free(tmpbuf); + msg_debug("added userinfo for the map from the configuration: %s", map_line); + } + } + } + } + } + } + + hdata->cache = rspamd_mempool_alloc0_shared(cfg->cfg_pool, + sizeof(*hdata->cache)); + + bk->data.hd = hdata; + } + else if (bk->protocol == MAP_PROTO_STATIC) { + sdata = g_malloc0(sizeof(*sdata)); + bk->data.sd = sdata; + } + + bk->id = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_T1HA, + bk->uri, strlen(bk->uri), + 0xdeadbabe); + + return bk; + +err: + MAP_RELEASE(bk, "rspamd_map_backend"); + + if (hdata) { + g_free(hdata); + } + + if (fdata) { + g_free(fdata); + } + + if (sdata) { + g_free(sdata); + } + + return NULL; +} + +static void +rspamd_map_calculate_hash(struct rspamd_map *map) +{ + struct rspamd_map_backend *bk; + guint i; + rspamd_cryptobox_hash_state_t st; + gchar *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES]; + + rspamd_cryptobox_hash_init(&st, NULL, 0); + + for (i = 0; i < map->backends->len; i++) { + bk = g_ptr_array_index(map->backends, i); + rspamd_cryptobox_hash_update(&st, bk->uri, strlen(bk->uri)); + } + + rspamd_cryptobox_hash_final(&st, cksum); + cksum_encoded = rspamd_encode_base32(cksum, sizeof(cksum), RSPAMD_BASE32_DEFAULT); + rspamd_strlcpy(map->tag, cksum_encoded, sizeof(map->tag)); + g_free(cksum_encoded); +} + +static gboolean +rspamd_map_add_static_string(struct rspamd_config *cfg, + const ucl_object_t *elt, + GString *target) +{ + gsize sz; + const gchar *dline; + + if (ucl_object_type(elt) != UCL_STRING) { + msg_err_config("map has static backend but `data` is " + "not string like: %s", + ucl_object_type_to_string(elt->type)); + return FALSE; + } + + /* Otherwise, we copy data to the backend */ + dline = ucl_object_tolstring(elt, &sz); + + if (sz == 0) { + msg_err_config("map has static backend but empty no data"); + return FALSE; + } + + g_string_append_len(target, dline, sz); + g_string_append_c(target, '\n'); + + return TRUE; +} + +struct rspamd_map * +rspamd_map_add(struct rspamd_config *cfg, + const gchar *map_line, + const gchar *description, + map_cb_t read_callback, + map_fin_cb_t fin_callback, + map_dtor_t dtor, + void **user_data, + struct rspamd_worker *worker, + int flags) +{ + struct rspamd_map *map; + struct rspamd_map_backend *bk; + + bk = rspamd_map_parse_backend(cfg, map_line); + if (bk == NULL) { + return NULL; + } + + if (bk->is_fallback) { + msg_err_config("cannot add map with fallback only backend: %s", bk->uri); + REF_RELEASE(bk); + + return NULL; + } + + map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map)); + map->read_callback = read_callback; + map->fin_callback = fin_callback; + map->dtor = dtor; + map->user_data = user_data; + map->cfg = cfg; + map->id = rspamd_random_uint64_fast(); + map->locked = + rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(gint)); + map->backends = g_ptr_array_sized_new(1); + map->wrk = worker; + rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, + map->backends); + g_ptr_array_add(map->backends, bk); + map->name = rspamd_mempool_strdup(cfg->cfg_pool, map_line); + map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ); + + if (bk->protocol == MAP_PROTO_FILE) { + map->poll_timeout = (cfg->map_timeout * cfg->map_file_watch_multiplier); + } + else { + map->poll_timeout = cfg->map_timeout; + } + + if (description != NULL) { + map->description = rspamd_mempool_strdup(cfg->cfg_pool, description); + } + + rspamd_map_calculate_hash(map); + msg_info_map("added map %s", bk->uri); + bk->map = map; + + cfg->maps = g_list_prepend(cfg->maps, map); + + return map; +} + +struct rspamd_map * +rspamd_map_add_fake(struct rspamd_config *cfg, + const gchar *description, + const gchar *name) +{ + struct rspamd_map *map; + + map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map)); + map->cfg = cfg; + map->id = rspamd_random_uint64_fast(); + map->name = rspamd_mempool_strdup(cfg->cfg_pool, name); + map->user_data = (void **) ↦ /* to prevent null pointer dereferencing */ + + if (description != NULL) { + map->description = rspamd_mempool_strdup(cfg->cfg_pool, description); + } + + return map; +} + +static inline void +rspamd_map_add_backend(struct rspamd_map *map, struct rspamd_map_backend *bk) +{ + if (bk->is_fallback) { + if (map->fallback_backend) { + msg_warn_map("redefining fallback backend from %s to %s", + map->fallback_backend->uri, bk->uri); + } + + map->fallback_backend = bk; + } + else { + g_ptr_array_add(map->backends, bk); + } + + bk->map = map; +} + +struct rspamd_map * +rspamd_map_add_from_ucl(struct rspamd_config *cfg, + const ucl_object_t *obj, + const gchar *description, + map_cb_t read_callback, + map_fin_cb_t fin_callback, + map_dtor_t dtor, + void **user_data, + struct rspamd_worker *worker, + gint flags) +{ + ucl_object_iter_t it = NULL; + const ucl_object_t *cur, *elt; + struct rspamd_map *map; + struct rspamd_map_backend *bk; + guint i; + + g_assert(obj != NULL); + + if (ucl_object_type(obj) == UCL_STRING) { + /* Just a plain string */ + return rspamd_map_add(cfg, ucl_object_tostring(obj), description, + read_callback, fin_callback, dtor, user_data, worker, flags); + } + + map = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct rspamd_map)); + map->read_callback = read_callback; + map->fin_callback = fin_callback; + map->dtor = dtor; + map->user_data = user_data; + map->cfg = cfg; + map->id = rspamd_random_uint64_fast(); + map->locked = + rspamd_mempool_alloc0_shared(cfg->cfg_pool, sizeof(gint)); + map->backends = g_ptr_array_new(); + map->wrk = worker; + map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ); + rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard, + map->backends); + map->poll_timeout = cfg->map_timeout; + + if (description) { + map->description = rspamd_mempool_strdup(cfg->cfg_pool, description); + } + + if (ucl_object_type(obj) == UCL_ARRAY) { + /* Add array of maps as multiple backends */ + while ((cur = ucl_object_iterate(obj, &it, true)) != NULL) { + if (ucl_object_type(cur) == UCL_STRING) { + bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur)); + + if (bk != NULL) { + rspamd_map_add_backend(map, bk); + + if (!map->name) { + map->name = rspamd_mempool_strdup(cfg->cfg_pool, + ucl_object_tostring(cur)); + } + } + } + else { + msg_err_config("bad map element type: %s", + ucl_object_type_to_string(ucl_object_type(cur))); + } + } + + if (map->backends->len == 0) { + msg_err_config("map has no urls to be loaded: empty list"); + goto err; + } + } + else if (ucl_object_type(obj) == UCL_OBJECT) { + elt = ucl_object_lookup(obj, "name"); + if (elt && ucl_object_type(elt) == UCL_STRING) { + map->name = rspamd_mempool_strdup(cfg->cfg_pool, + ucl_object_tostring(elt)); + } + + elt = ucl_object_lookup(obj, "description"); + if (elt && ucl_object_type(elt) == UCL_STRING) { + map->description = rspamd_mempool_strdup(cfg->cfg_pool, + ucl_object_tostring(elt)); + } + + elt = ucl_object_lookup_any(obj, "timeout", "poll", "poll_time", + "watch_interval", NULL); + if (elt) { + map->poll_timeout = ucl_object_todouble(elt); + } + + elt = ucl_object_lookup_any(obj, "upstreams", "url", "urls", NULL); + if (elt == NULL) { + msg_err_config("map has no urls to be loaded: no elt"); + goto err; + } + + if (ucl_object_type(elt) == UCL_ARRAY) { + /* Add array of maps as multiple backends */ + it = ucl_object_iterate_new(elt); + + while ((cur = ucl_object_iterate_safe(it, true)) != NULL) { + if (ucl_object_type(cur) == UCL_STRING) { + bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(cur)); + + if (bk != NULL) { + rspamd_map_add_backend(map, bk); + + if (!map->name) { + map->name = rspamd_mempool_strdup(cfg->cfg_pool, + ucl_object_tostring(cur)); + } + } + } + else { + msg_err_config("bad map element type: %s", + ucl_object_type_to_string(ucl_object_type(cur))); + ucl_object_iterate_free(it); + goto err; + } + } + + ucl_object_iterate_free(it); + + if (map->backends->len == 0) { + msg_err_config("map has no urls to be loaded: empty object list"); + goto err; + } + } + else if (ucl_object_type(elt) == UCL_STRING) { + bk = rspamd_map_parse_backend(cfg, ucl_object_tostring(elt)); + + if (bk != NULL) { + rspamd_map_add_backend(map, bk); + + if (!map->name) { + map->name = rspamd_mempool_strdup(cfg->cfg_pool, + ucl_object_tostring(elt)); + } + } + } + + if (!map->backends || map->backends->len == 0) { + msg_err_config("map has no urls to be loaded: no valid backends"); + goto err; + } + } + else { + msg_err_config("map has invalid type for value: %s", + ucl_object_type_to_string(ucl_object_type(obj))); + goto err; + } + + gboolean all_local = TRUE; + + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + if (bk->protocol == MAP_PROTO_STATIC) { + GString *map_data; + /* We need data field in ucl */ + elt = ucl_object_lookup(obj, "data"); + + if (elt == NULL) { + msg_err_config("map has static backend but no `data` field"); + goto err; + } + + + if (ucl_object_type(elt) == UCL_STRING) { + map_data = g_string_sized_new(32); + + if (rspamd_map_add_static_string(cfg, elt, map_data)) { + bk->data.sd->data = map_data->str; + bk->data.sd->len = map_data->len; + g_string_free(map_data, FALSE); + } + else { + g_string_free(map_data, TRUE); + msg_err_config("map has static backend with invalid `data` field"); + goto err; + } + } + else if (ucl_object_type(elt) == UCL_ARRAY) { + map_data = g_string_sized_new(32); + it = ucl_object_iterate_new(elt); + + while ((cur = ucl_object_iterate_safe(it, true))) { + if (!rspamd_map_add_static_string(cfg, cur, map_data)) { + g_string_free(map_data, TRUE); + msg_err_config("map has static backend with invalid " + "`data` field"); + ucl_object_iterate_free(it); + goto err; + } + } + + ucl_object_iterate_free(it); + bk->data.sd->data = map_data->str; + bk->data.sd->len = map_data->len; + g_string_free(map_data, FALSE); + } + } + else if (bk->protocol != MAP_PROTO_FILE) { + all_local = FALSE; + } + } + + if (all_local) { + map->poll_timeout = (map->poll_timeout * + cfg->map_file_watch_multiplier); + } + + rspamd_map_calculate_hash(map); + msg_debug_map("added map from ucl"); + + cfg->maps = g_list_prepend(cfg->maps, map); + + return map; + +err: + + if (map) { + PTR_ARRAY_FOREACH(map->backends, i, bk) + { + MAP_RELEASE(bk, "rspamd_map_backend"); + } + } + + return NULL; +} + +rspamd_map_traverse_function +rspamd_map_get_traverse_function(struct rspamd_map *map) +{ + if (map) { + return map->traverse_function; + } + + return NULL; +} + +void rspamd_map_traverse(struct rspamd_map *map, rspamd_map_traverse_cb cb, + gpointer cbdata, gboolean reset_hits) +{ + if (*map->user_data && map->traverse_function) { + map->traverse_function(*map->user_data, cb, cbdata, reset_hits); + } +} + +void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_function cb, + gpointer cbdata, GDestroyNotify dtor) +{ + if (map) { + map->on_load_function = cb; + map->on_load_ud = cbdata; + map->on_load_ud_dtor = dtor; + } +} -- cgit v1.2.3