diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 08:41:51 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-13 08:41:51 +0000 |
commit | 3e160e27e4686620d16477a9ea9cf00141e52ce7 (patch) | |
tree | 884561d26afa36d7653aa4dc43410e1ae479d43e /src/global | |
parent | Adding upstream version 3.8.6. (diff) | |
download | postfix-88041ab92eed134a20655cccac118a666f236db5.tar.xz postfix-88041ab92eed134a20655cccac118a666f236db5.zip |
Adding upstream version 3.9.0.upstream/3.9.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/global')
-rw-r--r-- | src/global/Makefile.in | 31 | ||||
-rw-r--r-- | src/global/dict_ldap.c | 4 | ||||
-rw-r--r-- | src/global/dict_mongodb.c | 570 | ||||
-rwxr-xr-x | src/global/dict_mongodb.h | 43 | ||||
-rw-r--r-- | src/global/dict_mysql.c | 87 | ||||
-rw-r--r-- | src/global/dict_pgsql.c | 73 | ||||
-rw-r--r-- | src/global/dict_sqlite.c | 2 | ||||
-rw-r--r-- | src/global/mail_addr_find.c | 4 | ||||
-rw-r--r-- | src/global/mail_date.c | 9 | ||||
-rw-r--r-- | src/global/mail_dict.c | 4 | ||||
-rw-r--r-- | src/global/mail_params.c | 19 | ||||
-rw-r--r-- | src/global/mail_params.h | 32 | ||||
-rw-r--r-- | src/global/mail_proto.h | 7 | ||||
-rw-r--r-- | src/global/mail_version.h | 4 | ||||
-rw-r--r-- | src/global/maillog_client.c | 2 | ||||
-rw-r--r-- | src/global/maps.c | 12 | ||||
-rw-r--r-- | src/global/wildcard_inet_addr.c | 2 |
17 files changed, 811 insertions, 94 deletions
diff --git a/src/global/Makefile.in b/src/global/Makefile.in index 86390ed..c7a1d36 100644 --- a/src/global/Makefile.in +++ b/src/global/Makefile.in @@ -3,7 +3,7 @@ SRCS = abounce.c anvil_clnt.c been_here.c bounce.c bounce_log.c \ canon_addr.c cfg_parser.c cleanup_strerror.c cleanup_strflags.c \ clnt_stream.c conv_time.c db_common.c debug_peer.c debug_process.c \ defer.c deliver_completed.c deliver_flock.c deliver_pass.c \ - deliver_request.c dict_ldap.c dict_mysql.c dict_pgsql.c \ + deliver_request.c dict_ldap.c dict_mongodb.c dict_mysql.c dict_pgsql.c \ dict_proxy.c dict_sqlite.c domain_list.c dot_lockfile.c dot_lockfile_as.c \ dsb_scan.c dsn.c dsn_buf.c dsn_mask.c dsn_print.c dsn_util.c \ ehlo_mask.c ext_prop.c file_id.c flush_clnt.c header_opts.c \ @@ -80,13 +80,13 @@ OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \ # MAP_OBJ is for maps that may be dynamically loaded with dynamicmaps.cf. # When hard-linking these maps, makedefs sets NON_PLUGIN_MAP_OBJ=$(MAP_OBJ), # otherwise it sets the PLUGIN_* macros. -MAP_OBJ = dict_ldap.o dict_mysql.o dict_pgsql.o dict_sqlite.o +MAP_OBJ = dict_ldap.o dict_mysql.o dict_pgsql.o dict_sqlite.o dict_mongodb.o HDRS = abounce.h anvil_clnt.h been_here.h bounce.h bounce_log.h \ canon_addr.h cfg_parser.h cleanup_user.h clnt_stream.h config.h \ conv_time.h db_common.h debug_peer.h debug_process.h defer.h \ deliver_completed.h deliver_flock.h deliver_pass.h deliver_request.h \ - dict_ldap.h dict_mysql.h dict_pgsql.h dict_proxy.h dict_sqlite.h domain_list.h \ + dict_ldap.h dict_mysql.h dict_pgsql.h dict_mongodb.h dict_proxy.h dict_sqlite.h domain_list.h \ dot_lockfile.h dot_lockfile_as.h dsb_scan.h dsn.h dsn_buf.h \ dsn_mask.h dsn_print.h dsn_util.h ehlo_mask.h ext_prop.h \ file_id.h flush_clnt.h header_opts.h header_token.h input_transp.h \ @@ -136,7 +136,7 @@ LIBS = ../../lib/lib$(LIB_PREFIX)util$(LIB_SUFFIX) LIB_DIR = ../../lib INC_DIR = ../../include PLUGIN_MAP_SO = $(LIB_PREFIX)ldap$(LIB_SUFFIX) $(LIB_PREFIX)mysql$(LIB_SUFFIX) \ - $(LIB_PREFIX)pgsql$(LIB_SUFFIX) $(LIB_PREFIX)sqlite$(LIB_SUFFIX) + $(LIB_PREFIX)pgsql$(LIB_SUFFIX) $(LIB_PREFIX)sqlite$(LIB_SUFFIX) $(LIB_PREFIX)mongodb$(LIB_SUFFIX) MAKES = .c.o:; $(CC) $(SHLIB_CFLAGS) $(CFLAGS) -c $*.c @@ -173,6 +173,9 @@ $(LIB_PREFIX)pgsql$(LIB_SUFFIX): dict_pgsql.o $(LIB_PREFIX)sqlite$(LIB_SUFFIX): dict_sqlite.o $(PLUGIN_LD) $(SHLIB_RPATH) -o $@ dict_sqlite.o $(AUXLIBS_SQLITE) +$(LIB_PREFIX)mongodb$(LIB_SUFFIX): dict_mongodb.o + $(PLUGIN_LD) $(SHLIB_RPATH) -o $@ dict_mongodb.o $(AUXLIBS_MONGODB) + update: $(LIB_DIR)/$(LIB) $(HDRS) $(PLUGIN_MAP_SO_UPDATE) -for i in $(HDRS); \ do \ @@ -1210,6 +1213,24 @@ dict_memcache.o: dict_memcache.c dict_memcache.o: dict_memcache.h dict_memcache.o: memcache_proto.h dict_memcache.o: string_list.h +dict_mongodb.o: ../../include/argv.h +dict_mongodb.o: ../../include/auto_clnt.h +dict_mongodb.o: ../../include/check_arg.h +dict_mongodb.o: ../../include/dict.h +dict_mongodb.o: ../../include/match_list.h +dict_mongodb.o: ../../include/msg.h +dict_mongodb.o: ../../include/myflock.h +dict_mongodb.o: ../../include/mymalloc.h +dict_mongodb.o: ../../include/stringops.h +dict_mongodb.o: ../../include/sys_defs.h +dict_mongodb.o: ../../include/vbuf.h +dict_mongodb.o: ../../include/vstream.h +dict_mongodb.o: ../../include/vstring.h +dict_mongodb.o: cfg_parser.h +dict_mongodb.o: db_common.h +dict_mongodb.o: dict_mongodb.c +dict_mongodb.o: dict_mongodb.h +dict_mongodb.o: string_list.h dict_mysql.o: ../../include/argv.h dict_mysql.o: ../../include/check_arg.h dict_mysql.o: ../../include/dict.h @@ -1861,6 +1882,7 @@ mail_dict.o: ../../include/vstream.h mail_dict.o: ../../include/vstring.h mail_dict.o: dict_ldap.h mail_dict.o: dict_memcache.h +mail_dict.o: dict_mongodb.h mail_dict.o: dict_mysql.h mail_dict.o: dict_pgsql.h mail_dict.o: dict_proxy.h @@ -1911,6 +1933,7 @@ mail_params.o: ../../include/htable.h mail_params.o: ../../include/inet_addr_list.h mail_params.o: ../../include/inet_proto.h mail_params.o: ../../include/iostuff.h +mail_params.o: ../../include/logwriter.h mail_params.o: ../../include/midna_domain.h mail_params.o: ../../include/mkmap.h mail_params.o: ../../include/msg.h diff --git a/src/global/dict_ldap.c b/src/global/dict_ldap.c index a078721..7310a96 100644 --- a/src/global/dict_ldap.c +++ b/src/global/dict_ldap.c @@ -904,7 +904,7 @@ static int attrdesc_subtype(const char *a1, const char *a2) /* url_attrs - attributes we want from LDAP URL */ -static char **url_attrs(DICT_LDAP *dict_ldap, LDAPURLDesc * url) +static char **url_attrs(DICT_LDAP *dict_ldap, LDAPURLDesc *url) { static ARGV *attrs; char **a1; @@ -1234,7 +1234,7 @@ static const char *dict_ldap_lookup(DICT *dict, const char *name) * Don't frustrate future attempts to make Postfix UTF-8 transparent. */ if ((dict->flags & DICT_FLAG_UTF8_ACTIVE) == 0 - && !valid_utf8_string(name, strlen(name))) { + && !valid_utf8_stringz(name)) { if (msg_verbose) msg_info("%s: %s: Skipping lookup of non-UTF-8 key '%s'", myname, dict_ldap->parser->name, name); diff --git a/src/global/dict_mongodb.c b/src/global/dict_mongodb.c new file mode 100644 index 0000000..18144b7 --- /dev/null +++ b/src/global/dict_mongodb.c @@ -0,0 +1,570 @@ +/*++ +/* NAME +/* dict_mongodb 3 +/* SUMMARY +/* dictionary interface to mongodb, compatible with libmongoc-1.0 +/* SYNOPSIS +/* #include <dict_mongodb.h> +/* +/* DICT *dict_mongodb_open(name, open_flags, dict_flags) +/* const char *name; +/* int open_flags; +/* int dict_flags; +/* DESCRIPTION +/* dict_mongodb_open() opens a MongoDB database, providing a +/* dictionary interface for Postfix mappings. The result is a +/* pointer to the installed dictionary. +/* +/* Configuration parameters are described in mongodb_table(5). +/* +/* Arguments: +/* .IP name +/* Either the path to the MongoDB configuration file (if it +/* starts with '/' or '.'), or the prefix which will be used +/* to obtain main.cf configuration parameters for this search. +/* +/* In the first case, configuration parameters are specified +/* in the file as \fIname\fR=\fIvalue\fR pairs. +/* +/* In the second case, the configuration parameters are prefixed +/* with the value of \fIname\fR and an underscore, and they +/* are specified in main.cf. For example, if this value is +/* \fImongodbconf\fR, the parameters would look like +/* \fImongodbconf_uri\fR, \fImongodbconf_collection\fR, and +/* so on. +/* .IP open_flags +/* Must be O_RDONLY +/* .IP dict_flags +/* See dict_open(3). +/* SEE ALSO +/* dict(3) generic dictionary manager +/* HISTORY +/* .ad +/* .fi +/* MongoDB support was added in Postfix 3.9. +/* AUTHOR(S) +/* Hamid Maadani (hamid@dexo.tech) +/* Dextrous Technologies, LLC +/* +/* Edited by: +/* Wietse Venema +/* porcupine.org +/* +/* Based on prior work by: +/* Stephan Ferraro +/* Aionda GmbH +/*--*/ + + /* + * System library. + */ +#include <sys_defs.h> +#ifdef HAS_MONGODB +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <ctype.h> +#include <inttypes.h> /* C99 PRId64 */ + +#include <bson/bson.h> +#include <mongoc/mongoc.h> + + /* + * Utility library. + */ +#include <dict.h> +#include <msg.h> +#include <mymalloc.h> +#include <vstring.h> +#include <stringops.h> +#include <auto_clnt.h> +#include <vstream.h> + + /* + * Global library. + */ +#include <cfg_parser.h> +#include <db_common.h> + + /* + * Application-specific. + */ +#include <dict_mongodb.h> + + /* + * Initial size for dynamically-allocated buffers. + */ +#ifndef BUFFER_SIZE +#define BUFFER_SIZE 1024 +#endif + +#define INIT_VSTR(buf, len) do { \ + if (buf == 0) \ + buf = vstring_alloc(len); \ + VSTRING_RESET(buf); \ + VSTRING_TERMINATE(buf); \ + } while (0) + +/* Structure of one mongodb dictionary handle. */ +typedef struct { + /* Initialized by dict_mongodb_open(). */ + DICT dict; /* Parent class */ + CFG_PARSER *parser; /* Configuration file parser */ + mongoc_client_t *client; /* Mongo C client handle */ + /* Initialized by mongodb_parse_config(). */ + char *uri; /* mongodb+srv:/*localhost:27017 */ + char *dbname; /* Database name */ + char *collection; /* Collection name */ + char *query_filter; /* db_common_expand() query template */ + char *projection; /* Advanced MongoDB projection */ + char *result_attribute; /* The key(s) to return the data for */ + char *result_format; /* db_common_expand() result_template */ + int expansion_limit; /* Result expansion limit */ + void *ctx; /* db_common handle */ +} DICT_MONGODB; + +/* Per-process initialization. */ +static bool init_done = false; + +/* itoa - int64_t to string */ + +static char *itoa(int64_t val) +{ + static char buf[21] = {0}; + int ret; + + /* + * XXX(Wietse) replaced custom code with standard library calls that + * handle zero, and negative values. + */ +#define PRId64_FORMAT "%" PRId64 + + ret = snprintf(buf, sizeof(buf), PRId64_FORMAT, val); + if (ret < 0) + msg_panic("itoa: output error for '%s'", PRId64_FORMAT); + if (ret >= sizeof(buf)) + msg_panic("itoa: output for '%s' exceeds space %ld", + PRId64_FORMAT, sizeof(buf)); + return (buf); +} + +/* mongodb_parse_config - parse mongodb configuration file */ + +static void mongodb_parse_config(DICT_MONGODB *dict_mongodb, + const char *mongodbcf) +{ + CFG_PARSER *p = dict_mongodb->parser; + + /* + * Parse the configuration file. + */ + dict_mongodb->uri = cfg_get_str(p, "uri", NULL, 1, 0); + dict_mongodb->dbname = cfg_get_str(p, "dbname", NULL, 1, 0); + dict_mongodb->collection = cfg_get_str(p, "collection", NULL, 1, 0); + dict_mongodb->query_filter = cfg_get_str(p, "query_filter", NULL, 1, 0); + + /* + * One of projection and result_attribute must be specified. That is + * enforced in the caller. + */ + dict_mongodb->projection = cfg_get_str(p, "projection", NULL, 0, 0); + dict_mongodb->result_attribute + = cfg_get_str(p, "result_attribute", NULL, 0, 0); + dict_mongodb->result_format + = cfg_get_str(dict_mongodb->parser, "result_format", "%s", 1, 0); + dict_mongodb->expansion_limit + = cfg_get_int(dict_mongodb->parser, "expansion_limit", 10, 0, 100); + + /* + * db_common query parsing and domain pattern lookup. + */ + dict_mongodb->ctx = 0; + (void) db_common_parse(&dict_mongodb->dict, &dict_mongodb->ctx, + dict_mongodb->query_filter, 1); + db_common_parse_domain(dict_mongodb->parser, dict_mongodb->ctx); +} + +/* expand_value - expand lookup result value */ + +static bool expand_value(DICT_MONGODB *dict_mongodb, const char *p, + const char *lookup_name, + VSTRING *resultString, + int *expansion, const char *key) +{ + + /* + * If a lookup result cannot be processed due to an expansion limit + * error, return a DICT_ERR_RETRY error code and a 'false' result value. + * As documented for many dict_xxx() implementations, and expansion limit + * error is considered a temporary error. + */ + if (dict_mongodb->expansion_limit > 0 + && ++(*expansion) > dict_mongodb->expansion_limit) { + msg_warn("%s:%s: expansion limit exceeded for key: '%s'", + dict_mongodb->dict.type, dict_mongodb->dict.name, key); + dict_mongodb->dict.error = DICT_ERR_RETRY; + return (false); + } + + /* + * XXX(Wietse) Added the dict_mongodb_lookup() lookup_name argument, + * because it selects code paths inside db_common_expand() that are + * specifically for lookup results instead of lookup keys, including + * %[SUD] substitution. + */ + db_common_expand(dict_mongodb->ctx, dict_mongodb->result_format, p, + lookup_name, resultString, 0); + return (true); +} + +/* get_result_string - convert lookup result to string, or set dict.error */ + +static char *get_result_string(DICT_MONGODB *dict_mongodb, + VSTRING *resultString, + bson_iter_t *iter, + const char *lookup_name, + int *expansion, + const char *key) +{ + char *p = NULL; + bool got_one_result = false; + + /* + * If a lookup result cannot be processed due to an error, return a + * non-zero error code and a NULL result value. + */ + INIT_VSTR(resultString, BUFFER_SIZE); + while (dict_mongodb->dict.error == DICT_ERR_NONE && bson_iter_next(iter)) { + switch (bson_iter_type(iter)) { + case BSON_TYPE_UTF8: + p = (char *) bson_iter_utf8(iter, NULL); + if (!bson_utf8_validate(p, strlen(p), true)) { + msg_warn("%s:%s: invalid UTF-8 in lookup result '%s'", + dict_mongodb->dict.type, dict_mongodb->dict.name, p); + dict_mongodb->dict.error = DICT_ERR_RETRY; + break; + } + got_one_result |= expand_value(dict_mongodb, p, lookup_name, + resultString, expansion, key); + break; + case BSON_TYPE_INT64: + case BSON_TYPE_INT32: + p = itoa(bson_iter_as_int64(iter)); + got_one_result |= expand_value(dict_mongodb, p, lookup_name, + resultString, expansion, key); + break; + case BSON_TYPE_ARRAY: + ; /* For pre-C23 Clang. */ + const uint8_t *dataBuffer = NULL; + unsigned int len = 0; + bson_iter_t dataIter; + bson_t *data = NULL; + + /* + * XXX(Wietse) are there any non-error cases, such as a valid but + * empty array, where bson_new_from_data() or bson_iter_init() + * would return null or false? If there are no such cases then we + * must handle null/false as an error. + */ + bson_iter_array(iter, &len, &dataBuffer); + if ((data = bson_new_from_data(dataBuffer, len)) != 0 + && bson_iter_init(&dataIter, data)) { + VSTRING *iterResult = vstring_alloc(BUFFER_SIZE); + + if ((p = get_result_string(dict_mongodb, iterResult, &dataIter, + lookup_name, expansion, key)) != 0) { + vstring_sprintf_append(resultString, (got_one_result) ? + ",%s" : "%s", p); + got_one_result |= true; + } + vstring_free(iterResult); + } + bson_destroy(data); + break; + default: + /* Unexpected field type. As documented, warn and ignore. */ + msg_warn("%s:%s: failed to retrieve value of '%s', " + "Unknown result type %d.", dict_mongodb->dict.type, + dict_mongodb->dict.name, bson_iter_key(iter), + bson_iter_type(iter)); + break; + } + } + if (dict_mongodb->dict.error != DICT_ERR_NONE || !got_one_result) + return (0); + return (vstring_str(resultString)); +} + +/* dict_mongdb_quote - quote json string */ + +static void dict_mongdb_quote(DICT *dict, const char *name, VSTRING *result) +{ + /* quote_for_json_append() will resize the result buffer as needed. */ + (void) quote_for_json_append(result, name, -1); +} + +/* dict_mongdb_append_result_attributes - projection builder */ + +static int dict_mongdb_append_result_attribute(bson_t * projection, + const char *result_attribute) +{ + char *ra = mystrdup(result_attribute); + char *pp = ra; + char *cp; + int ok = 1; + + while (ok && (cp = mystrtok(&pp, CHARS_COMMA_SP)) != 0) + ok = BSON_APPEND_INT32(projection, cp, 1); + myfree(ra); + return (ok); +} + +/* dict_mongodb_lookup - find database entry using mongo query language */ + +static const char *dict_mongodb_lookup(DICT *dict, const char *name) +{ + DICT_MONGODB *dict_mongodb = (DICT_MONGODB *) dict; + mongoc_collection_t *coll = NULL; + mongoc_cursor_t *cursor = NULL; + bson_iter_t iter; + const bson_t *doc = NULL; + bson_t *query = NULL; + bson_t *options = NULL; + bson_t *projection = NULL; + bson_error_t error; + char *result = NULL; + static VSTRING *queryString = NULL; + static VSTRING *resultString = NULL; + int domain_rc; + int expansion = 0; + + dict_mongodb->dict.error = DICT_ERR_NONE; + + /* + * If they specified a domain list for this map, then only search for + * addresses in domains on the list. This can significantly reduce the + * load on the database. + */ + if ((domain_rc = db_common_check_domain(dict_mongodb->ctx, name)) == 0) { + if (msg_verbose) + msg_info("%s:%s: skipping lookup of '%s': domain mismatch", + dict_mongodb->dict.type, dict_mongodb->dict.name, name); + return (0); + } else if (domain_rc < 0) { + DICT_ERR_VAL_RETURN(dict, domain_rc, (char *) 0); + } + + /* + * Ugly macros to make error and non-error handling code more readable. + * If code size is a concern, them an optimizing compiler can eliminate + * dead code or duplicated code. + */ + + /* Set an error code, and return null. */ +#define DICT_MONGODB_LOOKUP_ERR_RETURN(err) do { \ + dict_mongodb->dict.error = (err); \ + DICT_MONGODB_LOOKUP_RETURN((char *) 0); \ +} while (0); + + /* Pass through any error, and return the specified value. */ +#define DICT_MONGODB_LOOKUP_RETURN(val) do { \ + if (coll) mongoc_collection_destroy(coll); \ + if (cursor) mongoc_cursor_destroy(cursor); \ + if (query) bson_destroy(query); \ + if (options) bson_destroy(options); \ + if (projection) bson_destroy(projection); \ + return (val); \ + } while (0) + + coll = mongoc_client_get_collection(dict_mongodb->client, + dict_mongodb->dbname, + dict_mongodb->collection); + if (!coll) { + msg_warn("%s:%s: failed to get collection [%s] from [%s]", + dict_mongodb->dict.type, dict_mongodb->dict.name, + dict_mongodb->collection, dict_mongodb->dbname); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + + /* + * Use the specified result projection, or craft one from the + * result_attribute. Exclude the _id field from the result. + */ + options = bson_new(); + if (dict_mongodb->projection) { + projection = bson_new_from_json((uint8_t *) dict_mongodb->projection, + -1, &error); + if (!projection) { + msg_warn("%s:%s: failed to create a projection from '%s': %s", + dict_mongodb->dict.type, dict_mongodb->dict.name, + dict_mongodb->projection, error.message); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + if (!BSON_APPEND_INT32(projection, "_id", 0) + || !BSON_APPEND_DOCUMENT(options, "projection", projection)) { + msg_warn("%s:%s: failed to append a projection from '%s'", + dict_mongodb->dict.type, dict_mongodb->dict.name, + dict_mongodb->projection); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + } else if (dict_mongodb->result_attribute) { + bson_t res_attr; + + if (!BSON_APPEND_DOCUMENT_BEGIN(options, "projection", &res_attr) + || !BSON_APPEND_INT32(&res_attr, "_id", 0) + || !dict_mongdb_append_result_attribute(&res_attr, + dict_mongodb->result_attribute) + || !bson_append_document_end(options, &res_attr)) { + msg_warn("%s:%s: failed to append a projection from '%s'", + dict_mongodb->dict.type, dict_mongodb->dict.name, + dict_mongodb->result_attribute); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + } else { + /* Can't happen. The configuration parser should reject this. */ + msg_panic("%s:%s: empty 'projection' and 'result_attribute'", + dict_mongodb->dict.type, dict_mongodb->dict.name); + } + + /* + * Expand filter template. This uses a quoting function to prevent + * metacharacter injection with parts from a crafted email address. + */ + INIT_VSTR(queryString, BUFFER_SIZE); + if (!db_common_expand(dict_mongodb->ctx, dict_mongodb->query_filter, + name, 0, queryString, dict_mongdb_quote)) + /* Suppress the actual lookup if the expansion is empty. */ + DICT_MONGODB_LOOKUP_RETURN(0); + + /* Create the query from the expanded query template. */ + query = bson_new_from_json((uint8_t *) vstring_str(queryString), + -1, &error); + if (!query) { + msg_warn("%s:%s: failed to create a query from '%s': %s", + dict_mongodb->dict.type, dict_mongodb->dict.name, + vstring_str(queryString), error.message); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + /* Run the query. */ + cursor = mongoc_collection_find_with_opts(coll, query, options, NULL); + if (mongoc_cursor_error(cursor, &error)) { + msg_warn("%s:%s: cursor error for '%s': %s", + dict_mongodb->dict.type, dict_mongodb->dict.name, + vstring_str(queryString), error.message); + DICT_MONGODB_LOOKUP_ERR_RETURN(DICT_ERR_RETRY); + } + /* Convert the lookup result to C string. */ + INIT_VSTR(resultString, BUFFER_SIZE); + while (mongoc_cursor_next(cursor, &doc)) { + if (bson_iter_init(&iter, doc)) { + result = get_result_string(dict_mongodb, resultString, &iter, + name, &expansion, name); + } + } + DICT_MONGODB_LOOKUP_RETURN(result); +} + +/* dict_mongodb_close - close MongoDB database */ + +static void dict_mongodb_close(DICT *dict) +{ + DICT_MONGODB *dict_mongodb = (DICT_MONGODB *) dict; + + cfg_parser_free(dict_mongodb->parser); + if (dict_mongodb->ctx) { + db_common_free_ctx(dict_mongodb->ctx); + } + myfree(dict_mongodb->uri); + myfree(dict_mongodb->dbname); + myfree(dict_mongodb->collection); + myfree(dict_mongodb->query_filter); + + if (dict_mongodb->result_attribute) { + myfree(dict_mongodb->result_attribute); + } + if (dict_mongodb->result_format) { + myfree(dict_mongodb->result_format); + } + if (dict_mongodb->projection) { + myfree(dict_mongodb->projection); + } + if (dict_mongodb->client) { + mongoc_client_destroy(dict_mongodb->client); + } + dict_free(dict); +} + +/* dict_mongodb_open - open MongoDB database connection */ + +DICT *dict_mongodb_open(const char *name, int open_flags, int dict_flags) +{ + DICT_MONGODB *dict_mongodb; + CFG_PARSER *parser; + mongoc_uri_t *uri = 0; + bson_error_t error; + + /* Sanity checks. */ + if (open_flags != O_RDONLY) { + return (dict_surrogate(DICT_TYPE_MONGODB, name, open_flags, dict_flags, + "%s:%s: map requires O_RDONLY access mode", + DICT_TYPE_MONGODB, name)); + } + /* Open the configuration file. */ + if ((parser = cfg_parser_alloc(name)) == 0) { + return (dict_surrogate(DICT_TYPE_MONGODB, name, open_flags, dict_flags, + "open %s: %m", name)); + } + /* Create the dictionary object. */ + dict_mongodb = (DICT_MONGODB *) dict_alloc(DICT_TYPE_MONGODB, name, + sizeof(*dict_mongodb)); + dict_mongodb->dict.lookup = dict_mongodb_lookup; + dict_mongodb->dict.close = dict_mongodb_close; + dict_mongodb->dict.flags = dict_flags; + dict_mongodb->parser = parser; + dict_mongodb->dict.owner = cfg_get_owner(dict_mongodb->parser); + dict_mongodb->client = NULL; + + /* Parse config. */ + mongodb_parse_config(dict_mongodb, name); + if (!dict_mongodb->projection == !dict_mongodb->result_attribute) { + dict_mongodb_close(&dict_mongodb->dict); + return (dict_surrogate(DICT_TYPE_MONGODB, name, open_flags, dict_flags, + "%s:%s: specify exactly one of 'projection' or 'result_attribute'", + DICT_TYPE_MONGODB, name)); + } + /* One-time initialization of libmongoc 's internals. */ + if (!init_done) { + mongoc_init(); + init_done = true; + } +#define DICT_MONGODB_OPEN_ERR_RETURN(d) do { \ + DICT *_d = (d); \ + if (uri) mongoc_uri_destroy(uri); \ + dict_mongodb_close(&dict_mongodb->dict); \ + return (_d); \ + } while (0); + + uri = mongoc_uri_new_with_error(dict_mongodb->uri, &error); + if (!uri) + DICT_MONGODB_OPEN_ERR_RETURN(dict_surrogate(DICT_TYPE_MONGODB, name, + open_flags, dict_flags, + "%s:%s: failed to parse URI '%s': %s", + DICT_TYPE_MONGODB, name, + dict_mongodb->uri, error.message)); + + dict_mongodb->client = mongoc_client_new_from_uri_with_error(uri, &error); + if (!dict_mongodb->client) + DICT_MONGODB_OPEN_ERR_RETURN(dict_surrogate(DICT_TYPE_MONGODB, name, + open_flags, dict_flags, + "%s:%s: failed to create client for '%s': %s", + DICT_TYPE_MONGODB, name, + dict_mongodb->uri, + error.message)); + + mongoc_uri_destroy(uri); + mongoc_client_set_error_api(dict_mongodb->client, MONGOC_ERROR_API_VERSION_2); + return (DICT_DEBUG (&dict_mongodb->dict)); +} + +#endif diff --git a/src/global/dict_mongodb.h b/src/global/dict_mongodb.h new file mode 100755 index 0000000..d5120cb --- /dev/null +++ b/src/global/dict_mongodb.h @@ -0,0 +1,43 @@ +#ifndef _DICT_MONGODB_INCLUDED_ +#define _DICT_MONGODB_INCLUDED_ + +/*++ +/* NAME +/* dict_mongodb 3h +/* SUMMARY +/* dictionary interface to mongodb databases +/* SYNOPSIS +/* #include <dict_mongodb.h> +/* DESCRIPTION +/* .nf + + /* + * Utility library. + */ +#include <dict.h> + + /* + * External interface. + */ +#define DICT_TYPE_MONGODB "mongodb" + +extern DICT *dict_mongodb_open(const char *, int, int); + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Hamid Maadani (hamid@dexo.tech) +/* Dextrous Technologies, LLC +/* +/* Edited by: +/* Wietse Venema +/* porcupine.org +/* +/* Based on prior work by: +/* Stephan Ferraro +/* Aionda GmbH +/*--*/ + +#endif diff --git a/src/global/dict_mysql.c b/src/global/dict_mysql.c index 3c8fe4f..133cc0d 100644 --- a/src/global/dict_mysql.c +++ b/src/global/dict_mysql.c @@ -83,6 +83,10 @@ #include <limits.h> #include <errno.h> +#if !defined(MYSQL_VERSION_ID) || MYSQL_VERSION_ID < 40000 +#error "MySQL versions <4 are no longer supported" +#endif + #ifdef STRCASECMP_IN_STRINGS_H #include <strings.h> #endif @@ -147,9 +151,11 @@ typedef struct { char *username; char *password; char *dbname; + char *charset; + int retry_interval; + int idle_interval; ARGV *hosts; PLMYSQL *pldb; -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 HOST *active_host; char *tls_cert_file; char *tls_key_file; @@ -159,7 +165,6 @@ typedef struct { #if defined(DICT_MYSQL_SSL_VERIFY_SERVER_CERT) int tls_verify_cert; #endif -#endif int require_result_set; } DICT_MYSQL; @@ -171,15 +176,15 @@ typedef struct { #define TYPEINET (1<<1) #define RETRY_CONN_MAX 100 -#define RETRY_CONN_INTV 60 /* 1 minute */ -#define IDLE_CONN_INTV 60 /* 1 minute */ +#define DEF_RETRY_INTV 60 /* 1 minute */ +#define DEF_IDLE_INTV 60 /* 1 minute */ /* internal function declarations */ static PLMYSQL *plmysql_init(ARGV *); static int plmysql_query(DICT_MYSQL *, const char *, VSTRING *, MYSQL_RES **); static void plmysql_dealloc(PLMYSQL *); static void plmysql_close_host(HOST *); -static void plmysql_down_host(HOST *); +static void plmysql_down_host(HOST *, int); static void plmysql_connect_single(DICT_MYSQL *, HOST *); static const char *dict_mysql_lookup(DICT *, const char *); DICT *dict_mysql_open(const char *, int, int); @@ -205,13 +210,21 @@ static void dict_mysql_quote(DICT *dict, const char *name, VSTRING *result) buflen = 2 * len + 1; VSTRING_SPACE(result, buflen); -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 - if (dict_mysql->active_host) - mysql_real_escape_string(dict_mysql->active_host->db, - vstring_end(result), name, len); - else + if (dict_mysql->active_host == 0) + msg_panic("dict_mysql_quote: no active host"); +#if MYSQL_VERSION_ID >= 50706 && !defined(MARIADB_VERSION_ID) + mysql_real_escape_string_quote(dict_mysql->active_host->db, + vstring_end(result), name, len, '\''); +#else + if (mysql_real_escape_string(dict_mysql->active_host->db, + vstring_end(result), name, len) == + (unsigned long) -1) { + msg_warn("dict_mysql: host (%s) cannot escape input string: >%s<", + dict_mysql->active_host->hostname, + mysql_error(dict_mysql->active_host->db)); + dict_mysql->active_host->stat = STATFAIL; + } #endif - mysql_escape_string(vstring_end(result), name, len); VSTRING_SKIP(result); } @@ -231,7 +244,6 @@ static const char *dict_mysql_lookup(DICT *dict, const char *name) int numrows; int expansion; const char *r; - db_quote_callback_t quote_func = dict_mysql_quote; int domain_rc; dict->error = 0; @@ -241,7 +253,7 @@ static const char *dict_mysql_lookup(DICT *dict, const char *name) */ #ifdef SNAPSHOT if ((dict->flags & DICT_FLAG_UTF8_ACTIVE) == 0 - && !valid_utf8_string(name, strlen(name))) { + && !valid_utf8_stringz(name)) { if (msg_verbose) msg_info("%s: %s: Skipping lookup of non-UTF-8 key '%s'", myname, dict_mysql->parser->name, name); @@ -291,11 +303,8 @@ static const char *dict_mysql_lookup(DICT *dict, const char *name) * quoting happens separately for each connection, we don't bother with * quoting... */ -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 - quote_func = 0; -#endif if (!db_common_expand(dict_mysql->ctx, dict_mysql->query, - name, 0, query, quote_func)) + name, 0, query, (db_quote_callback_t) 0)) return (0); /* do the query - set dict->error & cleanup if there's an error */ @@ -439,8 +448,12 @@ static int plmysql_query(DICT_MYSQL *dict_mysql, { HOST *host; MYSQL_RES *first_result = 0; + + /* In case all hosts are down. */ int query_error = 1; + errno = ENOTSUP; + /* * Helper to avoid spamming the log with warnings. */ @@ -454,8 +467,6 @@ static int plmysql_query(DICT_MYSQL *dict_mysql, while ((host = dict_mysql_get_active(dict_mysql)) != NULL) { -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 - /* * The active host is used to escape strings in the context of the * active connection's character encoding. @@ -465,8 +476,14 @@ static int plmysql_query(DICT_MYSQL *dict_mysql, VSTRING_TERMINATE(query); db_common_expand(dict_mysql->ctx, dict_mysql->query, name, 0, query, dict_mysql_quote); + /* Check for potential dict_mysql_quote() failure. */ + if (host->stat == STATFAIL) { + plmysql_down_host(host, dict_mysql->retry_interval); + continue; + } + if (msg_verbose) + msg_info("expanded and quoted query: >%s<", vstring_str(query)); dict_mysql->active_host = 0; -#endif query_error = 0; errno = 0; @@ -546,7 +563,7 @@ static int plmysql_query(DICT_MYSQL *dict_mysql, * See what we got. */ if (query_error) { - plmysql_down_host(host); + plmysql_down_host(host, dict_mysql->retry_interval); if (errno == 0) errno = ENOTSUP; if (first_result) { @@ -559,7 +576,7 @@ static int plmysql_query(DICT_MYSQL *dict_mysql, dict_mysql->dict.type, dict_mysql->dict.name, host->hostname); event_request_timer(dict_mysql_event, (void *) host, - IDLE_CONN_INTV); + dict_mysql->idle_interval); break; } } @@ -581,7 +598,6 @@ static void plmysql_connect_single(DICT_MYSQL *dict_mysql, HOST *host) mysql_options(host->db, MYSQL_READ_DEFAULT_FILE, dict_mysql->option_file); if (dict_mysql->option_group && dict_mysql->option_group[0]) mysql_options(host->db, MYSQL_READ_DEFAULT_GROUP, dict_mysql->option_group); -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 if (dict_mysql->tls_key_file || dict_mysql->tls_cert_file || dict_mysql->tls_CAfile || dict_mysql->tls_CApath || dict_mysql->tls_ciphers) mysql_ssl_set(host->db, @@ -593,7 +609,6 @@ static void plmysql_connect_single(DICT_MYSQL *dict_mysql, HOST *host) mysql_options(host->db, DICT_MYSQL_SSL_VERIFY_SERVER_CERT, &dict_mysql->tls_verify_cert); #endif -#endif if (mysql_real_connect(host->db, (host->type == TYPEINET ? host->name : 0), dict_mysql->username, @@ -602,6 +617,12 @@ static void plmysql_connect_single(DICT_MYSQL *dict_mysql, HOST *host) host->port, (host->type == TYPEUNIX ? host->name : 0), CLIENT_MULTI_RESULTS)) { + if (mysql_set_character_set(host->db, dict_mysql->charset) != 0) { + msg_warn("dict_mysql: mysql_set_character_set '%s' failed: %s", + dict_mysql->charset, mysql_error(host->db)); + plmysql_down_host(host, dict_mysql->retry_interval); + return; + } if (msg_verbose) msg_info("dict_mysql: successful connection to host %s", host->hostname); @@ -609,7 +630,7 @@ static void plmysql_connect_single(DICT_MYSQL *dict_mysql, HOST *host) } else { msg_warn("connect to mysql server %s: %s", host->hostname, mysql_error(host->db)); - plmysql_down_host(host); + plmysql_down_host(host, dict_mysql->retry_interval); } } @@ -625,11 +646,11 @@ static void plmysql_close_host(HOST *host) * plmysql_down_host - close a failed connection AND set a "stay away from * this host" timer */ -static void plmysql_down_host(HOST *host) +static void plmysql_down_host(HOST *host, int retry_interval) { mysql_close(host->db); host->db = 0; - host->ts = time((time_t *) 0) + RETRY_CONN_INTV; + host->ts = time((time_t *) 0) + retry_interval; host->stat = STATFAIL; event_cancel_timer(dict_mysql_event, (void *) host); } @@ -646,10 +667,14 @@ static void mysql_parse_config(DICT_MYSQL *dict_mysql, const char *mysqlcf) dict_mysql->username = cfg_get_str(p, "user", "", 0, 0); dict_mysql->password = cfg_get_str(p, "password", "", 0, 0); dict_mysql->dbname = cfg_get_str(p, "dbname", "", 1, 0); + dict_mysql->charset = cfg_get_str(p, "charset", "utf8mb4", 1, 0); + dict_mysql->retry_interval = cfg_get_int(p, "retry_interval", + DEF_RETRY_INTV, 1, 0); + dict_mysql->idle_interval = cfg_get_int(p, "idle_interval", + DEF_IDLE_INTV, 1, 0); dict_mysql->result_format = cfg_get_str(p, "result_format", "%s", 1, 0); dict_mysql->option_file = cfg_get_str(p, "option_file", NULL, 0, 0); dict_mysql->option_group = cfg_get_str(p, "option_group", "client", 0, 0); -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 dict_mysql->tls_key_file = cfg_get_str(p, "tls_key_file", NULL, 0, 0); dict_mysql->tls_cert_file = cfg_get_str(p, "tls_cert_file", NULL, 0, 0); dict_mysql->tls_CAfile = cfg_get_str(p, "tls_CAfile", NULL, 0, 0); @@ -658,7 +683,6 @@ static void mysql_parse_config(DICT_MYSQL *dict_mysql, const char *mysqlcf) #if defined(DICT_MYSQL_SSL_VERIFY_SERVER_CERT) dict_mysql->tls_verify_cert = cfg_get_bool(p, "tls_verify_cert", -1); #endif -#endif dict_mysql->require_result_set = cfg_get_bool(p, "require_result_set", 1); /* @@ -741,9 +765,7 @@ DICT *dict_mysql_open(const char *name, int open_flags, int dict_flags) dict_mysql->dict.flags = dict_flags; dict_mysql->parser = parser; mysql_parse_config(dict_mysql, name); -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 dict_mysql->active_host = 0; -#endif dict_mysql->pldb = plmysql_init(dict_mysql->hosts); if (dict_mysql->pldb == NULL) msg_fatal("couldn't initialize pldb!\n"); @@ -826,13 +848,13 @@ static void dict_mysql_close(DICT *dict) myfree(dict_mysql->username); myfree(dict_mysql->password); myfree(dict_mysql->dbname); + myfree(dict_mysql->charset); myfree(dict_mysql->query); myfree(dict_mysql->result_format); if (dict_mysql->option_file) myfree(dict_mysql->option_file); if (dict_mysql->option_group) myfree(dict_mysql->option_group); -#if defined(MYSQL_VERSION_ID) && MYSQL_VERSION_ID >= 40000 if (dict_mysql->tls_key_file) myfree(dict_mysql->tls_key_file); if (dict_mysql->tls_cert_file) @@ -843,7 +865,6 @@ static void dict_mysql_close(DICT *dict) myfree(dict_mysql->tls_CApath); if (dict_mysql->tls_ciphers) myfree(dict_mysql->tls_ciphers); -#endif if (dict_mysql->hosts) argv_free(dict_mysql->hosts); if (dict_mysql->ctx) diff --git a/src/global/dict_pgsql.c b/src/global/dict_pgsql.c index 5992135..c626854 100644 --- a/src/global/dict_pgsql.c +++ b/src/global/dict_pgsql.c @@ -108,18 +108,18 @@ #define TYPEUNIX (1<<0) #define TYPEINET (1<<1) -#define TYPECONNSTRING (1<<2) +#define TYPECONNSTR (1<<2) #define RETRY_CONN_MAX 100 -#define RETRY_CONN_INTV 60 /* 1 minute */ -#define IDLE_CONN_INTV 60 /* 1 minute */ +#define DEF_RETRY_INTV 60 /* 1 minute */ +#define DEF_IDLE_INTV 60 /* 1 minute */ typedef struct { PGconn *db; char *hostname; char *name; char *port; - unsigned type; /* TYPEUNIX | TYPEINET | TYPECONNSTRING */ + unsigned type; /* TYPEUNIX | TYPEINET | TYPECONNSTR */ unsigned stat; /* STATUNTRIED | STATFAIL | STATCUR */ time_t ts; /* used for attempting reconnection */ } HOST; @@ -140,6 +140,8 @@ typedef struct { char *password; char *dbname; char *encoding; + int retry_interval; + int idle_interval; char *table; ARGV *hosts; PLPGSQL *pldb; @@ -152,12 +154,11 @@ typedef struct { /* internal function declarations */ static PLPGSQL *plpgsql_init(ARGV *); -static PGSQL_RES *plpgsql_query(DICT_PGSQL *, const char *, VSTRING *, char *, - char *, char *, char *); +static PGSQL_RES *plpgsql_query(DICT_PGSQL *, const char *, VSTRING *); static void plpgsql_dealloc(PLPGSQL *); static void plpgsql_close_host(HOST *); -static void plpgsql_down_host(HOST *); -static void plpgsql_connect_single(HOST *, char *, char *, char *, char *); +static void plpgsql_down_host(HOST *, int); +static void plpgsql_connect_single(DICT_PGSQL *, HOST *); static const char *dict_pgsql_lookup(DICT *, const char *); DICT *dict_pgsql_open(const char *, int, int); static void dict_pgsql_close(DICT *); @@ -280,7 +281,7 @@ static const char *dict_pgsql_lookup(DICT *dict, const char *name) */ #ifdef SNAPSHOT if ((dict->flags & DICT_FLAG_UTF8_ACTIVE) == 0 - && !valid_utf8_string(name, strlen(name))) { + && !valid_utf8_stringz(name)) { if (msg_verbose) msg_info("%s: %s: Skipping lookup of non-UTF-8 key '%s'", myname, dict_pgsql->parser->name, name); @@ -324,11 +325,7 @@ static const char *dict_pgsql_lookup(DICT *dict, const char *name) return (0); /* do the query - set dict->error & cleanup if there's an error */ - if ((query_res = plpgsql_query(dict_pgsql, name, query, - dict_pgsql->dbname, - dict_pgsql->encoding, - dict_pgsql->username, - dict_pgsql->password)) == 0) { + if ((query_res = plpgsql_query(dict_pgsql, name, query)) == 0) { dict->error = DICT_ERR_RETRY; return 0; } @@ -404,8 +401,7 @@ static HOST *dict_pgsql_find_host(PLPGSQL *PLDB, unsigned stat, unsigned type) /* dict_pgsql_get_active - get an active connection */ -static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding, - char *username, char *password) +static HOST *dict_pgsql_get_active(DICT_PGSQL *dict_pgsql, PLPGSQL *PLDB) { const char *myname = "dict_pgsql_get_active"; HOST *host; @@ -414,7 +410,7 @@ static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding, /* try the active connections first; prefer the ones to UNIX sockets */ if ((host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPEUNIX)) != NULL || (host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPEINET)) != NULL || - (host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPECONNSTRING)) != NULL) { + (host = dict_pgsql_find_host(PLDB, STATACTIVE, TYPECONNSTR)) != NULL) { if (msg_verbose) msg_info("%s: found active connection to host %s", myname, host->hostname); @@ -432,11 +428,11 @@ static HOST *dict_pgsql_get_active(PLPGSQL *PLDB, char *dbname, char *encoding, (host = dict_pgsql_find_host(PLDB, STATUNTRIED | STATFAIL, TYPEINET)) != NULL || (host = dict_pgsql_find_host(PLDB, STATUNTRIED | STATFAIL, - TYPECONNSTRING)) != NULL)) { + TYPECONNSTR)) != NULL)) { if (msg_verbose) msg_info("%s: attempting to connect to host %s", myname, host->hostname); - plpgsql_connect_single(host, dbname, encoding, username, password); + plpgsql_connect_single(dict_pgsql, host); if (host->stat == STATACTIVE) return host; } @@ -464,18 +460,14 @@ static void dict_pgsql_event(int unused_event, void *context) static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql, const char *name, - VSTRING *query, - char *dbname, - char *encoding, - char *username, - char *password) + VSTRING *query) { PLPGSQL *PLDB = dict_pgsql->pldb; HOST *host; PGSQL_RES *res = 0; ExecStatusType status; - while ((host = dict_pgsql_get_active(PLDB, dbname, encoding, username, password)) != NULL) { + while ((host = dict_pgsql_get_active(dict_pgsql, PLDB)) != NULL) { /* * The active host is used to escape strings in the context of the @@ -490,7 +482,7 @@ static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql, /* Check for potential dict_pgsql_quote() failure. */ if (host->stat == STATFAIL) { - plpgsql_down_host(host); + plpgsql_down_host(host, dict_pgsql->retry_interval); continue; } @@ -528,7 +520,7 @@ static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql, msg_info("dict_pgsql: successful query from host %s", host->hostname); event_request_timer(dict_pgsql_event, (void *) host, - IDLE_CONN_INTV); + dict_pgsql->idle_interval); return (res); case PGRES_FATAL_ERROR: msg_warn("pgsql query failed: fatal error from host %s: %s", @@ -559,7 +551,7 @@ static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql, */ if (res != 0) PQclear(res); - plpgsql_down_host(host); + plpgsql_down_host(host, dict_pgsql->retry_interval); } return (0); @@ -570,24 +562,25 @@ static PGSQL_RES *plpgsql_query(DICT_PGSQL *dict_pgsql, * used to reconnect to a single database when one is down or none is * connected yet. Log all errors and set the stat field of host accordingly */ -static void plpgsql_connect_single(HOST *host, char *dbname, char *encoding, char *username, char *password) +static void plpgsql_connect_single(DICT_PGSQL *dict_pgsql, HOST *host) { - if (host->type == TYPECONNSTRING) { + if (host->type == TYPECONNSTR) { host->db = PQconnectdb(host->name); } else { host->db = PQsetdbLogin(host->name, host->port, NULL, NULL, - dbname, username, password); + dict_pgsql->dbname, dict_pgsql->username, + dict_pgsql->password); } if (host->db == NULL || PQstatus(host->db) != CONNECTION_OK) { msg_warn("connect to pgsql server %s: %s", host->hostname, PQerrorMessage(host->db)); - plpgsql_down_host(host); + plpgsql_down_host(host, dict_pgsql->retry_interval); return; } - if (PQsetClientEncoding(host->db, encoding) != 0) { + if (PQsetClientEncoding(host->db, dict_pgsql->encoding) != 0) { msg_warn("dict_pgsql: cannot set the encoding to %s, skipping %s", - encoding, host->hostname); - plpgsql_down_host(host); + dict_pgsql->encoding, host->hostname); + plpgsql_down_host(host, dict_pgsql->retry_interval); return; } if (msg_verbose) @@ -611,12 +604,12 @@ static void plpgsql_close_host(HOST *host) * plpgsql_down_host - close a failed connection AND set a "stay away from * this host" timer. */ -static void plpgsql_down_host(HOST *host) +static void plpgsql_down_host(HOST *host, int retry_interval) { if (host->db) PQfinish(host->db); host->db = 0; - host->ts = time((time_t *) 0) + RETRY_CONN_INTV; + host->ts = time((time_t *) 0) + retry_interval; host->stat = STATFAIL; event_cancel_timer(dict_pgsql_event, (void *) host); } @@ -635,6 +628,10 @@ static void pgsql_parse_config(DICT_PGSQL *dict_pgsql, const char *pgsqlcf) dict_pgsql->password = cfg_get_str(p, "password", "", 0, 0); dict_pgsql->dbname = cfg_get_str(p, "dbname", "", 1, 0); dict_pgsql->encoding = cfg_get_str(p, "encoding", "UTF8", 1, 0); + dict_pgsql->retry_interval = cfg_get_int(p, "retry_interval", + DEF_RETRY_INTV, 1, 0); + dict_pgsql->idle_interval = cfg_get_int(p, "idle_interval", + DEF_IDLE_INTV, 1, 0); dict_pgsql->result_format = cfg_get_str(p, "result_format", "%s", 1, 0); /* @@ -764,7 +761,7 @@ static HOST *host_init(const char *hostname) * Modern syntax: "postgresql://connection-info". */ if (strncmp(d, "postgresql:", 11) == 0) { - host->type = TYPECONNSTRING; + host->type = TYPECONNSTR; host->name = mystrdup(d); host->port = 0; } diff --git a/src/global/dict_sqlite.c b/src/global/dict_sqlite.c index 677d05a..7d6608a 100644 --- a/src/global/dict_sqlite.c +++ b/src/global/dict_sqlite.c @@ -149,7 +149,7 @@ static const char *dict_sqlite_lookup(DICT *dict, const char *name) * Don't frustrate future attempts to make Postfix UTF-8 transparent. */ if ((dict->flags & DICT_FLAG_UTF8_ACTIVE) == 0 - && !valid_utf8_string(name, strlen(name))) { + && !valid_utf8_stringz(name)) { if (msg_verbose) msg_info("%s: %s: Skipping lookup of non-UTF-8 key '%s'", myname, dict_sqlite->parser->name, name); diff --git a/src/global/mail_addr_find.c b/src/global/mail_addr_find.c index afbccd5..c7e5545 100644 --- a/src/global/mail_addr_find.c +++ b/src/global/mail_addr_find.c @@ -442,8 +442,8 @@ const char *mail_addr_find_opt(MAPS *path, const char *address, char **extp, /* * Try localpart@ even if the domain is not local. */ - if ((strategy & MA_FIND_LOCALPART_AT) != 0 \ - &&result == 0 && path->error == 0) + if ((strategy & MA_FIND_LOCALPART_AT) != 0 + && result == 0 && path->error == 0) result = find_local(path, ratsign, 1, int_full_key, int_bare_key, query_form, extp, &saved_ext, ext_addr_buf); diff --git a/src/global/mail_date.c b/src/global/mail_date.c index 55d8907..439a0ea 100644 --- a/src/global/mail_date.c +++ b/src/global/mail_date.c @@ -10,7 +10,7 @@ /* time_t when; /* DESCRIPTION /* mail_date() converts the time specified in \fIwhen\fR to the -/* form: "Mon, 9 Dec 1996 05:38:26 -0500 (EST)" and returns +/* form: "Mon, 09 Dec 1996 05:38:26 -0500 (EST)" and returns /* a pointer to the result. The result is overwritten upon /* each call. /* DIAGNOSTICS @@ -98,8 +98,13 @@ const char *mail_date(time_t when) * First, format the date and wall-clock time. XXX The %e format (day of * month, leading zero replaced by blank) isn't in my POSIX book, but * many vendors seem to support it. + * + * The RFC 5322 Date and Time Specification recommends (i.e., should) "that + * a single space be used in each place that FWS appears". To avoid a + * potentially breaking change, we prefer the %d (two-digit day) format, + * i.e. days 1-9 now have a leading zero instead of a leading space. */ -#ifdef MISSING_STRFTIME_E +#if defined(MISSING_STRFTIME_E) || defined(TWO_DIGIT_DAY_IN_DATE_TIME) #define STRFTIME_FMT "%a, %d %b %Y %H:%M:%S " #else #define STRFTIME_FMT "%a, %e %b %Y %H:%M:%S " diff --git a/src/global/mail_dict.c b/src/global/mail_dict.c index c640a80..55ac5dc 100644 --- a/src/global/mail_dict.c +++ b/src/global/mail_dict.c @@ -52,6 +52,7 @@ #include <dict_pgsql.h> #include <dict_sqlite.h> #include <dict_memcache.h> +#include <dict_mongodb.h> #include <mail_dict.h> #include <mail_params.h> #include <mail_dict.h> @@ -71,6 +72,9 @@ static const DICT_OPEN_INFO dict_open_info[] = { #ifdef HAS_SQLITE DICT_TYPE_SQLITE, dict_sqlite_open, 0, #endif +#ifdef HAS_MONGODB + DICT_TYPE_MONGODB, dict_mongodb_open, 0, +#endif #endif /* !USE_DYNAMIC_MAPS */ DICT_TYPE_MEMCACHE, dict_memcache_open, 0, 0, diff --git a/src/global/mail_params.c b/src/global/mail_params.c index 81aee73..2a7f84c 100644 --- a/src/global/mail_params.c +++ b/src/global/mail_params.c @@ -159,6 +159,7 @@ /* char *var_maillog_file_pfxs; /* char *var_maillog_file_comp; /* char *var_maillog_file_stamp; +/* char *var_maillog_file_perms; /* char *var_postlog_service; /* /* char *var_dnssec_probe; @@ -226,6 +227,7 @@ #include <vstring_vstream.h> #include <iostuff.h> #include <midna_domain.h> +#include <logwriter.h> /* Global library. */ @@ -375,6 +377,7 @@ char *var_maillog_file; char *var_maillog_file_pfxs; char *var_maillog_file_comp; char *var_maillog_file_stamp; +char *var_maillog_file_perms; char *var_postlog_service; char *var_dnssec_probe; @@ -515,9 +518,11 @@ static void check_mail_owner(void) */ if ((pwd = getpwuid(var_owner_uid)) != 0 && strcmp(pwd->pw_name, var_mail_owner) != 0) - msg_fatal("file %s/%s: parameter %s: user %s has same user ID as %s", + msg_fatal("file %s/%s: parameter %s: user %s has the same" + " user ID %ld as user %s", var_config_dir, MAIN_CONF_FILE, - VAR_MAIL_OWNER, var_mail_owner, pwd->pw_name); + VAR_MAIL_OWNER, var_mail_owner, + (long) var_owner_uid, pwd->pw_name); } /* check_sgid_group - lookup setgid group attributes and validate */ @@ -542,9 +547,11 @@ static void check_sgid_group(void) */ if ((grp = getgrgid(var_sgid_gid)) != 0 && strcmp(grp->gr_name, var_sgid_group) != 0) - msg_fatal("file %s/%s: parameter %s: group %s has same group ID as %s", + msg_fatal("file %s/%s: parameter %s: group %s has the same" + " group ID %ld as group %s", var_config_dir, MAIN_CONF_FILE, - VAR_SGID_GROUP, var_sgid_group, grp->gr_name); + VAR_SGID_GROUP, var_sgid_group, + (long) var_sgid_gid, grp->gr_name); } /* check_overlap - disallow UID or GID sharing */ @@ -729,6 +736,7 @@ void mail_params_init() VAR_MAILLOG_FILE_PFXS, DEF_MAILLOG_FILE_PFXS, &var_maillog_file_pfxs, 1, 0, VAR_MAILLOG_FILE_COMP, DEF_MAILLOG_FILE_COMP, &var_maillog_file_comp, 1, 0, VAR_MAILLOG_FILE_STAMP, DEF_MAILLOG_FILE_STAMP, &var_maillog_file_stamp, 1, 0, + VAR_MAILLOG_FILE_PERMS, DEF_MAILLOG_FILE_PERMS, &var_maillog_file_perms, 1, 0, VAR_POSTLOG_SERVICE, DEF_POSTLOG_SERVICE, &var_postlog_service, 1, 0, VAR_DNSSEC_PROBE, DEF_DNSSEC_PROBE, &var_dnssec_probe, 0, 0, VAR_KNOWN_TCP_PORTS, DEF_KNOWN_TCP_PORTS, &var_known_tcp_ports, 0, 0, @@ -979,6 +987,9 @@ void mail_params_init() dict_db_cache_size = var_db_read_buf; dict_lmdb_map_size = var_lmdb_map_size; inet_windowsize = var_inet_windowsize; + if (set_logwriter_create_perms(var_maillog_file_perms) < 0) + msg_warn("ignoring bad permissions: %s = %s", + VAR_MAILLOG_FILE_PERMS, var_maillog_file_perms); /* * Variables whose defaults are determined at runtime, after other diff --git a/src/global/mail_params.h b/src/global/mail_params.h index 3064b01..1f03b0b 100644 --- a/src/global/mail_params.h +++ b/src/global/mail_params.h @@ -1321,6 +1321,10 @@ extern bool var_smtpd_tls_ask_ccert; #define DEF_SMTPD_TLS_RCERT 0 extern bool var_smtpd_tls_req_ccert; +#define VAR_SMTPD_TLS_ENABLE_RPK "smtpd_tls_enable_rpk" +#define DEF_SMTPD_TLS_ENABLE_RPK 0 +extern bool var_smtpd_tls_enable_rpk; + #define VAR_SMTPD_TLS_CCERT_VD "smtpd_tls_ccert_verifydepth" #define DEF_SMTPD_TLS_CCERT_VD 9 extern int var_smtpd_tls_ccert_vd; @@ -1555,6 +1559,12 @@ extern char *var_smtp_tls_mand_excl; "{md5} : {sha256}}" extern char *var_smtp_tls_fpt_dgst; +#define VAR_SMTP_TLS_ENABLE_RPK "smtp_tls_enable_rpk" +#define DEF_SMTP_TLS_ENABLE_RPK 0 +#define VAR_LMTP_TLS_ENABLE_RPK "lmtp_tls_enable_rpk" +#define DEF_LMTP_TLS_ENABLE_RPK 0 +extern bool var_smtp_tls_enable_rpk; + #define VAR_SMTP_TLS_TAFILE "smtp_tls_trust_anchor_file" #define DEF_SMTP_TLS_TAFILE "" #define VAR_LMTP_TLS_TAFILE "lmtp_tls_trust_anchor_file" @@ -1745,6 +1755,12 @@ extern bool var_smtp_sasl_enable; #define DEF_SMTP_SASL_PASSWD "" extern char *var_smtp_sasl_passwd; +#define VAR_SMTP_SASL_PASSWD_RES_DELIM "smtp_sasl_password_result_delimiter" +#define DEF_SMTP_SASL_PASSWD_RES_DELIM ":" +#define VAR_LMTP_SASL_PASSWD_RES_DELIM "lmtp_sasl_password_result_delimiter" +#define DEF_LMTP_SASL_PASSWD_RES_DELIM DEF_SMTP_SASL_PASSWD_RES_DELIM +extern char *var_smtp_sasl_passwd_res_delim; + #define VAR_SMTP_SASL_OPTS "smtp_sasl_security_options" #define DEF_SMTP_SASL_OPTS "noplaintext, noanonymous" extern char *var_smtp_sasl_opts; @@ -2437,7 +2453,7 @@ extern char *var_smtpd_exp_filter; extern bool var_smtpd_peername_lookup; #define VAR_SMTPD_FORBID_UNAUTH_PIPE "smtpd_forbid_unauth_pipelining" -#define DEF_SMTPD_FORBID_UNAUTH_PIPE 0 +#define DEF_SMTPD_FORBID_UNAUTH_PIPE 1 extern bool var_smtpd_forbid_unauth_pipe; /* @@ -3072,6 +3088,10 @@ extern bool var_disable_mime_input; #define DEF_DISABLE_MIME_OCONV 0 extern bool var_disable_mime_oconv; +#define VAR_FORCE_MIME_ICONV "force_mime_input_conversion" +#define DEF_FORCE_MIME_ICONV 0 +extern bool var_force_mime_iconv; + #define VAR_STRICT_8BITMIME "strict_8bitmime" #define DEF_STRICT_8BITMIME 0 extern bool var_strict_8bitmime; @@ -3982,6 +4002,10 @@ extern bool var_tlsp_tls_ask_ccert; #define DEF_TLSP_TLS_RCERT "$" VAR_SMTPD_TLS_RCERT extern bool var_tlsp_tls_req_ccert; +#define VAR_TLSP_TLS_ENABLE_RPK "tlsproxy_tls_enable_rpk" +#define DEF_TLSP_TLS_ENABLE_RPK "$" VAR_SMTPD_TLS_ENABLE_RPK +extern bool var_tlsp_tls_enable_rpk; + #define VAR_TLSP_TLS_CCERT_VD "tlsproxy_tls_ccert_verifydepth" #define DEF_TLSP_TLS_CCERT_VD "$" VAR_SMTPD_TLS_CCERT_VD extern int var_tlsp_tls_ccert_vd; @@ -4282,7 +4306,7 @@ extern char *var_smtpd_dns_re_filter; * Backwards compatibility. */ #define VAR_SMTPD_FORBID_BARE_LF "smtpd_forbid_bare_newline" -#define DEF_SMTPD_FORBID_BARE_LF "no" +#define DEF_SMTPD_FORBID_BARE_LF "normalize" #define VAR_SMTPD_FORBID_BARE_LF_EXCL "smtpd_forbid_bare_newline_exclusions" #define DEF_SMTPD_FORBID_BARE_LF_EXCL "$" VAR_MYNETWORKS @@ -4379,6 +4403,10 @@ extern char *var_maillog_file_comp; #define DEF_MAILLOG_FILE_STAMP "%Y%m%d-%H%M%S" extern char *var_maillog_file_stamp; +#define VAR_MAILLOG_FILE_PERMS "maillog_file_permissions" +#define DEF_MAILLOG_FILE_PERMS "0600" +extern char *var_maillog_file_perms; + #define VAR_POSTLOG_SERVICE "postlog_service_name" #define DEF_POSTLOG_SERVICE MAIL_SERVICE_POSTLOG extern char *var_postlog_service; diff --git a/src/global/mail_proto.h b/src/global/mail_proto.h index 315a2e1..bea0886 100644 --- a/src/global/mail_proto.h +++ b/src/global/mail_proto.h @@ -63,6 +63,13 @@ #define MAIL_SERVICE_POSTLOG "postlog" /* + * Process names: convention is to use the basename of an executable file, + * but there is nothing to enforce that. + */ +#define MAIL_PROC_NAME_SMTP "smtp" +#define MAIL_PROC_NAME_LMTP "lmtp" + + /* * Mail source classes. Used to specify policy decisions for content * inspection and SMTPUTF8 detection. */ diff --git a/src/global/mail_version.h b/src/global/mail_version.h index 9eda667..9e08896 100644 --- a/src/global/mail_version.h +++ b/src/global/mail_version.h @@ -20,8 +20,8 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20240304" -#define MAIL_VERSION_NUMBER "3.8.6" +#define MAIL_RELEASE_DATE "20240306" +#define MAIL_VERSION_NUMBER "3.9" #ifdef SNAPSHOT #define MAIL_VERSION_DATE "-" MAIL_RELEASE_DATE diff --git a/src/global/maillog_client.c b/src/global/maillog_client.c index 7f79a1f..34952ef 100644 --- a/src/global/maillog_client.c +++ b/src/global/maillog_client.c @@ -58,7 +58,7 @@ /* unitialized and the process environment does not specify /* POSTLOG_SERVICE, the program will log to the syslog service /* instead. -/* .IP "myhostname (default: see postconf -d output)" +/* .IP "myhostname (default: see 'postconf -d' output)" /* The internet hostname of this mail system. /* .IP "postlog_service_name (postlog)" /* The name of the internal postlog logging service. diff --git a/src/global/maps.c b/src/global/maps.c index 790396b..d237002 100644 --- a/src/global/maps.c +++ b/src/global/maps.c @@ -195,8 +195,12 @@ const char *maps_find(MAPS *maps, const char *name, int flags) for (map_name = maps->argv->argv; *map_name; map_name++) { if ((dict = dict_handle(*map_name)) == 0) msg_panic("%s: dictionary not found: %s", myname, *map_name); - if (flags != 0 && (dict->flags & flags) == 0) + if (flags != 0 && (dict->flags & flags) == 0) { + if (msg_verbose) + msg_info("%s: %s: skipping %s lookup for %s", + myname, maps->title, *map_name, name); continue; + } if ((expansion = dict_get(dict, name)) != 0) { if (*expansion == 0) { msg_warn("%s lookup of %s returns an empty string result", @@ -252,8 +256,12 @@ const char *maps_file_find(MAPS *maps, const char *name, int flags) if ((dict->flags & DICT_FLAG_SRC_RHS_IS_FILE) == 0) msg_panic("%s: %s: opened without DICT_FLAG_SRC_RHS_IS_FILE", myname, maps->title); - if (flags != 0 && (dict->flags & flags) == 0) + if (flags != 0 && (dict->flags & flags) == 0) { + if (msg_verbose) + msg_info("%s: %s: skipping %s lookup for %s", + myname, maps->title, *map_name, name); continue; + } if ((expansion = dict_get(dict, name)) != 0) { if (*expansion == 0) { msg_warn("%s lookup of %s returns an empty string result", diff --git a/src/global/wildcard_inet_addr.c b/src/global/wildcard_inet_addr.c index 97f6c46..0a3c37a 100644 --- a/src/global/wildcard_inet_addr.c +++ b/src/global/wildcard_inet_addr.c @@ -11,7 +11,7 @@ /* wildcard_inet_addr() determines all wild-card addresses /* for all supported address families. /* DIAGNOSTICS -/* Fatal errors: out of memory. +/* Fatal errors: out of memory; no wildcard addresses. /* SEE ALSO /* inet_addr_list(3) address list management /* LICENSE |