diff options
Diffstat (limited to 'src/util/dict_cache.c')
| -rw-r--r-- | src/util/dict_cache.c | 1121 |
1 files changed, 1121 insertions, 0 deletions
diff --git a/src/util/dict_cache.c b/src/util/dict_cache.c new file mode 100644 index 0000000..2259f12 --- /dev/null +++ b/src/util/dict_cache.c @@ -0,0 +1,1121 @@ +/*++ +/* NAME +/* dict_cache 3 +/* SUMMARY +/* External cache manager +/* SYNOPSIS +/* #include <dict_cache.h> +/* +/* DICT_CACHE *dict_cache_open(dbname, open_flags, dict_flags) +/* const char *dbname; +/* int open_flags; +/* int dict_flags; +/* +/* void dict_cache_close(cache) +/* DICT_CACHE *cache; +/* +/* const char *dict_cache_lookup(cache, cache_key) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* +/* int dict_cache_update(cache, cache_key, cache_val) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* const char *cache_val; +/* +/* int dict_cache_delete(cache, cache_key) +/* DICT_CACHE *cache; +/* const char *cache_key; +/* +/* int dict_cache_sequence(cache, first_next, cache_key, cache_val) +/* DICT_CACHE *cache; +/* int first_next; +/* const char **cache_key; +/* const char **cache_val; +/* AUXILIARY FUNCTIONS +/* void dict_cache_control(cache, name, value, ...) +/* DICT_CACHE *cache; +/* int name; +/* +/* typedef int (*DICT_CACHE_VALIDATOR_FN) (const char *cache_key, +/* const char *cache_val, void *context); +/* +/* const char *dict_cache_name(cache) +/* DICT_CACHE *cache; +/* DESCRIPTION +/* This module maintains external cache files with support +/* for expiration. The underlying table must implement the +/* "lookup", "update", "delete" and "sequence" operations. +/* +/* Although this API is similar to the one documented in +/* dict_open(3), there are subtle differences in the interaction +/* between the iterators that access all cache elements, and +/* other operations that access individual cache elements. +/* +/* In particular, when a "sequence" or "cleanup" operation is +/* in progress the cache intercepts requests to delete the +/* "current" entry, as this would cause some databases to +/* mis-behave. Instead, the cache implements a "delete behind" +/* strategy, and deletes such an entry after the "sequence" +/* or "cleanup" operation moves on to the next cache element. +/* The "delete behind" strategy also affects the cache lookup +/* and update operations as detailed below. +/* +/* dict_cache_open() is a wrapper around the dict_open() +/* function. It opens the specified cache and returns a handle +/* that must be used for subsequent access. This function does +/* not return in case of error. +/* +/* dict_cache_close() closes the specified cache and releases +/* memory that was allocated by dict_cache_open(), and terminates +/* any thread that was started with dict_cache_control(). +/* +/* dict_cache_lookup() looks up the specified cache entry. +/* The result value is a null pointer when the cache entry was +/* not found, or when the entry is scheduled for "delete +/* behind". +/* +/* dict_cache_update() updates the specified cache entry. If +/* the entry is scheduled for "delete behind", the delete +/* operation is canceled (because of this, the cache must be +/* opened with DICT_FLAG_DUP_REPLACE). This function does not +/* return in case of error. +/* +/* dict_cache_delete() removes the specified cache entry. If +/* this is the "current" entry of a "sequence" operation, the +/* entry is scheduled for "delete behind". The result value +/* is zero when the entry was found. +/* +/* dict_cache_sequence() iterates over the specified cache and +/* returns each entry in an implementation-defined order. The +/* result value is zero when a cache entry was found. +/* +/* Important: programs must not use both dict_cache_sequence() +/* and the built-in cache cleanup feature. +/* +/* dict_cache_control() provides control over the built-in +/* cache cleanup feature and logging. The arguments are a list +/* of macros with zero or more arguments, terminated with +/* CA_DICT_CACHE_CTL_END which has none. The following lists +/* the macros and corresponding argument types. +/* .IP "CA_DICT_CACHE_CTL_FLAGS(int flags)" +/* The arguments to this command are the bit-wise OR of zero +/* or more of the following: +/* .RS +/* .IP CA_DICT_CACHE_CTL_FLAG_VERBOSE +/* Enable verbose logging of cache activity. +/* .IP CA_DICT_CACHE_CTL_FLAG_EXP_SUMMARY +/* Log cache statistics after each cache cleanup run. +/* .RE +/* .IP "CA_DICT_CACHE_CTL_INTERVAL(int interval)" +/* The interval between cache cleanup runs. Specify a null +/* validator or interval to stop cache cleanup. +/* .IP "CA_DICT_CACHE_CTL_VALIDATOR(DICT_CACHE_VALIDATOR_FN validator)" +/* An application call-back routine that returns non-zero when +/* a cache entry should be kept. The call-back function should +/* not make changes to the cache. Specify a null validator or +/* interval to stop cache cleanup. +/* .IP "CA_DICT_CACHE_CTL_CONTEXT(void *context)" +/* Application context that is passed to the validator function. +/* .RE +/* .PP +/* dict_cache_name() returns the name of the specified cache. +/* +/* Arguments: +/* .IP "dbname, open_flags, dict_flags" +/* These are passed unchanged to dict_open(). The cache must +/* be opened with DICT_FLAG_DUP_REPLACE. +/* .IP cache +/* Cache handle created with dict_cache_open(). +/* .IP cache_key +/* Cache lookup key. +/* .IP cache_val +/* Information that is stored under a cache lookup key. +/* .IP first_next +/* One of DICT_SEQ_FUN_FIRST (first cache element) or +/* DICT_SEQ_FUN_NEXT (next cache element). +/* .sp +/* Note: there is no "stop" request. To ensure that the "delete +/* behind" strategy does not interfere with database access, +/* allow dict_cache_sequence() to run to completion. +/* .IP table +/* A bare dictonary handle. +/* DIAGNOSTICS +/* When a request is satisfied, the lookup routine returns +/* non-null, and the update, delete and sequence routines +/* return zero. The cache->error value is zero when a request +/* could not be satisfied because an item did not exist (delete, +/* sequence) or if it could not be updated. The cache->error +/* value is non-zero only when a request could not be satisfied, +/* and the cause was a database error. +/* +/* Cache access errors are logged with a warning message. To +/* avoid spamming the log, each type of operation logs no more +/* than one cache access error per second, per cache. Specify +/* the DICT_CACHE_FLAG_VERBOSE flag (see above) to log all +/* warnings. +/* BUGS +/* There should be a way to suspend automatic program suicide +/* until a cache cleanup run is completed. Some entries may +/* never be removed when the process max_idle time is less +/* than the time needed to make a full pass over the cache. +/* +/* The delete-behind strategy assumes that all updates are +/* made by a single process. Otherwise, delete-behind may +/* remove an entry that was updated after it was scheduled for +/* deletion. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* HISTORY +/* .ad +/* .fi +/* A predecessor of this code was written first for the Postfix +/* tlsmgr(8) daemon. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include <sys_defs.h> +#include <string.h> +#include <stdlib.h> + +/* Utility library. */ + +#include <msg.h> +#include <dict.h> +#include <mymalloc.h> +#include <events.h> +#include <dict_cache.h> + +/* Application-specific. */ + + /* + * XXX Deleting entries while enumerating a map can he tricky. Some map + * types have a concept of cursor and support a "delete the current element" + * operation. Some map types without cursors don't behave well when the + * current first/next entry is deleted (example: with Berkeley DB < 2, the + * "next" operation produces garbage). To avoid trouble, we delete an entry + * after advancing the current first/next position beyond it; we use the + * same strategy with application requests to delete the current entry. + */ + + /* + * Opaque data structure. Use dict_cache_name() to access the name of the + * underlying database. + */ +struct DICT_CACHE { + char *name; /* full name including proxy: */ + int cache_flags; /* see below */ + int user_flags; /* logging */ + DICT *db; /* database handle */ + int error; /* last operation only */ + + /* Delete-behind support. */ + char *saved_curr_key; /* "current" cache lookup key */ + char *saved_curr_val; /* "current" cache lookup result */ + + /* Cleanup support. */ + int exp_interval; /* time between cleanup runs */ + DICT_CACHE_VALIDATOR_FN exp_validator; /* expiration call-back */ + void *exp_context; /* call-back context */ + int retained; /* entries retained in cleanup run */ + int dropped; /* entries removed in cleanup run */ + + /* Rate-limited logging support. */ + int log_delay; + time_t upd_log_stamp; /* last update warning */ + time_t get_log_stamp; /* last lookup warning */ + time_t del_log_stamp; /* last delete warning */ + time_t seq_log_stamp; /* last sequence warning */ +}; + +#define DC_FLAG_DEL_SAVED_CURRENT_KEY (1<<0) /* delete-behind is scheduled */ + + /* + * Don't log cache access errors more than once per second. + */ +#define DC_DEF_LOG_DELAY 1 + + /* + * Macros to make obscure code more readable. + */ +#define DC_SCHEDULE_FOR_DELETE_BEHIND(cp) \ + ((cp)->cache_flags |= DC_FLAG_DEL_SAVED_CURRENT_KEY) + +#define DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key) \ + ((cp)->saved_curr_key && strcmp((cp)->saved_curr_key, (cache_key)) == 0) + +#define DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) \ + (/* NOT: (cp)->saved_curr_key && */ \ + ((cp)->cache_flags & DC_FLAG_DEL_SAVED_CURRENT_KEY) != 0) + +#define DC_CANCEL_DELETE_BEHIND(cp) \ + ((cp)->cache_flags &= ~DC_FLAG_DEL_SAVED_CURRENT_KEY) + + /* + * Special key to store the time of the last cache cleanup run completion. + */ +#define DC_LAST_CACHE_CLEANUP_COMPLETED "_LAST_CACHE_CLEANUP_COMPLETED_" + +/* dict_cache_lookup - load entry from cache */ + +const char *dict_cache_lookup(DICT_CACHE *cp, const char *cache_key) +{ + const char *myname = "dict_cache_lookup"; + const char *cache_val; + DICT *db = cp->db; + + /* + * Search for the cache entry. Don't return an entry that is scheduled + * for delete-behind. + */ + if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) + && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s (pretend not found - scheduled for deletion)", + myname, cache_key); + DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, (char *) 0); + } else { + cache_val = dict_get(db, cache_key); + if (cache_val == 0 && db->error != 0) + msg_rate_delay(&cp->get_log_stamp, cp->log_delay, msg_warn, + "%s: cache lookup for '%s' failed due to error", + cp->name, cache_key); + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s value=%s", myname, cache_key, + cache_val ? cache_val : db->error ? + "error" : "(not found)"); + DICT_ERR_VAL_RETURN(cp, db->error, cache_val); + } +} + +/* dict_cache_update - save entry to cache */ + +int dict_cache_update(DICT_CACHE *cp, const char *cache_key, + const char *cache_val) +{ + const char *myname = "dict_cache_update"; + DICT *db = cp->db; + int put_res; + + /* + * Store the cache entry and cancel the delete-behind operation. + */ + if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) + && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: cancel delete-behind for key=%s", myname, cache_key); + DC_CANCEL_DELETE_BEHIND(cp); + } + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s value=%s", myname, cache_key, cache_val); + put_res = dict_put(db, cache_key, cache_val); + if (put_res != 0) + msg_rate_delay(&cp->upd_log_stamp, cp->log_delay, msg_warn, + "%s: could not update entry for %s", cp->name, cache_key); + DICT_ERR_VAL_RETURN(cp, db->error, put_res); +} + +/* dict_cache_delete - delete entry from cache */ + +int dict_cache_delete(DICT_CACHE *cp, const char *cache_key) +{ + const char *myname = "dict_cache_delete"; + int del_res; + DICT *db = cp->db; + + /* + * Delete the entry, unless we would delete the current first/next entry. + * In that case, schedule the "current" entry for delete-behind to avoid + * mis-behavior by some databases. + */ + if (DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { + DC_SCHEDULE_FOR_DELETE_BEHIND(cp); + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s (current entry - schedule for delete-behind)", + myname, cache_key); + DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, DICT_STAT_SUCCESS); + } else { + del_res = dict_del(db, cache_key); + if (del_res != 0) + msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, + "%s: could not delete entry for %s", cp->name, cache_key); + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s (%s)", myname, cache_key, + del_res == 0 ? "found" : + db->error ? "error" : "not found"); + DICT_ERR_VAL_RETURN(cp, db->error, del_res); + } +} + +/* dict_cache_sequence - look up the first/next cache entry */ + +int dict_cache_sequence(DICT_CACHE *cp, int first_next, + const char **cache_key, + const char **cache_val) +{ + const char *myname = "dict_cache_sequence"; + int seq_res; + const char *raw_cache_key; + const char *raw_cache_val; + char *previous_curr_key; + char *previous_curr_val; + DICT *db = cp->db; + + /* + * Find the first or next database entry. Hide the record with the cache + * cleanup completion time stamp. + */ + seq_res = dict_seq(db, first_next, &raw_cache_key, &raw_cache_val); + if (seq_res == 0 + && strcmp(raw_cache_key, DC_LAST_CACHE_CLEANUP_COMPLETED) == 0) + seq_res = + dict_seq(db, DICT_SEQ_FUN_NEXT, &raw_cache_key, &raw_cache_val); + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: key=%s value=%s", myname, + seq_res == 0 ? raw_cache_key : db->error ? + "(error)" : "(not found)", + seq_res == 0 ? raw_cache_val : db->error ? + "(error)" : "(not found)"); + if (db->error) + msg_rate_delay(&cp->seq_log_stamp, cp->log_delay, msg_warn, + "%s: sequence error", cp->name); + + /* + * Save the current cache_key and cache_val before they are clobbered by + * our own delete operation below. This also prevents surprises when the + * application accesses the database after this function returns. + * + * We also use the saved cache_key to protect the current entry against + * application delete requests. + */ + previous_curr_key = cp->saved_curr_key; + previous_curr_val = cp->saved_curr_val; + if (seq_res == 0) { + cp->saved_curr_key = mystrdup(raw_cache_key); + cp->saved_curr_val = mystrdup(raw_cache_val); + } else { + cp->saved_curr_key = 0; + cp->saved_curr_val = 0; + } + + /* + * Delete behind. + */ + if (db->error == 0 && DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp)) { + DC_CANCEL_DELETE_BEHIND(cp); + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: delete-behind key=%s value=%s", + myname, previous_curr_key, previous_curr_val); + if (dict_del(db, previous_curr_key) != 0) + msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, + "%s: could not delete entry for %s", + cp->name, previous_curr_key); + } + + /* + * Clean up previous iteration key and value. + */ + if (previous_curr_key) + myfree(previous_curr_key); + if (previous_curr_val) + myfree(previous_curr_val); + + /* + * Return the result. + */ + *cache_key = (cp)->saved_curr_key; + *cache_val = (cp)->saved_curr_val; + DICT_ERR_VAL_RETURN(cp, db->error, seq_res); +} + +/* dict_cache_delete_behind_reset - reset "delete behind" state */ + +static void dict_cache_delete_behind_reset(DICT_CACHE *cp) +{ +#define FREE_AND_WIPE(s) do { if (s) { myfree(s); (s) = 0; } } while (0) + + DC_CANCEL_DELETE_BEHIND(cp); + FREE_AND_WIPE(cp->saved_curr_key); + FREE_AND_WIPE(cp->saved_curr_val); +} + +/* dict_cache_clean_stat_log_reset - log and reset cache cleanup statistics */ + +static void dict_cache_clean_stat_log_reset(DICT_CACHE *cp, + const char *full_partial) +{ + if (cp->user_flags & DICT_CACHE_FLAG_STATISTICS) + msg_info("cache %s %s cleanup: retained=%d dropped=%d entries", + cp->name, full_partial, cp->retained, cp->dropped); + cp->retained = cp->dropped = 0; +} + +/* dict_cache_clean_event - examine one cache entry */ + +static void dict_cache_clean_event(int unused_event, void *cache_context) +{ + const char *myname = "dict_cache_clean_event"; + DICT_CACHE *cp = (DICT_CACHE *) cache_context; + const char *cache_key; + const char *cache_val; + int next_interval; + VSTRING *stamp_buf; + int first_next; + + /* + * We interleave cache cleanup with other processing, so that the + * application's service remains available, with perhaps increased + * latency. + */ + + /* + * Start a new cache cleanup run. + */ + if (cp->saved_curr_key == 0) { + cp->retained = cp->dropped = 0; + first_next = DICT_SEQ_FUN_FIRST; + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: start %s cache cleanup", myname, cp->name); + } + + /* + * Continue a cache cleanup run in progress. + */ + else { + first_next = DICT_SEQ_FUN_NEXT; + } + + /* + * Examine one cache entry. + */ + if (dict_cache_sequence(cp, first_next, &cache_key, &cache_val) == 0) { + if (cp->exp_validator(cache_key, cache_val, cp->exp_context) == 0) { + DC_SCHEDULE_FOR_DELETE_BEHIND(cp); + cp->dropped++; + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: drop %s cache entry for %s", + myname, cp->name, cache_key); + } else { + cp->retained++; + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: keep %s cache entry for %s", + myname, cp->name, cache_key); + } + next_interval = 0; + } + + /* + * Cache cleanup completed. Report vital statistics. + */ + else if (cp->error != 0) { + msg_warn("%s: cache cleanup scan terminated due to error", cp->name); + dict_cache_clean_stat_log_reset(cp, "partial"); + next_interval = cp->exp_interval; + } else { + if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) + msg_info("%s: done %s cache cleanup scan", myname, cp->name); + dict_cache_clean_stat_log_reset(cp, "full"); + stamp_buf = vstring_alloc(100); + vstring_sprintf(stamp_buf, "%ld", (long) event_time()); + dict_put(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED, + vstring_str(stamp_buf)); + vstring_free(stamp_buf); + next_interval = cp->exp_interval; + } + event_request_timer(dict_cache_clean_event, cache_context, next_interval); +} + +/* dict_cache_control - schedule or stop the cache cleanup thread */ + +void dict_cache_control(DICT_CACHE *cp,...) +{ + const char *myname = "dict_cache_control"; + const char *last_done; + time_t next_interval; + int cache_cleanup_is_active = (cp->exp_validator && cp->exp_interval); + va_list ap; + int name; + + /* + * Update the control settings. + */ + va_start(ap, cp); + while ((name = va_arg(ap, int)) > 0) { + switch (name) { + case DICT_CACHE_CTL_END: + break; + case DICT_CACHE_CTL_FLAGS: + cp->user_flags = va_arg(ap, int); + cp->log_delay = (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) ? + 0 : DC_DEF_LOG_DELAY; + break; + case DICT_CACHE_CTL_INTERVAL: + cp->exp_interval = va_arg(ap, int); + if (cp->exp_interval < 0) + msg_panic("%s: bad %s cache cleanup interval %d", + myname, cp->name, cp->exp_interval); + break; + case DICT_CACHE_CTL_VALIDATOR: + cp->exp_validator = va_arg(ap, DICT_CACHE_VALIDATOR_FN); + break; + case DICT_CACHE_CTL_CONTEXT: + cp->exp_context = va_arg(ap, void *); + break; + default: + msg_panic("%s: bad command: %d", myname, name); + } + } + va_end(ap); + + /* + * Schedule the cache cleanup thread. + */ + if (cp->exp_interval && cp->exp_validator) { + + /* + * Sanity checks. + */ + if (cache_cleanup_is_active) + msg_panic("%s: %s cache cleanup is already scheduled", + myname, cp->name); + + /* + * The next start time depends on the last completion time. + */ +#define NEXT_START(last, delta) ((delta) + (unsigned long) atol(last)) +#define NOW (time((time_t *) 0)) /* NOT: event_time() */ + + if ((last_done = dict_get(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED)) == 0 + || (next_interval = (NEXT_START(last_done, cp->exp_interval) - NOW)) < 0) + next_interval = 0; + if (next_interval > cp->exp_interval) + next_interval = cp->exp_interval; + if ((cp->user_flags & DICT_CACHE_FLAG_VERBOSE) && next_interval > 0) + msg_info("%s cache cleanup will start after %ds", + cp->name, (int) next_interval); + event_request_timer(dict_cache_clean_event, (void *) cp, + (int) next_interval); + } + + /* + * Cancel the cache cleanup thread. + */ + else if (cache_cleanup_is_active) { + if (cp->retained || cp->dropped) + dict_cache_clean_stat_log_reset(cp, "partial"); + dict_cache_delete_behind_reset(cp); + event_cancel_timer(dict_cache_clean_event, (void *) cp); + } +} + +/* dict_cache_open - open cache file */ + +DICT_CACHE *dict_cache_open(const char *dbname, int open_flags, int dict_flags) +{ + DICT_CACHE *cp; + DICT *dict; + + /* + * Open the database as requested. Don't attempt to second-guess the + * application. + */ + dict = dict_open(dbname, open_flags, dict_flags); + + /* + * Create the DICT_CACHE object. + */ + cp = (DICT_CACHE *) mymalloc(sizeof(*cp)); + cp->name = mystrdup(dbname); + cp->cache_flags = 0; + cp->user_flags = 0; + cp->db = dict; + cp->saved_curr_key = 0; + cp->saved_curr_val = 0; + cp->exp_interval = 0; + cp->exp_validator = 0; + cp->exp_context = 0; + cp->retained = 0; + cp->dropped = 0; + cp->log_delay = DC_DEF_LOG_DELAY; + cp->upd_log_stamp = cp->get_log_stamp = + cp->del_log_stamp = cp->seq_log_stamp = 0; + + return (cp); +} + +/* dict_cache_close - close cache file */ + +void dict_cache_close(DICT_CACHE *cp) +{ + + /* + * Destroy the DICT_CACHE object. + */ + dict_cache_control(cp, DICT_CACHE_CTL_INTERVAL, 0, DICT_CACHE_CTL_END); + myfree(cp->name); + dict_close(cp->db); + if (cp->saved_curr_key) + myfree(cp->saved_curr_key); + if (cp->saved_curr_val) + myfree(cp->saved_curr_val); + myfree((void *) cp); +} + +/* dict_cache_name - get the cache name */ + +const char *dict_cache_name(DICT_CACHE *cp) +{ + + /* + * This is used for verbose logging or warning messages, so the cost of + * call is only made where needed (well sort off - code that does not + * execute still presents overhead for the processor pipeline, processor + * cache, etc). + */ + return (cp->name); +} + + /* + * Test driver with support for interleaved access. First, enter a number of + * requests to look up, update or delete a sequence of cache entries, then + * interleave those sequences with the "run" command. + */ +#ifdef TEST +#include <msg_vstream.h> +#include <vstring_vstream.h> +#include <argv.h> +#include <stringops.h> + +#define DELIMS " " +#define USAGE "\n\tTo manage settings:" \ + "\n\tverbose <level> (verbosity level)" \ + "\n\telapsed <level> (0=don't show elapsed time)" \ + "\n\tlmdb_map_size <limit> (initial LMDB size limit)" \ + "\n\tcache <type>:<name> (switch to named database)" \ + "\n\tstatus (show map size, cache, pending requests)" \ + "\n\n\tTo manage pending requests:" \ + "\n\treset (discard pending requests)" \ + "\n\trun (execute pending requests in interleaved order)" \ + "\n\n\tTo add a pending request:" \ + "\n\tquery <key-suffix> <count> (negative to reverse order)" \ + "\n\tupdate <key-suffix> <count> (negative to reverse order)" \ + "\n\tdelete <key-suffix> <count> (negative to reverse order)" \ + "\n\tpurge <key-suffix>" \ + "\n\tcount <key-suffix>" + + /* + * For realism, open the cache with the same flags as postscreen(8) and + * verify(8). + */ +#define DICT_CACHE_OPEN_FLAGS (DICT_FLAG_DUP_REPLACE | DICT_FLAG_SYNC_UPDATE | \ + DICT_FLAG_OPEN_LOCK) + + /* + * Storage for one request to access a sequence of cache entries. + */ +typedef struct DICT_CACHE_SREQ { + int flags; /* per-request: reverse, purge */ + char *cmd; /* command for status report */ + void (*action) (struct DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); + char *suffix; /* key suffix */ + int done; /* progress indicator */ + int todo; /* number of entries to process */ + int first_next; /* first/next */ +} DICT_CACHE_SREQ; + +#define DICT_CACHE_SREQ_FLAG_PURGE (1<<1) /* purge instead of count */ +#define DICT_CACHE_SREQ_FLAG_REVERSE (1<<2) /* reverse instead of forward */ + +#define DICT_CACHE_SREQ_LIMIT 10 + + /* + * All test requests combined. + */ +typedef struct DICT_CACHE_TEST { + int flags; /* exclusion flags */ + int size; /* allocated slots */ + int used; /* used slots */ + DICT_CACHE_SREQ job_list[1]; /* actually, a bunch */ +} DICT_CACHE_TEST; + +#define DICT_CACHE_TEST_FLAG_ITER (1<<0) /* count or purge */ + +#define STR(x) vstring_str(x) + +int show_elapsed = 1; /* show elapsed time */ + +#ifdef HAS_LMDB +extern size_t dict_lmdb_map_size; /* LMDB-specific */ + +#endif + +/* usage - command-line usage message */ + +static NORETURN usage(const char *progname) +{ + msg_fatal("usage: %s (no argument)", progname); +} + +/* make_tagged_key - make tagged search key */ + +static void make_tagged_key(VSTRING *bp, DICT_CACHE_SREQ *cp) +{ + if (cp->done < 0) + msg_panic("make_tagged_key: bad done count: %d", cp->done); + if (cp->todo < 1) + msg_panic("make_tagged_key: bad todo count: %d", cp->todo); + vstring_sprintf(bp, "%d-%s", + (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? + cp->todo - cp->done - 1 : cp->done, cp->suffix); +} + +/* create_requests - create request list */ + +static DICT_CACHE_TEST *create_requests(int count) +{ + DICT_CACHE_TEST *tp; + DICT_CACHE_SREQ *cp; + + tp = (DICT_CACHE_TEST *) mymalloc(sizeof(DICT_CACHE_TEST) + + (count - 1) *sizeof(DICT_CACHE_SREQ)); + tp->flags = 0; + tp->size = count; + tp->used = 0; + for (cp = tp->job_list; cp < tp->job_list + count; cp++) { + cp->flags = 0; + cp->cmd = 0; + cp->action = 0; + cp->suffix = 0; + cp->todo = 0; + cp->first_next = DICT_SEQ_FUN_FIRST; + } + return (tp); +} + +/* reset_requests - reset request list */ + +static void reset_requests(DICT_CACHE_TEST *tp) +{ + DICT_CACHE_SREQ *cp; + + tp->flags = 0; + tp->used = 0; + for (cp = tp->job_list; cp < tp->job_list + tp->size; cp++) { + cp->flags = 0; + if (cp->cmd) { + myfree(cp->cmd); + cp->cmd = 0; + } + cp->action = 0; + if (cp->suffix) { + myfree(cp->suffix); + cp->suffix = 0; + } + cp->todo = 0; + cp->first_next = DICT_SEQ_FUN_FIRST; + } +} + +/* free_requests - destroy request list */ + +static void free_requests(DICT_CACHE_TEST *tp) +{ + reset_requests(tp); + myfree((void *) tp); +} + +/* run_requests - execute pending requests in interleaved order */ + +static void run_requests(DICT_CACHE_TEST *tp, DICT_CACHE *dp, VSTRING *bp) +{ + DICT_CACHE_SREQ *cp; + int todo; + struct timeval start; + struct timeval finish; + struct timeval elapsed; + + if (dp == 0) { + msg_warn("no cache"); + return; + } + GETTIMEOFDAY(&start); + do { + todo = 0; + for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) { + if (cp->done < cp->todo) { + todo = 1; + cp->action(cp, dp, bp); + } + } + } while (todo); + GETTIMEOFDAY(&finish); + timersub(&finish, &start, &elapsed); + if (show_elapsed) + vstream_printf("Elapsed: %g\n", + elapsed.tv_sec + elapsed.tv_usec / 1000000.0); + + reset_requests(tp); +} + +/* show_status - show settings and pending requests */ + +static void show_status(DICT_CACHE_TEST *tp, DICT_CACHE *dp) +{ + DICT_CACHE_SREQ *cp; + +#ifdef HAS_LMDB + vstream_printf("lmdb_map_size\t%ld\n", (long) dict_lmdb_map_size); +#endif + vstream_printf("cache\t%s\n", dp ? dp->name : "(none)"); + + if (tp->used == 0) + vstream_printf("No pending requests\n"); + else + vstream_printf("%s\t%s\t%s\t%s\t%s\t%s\n", + "cmd", "dir", "suffix", "count", "done", "first/next"); + + for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) + if (cp->todo > 0) + vstream_printf("%s\t%s\t%s\t%d\t%d\t%d\n", + cp->cmd, + (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? + "reverse" : "forward", + cp->suffix ? cp->suffix : "(null)", cp->todo, + cp->done, cp->first_next); +} + +/* query_action - lookup cache entry */ + +static void query_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) +{ + const char *lookup; + + make_tagged_key(bp, cp); + if ((lookup = dict_cache_lookup(dp, STR(bp))) == 0) { + if (dp->error) + msg_warn("query_action: query failed: %s: %m", STR(bp)); + else + msg_warn("query_action: query failed: %s", STR(bp)); + } else if (strcmp(STR(bp), lookup) != 0) { + msg_warn("lookup result \"%s\" differs from key \"%s\"", + lookup, STR(bp)); + } + cp->done += 1; +} + +/* update_action - update cache entry */ + +static void update_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) +{ + make_tagged_key(bp, cp); + if (dict_cache_update(dp, STR(bp), STR(bp)) != 0) { + if (dp->error) + msg_warn("update_action: update failed: %s: %m", STR(bp)); + else + msg_warn("update_action: update failed: %s", STR(bp)); + } + cp->done += 1; +} + +/* delete_action - delete cache entry */ + +static void delete_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) +{ + make_tagged_key(bp, cp); + if (dict_cache_delete(dp, STR(bp)) != 0) { + if (dp->error) + msg_warn("delete_action: delete failed: %s: %m", STR(bp)); + else + msg_warn("delete_action: delete failed: %s", STR(bp)); + } + cp->done += 1; +} + +/* iter_action - iterate over cache and act on entries with given suffix */ + +static void iter_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) +{ + const char *cache_key; + const char *cache_val; + const char *what; + const char *suffix; + + if (dict_cache_sequence(dp, cp->first_next, &cache_key, &cache_val) == 0) { + if (strcmp(cache_key, cache_val) != 0) + msg_warn("value \"%s\" differs from key \"%s\"", + cache_val, cache_key); + suffix = cache_key + strspn(cache_key, "0123456789"); + if (suffix[0] == '-' && strcmp(suffix + 1, cp->suffix) == 0) { + cp->done += 1; + cp->todo = cp->done + 1; /* XXX */ + if ((cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) + && dict_cache_delete(dp, cache_key) != 0) { + if (dp->error) + msg_warn("purge_action: delete failed: %s: %m", STR(bp)); + else + msg_warn("purge_action: delete failed: %s", STR(bp)); + } + } + cp->first_next = DICT_SEQ_FUN_NEXT; + } else { + what = (cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) ? "purge" : "count"; + if (dp->error) + msg_warn("%s error after %d: %m", what, cp->done); + else + vstream_printf("suffix=%s %s=%d\n", cp->suffix, what, cp->done); + cp->todo = 0; + } +} + + /* + * Table-driven support. + */ +typedef struct DICT_CACHE_SREQ_INFO { + const char *name; + int argc; + void (*action) (DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); + int test_flags; + int req_flags; +} DICT_CACHE_SREQ_INFO; + +static DICT_CACHE_SREQ_INFO req_info[] = { + {"query", 3, query_action}, + {"update", 3, update_action}, + {"delete", 3, delete_action}, + {"count", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER}, + {"purge", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER, DICT_CACHE_SREQ_FLAG_PURGE}, + 0, +}; + +/* add_request - add a request to the list */ + +static void add_request(DICT_CACHE_TEST *tp, ARGV *argv) +{ + DICT_CACHE_SREQ_INFO *rp; + DICT_CACHE_SREQ *cp; + int req_flags; + int count; + char *cmd = argv->argv[0]; + char *suffix = (argv->argc > 1 ? argv->argv[1] : 0); + char *todo = (argv->argc > 2 ? argv->argv[2] : "1"); /* XXX */ + + if (tp->used >= tp->size) { + msg_warn("%s: request list is full", cmd); + return; + } + for (rp = req_info; /* See below */ ; rp++) { + if (rp->name == 0) { + vstream_printf("usage: %s\n", USAGE); + return; + } + if (strcmp(rp->name, argv->argv[0]) == 0 + && rp->argc == argv->argc) + break; + } + req_flags = rp->req_flags; + if (todo[0] == '-') { + req_flags |= DICT_CACHE_SREQ_FLAG_REVERSE; + todo += 1; + } + if (!alldig(todo) || (count = atoi(todo)) == 0) { + msg_warn("%s: bad count: %s", cmd, todo); + return; + } + if (tp->flags & rp->test_flags) { + msg_warn("%s: command conflicts with other command", cmd); + return; + } + tp->flags |= rp->test_flags; + cp = tp->job_list + tp->used; + cp->cmd = mystrdup(cmd); + cp->action = rp->action; + if (suffix) + cp->suffix = mystrdup(suffix); + cp->done = 0; + cp->flags = req_flags; + cp->todo = count; + tp->used += 1; +} + +/* main - main program */ + +int main(int argc, char **argv) +{ + DICT_CACHE_TEST *test_job; + VSTRING *inbuf = vstring_alloc(100); + char *bufp; + ARGV *args; + DICT_CACHE *cache = 0; + int stdin_is_tty; + + msg_vstream_init(argv[0], VSTREAM_ERR); + if (argc != 1) + usage(argv[0]); + + + test_job = create_requests(DICT_CACHE_SREQ_LIMIT); + + stdin_is_tty = isatty(0); + + for (;;) { + if (stdin_is_tty) { + vstream_printf("> "); + vstream_fflush(VSTREAM_OUT); + } + if (vstring_fgets_nonl(inbuf, VSTREAM_IN) == 0) + break; + bufp = vstring_str(inbuf); + if (!stdin_is_tty) { + vstream_printf("> %s\n", bufp); + vstream_fflush(VSTREAM_OUT); + } + if (*bufp == '#') + continue; + args = argv_split(bufp, DELIMS); + if (argc == 0) { + vstream_printf("usage: %s\n", USAGE); + vstream_fflush(VSTREAM_OUT); + continue; + } + if (strcmp(args->argv[0], "verbose") == 0 && args->argc == 2) { + msg_verbose = atoi(args->argv[1]); + } else if (strcmp(args->argv[0], "elapsed") == 0 && args->argc == 2) { + show_elapsed = atoi(args->argv[1]); +#ifdef HAS_LMDB + } else if (strcmp(args->argv[0], "lmdb_map_size") == 0 && args->argc == 2) { + dict_lmdb_map_size = atol(args->argv[1]); +#endif + } else if (strcmp(args->argv[0], "cache") == 0 && args->argc == 2) { + if (cache) + dict_cache_close(cache); + cache = dict_cache_open(args->argv[1], O_CREAT | O_RDWR, + DICT_CACHE_OPEN_FLAGS); + } else if (strcmp(args->argv[0], "reset") == 0 && args->argc == 1) { + reset_requests(test_job); + } else if (strcmp(args->argv[0], "run") == 0 && args->argc == 1) { + run_requests(test_job, cache, inbuf); + } else if (strcmp(args->argv[0], "status") == 0 && args->argc == 1) { + show_status(test_job, cache); + } else { + add_request(test_job, args); + } + vstream_fflush(VSTREAM_OUT); + argv_free(args); + } + + vstring_free(inbuf); + free_requests(test_job); + if (cache) + dict_cache_close(cache); + return (0); +} + +#endif |