/*++ /* NAME /* dict_cache 3 /* SUMMARY /* External cache manager /* SYNOPSIS /* #include /* /* DICT_CACHE *dict_cache_open(dbname, open_flags, dict_flags) /* const char *dbname; /* int open_flags; /* int dict_flags; /* /* void dict_cache_close(cache) /* DICT_CACHE *cache; /* /* const char *dict_cache_lookup(cache, cache_key) /* DICT_CACHE *cache; /* const char *cache_key; /* /* int dict_cache_update(cache, cache_key, cache_val) /* DICT_CACHE *cache; /* const char *cache_key; /* const char *cache_val; /* /* int dict_cache_delete(cache, cache_key) /* DICT_CACHE *cache; /* const char *cache_key; /* /* int dict_cache_sequence(cache, first_next, cache_key, cache_val) /* DICT_CACHE *cache; /* int first_next; /* const char **cache_key; /* const char **cache_val; /* AUXILIARY FUNCTIONS /* void dict_cache_control(cache, name, value, ...) /* DICT_CACHE *cache; /* int name; /* /* typedef int (*DICT_CACHE_VALIDATOR_FN) (const char *cache_key, /* const char *cache_val, void *context); /* /* const char *dict_cache_name(cache) /* DICT_CACHE *cache; /* DESCRIPTION /* This module maintains external cache files with support /* for expiration. The underlying table must implement the /* "lookup", "update", "delete" and "sequence" operations. /* /* Although this API is similar to the one documented in /* dict_open(3), there are subtle differences in the interaction /* between the iterators that access all cache elements, and /* other operations that access individual cache elements. /* /* In particular, when a "sequence" or "cleanup" operation is /* in progress the cache intercepts requests to delete the /* "current" entry, as this would cause some databases to /* mis-behave. Instead, the cache implements a "delete behind" /* strategy, and deletes such an entry after the "sequence" /* or "cleanup" operation moves on to the next cache element. /* The "delete behind" strategy also affects the cache lookup /* and update operations as detailed below. /* /* dict_cache_open() is a wrapper around the dict_open() /* function. It opens the specified cache and returns a handle /* that must be used for subsequent access. This function does /* not return in case of error. /* /* dict_cache_close() closes the specified cache and releases /* memory that was allocated by dict_cache_open(), and terminates /* any thread that was started with dict_cache_control(). /* /* dict_cache_lookup() looks up the specified cache entry. /* The result value is a null pointer when the cache entry was /* not found, or when the entry is scheduled for "delete /* behind". /* /* dict_cache_update() updates the specified cache entry. If /* the entry is scheduled for "delete behind", the delete /* operation is canceled (because of this, the cache must be /* opened with DICT_FLAG_DUP_REPLACE). This function does not /* return in case of error. /* /* dict_cache_delete() removes the specified cache entry. If /* this is the "current" entry of a "sequence" operation, the /* entry is scheduled for "delete behind". The result value /* is zero when the entry was found. /* /* dict_cache_sequence() iterates over the specified cache and /* returns each entry in an implementation-defined order. The /* result value is zero when a cache entry was found. /* /* Important: programs must not use both dict_cache_sequence() /* and the built-in cache cleanup feature. /* /* dict_cache_control() provides control over the built-in /* cache cleanup feature and logging. The arguments are a list /* of macros with zero or more arguments, terminated with /* CA_DICT_CACHE_CTL_END which has none. The following lists /* the macros and corresponding argument types. /* .IP "CA_DICT_CACHE_CTL_FLAGS(int flags)" /* The arguments to this command are the bit-wise OR of zero /* or more of the following: /* .RS /* .IP CA_DICT_CACHE_CTL_FLAG_VERBOSE /* Enable verbose logging of cache activity. /* .IP CA_DICT_CACHE_CTL_FLAG_EXP_SUMMARY /* Log cache statistics after each cache cleanup run. /* .RE /* .IP "CA_DICT_CACHE_CTL_INTERVAL(int interval)" /* The interval between cache cleanup runs. Specify a null /* validator or interval to stop cache cleanup. /* .IP "CA_DICT_CACHE_CTL_VALIDATOR(DICT_CACHE_VALIDATOR_FN validator)" /* An application call-back routine that returns non-zero when /* a cache entry should be kept. The call-back function should /* not make changes to the cache. Specify a null validator or /* interval to stop cache cleanup. /* .IP "CA_DICT_CACHE_CTL_CONTEXT(void *context)" /* Application context that is passed to the validator function. /* .RE /* .PP /* dict_cache_name() returns the name of the specified cache. /* /* Arguments: /* .IP "dbname, open_flags, dict_flags" /* These are passed unchanged to dict_open(). The cache must /* be opened with DICT_FLAG_DUP_REPLACE. /* .IP cache /* Cache handle created with dict_cache_open(). /* .IP cache_key /* Cache lookup key. /* .IP cache_val /* Information that is stored under a cache lookup key. /* .IP first_next /* One of DICT_SEQ_FUN_FIRST (first cache element) or /* DICT_SEQ_FUN_NEXT (next cache element). /* .sp /* Note: there is no "stop" request. To ensure that the "delete /* behind" strategy does not interfere with database access, /* allow dict_cache_sequence() to run to completion. /* .IP table /* A bare dictionary handle. /* DIAGNOSTICS /* When a request is satisfied, the lookup routine returns /* non-null, and the update, delete and sequence routines /* return zero. The cache->error value is zero when a request /* could not be satisfied because an item did not exist (delete, /* sequence) or if it could not be updated. The cache->error /* value is non-zero only when a request could not be satisfied, /* and the cause was a database error. /* /* Cache access errors are logged with a warning message. To /* avoid spamming the log, each type of operation logs no more /* than one cache access error per second, per cache. Specify /* the DICT_CACHE_FLAG_VERBOSE flag (see above) to log all /* warnings. /* BUGS /* There should be a way to suspend automatic program suicide /* until a cache cleanup run is completed. Some entries may /* never be removed when the process max_idle time is less /* than the time needed to make a full pass over the cache. /* /* The delete-behind strategy assumes that all updates are /* made by a single process. Otherwise, delete-behind may /* remove an entry that was updated after it was scheduled for /* deletion. /* LICENSE /* .ad /* .fi /* The Secure Mailer license must be distributed with this software. /* HISTORY /* .ad /* .fi /* A predecessor of this code was written first for the Postfix /* tlsmgr(8) daemon. /* AUTHOR(S) /* Wietse Venema /* IBM T.J. Watson Research /* P.O. Box 704 /* Yorktown Heights, NY 10598, USA /*--*/ /* System library. */ #include #include #include /* Utility library. */ #include #include #include #include #include /* Application-specific. */ /* * XXX Deleting entries while enumerating a map can he tricky. Some map * types have a concept of cursor and support a "delete the current element" * operation. Some map types without cursors don't behave well when the * current first/next entry is deleted (example: with Berkeley DB < 2, the * "next" operation produces garbage). To avoid trouble, we delete an entry * after advancing the current first/next position beyond it; we use the * same strategy with application requests to delete the current entry. */ /* * Opaque data structure. Use dict_cache_name() to access the name of the * underlying database. */ struct DICT_CACHE { char *name; /* full name including proxy: */ int cache_flags; /* see below */ int user_flags; /* logging */ DICT *db; /* database handle */ int error; /* last operation only */ /* Delete-behind support. */ char *saved_curr_key; /* "current" cache lookup key */ char *saved_curr_val; /* "current" cache lookup result */ /* Cleanup support. */ int exp_interval; /* time between cleanup runs */ DICT_CACHE_VALIDATOR_FN exp_validator; /* expiration call-back */ void *exp_context; /* call-back context */ int retained; /* entries retained in cleanup run */ int dropped; /* entries removed in cleanup run */ /* Rate-limited logging support. */ int log_delay; time_t upd_log_stamp; /* last update warning */ time_t get_log_stamp; /* last lookup warning */ time_t del_log_stamp; /* last delete warning */ time_t seq_log_stamp; /* last sequence warning */ }; #define DC_FLAG_DEL_SAVED_CURRENT_KEY (1<<0) /* delete-behind is scheduled */ /* * Don't log cache access errors more than once per second. */ #define DC_DEF_LOG_DELAY 1 /* * Macros to make obscure code more readable. */ #define DC_SCHEDULE_FOR_DELETE_BEHIND(cp) \ ((cp)->cache_flags |= DC_FLAG_DEL_SAVED_CURRENT_KEY) #define DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key) \ ((cp)->saved_curr_key && strcmp((cp)->saved_curr_key, (cache_key)) == 0) #define DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) \ (/* NOT: (cp)->saved_curr_key && */ \ ((cp)->cache_flags & DC_FLAG_DEL_SAVED_CURRENT_KEY) != 0) #define DC_CANCEL_DELETE_BEHIND(cp) \ ((cp)->cache_flags &= ~DC_FLAG_DEL_SAVED_CURRENT_KEY) /* * Special key to store the time of the last cache cleanup run completion. */ #define DC_LAST_CACHE_CLEANUP_COMPLETED "_LAST_CACHE_CLEANUP_COMPLETED_" /* dict_cache_lookup - load entry from cache */ const char *dict_cache_lookup(DICT_CACHE *cp, const char *cache_key) { const char *myname = "dict_cache_lookup"; const char *cache_val; DICT *db = cp->db; /* * Search for the cache entry. Don't return an entry that is scheduled * for delete-behind. */ if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s (pretend not found - scheduled for deletion)", myname, cache_key); DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, (char *) 0); } else { cache_val = dict_get(db, cache_key); if (cache_val == 0 && db->error != 0) msg_rate_delay(&cp->get_log_stamp, cp->log_delay, msg_warn, "%s: cache lookup for '%s' failed due to error", cp->name, cache_key); if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s value=%s", myname, cache_key, cache_val ? cache_val : db->error ? "error" : "(not found)"); DICT_ERR_VAL_RETURN(cp, db->error, cache_val); } } /* dict_cache_update - save entry to cache */ int dict_cache_update(DICT_CACHE *cp, const char *cache_key, const char *cache_val) { const char *myname = "dict_cache_update"; DICT *db = cp->db; int put_res; /* * Store the cache entry and cancel the delete-behind operation. */ if (DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp) && DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: cancel delete-behind for key=%s", myname, cache_key); DC_CANCEL_DELETE_BEHIND(cp); } if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s value=%s", myname, cache_key, cache_val); put_res = dict_put(db, cache_key, cache_val); if (put_res != 0) msg_rate_delay(&cp->upd_log_stamp, cp->log_delay, msg_warn, "%s: could not update entry for %s", cp->name, cache_key); DICT_ERR_VAL_RETURN(cp, db->error, put_res); } /* dict_cache_delete - delete entry from cache */ int dict_cache_delete(DICT_CACHE *cp, const char *cache_key) { const char *myname = "dict_cache_delete"; int del_res; DICT *db = cp->db; /* * Delete the entry, unless we would delete the current first/next entry. * In that case, schedule the "current" entry for delete-behind to avoid * mis-behavior by some databases. */ if (DC_MATCH_SAVED_CURRENT_KEY(cp, cache_key)) { DC_SCHEDULE_FOR_DELETE_BEHIND(cp); if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s (current entry - schedule for delete-behind)", myname, cache_key); DICT_ERR_VAL_RETURN(cp, DICT_ERR_NONE, DICT_STAT_SUCCESS); } else { del_res = dict_del(db, cache_key); if (del_res != 0) msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, "%s: could not delete entry for %s", cp->name, cache_key); if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s (%s)", myname, cache_key, del_res == 0 ? "found" : db->error ? "error" : "not found"); DICT_ERR_VAL_RETURN(cp, db->error, del_res); } } /* dict_cache_sequence - look up the first/next cache entry */ int dict_cache_sequence(DICT_CACHE *cp, int first_next, const char **cache_key, const char **cache_val) { const char *myname = "dict_cache_sequence"; int seq_res; const char *raw_cache_key; const char *raw_cache_val; char *previous_curr_key; char *previous_curr_val; DICT *db = cp->db; /* * Find the first or next database entry. Hide the record with the cache * cleanup completion time stamp. */ seq_res = dict_seq(db, first_next, &raw_cache_key, &raw_cache_val); if (seq_res == 0 && strcmp(raw_cache_key, DC_LAST_CACHE_CLEANUP_COMPLETED) == 0) seq_res = dict_seq(db, DICT_SEQ_FUN_NEXT, &raw_cache_key, &raw_cache_val); if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: key=%s value=%s", myname, seq_res == 0 ? raw_cache_key : db->error ? "(error)" : "(not found)", seq_res == 0 ? raw_cache_val : db->error ? "(error)" : "(not found)"); if (db->error) msg_rate_delay(&cp->seq_log_stamp, cp->log_delay, msg_warn, "%s: sequence error", cp->name); /* * Save the current cache_key and cache_val before they are clobbered by * our own delete operation below. This also prevents surprises when the * application accesses the database after this function returns. * * We also use the saved cache_key to protect the current entry against * application delete requests. */ previous_curr_key = cp->saved_curr_key; previous_curr_val = cp->saved_curr_val; if (seq_res == 0) { cp->saved_curr_key = mystrdup(raw_cache_key); cp->saved_curr_val = mystrdup(raw_cache_val); } else { cp->saved_curr_key = 0; cp->saved_curr_val = 0; } /* * Delete behind. */ if (db->error == 0 && DC_IS_SCHEDULED_FOR_DELETE_BEHIND(cp)) { DC_CANCEL_DELETE_BEHIND(cp); if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: delete-behind key=%s value=%s", myname, previous_curr_key, previous_curr_val); if (dict_del(db, previous_curr_key) != 0) msg_rate_delay(&cp->del_log_stamp, cp->log_delay, msg_warn, "%s: could not delete entry for %s", cp->name, previous_curr_key); } /* * Clean up previous iteration key and value. */ if (previous_curr_key) myfree(previous_curr_key); if (previous_curr_val) myfree(previous_curr_val); /* * Return the result. */ *cache_key = (cp)->saved_curr_key; *cache_val = (cp)->saved_curr_val; DICT_ERR_VAL_RETURN(cp, db->error, seq_res); } /* dict_cache_delete_behind_reset - reset "delete behind" state */ static void dict_cache_delete_behind_reset(DICT_CACHE *cp) { #define FREE_AND_WIPE(s) do { if (s) { myfree(s); (s) = 0; } } while (0) DC_CANCEL_DELETE_BEHIND(cp); FREE_AND_WIPE(cp->saved_curr_key); FREE_AND_WIPE(cp->saved_curr_val); } /* dict_cache_clean_stat_log_reset - log and reset cache cleanup statistics */ static void dict_cache_clean_stat_log_reset(DICT_CACHE *cp, const char *full_partial) { if (cp->user_flags & DICT_CACHE_FLAG_STATISTICS) msg_info("cache %s %s cleanup: retained=%d dropped=%d entries", cp->name, full_partial, cp->retained, cp->dropped); cp->retained = cp->dropped = 0; } /* dict_cache_clean_event - examine one cache entry */ static void dict_cache_clean_event(int unused_event, void *cache_context) { const char *myname = "dict_cache_clean_event"; DICT_CACHE *cp = (DICT_CACHE *) cache_context; const char *cache_key; const char *cache_val; int next_interval; VSTRING *stamp_buf; int first_next; /* * We interleave cache cleanup with other processing, so that the * application's service remains available, with perhaps increased * latency. */ /* * Start a new cache cleanup run. */ if (cp->saved_curr_key == 0) { cp->retained = cp->dropped = 0; first_next = DICT_SEQ_FUN_FIRST; if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: start %s cache cleanup", myname, cp->name); } /* * Continue a cache cleanup run in progress. */ else { first_next = DICT_SEQ_FUN_NEXT; } /* * Examine one cache entry. */ if (dict_cache_sequence(cp, first_next, &cache_key, &cache_val) == 0) { if (cp->exp_validator(cache_key, cache_val, cp->exp_context) == 0) { DC_SCHEDULE_FOR_DELETE_BEHIND(cp); cp->dropped++; if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: drop %s cache entry for %s", myname, cp->name, cache_key); } else { cp->retained++; if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: keep %s cache entry for %s", myname, cp->name, cache_key); } next_interval = 0; } /* * Cache cleanup completed. Report vital statistics. */ else if (cp->error != 0) { msg_warn("%s: cache cleanup scan terminated due to error", cp->name); dict_cache_clean_stat_log_reset(cp, "partial"); next_interval = cp->exp_interval; } else { if (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) msg_info("%s: done %s cache cleanup scan", myname, cp->name); dict_cache_clean_stat_log_reset(cp, "full"); stamp_buf = vstring_alloc(100); vstring_sprintf(stamp_buf, "%ld", (long) event_time()); dict_put(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED, vstring_str(stamp_buf)); vstring_free(stamp_buf); next_interval = cp->exp_interval; } event_request_timer(dict_cache_clean_event, cache_context, next_interval); } /* dict_cache_control - schedule or stop the cache cleanup thread */ void dict_cache_control(DICT_CACHE *cp,...) { const char *myname = "dict_cache_control"; const char *last_done; time_t next_interval; int cache_cleanup_is_active = (cp->exp_validator && cp->exp_interval); va_list ap; int name; /* * Update the control settings. */ va_start(ap, cp); while ((name = va_arg(ap, int)) > 0) { switch (name) { case DICT_CACHE_CTL_END: break; case DICT_CACHE_CTL_FLAGS: cp->user_flags = va_arg(ap, int); cp->log_delay = (cp->user_flags & DICT_CACHE_FLAG_VERBOSE) ? 0 : DC_DEF_LOG_DELAY; break; case DICT_CACHE_CTL_INTERVAL: cp->exp_interval = va_arg(ap, int); if (cp->exp_interval < 0) msg_panic("%s: bad %s cache cleanup interval %d", myname, cp->name, cp->exp_interval); break; case DICT_CACHE_CTL_VALIDATOR: cp->exp_validator = va_arg(ap, DICT_CACHE_VALIDATOR_FN); break; case DICT_CACHE_CTL_CONTEXT: cp->exp_context = va_arg(ap, void *); break; default: msg_panic("%s: bad command: %d", myname, name); } } va_end(ap); /* * Schedule the cache cleanup thread. */ if (cp->exp_interval && cp->exp_validator) { /* * Sanity checks. */ if (cache_cleanup_is_active) msg_panic("%s: %s cache cleanup is already scheduled", myname, cp->name); /* * The next start time depends on the last completion time. */ #define NEXT_START(last, delta) ((delta) + (unsigned long) atol(last)) #define NOW (time((time_t *) 0)) /* NOT: event_time() */ if ((last_done = dict_get(cp->db, DC_LAST_CACHE_CLEANUP_COMPLETED)) == 0 || (next_interval = (NEXT_START(last_done, cp->exp_interval) - NOW)) < 0) next_interval = 0; if (next_interval > cp->exp_interval) next_interval = cp->exp_interval; if ((cp->user_flags & DICT_CACHE_FLAG_VERBOSE) && next_interval > 0) msg_info("%s cache cleanup will start after %ds", cp->name, (int) next_interval); event_request_timer(dict_cache_clean_event, (void *) cp, (int) next_interval); } /* * Cancel the cache cleanup thread. */ else if (cache_cleanup_is_active) { if (cp->retained || cp->dropped) dict_cache_clean_stat_log_reset(cp, "partial"); dict_cache_delete_behind_reset(cp); event_cancel_timer(dict_cache_clean_event, (void *) cp); } } /* dict_cache_open - open cache file */ DICT_CACHE *dict_cache_open(const char *dbname, int open_flags, int dict_flags) { DICT_CACHE *cp; DICT *dict; /* * Open the database as requested. Don't attempt to second-guess the * application. */ dict = dict_open(dbname, open_flags, dict_flags); /* * Create the DICT_CACHE object. */ cp = (DICT_CACHE *) mymalloc(sizeof(*cp)); cp->name = mystrdup(dbname); cp->cache_flags = 0; cp->user_flags = 0; cp->db = dict; cp->saved_curr_key = 0; cp->saved_curr_val = 0; cp->exp_interval = 0; cp->exp_validator = 0; cp->exp_context = 0; cp->retained = 0; cp->dropped = 0; cp->log_delay = DC_DEF_LOG_DELAY; cp->upd_log_stamp = cp->get_log_stamp = cp->del_log_stamp = cp->seq_log_stamp = 0; return (cp); } /* dict_cache_close - close cache file */ void dict_cache_close(DICT_CACHE *cp) { /* * Destroy the DICT_CACHE object. */ dict_cache_control(cp, DICT_CACHE_CTL_INTERVAL, 0, DICT_CACHE_CTL_END); myfree(cp->name); dict_close(cp->db); if (cp->saved_curr_key) myfree(cp->saved_curr_key); if (cp->saved_curr_val) myfree(cp->saved_curr_val); myfree((void *) cp); } /* dict_cache_name - get the cache name */ const char *dict_cache_name(DICT_CACHE *cp) { /* * This is used for verbose logging or warning messages, so the cost of * call is only made where needed (well sort off - code that does not * execute still presents overhead for the processor pipeline, processor * cache, etc). */ return (cp->name); } /* * Test driver with support for interleaved access. First, enter a number of * requests to look up, update or delete a sequence of cache entries, then * interleave those sequences with the "run" command. */ #ifdef TEST #include #include #include #include #define DELIMS " " #define USAGE "\n\tTo manage settings:" \ "\n\tverbose (verbosity level)" \ "\n\telapsed (0=don't show elapsed time)" \ "\n\tlmdb_map_size (initial LMDB size limit)" \ "\n\tcache : (switch to named database)" \ "\n\tstatus (show map size, cache, pending requests)" \ "\n\n\tTo manage pending requests:" \ "\n\treset (discard pending requests)" \ "\n\trun (execute pending requests in interleaved order)" \ "\n\n\tTo add a pending request:" \ "\n\tquery (negative to reverse order)" \ "\n\tupdate (negative to reverse order)" \ "\n\tdelete (negative to reverse order)" \ "\n\tpurge " \ "\n\tcount " /* * For realism, open the cache with the same flags as postscreen(8) and * verify(8). */ #define DICT_CACHE_OPEN_FLAGS (DICT_FLAG_DUP_REPLACE | DICT_FLAG_SYNC_UPDATE | \ DICT_FLAG_OPEN_LOCK) /* * Storage for one request to access a sequence of cache entries. */ typedef struct DICT_CACHE_SREQ { int flags; /* per-request: reverse, purge */ char *cmd; /* command for status report */ void (*action) (struct DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); char *suffix; /* key suffix */ int done; /* progress indicator */ int todo; /* number of entries to process */ int first_next; /* first/next */ } DICT_CACHE_SREQ; #define DICT_CACHE_SREQ_FLAG_PURGE (1<<1) /* purge instead of count */ #define DICT_CACHE_SREQ_FLAG_REVERSE (1<<2) /* reverse instead of forward */ #define DICT_CACHE_SREQ_LIMIT 10 /* * All test requests combined. */ typedef struct DICT_CACHE_TEST { int flags; /* exclusion flags */ int size; /* allocated slots */ int used; /* used slots */ DICT_CACHE_SREQ job_list[1]; /* actually, a bunch */ } DICT_CACHE_TEST; #define DICT_CACHE_TEST_FLAG_ITER (1<<0) /* count or purge */ #define STR(x) vstring_str(x) int show_elapsed = 1; /* show elapsed time */ #ifdef HAS_LMDB extern size_t dict_lmdb_map_size; /* LMDB-specific */ #endif /* usage - command-line usage message */ static NORETURN usage(const char *progname) { msg_fatal("usage: %s (no argument)", progname); } /* make_tagged_key - make tagged search key */ static void make_tagged_key(VSTRING *bp, DICT_CACHE_SREQ *cp) { if (cp->done < 0) msg_panic("make_tagged_key: bad done count: %d", cp->done); if (cp->todo < 1) msg_panic("make_tagged_key: bad todo count: %d", cp->todo); vstring_sprintf(bp, "%d-%s", (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? cp->todo - cp->done - 1 : cp->done, cp->suffix); } /* create_requests - create request list */ static DICT_CACHE_TEST *create_requests(int count) { DICT_CACHE_TEST *tp; DICT_CACHE_SREQ *cp; tp = (DICT_CACHE_TEST *) mymalloc(sizeof(DICT_CACHE_TEST) + (count - 1) *sizeof(DICT_CACHE_SREQ)); tp->flags = 0; tp->size = count; tp->used = 0; for (cp = tp->job_list; cp < tp->job_list + count; cp++) { cp->flags = 0; cp->cmd = 0; cp->action = 0; cp->suffix = 0; cp->todo = 0; cp->first_next = DICT_SEQ_FUN_FIRST; } return (tp); } /* reset_requests - reset request list */ static void reset_requests(DICT_CACHE_TEST *tp) { DICT_CACHE_SREQ *cp; tp->flags = 0; tp->used = 0; for (cp = tp->job_list; cp < tp->job_list + tp->size; cp++) { cp->flags = 0; if (cp->cmd) { myfree(cp->cmd); cp->cmd = 0; } cp->action = 0; if (cp->suffix) { myfree(cp->suffix); cp->suffix = 0; } cp->todo = 0; cp->first_next = DICT_SEQ_FUN_FIRST; } } /* free_requests - destroy request list */ static void free_requests(DICT_CACHE_TEST *tp) { reset_requests(tp); myfree((void *) tp); } /* run_requests - execute pending requests in interleaved order */ static void run_requests(DICT_CACHE_TEST *tp, DICT_CACHE *dp, VSTRING *bp) { DICT_CACHE_SREQ *cp; int todo; struct timeval start; struct timeval finish; struct timeval elapsed; if (dp == 0) { msg_warn("no cache"); return; } GETTIMEOFDAY(&start); do { todo = 0; for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) { if (cp->done < cp->todo) { todo = 1; cp->action(cp, dp, bp); } } } while (todo); GETTIMEOFDAY(&finish); timersub(&finish, &start, &elapsed); if (show_elapsed) vstream_printf("Elapsed: %g\n", elapsed.tv_sec + elapsed.tv_usec / 1000000.0); reset_requests(tp); } /* show_status - show settings and pending requests */ static void show_status(DICT_CACHE_TEST *tp, DICT_CACHE *dp) { DICT_CACHE_SREQ *cp; #ifdef HAS_LMDB vstream_printf("lmdb_map_size\t%ld\n", (long) dict_lmdb_map_size); #endif vstream_printf("cache\t%s\n", dp ? dp->name : "(none)"); if (tp->used == 0) vstream_printf("No pending requests\n"); else vstream_printf("%s\t%s\t%s\t%s\t%s\t%s\n", "cmd", "dir", "suffix", "count", "done", "first/next"); for (cp = tp->job_list; cp < tp->job_list + tp->used; cp++) if (cp->todo > 0) vstream_printf("%s\t%s\t%s\t%d\t%d\t%d\n", cp->cmd, (cp->flags & DICT_CACHE_SREQ_FLAG_REVERSE) ? "reverse" : "forward", cp->suffix ? cp->suffix : "(null)", cp->todo, cp->done, cp->first_next); } /* query_action - lookup cache entry */ static void query_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) { const char *lookup; make_tagged_key(bp, cp); if ((lookup = dict_cache_lookup(dp, STR(bp))) == 0) { if (dp->error) msg_warn("query_action: query failed: %s: %m", STR(bp)); else msg_warn("query_action: query failed: %s", STR(bp)); } else if (strcmp(STR(bp), lookup) != 0) { msg_warn("lookup result \"%s\" differs from key \"%s\"", lookup, STR(bp)); } cp->done += 1; } /* update_action - update cache entry */ static void update_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) { make_tagged_key(bp, cp); if (dict_cache_update(dp, STR(bp), STR(bp)) != 0) { if (dp->error) msg_warn("update_action: update failed: %s: %m", STR(bp)); else msg_warn("update_action: update failed: %s", STR(bp)); } cp->done += 1; } /* delete_action - delete cache entry */ static void delete_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) { make_tagged_key(bp, cp); if (dict_cache_delete(dp, STR(bp)) != 0) { if (dp->error) msg_warn("delete_action: delete failed: %s: %m", STR(bp)); else msg_warn("delete_action: delete failed: %s", STR(bp)); } cp->done += 1; } /* iter_action - iterate over cache and act on entries with given suffix */ static void iter_action(DICT_CACHE_SREQ *cp, DICT_CACHE *dp, VSTRING *bp) { const char *cache_key; const char *cache_val; const char *what; const char *suffix; if (dict_cache_sequence(dp, cp->first_next, &cache_key, &cache_val) == 0) { if (strcmp(cache_key, cache_val) != 0) msg_warn("value \"%s\" differs from key \"%s\"", cache_val, cache_key); suffix = cache_key + strspn(cache_key, "0123456789"); if (suffix[0] == '-' && strcmp(suffix + 1, cp->suffix) == 0) { cp->done += 1; cp->todo = cp->done + 1; /* XXX */ if ((cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) && dict_cache_delete(dp, cache_key) != 0) { if (dp->error) msg_warn("purge_action: delete failed: %s: %m", STR(bp)); else msg_warn("purge_action: delete failed: %s", STR(bp)); } } cp->first_next = DICT_SEQ_FUN_NEXT; } else { what = (cp->flags & DICT_CACHE_SREQ_FLAG_PURGE) ? "purge" : "count"; if (dp->error) msg_warn("%s error after %d: %m", what, cp->done); else vstream_printf("suffix=%s %s=%d\n", cp->suffix, what, cp->done); cp->todo = 0; } } /* * Table-driven support. */ typedef struct DICT_CACHE_SREQ_INFO { const char *name; int argc; void (*action) (DICT_CACHE_SREQ *, DICT_CACHE *, VSTRING *); int test_flags; int req_flags; } DICT_CACHE_SREQ_INFO; static DICT_CACHE_SREQ_INFO req_info[] = { {"query", 3, query_action}, {"update", 3, update_action}, {"delete", 3, delete_action}, {"count", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER}, {"purge", 2, iter_action, DICT_CACHE_TEST_FLAG_ITER, DICT_CACHE_SREQ_FLAG_PURGE}, 0, }; /* add_request - add a request to the list */ static void add_request(DICT_CACHE_TEST *tp, ARGV *argv) { DICT_CACHE_SREQ_INFO *rp; DICT_CACHE_SREQ *cp; int req_flags; int count; char *cmd = argv->argv[0]; char *suffix = (argv->argc > 1 ? argv->argv[1] : 0); char *todo = (argv->argc > 2 ? argv->argv[2] : "1"); /* XXX */ if (tp->used >= tp->size) { msg_warn("%s: request list is full", cmd); return; } for (rp = req_info; /* See below */ ; rp++) { if (rp->name == 0) { vstream_printf("usage: %s\n", USAGE); return; } if (strcmp(rp->name, argv->argv[0]) == 0 && rp->argc == argv->argc) break; } req_flags = rp->req_flags; if (todo[0] == '-') { req_flags |= DICT_CACHE_SREQ_FLAG_REVERSE; todo += 1; } if (!alldig(todo) || (count = atoi(todo)) == 0) { msg_warn("%s: bad count: %s", cmd, todo); return; } if (tp->flags & rp->test_flags) { msg_warn("%s: command conflicts with other command", cmd); return; } tp->flags |= rp->test_flags; cp = tp->job_list + tp->used; cp->cmd = mystrdup(cmd); cp->action = rp->action; if (suffix) cp->suffix = mystrdup(suffix); cp->done = 0; cp->flags = req_flags; cp->todo = count; tp->used += 1; } /* main - main program */ int main(int argc, char **argv) { DICT_CACHE_TEST *test_job; VSTRING *inbuf = vstring_alloc(100); char *bufp; ARGV *args; DICT_CACHE *cache = 0; int stdin_is_tty; msg_vstream_init(argv[0], VSTREAM_ERR); if (argc != 1) usage(argv[0]); test_job = create_requests(DICT_CACHE_SREQ_LIMIT); stdin_is_tty = isatty(0); for (;;) { if (stdin_is_tty) { vstream_printf("> "); vstream_fflush(VSTREAM_OUT); } if (vstring_fgets_nonl(inbuf, VSTREAM_IN) == 0) break; bufp = vstring_str(inbuf); if (!stdin_is_tty) { vstream_printf("> %s\n", bufp); vstream_fflush(VSTREAM_OUT); } if (*bufp == '#') continue; args = argv_split(bufp, DELIMS); if (argc == 0) { vstream_printf("usage: %s\n", USAGE); vstream_fflush(VSTREAM_OUT); continue; } if (strcmp(args->argv[0], "verbose") == 0 && args->argc == 2) { msg_verbose = atoi(args->argv[1]); } else if (strcmp(args->argv[0], "elapsed") == 0 && args->argc == 2) { show_elapsed = atoi(args->argv[1]); #ifdef HAS_LMDB } else if (strcmp(args->argv[0], "lmdb_map_size") == 0 && args->argc == 2) { dict_lmdb_map_size = atol(args->argv[1]); #endif } else if (strcmp(args->argv[0], "cache") == 0 && args->argc == 2) { if (cache) dict_cache_close(cache); cache = dict_cache_open(args->argv[1], O_CREAT | O_RDWR, DICT_CACHE_OPEN_FLAGS); } else if (strcmp(args->argv[0], "reset") == 0 && args->argc == 1) { reset_requests(test_job); } else if (strcmp(args->argv[0], "run") == 0 && args->argc == 1) { run_requests(test_job, cache, inbuf); } else if (strcmp(args->argv[0], "status") == 0 && args->argc == 1) { show_status(test_job, cache); } else { add_request(test_job, args); } vstream_fflush(VSTREAM_OUT); argv_free(args); } vstring_free(inbuf); free_requests(test_job); if (cache) dict_cache_close(cache); return (0); } #endif