diff options
Diffstat (limited to 'runtime/lookup.c')
| -rw-r--r-- | runtime/lookup.c | 1093 |
1 files changed, 1093 insertions, 0 deletions
diff --git a/runtime/lookup.c b/runtime/lookup.c new file mode 100644 index 0000000..4408581 --- /dev/null +++ b/runtime/lookup.c @@ -0,0 +1,1093 @@ +/* lookup.c + * Support for lookup tables in RainerScript. + * + * Copyright 2013-2023 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <json.h> +#include <assert.h> + +#include "rsyslog.h" +#include "srUtils.h" +#include "errmsg.h" +#include "lookup.h" +#include "msg.h" +#include "rsconf.h" +#include "dirty.h" +#include "unicode-helper.h" + +PRAGMA_IGNORE_Wdeprecated_declarations +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) + +/* forward definitions */ +static rsRetVal lookupReadFile(lookup_t *pThis, const uchar* name, const uchar* filename); +static void lookupDestruct(lookup_t *pThis); + +/* static data */ +/* tables for interfacing with the v6 config system (as far as we need to) */ +static struct cnfparamdescr modpdescr[] = { + { "name", eCmdHdlrString, CNFPARAM_REQUIRED }, + { "file", eCmdHdlrString, CNFPARAM_REQUIRED }, + { "reloadOnHUP", eCmdHdlrBinary, 0 } +}; +static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, + sizeof(modpdescr)/sizeof(struct cnfparamdescr), + modpdescr + }; + +/* internal data-types */ +typedef struct uint32_index_val_s { + uint32_t index; + uchar *val; +} uint32_index_val_t; + +const char * reloader_prefix = "lkp_tbl_reloader:"; + +static void * +lookupTableReloader(void *self); + +static void +lookupStopReloader(lookup_ref_t *pThis); + + +/* create a new lookup table object AND include it in our list of + * lookup tables. + */ +static rsRetVal +lookupNew(lookup_ref_t **ppThis) +{ + lookup_ref_t *pThis = NULL; + lookup_t *t = NULL; + DEFiRet; + + CHKmalloc(pThis = calloc(1, sizeof(lookup_ref_t))); + CHKmalloc(t = calloc(1, sizeof(lookup_t))); + pThis->do_reload = pThis->do_stop = 0; + pThis->reload_on_hup = 1; /*DO reload on HUP (default)*/ + + pThis->next = NULL; + if(loadConf->lu_tabs.root == NULL) { + loadConf->lu_tabs.root = pThis; + } else { + loadConf->lu_tabs.last->next = pThis; + } + loadConf->lu_tabs.last = pThis; + + pThis->self = t; + + *ppThis = pThis; +finalize_it: + if(iRet != RS_RET_OK) { + LogError(errno, iRet, "a lookup table could not be initialized"); + free(t); + free(pThis); + } + RETiRet; +} + +/* activate a lookup table entry once rsyslog is ready to do so */ +static rsRetVal +lookupActivateTable(lookup_ref_t *pThis) +{ + DEFiRet; + int initialized = 0; + + DBGPRINTF("lookupActivateTable called\n"); + CHKiConcCtrl(pthread_rwlock_init(&pThis->rwlock, NULL)); + initialized++; /*1*/ + CHKiConcCtrl(pthread_mutex_init(&pThis->reloader_mut, NULL)); + initialized++; /*2*/ + CHKiConcCtrl(pthread_cond_init(&pThis->run_reloader, NULL)); + initialized++; /*3*/ + CHKiConcCtrl(pthread_attr_init(&pThis->reloader_thd_attr)); + initialized++; /*4*/ + pThis->do_reload = pThis->do_stop = 0; + CHKiConcCtrl(pthread_create(&pThis->reloader, &pThis->reloader_thd_attr, + lookupTableReloader, pThis)); + initialized++; /*5*/ + + +finalize_it: + if(iRet != RS_RET_OK) { + LogError(errno, iRet, "a lookup table could not be activated: " + "failed at init-step %d (please enable debug logs for details)", + initialized); + /* Can not happen with current code, but might occur in the future when + * an error-condition as added after step 5. If we leave it in, Coverity + * scan complains. So we comment it out but do not remove the code. + * Triggered by CID 185426 + if (initialized > 4) lookupStopReloader(pThis); + */ + if (initialized > 3) pthread_attr_destroy(&pThis->reloader_thd_attr); + if (initialized > 2) pthread_cond_destroy(&pThis->run_reloader); + if (initialized > 1) pthread_mutex_destroy(&pThis->reloader_mut); + if (initialized > 0) pthread_rwlock_destroy(&pThis->rwlock); + } + RETiRet; +} + +/*must be called with reloader_mut acquired*/ +static void ATTR_NONNULL() +freeStubValueForReloadFailure(lookup_ref_t *const pThis) +{ + if (pThis->stub_value_for_reload_failure != NULL) { + free(pThis->stub_value_for_reload_failure); + pThis->stub_value_for_reload_failure = NULL; + } +} + +static void +lookupStopReloader(lookup_ref_t *pThis) { + pthread_mutex_lock(&pThis->reloader_mut); + freeStubValueForReloadFailure(pThis); + pThis->do_reload = 0; + pThis->do_stop = 1; + pthread_cond_signal(&pThis->run_reloader); + pthread_mutex_unlock(&pThis->reloader_mut); + pthread_join(pThis->reloader, NULL); +} + +static void +lookupRefDestruct(lookup_ref_t *pThis) +{ + lookupStopReloader(pThis); + pthread_mutex_destroy(&pThis->reloader_mut); + pthread_cond_destroy(&pThis->run_reloader); + pthread_attr_destroy(&pThis->reloader_thd_attr); + + pthread_rwlock_destroy(&pThis->rwlock); + lookupDestruct(pThis->self); + free(pThis->name); + free(pThis->filename); + free(pThis); +} + +static void +destructTable_str(lookup_t *pThis) { + uint32_t i = 0; + lookup_string_tab_entry_t *entries = pThis->table.str->entries; + for (i = 0; i < pThis->nmemb; i++) { + free(entries[i].key); + } + free(entries); + free(pThis->table.str); +} + + +static void +destructTable_arr(lookup_t *pThis) { + free(pThis->table.arr->interned_val_refs); + free(pThis->table.arr); +} + +static void +destructTable_sparseArr(lookup_t *pThis) { + free(pThis->table.sprsArr->entries); + free(pThis->table.sprsArr); +} + +static void +lookupDestruct(lookup_t *pThis) { + uint32_t i; + + if (pThis == NULL) return; + + if (pThis->type == STRING_LOOKUP_TABLE) { + destructTable_str(pThis); + } else if (pThis->type == ARRAY_LOOKUP_TABLE) { + destructTable_arr(pThis); + } else if (pThis->type == SPARSE_ARRAY_LOOKUP_TABLE) { + destructTable_sparseArr(pThis); + } else if (pThis->type == STUBBED_LOOKUP_TABLE) { + /*nothing to be done*/ + } + + for (i = 0; i < pThis->interned_val_count; i++) { + free(pThis->interned_vals[i]); + } + free(pThis->interned_vals); + free(pThis->nomatch); + free(pThis); +} + +void +lookupInitCnf(lookup_tables_t *lu_tabs) +{ + lu_tabs->root = NULL; + lu_tabs->last = NULL; +} + +void +lookupDestroyCnf(void) +{ + lookup_ref_t *luref, *luref_next; + for(luref = runConf->lu_tabs.root ; luref != NULL ; ) { + luref_next = luref->next; + lookupRefDestruct(luref); + luref = luref_next; + } +} + +/* comparison function for qsort() */ +static int +qs_arrcmp_strtab(const void *s1, const void *s2) +{ + return ustrcmp(((lookup_string_tab_entry_t*)s1)->key, ((lookup_string_tab_entry_t*)s2)->key); +} + +static int +qs_arrcmp_ustrs(const void *s1, const void *s2) +{ + return ustrcmp(*(uchar**)s1, *(uchar**)s2); +} + +static int +qs_arrcmp_uint32_index_val(const void *s1, const void *s2) +{ + uint32_t first_value = ((uint32_index_val_t*)s1)->index; + uint32_t second_value = ((uint32_index_val_t*)s2)->index; + if (first_value < second_value) { + return -1; + } + return first_value - second_value; +} + +static int +qs_arrcmp_sprsArrtab(const void *s1, const void *s2) +{ + uint32_t first_value = ((lookup_sparseArray_tab_entry_t*)s1)->key; + uint32_t second_value = ((lookup_sparseArray_tab_entry_t*)s2)->key; + if (first_value < second_value) { + return -1; + } + return first_value - second_value; +} + +/* comparison function for bsearch() and string array compare + * this is for the string lookup table type + */ +static int +bs_arrcmp_strtab(const void *s1, const void *s2) +{ + return strcmp((char*)s1, (char*)((lookup_string_tab_entry_t*)s2)->key); +} + +static int +bs_arrcmp_str(const void *s1, const void *s2) +{ + return ustrcmp((uchar*)s1, *(uchar**)s2); +} + +static int +bs_arrcmp_sprsArrtab(const void *s1, const void *s2) +{ + uint32_t key = *(uint32_t*)s1; + uint32_t array_member_value = ((lookup_sparseArray_tab_entry_t*)s2)->key; + if (key < array_member_value) { + return -1; + } + return key - array_member_value; +} + +static inline const char* +defaultVal(lookup_t *pThis) { + return (pThis->nomatch == NULL) ? "" : (const char*) pThis->nomatch; +} + +/* lookup_fn for different types of tables */ +static es_str_t* +lookupKey_stub(lookup_t *pThis, lookup_key_t __attribute__((unused)) key) { + return es_newStrFromCStr((char*) pThis->nomatch, ustrlen(pThis->nomatch)); +} + +static es_str_t* +lookupKey_str(lookup_t *pThis, lookup_key_t key) { + lookup_string_tab_entry_t *entry; + const char *r; + if(pThis->nmemb == 0) { + entry = NULL; + } else { + assert(pThis->table.str->entries); + entry = bsearch(key.k_str, pThis->table.str->entries, pThis->nmemb, + sizeof(lookup_string_tab_entry_t), bs_arrcmp_strtab); + } + if(entry == NULL) { + r = defaultVal(pThis); + } else { + r = (const char*)entry->interned_val_ref; + } + return es_newStrFromCStr(r, strlen(r)); +} + +static es_str_t* +lookupKey_arr(lookup_t *pThis, lookup_key_t key) { + const char *r; + uint32_t uint_key = key.k_uint; + if ((pThis->nmemb == 0) || (uint_key < pThis->table.arr->first_key)) { + r = defaultVal(pThis); + } else { + uint32_t idx = uint_key - pThis->table.arr->first_key; + if (idx >= pThis->nmemb) { + r = defaultVal(pThis); + } else { + r = (char*) pThis->table.arr->interned_val_refs[idx]; + } + } + + return es_newStrFromCStr(r, strlen(r)); +} + +typedef int (comp_fn_t)(const void *s1, const void *s2); + +static void * +bsearch_lte(const void *key, const void *base, size_t nmemb, size_t size, comp_fn_t *comp_fn) +{ + size_t l, u, idx; + const void *p; + int comparison; + + l = 0; + u = nmemb; + if (l == u) { + return NULL; + } + while (l < u) { + idx = (l + u) / 2; + p = (void *) (((const char *) base) + (idx * size)); + comparison = (*comp_fn)(key, p); + if (comparison < 0) + u = idx; + else if (comparison > 0) + l = idx + 1; + else + return (void *) p; + } + if (comparison < 0) { + if (idx == 0) { + return NULL; + } + idx--; + } + return (void *) (((const char *) base) + ( idx * size)); +} + +static es_str_t* +lookupKey_sprsArr(lookup_t *pThis, lookup_key_t key) { + lookup_sparseArray_tab_entry_t *entry; + const char *r; + if (pThis->nmemb == 0) { + entry = NULL; + } else { + entry = bsearch_lte(&key.k_uint, pThis->table.sprsArr->entries, pThis->nmemb, + sizeof(lookup_sparseArray_tab_entry_t), bs_arrcmp_sprsArrtab); + } + + if(entry == NULL) { + r = defaultVal(pThis); + } else { + r = (const char*)entry->interned_val_ref; + } + return es_newStrFromCStr(r, strlen(r)); +} + +/* builders for different table-types */ + +#define NO_INDEX_ERROR(type, name) \ + LogError(0, RS_RET_INVALID_VALUE, "'%s' lookup table named: '%s' has record(s) without 'index' "\ +"field", type, name); \ + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + +static rsRetVal +build_StringTable(lookup_t *pThis, struct json_object *jtab, const uchar* name) { + uint32_t i; + struct json_object *jrow, *jindex, *jvalue; + uchar *value, *canonicalValueRef; + DEFiRet; + + pThis->table.str = NULL; + CHKmalloc(pThis->table.str = calloc(1, sizeof(lookup_string_tab_t))); + if (pThis->nmemb > 0) { + CHKmalloc(pThis->table.str->entries = calloc(pThis->nmemb, sizeof(lookup_string_tab_entry_t))); + + for(i = 0; i < pThis->nmemb; i++) { + jrow = json_object_array_get_idx(jtab, i); + jindex = json_object_object_get(jrow, "index"); + jvalue = json_object_object_get(jrow, "value"); + if (jindex == NULL || json_object_is_type(jindex, json_type_null)) { + NO_INDEX_ERROR("string", name); + } + CHKmalloc(pThis->table.str->entries[i].key = ustrdup((uchar*) json_object_get_string(jindex))); + value = (uchar*) json_object_get_string(jvalue); + uchar **found = (uchar**) bsearch(value, pThis->interned_vals, + pThis->interned_val_count, sizeof(uchar*), bs_arrcmp_str); + if(found == NULL) { + LogError(0, RS_RET_INTERNAL_ERROR, "lookup.c:build_StringTable(): " + "internal error, bsearch returned NULL for '%s'", value); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } + // I give up, I see no way to remove false positive -- rgerhards, 2017-10-24 + #ifndef __clang_analyzer__ + canonicalValueRef = *found; + if(canonicalValueRef == NULL) { + LogError(0, RS_RET_INTERNAL_ERROR, "lookup.c:build_StringTable(): " + "internal error, canonicalValueRef returned from bsearch " + "is NULL for '%s'", value); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + } + pThis->table.str->entries[i].interned_val_ref = canonicalValueRef; + #endif + } + qsort(pThis->table.str->entries, pThis->nmemb, sizeof(lookup_string_tab_entry_t), qs_arrcmp_strtab); + } + + pThis->lookup = lookupKey_str; + pThis->key_type = LOOKUP_KEY_TYPE_STRING; +finalize_it: + RETiRet; +} + +static rsRetVal +build_ArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar *name) { + uint32_t i; + struct json_object *jrow, *jindex, *jvalue; + uchar *canonicalValueRef; + uint32_t prev_index, _index; + uint8_t prev_index_set; + uint32_index_val_t *indexes = NULL; + DEFiRet; + + prev_index_set = 0; + + pThis->table.arr = NULL; + CHKmalloc(pThis->table.arr = calloc(1, sizeof(lookup_array_tab_t))); + if (pThis->nmemb > 0) { + CHKmalloc(indexes = calloc(pThis->nmemb, sizeof(uint32_index_val_t))); + CHKmalloc(pThis->table.arr->interned_val_refs = calloc(pThis->nmemb, sizeof(uchar*))); + + for(i = 0; i < pThis->nmemb; i++) { + jrow = json_object_array_get_idx(jtab, i); + jindex = json_object_object_get(jrow, "index"); + jvalue = json_object_object_get(jrow, "value"); + if (jindex == NULL || json_object_is_type(jindex, json_type_null)) { + NO_INDEX_ERROR("array", name); + } + indexes[i].index = (uint32_t) json_object_get_int(jindex); + indexes[i].val = (uchar*) json_object_get_string(jvalue); + } + qsort(indexes, pThis->nmemb, sizeof(uint32_index_val_t), qs_arrcmp_uint32_index_val); + for(i = 0; i < pThis->nmemb; i++) { + _index = indexes[i].index; + if (prev_index_set == 0) { + prev_index = _index; + prev_index_set = 1; + pThis->table.arr->first_key = _index; + } else { + if (_index != ++prev_index) { + LogError(0, RS_RET_INVALID_VALUE, "'array' lookup table name: '%s' " + "has non-contiguous members between index '%d' and '%d'", + name, prev_index, _index); + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + } + } + uchar *const *const canonicalValueRef_ptr = bsearch(indexes[i].val, pThis->interned_vals, + pThis->interned_val_count, sizeof(uchar*), bs_arrcmp_str); + if(canonicalValueRef_ptr == NULL) { + LogError(0, RS_RET_ERR, "BUG: canonicalValueRef not found in " + "build_ArrayTable(), %s:%d", __FILE__, __LINE__); + ABORT_FINALIZE(RS_RET_ERR); + } + canonicalValueRef = *canonicalValueRef_ptr; + assert(canonicalValueRef != NULL); + pThis->table.arr->interned_val_refs[i] = canonicalValueRef; + } + } + + pThis->lookup = lookupKey_arr; + pThis->key_type = LOOKUP_KEY_TYPE_UINT; + +finalize_it: + free(indexes); + RETiRet; +} + +static rsRetVal +build_SparseArrayTable(lookup_t *pThis, struct json_object *jtab, const uchar* name) { + uint32_t i; + struct json_object *jrow, *jindex, *jvalue; + uchar *value, *canonicalValueRef; + DEFiRet; + + pThis->table.str = NULL; + CHKmalloc(pThis->table.sprsArr = calloc(1, sizeof(lookup_sparseArray_tab_t))); + if (pThis->nmemb > 0) { + CHKmalloc(pThis->table.sprsArr->entries = calloc(pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t))); + + for(i = 0; i < pThis->nmemb; i++) { + jrow = json_object_array_get_idx(jtab, i); + jindex = json_object_object_get(jrow, "index"); + jvalue = json_object_object_get(jrow, "value"); + if (jindex == NULL || json_object_is_type(jindex, json_type_null)) { + NO_INDEX_ERROR("sparseArray", name); + } + pThis->table.sprsArr->entries[i].key = (uint32_t) json_object_get_int(jindex); + value = (uchar*) json_object_get_string(jvalue); + uchar *const *const canonicalValueRef_ptr = bsearch(value, pThis->interned_vals, + pThis->interned_val_count, sizeof(uchar*), bs_arrcmp_str); + if(canonicalValueRef_ptr == NULL) { + LogError(0, RS_RET_ERR, "BUG: canonicalValueRef not found in " + "build_SparseArrayTable(), %s:%d", __FILE__, __LINE__); + ABORT_FINALIZE(RS_RET_ERR); + } + canonicalValueRef = *canonicalValueRef_ptr; + assert(canonicalValueRef != NULL); + pThis->table.sprsArr->entries[i].interned_val_ref = canonicalValueRef; + } + qsort(pThis->table.sprsArr->entries, pThis->nmemb, sizeof(lookup_sparseArray_tab_entry_t), + qs_arrcmp_sprsArrtab); + } + + pThis->lookup = lookupKey_sprsArr; + pThis->key_type = LOOKUP_KEY_TYPE_UINT; + +finalize_it: + RETiRet; +} + +static rsRetVal +lookupBuildStubbedTable(lookup_t *pThis, const uchar* stub_val) { + DEFiRet; + + CHKmalloc(pThis->nomatch = ustrdup(stub_val)); + pThis->lookup = lookupKey_stub; + pThis->type = STUBBED_LOOKUP_TABLE; + pThis->key_type = LOOKUP_KEY_TYPE_NONE; + +finalize_it: + RETiRet; +} + +static rsRetVal +lookupBuildTable_v1(lookup_t *pThis, struct json_object *jroot, const uchar* name) { + struct json_object *jnomatch, *jtype, *jtab; + struct json_object *jrow, *jvalue; + const char *table_type, *nomatch_value; + const uchar **all_values; + const uchar *curr, *prev; + uint32_t i, j; + uint32_t uniq_values; + + DEFiRet; + all_values = NULL; + + jnomatch = json_object_object_get(jroot, "nomatch"); + jtype = json_object_object_get(jroot, "type"); + jtab = json_object_object_get(jroot, "table"); + if (jtab == NULL || !json_object_is_type(jtab, json_type_array)) { + LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' has invalid table definition", name); + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + } + pThis->nmemb = json_object_array_length(jtab); + table_type = json_object_get_string(jtype); + if (table_type == NULL) { + table_type = "string"; + } + + CHKmalloc(all_values = malloc(pThis->nmemb * sizeof(uchar*))); + + /* before actual table can be loaded, prepare all-value list and remove duplicates*/ + for(i = 0; i < pThis->nmemb; i++) { + jrow = json_object_array_get_idx(jtab, i); + jvalue = json_object_object_get(jrow, "value"); + if (jvalue == NULL || json_object_is_type(jvalue, json_type_null)) { + LogError(0, RS_RET_INVALID_VALUE, "'%s' lookup table named: '%s' has record(s) " + "without 'value' field", table_type, name); + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + } + all_values[i] = (const uchar*) json_object_get_string(jvalue); + } + qsort(all_values, pThis->nmemb, sizeof(uchar*), qs_arrcmp_ustrs); + uniq_values = 1; + for(i = 1; i < pThis->nmemb; i++) { + curr = all_values[i]; + prev = all_values[i - 1]; + if (ustrcmp(prev, curr) != 0) { + uniq_values++; + } + } + + if (pThis->nmemb > 0) { + CHKmalloc(pThis->interned_vals = malloc(uniq_values * sizeof(uchar*))); + j = 0; + CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[0])); + for(i = 1; i < pThis->nmemb ; ++i) { + curr = all_values[i]; + prev = all_values[i - 1]; + if (ustrcmp(prev, curr) != 0) { + CHKmalloc(pThis->interned_vals[j++] = ustrdup(all_values[i])); + } + } + pThis->interned_val_count = uniq_values; + } + /* uniq values captured (sorted) */ + + nomatch_value = json_object_get_string(jnomatch); + if (nomatch_value != NULL) { + CHKmalloc(pThis->nomatch = (uchar*) strdup(nomatch_value)); + } + + if (strcmp(table_type, "array") == 0) { + pThis->type = ARRAY_LOOKUP_TABLE; + CHKiRet(build_ArrayTable(pThis, jtab, name)); + } else if (strcmp(table_type, "sparseArray") == 0) { + pThis->type = SPARSE_ARRAY_LOOKUP_TABLE; + CHKiRet(build_SparseArrayTable(pThis, jtab, name)); + } else if (strcmp(table_type, "string") == 0) { + pThis->type = STRING_LOOKUP_TABLE; + CHKiRet(build_StringTable(pThis, jtab, name)); + } else { + LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' uses unupported " + "type: '%s'", name, table_type); + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + } +finalize_it: + if (all_values != NULL) free(all_values); + RETiRet; +} + +static rsRetVal +lookupBuildTable(lookup_t *pThis, struct json_object *jroot, const uchar* name) +{ + struct json_object *jversion; + int version = 1; + + DEFiRet; + + jversion = json_object_object_get(jroot, "version"); + if (jversion != NULL && !json_object_is_type(jversion, json_type_null)) { + version = json_object_get_int(jversion); + } else { + LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' doesn't specify version " + "(will use default value: %d)", name, version); + } + if (version == 1) { + CHKiRet(lookupBuildTable_v1(pThis, jroot, name)); + } else { + LogError(0, RS_RET_INVALID_VALUE, "lookup table named: '%s' uses unsupported " + "version: %d", name, version); + ABORT_FINALIZE(RS_RET_INVALID_VALUE); + } + +finalize_it: + RETiRet; +} + + +/* find a lookup table. This is a naive O(n) algo, but this really + * doesn't matter as it is called only a few times during config + * load. The function returns either a pointer to the requested + * table or NULL, if not found. + */ +lookup_ref_t * ATTR_NONNULL() +lookupFindTable(uchar *name) +{ + lookup_ref_t *curr; + + for(curr = loadConf->lu_tabs.root ; curr != NULL ; curr = curr->next) { + if(!ustrcmp(curr->name, name)) + break; + } + return curr; +} + + +/* this reloads a lookup table. This is done while the engine is running, + * as such the function must ensure proper locking and proper order of + * operations (so that nothing can interfere). If the table cannot be loaded, + * the old table is continued to be used. + */ +static rsRetVal +lookupReloadOrStub(lookup_ref_t *pThis, const uchar* stub_val) { + lookup_t *newlu, *oldlu; /* dummy to be able to use support functions without + affecting current settings. */ + DEFiRet; + + oldlu = pThis->self; + newlu = NULL; + + DBGPRINTF("reload requested for lookup table '%s'\n", pThis->name); + CHKmalloc(newlu = calloc(1, sizeof(lookup_t))); + if (stub_val == NULL) { + CHKiRet(lookupReadFile(newlu, pThis->name, pThis->filename)); + } else { + CHKiRet(lookupBuildStubbedTable(newlu, stub_val)); + } + /* all went well, copy over data members */ + pthread_rwlock_wrlock(&pThis->rwlock); + pThis->self = newlu; + pthread_rwlock_unlock(&pThis->rwlock); +finalize_it: + if (iRet != RS_RET_OK) { + if (stub_val == NULL) { + LogError(0, RS_RET_INTERNAL_ERROR, + "lookup table '%s' could not be reloaded from file '%s'", + pThis->name, pThis->filename); + } else { + LogError(0, RS_RET_INTERNAL_ERROR, + "lookup table '%s' could not be stubbed with value '%s'", + pThis->name, stub_val); + } + lookupDestruct(newlu); + } else { + if (stub_val == NULL) { + LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' reloaded from file '%s'", + pThis->name, pThis->filename); + } else { + LogError(0, RS_RET_OK, "lookup table '%s' stubbed with value '%s'", + pThis->name, stub_val); + } + lookupDestruct(oldlu); + } + RETiRet; +} + +static rsRetVal +lookupDoStub(lookup_ref_t *pThis, const uchar* stub_val) +{ + int already_stubbed = 0; + DEFiRet; + pthread_rwlock_rdlock(&pThis->rwlock); + if (pThis->self->type == STUBBED_LOOKUP_TABLE && + ustrcmp(pThis->self->nomatch, stub_val) == 0) + already_stubbed = 1; + pthread_rwlock_unlock(&pThis->rwlock); + if (! already_stubbed) { + LogError(0, RS_RET_OK, "stubbing lookup table '%s' with value '%s'", + pThis->name, stub_val); + CHKiRet(lookupReloadOrStub(pThis, stub_val)); + } else { + LogError(0, RS_RET_OK, "lookup table '%s' is already stubbed with value '%s'", + pThis->name, stub_val); + } +finalize_it: + RETiRet; +} + +static uint8_t +lookupIsReloadPending(lookup_ref_t *pThis) { + uint8_t reload_pending; + pthread_mutex_lock(&pThis->reloader_mut); + reload_pending = pThis->do_reload; + pthread_mutex_unlock(&pThis->reloader_mut); + return reload_pending; +} + +/* note: stub_val_if_reload_fails may or may not be NULL */ +rsRetVal ATTR_NONNULL(1) +lookupReload(lookup_ref_t *const pThis, const uchar *const stub_val_if_reload_fails) +{ + uint8_t locked = 0; + int lock_errno = 0; + DEFiRet; + assert(pThis != NULL); + if ((lock_errno = pthread_mutex_trylock(&pThis->reloader_mut)) == 0) { + locked = 1; + /*so it doesn't leak memory in situation where 2 reload requests are issued back to back*/ + freeStubValueForReloadFailure(pThis); + if (stub_val_if_reload_fails != NULL) { + CHKmalloc(pThis->stub_value_for_reload_failure = ustrdup(stub_val_if_reload_fails)); + } + pThis->do_reload = 1; + pthread_cond_signal(&pThis->run_reloader); + } else { + LogError(lock_errno, RS_RET_INTERNAL_ERROR, "attempt to trigger " + "reload of lookup table '%s' failed (not stubbing)", pThis->name); + ABORT_FINALIZE(RS_RET_INTERNAL_ERROR); + /* we can choose to stub the table here, but it'll hurt because + the table reloader may take time to complete the reload + and stubbing because of a concurrent reload message may + not be desirable (except in very tightly controled environments + where reload-triggering messages pushed are timed accurately + and an idempotency-filter is used to reject re-deliveries) */ + } +finalize_it: + if (locked) { + pthread_mutex_unlock(&pThis->reloader_mut); + } + RETiRet; +} + +static rsRetVal ATTR_NONNULL() +lookupDoReload(lookup_ref_t *pThis) +{ + DEFiRet; + iRet = lookupReloadOrStub(pThis, NULL); + if ((iRet != RS_RET_OK) && + (pThis->stub_value_for_reload_failure != NULL)) { + iRet = lookupDoStub(pThis, pThis->stub_value_for_reload_failure); + } + freeStubValueForReloadFailure(pThis); + RETiRet; +} + +void * +lookupTableReloader(void *self) +{ + lookup_ref_t *pThis = (lookup_ref_t*) self; + pthread_mutex_lock(&pThis->reloader_mut); + while(1) { + if (pThis->do_stop) { + break; + } else if (pThis->do_reload) { + lookupDoReload(pThis); + pThis->do_reload = 0; + } else { + pthread_cond_wait(&pThis->run_reloader, &pThis->reloader_mut); + } + } + pthread_mutex_unlock(&pThis->reloader_mut); + return NULL; +} + +/* reload all lookup tables on HUP */ +void +lookupDoHUP(void) +{ + lookup_ref_t *luref; + for(luref = runConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { + if (luref->reload_on_hup) { + lookupReload(luref, NULL); + } + } +} + +/* activate lookup table system config + * most importantly, this means tarting the lookup table reloader thread in the + * right process space - it is a difference if we fork or not! + */ +void +lookupActivateConf(void) +{ + DBGPRINTF("lookup tables: activate config \n"); + lookup_ref_t *luref; + for(luref = runConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { + DBGPRINTF("lookup actiate: processing %p\n", luref); + lookupActivateTable(luref); + } + DBGPRINTF("lookup tables: activate done\n"); +} + +uint +lookupPendingReloadCount(void) +{ + uint pending_reload_count = 0; + lookup_ref_t *luref; + for(luref = runConf->lu_tabs.root ; luref != NULL ; luref = luref->next) { + if (lookupIsReloadPending(luref)) { + pending_reload_count++; + } + } + return pending_reload_count; +} + + +/* returns either a pointer to the value (read only!) or NULL + * if either the key could not be found or an error occurred. + * Note that an estr_t object is returned. The caller is + * responsible for freeing it. + */ +es_str_t * +lookupKey(lookup_ref_t *pThis, lookup_key_t key) +{ + es_str_t *estr; + lookup_t *t; + pthread_rwlock_rdlock(&pThis->rwlock); + t = pThis->self; + estr = t->lookup(t, key); + pthread_rwlock_unlock(&pThis->rwlock); + return estr; +} + + +/* note: widely-deployed json_c 0.9 does NOT support incremental + * parsing. In order to keep compatible with e.g. Ubuntu 12.04LTS, + * we read the file into one big memory buffer and parse it at once. + * While this is not very elegant, it will not pose any real issue + * for "reasonable" lookup tables (and "unreasonably" large ones + * will probably have other issues as well...). + */ +static rsRetVal ATTR_NONNULL() +lookupReadFile(lookup_t *const pThis, const uchar *const name, const uchar *const filename) +{ + struct json_tokener *tokener = NULL; + struct json_object *json = NULL; + char *iobuf = NULL; + int fd = -1; + ssize_t nread; + struct stat sb; + DEFiRet; + + + if((fd = open((const char*) filename, O_RDONLY)) == -1) { + LogError(errno, RS_RET_FILE_NOT_FOUND, + "lookup table file '%s' could not be opened", filename); + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + } + + if(fstat(fd, &sb) == -1) { + LogError(errno, RS_RET_FILE_NOT_FOUND, + "lookup table file '%s' stat failed", filename); + ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + } + + CHKmalloc(iobuf = malloc(sb.st_size)); + + tokener = json_tokener_new(); + nread = read(fd, iobuf, sb.st_size); + if(nread != (ssize_t) sb.st_size) { + LogError(errno, RS_RET_READ_ERR, + "lookup table file '%s' read error", filename); + ABORT_FINALIZE(RS_RET_READ_ERR); + } + + json = json_tokener_parse_ex(tokener, iobuf, sb.st_size); + if(json == NULL) { + LogError(0, RS_RET_JSON_PARSE_ERR, + "lookup table file '%s' json parsing error", + filename); + ABORT_FINALIZE(RS_RET_JSON_PARSE_ERR); + } + free(iobuf); /* early free to sever resources*/ + iobuf = NULL; /* make sure no double-free */ + + /* got json object, now populate our own in-memory structure */ + CHKiRet(lookupBuildTable(pThis, json, name)); + +finalize_it: + if (fd != -1) { + close(fd); + } + free(iobuf); + if(tokener != NULL) + json_tokener_free(tokener); + if(json != NULL) + json_object_put(json); + RETiRet; +} + + +rsRetVal +lookupTableDefProcessCnf(struct cnfobj *o) +{ + struct cnfparamvals *pvals; + lookup_ref_t *lu; + short i; +#ifdef HAVE_PTHREAD_SETNAME_NP + char *reloader_thd_name = NULL; + int thd_name_len = 0; +#endif + DEFiRet; + lu = NULL; + + pvals = nvlstGetParams(o->nvlst, &modpblk, NULL); + if(pvals == NULL) { + ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); + } + DBGPRINTF("lookupTableDefProcessCnf params:\n"); + cnfparamsPrint(&modpblk, pvals); + + CHKiRet(lookupNew(&lu)); + + for(i = 0 ; i < modpblk.nParams ; ++i) { + if(!pvals[i].bUsed) + continue; + if(!strcmp(modpblk.descr[i].name, "file")) { + CHKmalloc(lu->filename = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL)); + } else if(!strcmp(modpblk.descr[i].name, "name")) { + CHKmalloc(lu->name = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL)); + } else if(!strcmp(modpblk.descr[i].name, "reloadOnHUP")) { + lu->reload_on_hup = (pvals[i].val.d.n != 0); + } else { + dbgprintf("lookup_table: program error, non-handled " + "param '%s'\n", modpblk.descr[i].name); + } + } + const uchar *const lu_name = lu->name; /* we need a const to keep TSAN happy :-( */ + const uchar *const lu_filename = lu->filename; /* we need a const to keep TSAN happy :-( */ + if(lu_name == NULL || lu_filename == NULL) { + iRet = RS_RET_INTERNAL_ERROR; + LogError(0, iRet, "internal error: lookup table name not set albeit being mandatory"); + ABORT_FINALIZE(iRet); + } +#ifdef HAVE_PTHREAD_SETNAME_NP + thd_name_len = ustrlen(lu_name) + strlen(reloader_prefix) + 1; + CHKmalloc(reloader_thd_name = malloc(thd_name_len)); + strcpy(reloader_thd_name, reloader_prefix); + strcpy(reloader_thd_name + strlen(reloader_prefix), (char*) lu_name); + reloader_thd_name[thd_name_len - 1] = '\0'; +#if defined(__NetBSD__) + pthread_setname_np(lu->reloader, "%s", reloader_thd_name); +#elif defined(__APPLE__) + pthread_setname_np(reloader_thd_name); // must check +#else + pthread_setname_np(lu->reloader, reloader_thd_name); +#endif +#endif + CHKiRet(lookupReadFile(lu->self, lu_name, lu_filename)); + LogMsg(0, RS_RET_OK, LOG_INFO, "lookup table '%s' loaded from file '%s'", + lu_name, lu->filename); + +finalize_it: +#ifdef HAVE_PTHREAD_SETNAME_NP + free(reloader_thd_name); +#endif + cnfparamvalsDestruct(pvals, &modpblk); + if (iRet != RS_RET_OK) { + if (lu != NULL) { + lookupDestruct(lu->self); + lu->self = NULL; + } + } + RETiRet; +} + +void +lookupClassExit(void) +{ + objRelease(glbl, CORE_COMPONENT); +} + +rsRetVal +lookupClassInit(void) +{ + DEFiRet; + CHKiRet(objGetObjInterface(&obj)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); +finalize_it: + RETiRet; +} |