diff options
Diffstat (limited to 'libnetdata')
62 files changed, 29200 insertions, 812 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index 167d05caa..5962323e8 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -5,6 +5,7 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in SUBDIRS = \ adaptive_resortable_list \ + arrayalloc \ avl \ buffer \ clocks \ diff --git a/libnetdata/arrayalloc/Makefile.am b/libnetdata/arrayalloc/Makefile.am new file mode 100644 index 000000000..161784b8f --- /dev/null +++ b/libnetdata/arrayalloc/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/arrayalloc/README.md b/libnetdata/arrayalloc/README.md new file mode 100644 index 000000000..2f21bf3ff --- /dev/null +++ b/libnetdata/arrayalloc/README.md @@ -0,0 +1,7 @@ +<!-- +title: "Array Allocator" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/arrayalloc/README.md +--> + +# Array Allocator + diff --git a/libnetdata/arrayalloc/arrayalloc.c b/libnetdata/arrayalloc/arrayalloc.c new file mode 100644 index 000000000..bdf1384d4 --- /dev/null +++ b/libnetdata/arrayalloc/arrayalloc.c @@ -0,0 +1,335 @@ +#include "../libnetdata.h" +#include "arrayalloc.h" +#include "daemon/common.h" + +// max file size +#define ARAL_MAX_PAGE_SIZE_MMAP (1*1024*1024*1024) + +// max malloc size +#define ARAL_MAX_PAGE_SIZE_MALLOC (10*1024*1024) + +typedef struct arrayalloc_free { + size_t size; + struct arrayalloc_page *page; + struct arrayalloc_free *next; +} ARAL_FREE; + +typedef struct arrayalloc_page { + const char *filename; + size_t size; // the total size of the page + size_t used_elements; // the total number of used elements on this page + uint8_t *data; + ARAL_FREE *free_list; + struct arrayalloc_page *prev; // the prev page on the list + struct arrayalloc_page *next; // the next page on the list +} ARAL_PAGE; + +#define ARAL_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) +static inline size_t natural_alignment(size_t size, size_t alignment) { + if(unlikely(size % alignment)) + size = size + alignment - (size % alignment); + + return size; +} + +static void arrayalloc_init(ARAL *ar) { + static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER; + netdata_mutex_lock(&mutex); + + if(!ar->internal.initialized) { + netdata_mutex_init(&ar->internal.mutex); + + long int page_size = sysconf(_SC_PAGE_SIZE); + if (unlikely(page_size == -1)) + ar->internal.natural_page_size = 4096; + else + ar->internal.natural_page_size = page_size; + + // we need to add a page pointer after the element + // so, first align the element size to the pointer size + ar->internal.element_size = natural_alignment(ar->element_size, sizeof(uintptr_t)); + + // then add the size of a pointer to it + ar->internal.element_size += sizeof(uintptr_t); + + // make sure it is at least what we need for an ARAL_FREE slot + if (ar->internal.element_size < sizeof(ARAL_FREE)) + ar->internal.element_size = sizeof(ARAL_FREE); + + // and finally align it to the natural alignment + ar->internal.element_size = natural_alignment(ar->internal.element_size, ARAL_NATURAL_ALIGNMENT); + + // this is where we should write the pointer + ar->internal.page_ptr_offset = ar->internal.element_size - sizeof(uintptr_t); + + if(ar->element_size + sizeof(uintptr_t) > ar->internal.element_size) + fatal("ARRAYALLOC: failed to calculate properly page_ptr_offset: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu", + ar->element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset); + + //info("ARRAYALLOC: element size %zu, sizeof(uintptr_t) %zu, natural alignment %zu, final element size %zu, page_ptr_offset %zu", + // ar->element_size, sizeof(uintptr_t), ARAL_NATURAL_ALIGNMENT, ar->internal.element_size, ar->internal.page_ptr_offset); + + if (ar->elements < 10) + ar->elements = 10; + + ar->internal.mmap = (ar->use_mmap && ar->cache_dir && *ar->cache_dir) ? true : false; + ar->internal.max_alloc_size = ar->internal.mmap ? ARAL_MAX_PAGE_SIZE_MMAP : ARAL_MAX_PAGE_SIZE_MALLOC; + + if(ar->internal.max_alloc_size % ar->internal.natural_page_size) + ar->internal.max_alloc_size += ar->internal.natural_page_size - (ar->internal.max_alloc_size % ar->internal.natural_page_size) ; + + if(ar->internal.max_alloc_size % ar->internal.element_size) + ar->internal.max_alloc_size -= ar->internal.max_alloc_size % ar->internal.element_size; + + ar->internal.first_page = NULL; + ar->internal.last_page = NULL; + ar->internal.allocation_multiplier = 1; + ar->internal.file_number = 0; + + if(ar->internal.mmap) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/array_alloc.mmap", *ar->cache_dir); + int r = mkdir(filename, 0775); + if (r != 0 && errno != EEXIST) + fatal("Cannot create directory '%s'", filename); + } + + ar->internal.initialized = true; + } + + netdata_mutex_unlock(&mutex); +} + +#ifdef NETDATA_INTERNAL_CHECKS +static inline void arrayalloc_free_checks(ARAL *ar, ARAL_FREE *fr) { + if(fr->size < ar->internal.element_size) + fatal("ARRAYALLOC: free item of size %zu, less than the expected element size %zu", fr->size, ar->internal.element_size); + + if(fr->size % ar->internal.element_size) + fatal("ARRAYALLOC: free item of size %zu is not multiple to element size %zu", fr->size, ar->internal.element_size); +} +#else +#define arrayalloc_free_checks(ar, fr) debug_dummy() +#endif + +static inline void unlink_page(ARAL *ar, ARAL_PAGE *page) { + if(unlikely(!page)) return; + + if(page->next) + page->next->prev = page->prev; + + if(page->prev) + page->prev->next = page->next; + + if(page == ar->internal.first_page) + ar->internal.first_page = page->next; + + if(page == ar->internal.last_page) + ar->internal.last_page = page->prev; +} + +static inline void link_page_first(ARAL *ar, ARAL_PAGE *page) { + page->prev = NULL; + page->next = ar->internal.first_page; + if(page->next) page->next->prev = page; + + ar->internal.first_page = page; + + if(!ar->internal.last_page) + ar->internal.last_page = page; +} + +static inline void link_page_last(ARAL *ar, ARAL_PAGE *page) { + page->next = NULL; + page->prev = ar->internal.last_page; + if(page->prev) page->prev->next = page; + + ar->internal.last_page = page; + + if(!ar->internal.first_page) + ar->internal.first_page = page; +} + +static inline ARAL_PAGE *find_page_with_allocation(ARAL *ar, void *ptr) { + uintptr_t seeking = (uintptr_t)ptr; + ARAL_PAGE *page; + + for(page = ar->internal.first_page; page ; page = page->next) { + if(unlikely(seeking >= (uintptr_t)page->data && seeking < (uintptr_t)page->data + page->size)) + break; + } + + return page; +} + +static void arrayalloc_increase(ARAL *ar) { + if(unlikely(!ar->internal.initialized)) + arrayalloc_init(ar); + + ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE)); + page->size = ar->elements * ar->internal.element_size * ar->internal.allocation_multiplier; + if(page->size > ar->internal.max_alloc_size) + page->size = ar->internal.max_alloc_size; + else + ar->internal.allocation_multiplier *= 2; + + if(ar->internal.mmap) { + ar->internal.file_number++; + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/array_alloc.mmap/%s.%zu", *ar->cache_dir, ar->filename, ar->internal.file_number); + page->filename = strdupz(filename); + page->data = netdata_mmap(page->filename, page->size, MAP_SHARED, 0); + if (unlikely(!page->data)) + fatal("Cannot allocate arrayalloc buffer of size %zu on filename '%s'", page->size, page->filename); + } + else + page->data = mallocz(page->size); + + // link the free space to its page + ARAL_FREE *fr = (ARAL_FREE *)page->data; + fr->size = page->size; + fr->page = page; + fr->next = NULL; + page->free_list = fr; + + // link the new page at the front of the list of pages + link_page_first(ar, page); + + arrayalloc_free_checks(ar, fr); +} + +static void arrayalloc_lock(ARAL *ar) { + if(!ar->internal.lockless) + netdata_mutex_lock(&ar->internal.mutex); +} + +static void arrayalloc_unlock(ARAL *ar) { + if(!ar->internal.lockless) + netdata_mutex_unlock(&ar->internal.mutex); +} + +ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filename, char **cache_dir) { + ARAL *ar = callocz(1, sizeof(ARAL)); + ar->element_size = element_size; + ar->elements = elements; + ar->filename = filename; + ar->cache_dir = cache_dir; + return ar; +} + +void *arrayalloc_mallocz(ARAL *ar) { + arrayalloc_lock(ar); + + if(unlikely(!ar->internal.first_page || !ar->internal.first_page->free_list)) + arrayalloc_increase(ar); + + ARAL_PAGE *page = ar->internal.first_page; + ARAL_FREE *fr = page->free_list; + + if(unlikely(!fr)) + fatal("ARRAYALLOC: free item cannot be NULL."); + + if(unlikely(fr->size < ar->internal.element_size)) + fatal("ARRAYALLOC: free item size %zu is smaller than %zu", fr->size, ar->internal.element_size); + + if(fr->size - ar->internal.element_size <= ar->internal.element_size) { + // we are done with this page + page->free_list = NULL; + + if(page != ar->internal.last_page) { + unlink_page(ar, page); + link_page_last(ar, page); + } + } + else { + uint8_t *data = (uint8_t *)fr; + ARAL_FREE *fr2 = (ARAL_FREE *)&data[ar->internal.element_size]; + fr2->page = fr->page; + fr2->size = fr->size - ar->internal.element_size; + fr2->next = fr->next; + page->free_list = fr2; + + arrayalloc_free_checks(ar, fr2); + } + + fr->page->used_elements++; + + // put the page pointer after the element + uint8_t *data = (uint8_t *)fr; + ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->internal.page_ptr_offset]; + *page_ptr = page; + + arrayalloc_unlock(ar); + return (void *)fr; +} + +void arrayalloc_freez(ARAL *ar, void *ptr) { + if(!ptr) return; + arrayalloc_lock(ar); + + // get the page pointer + ARAL_PAGE *page; + { + uint8_t *data = (uint8_t *)ptr; + ARAL_PAGE **page_ptr = (ARAL_PAGE **)&data[ar->internal.page_ptr_offset]; + page = *page_ptr; + +#ifdef NETDATA_INTERNAL_CHECKS + // make it NULL so that we will fail on double free + // do not enable this on production, because the MMAP file + // will need to be saved again! + *page_ptr = NULL; +#endif + } + +#ifdef NETDATA_INTERNAL_CHECKS + { + // find the page ptr belongs + ARAL_PAGE *page2 = find_page_with_allocation(ar, ptr); + + if(unlikely(page != page2)) + fatal("ARRAYALLOC: page pointers do not match!"); + + if (unlikely(!page2)) + fatal("ARRAYALLOC: free of pointer %p is not in arrayalloc address space.", ptr); + } +#endif + + if(unlikely(!page)) + fatal("ARRAYALLOC: possible corruption or double free of pointer %p", ptr); + + if (unlikely(!page->used_elements)) + fatal("ARRAYALLOC: free of pointer %p is inside a page without any active allocations.", ptr); + + page->used_elements--; + + // make this element available + ARAL_FREE *fr = (ARAL_FREE *)ptr; + fr->page = page; + fr->size = ar->internal.element_size; + fr->next = page->free_list; + page->free_list = fr; + + // if the page is empty, release it + if(!page->used_elements) { + unlink_page(ar, page); + + // free it + if(ar->internal.mmap) { + munmap(page->data, page->size); + if (unlikely(unlink(page->filename) == 1)) + error("Cannot delete file '%s'", page->filename); + freez((void *)page->filename); + } + else + freez(page->data); + + freez(page); + } + else if(page != ar->internal.first_page) { + unlink_page(ar, page); + link_page_first(ar, page); + } + + arrayalloc_unlock(ar); +} diff --git a/libnetdata/arrayalloc/arrayalloc.h b/libnetdata/arrayalloc/arrayalloc.h new file mode 100644 index 000000000..e0e9e7f9f --- /dev/null +++ b/libnetdata/arrayalloc/arrayalloc.h @@ -0,0 +1,35 @@ + +#ifndef ARRAYALLOC_H +#define ARRAYALLOC_H 1 + +#include "../libnetdata.h" + +typedef struct arrayalloc { + size_t element_size; + size_t elements; + const char *filename; + char **cache_dir; + bool use_mmap; + + // private members - do not touch + struct { + bool mmap; + bool lockless; + bool initialized; + size_t element_size; + size_t page_ptr_offset; + size_t file_number; + size_t natural_page_size; + size_t allocation_multiplier; + size_t max_alloc_size; + netdata_mutex_t mutex; + struct arrayalloc_page *first_page; + struct arrayalloc_page *last_page; + } internal; +} ARAL; + +extern ARAL *arrayalloc_create(size_t element_size, size_t elements, const char *filename, char **cache_dir); +extern void *arrayalloc_mallocz(ARAL *ar); +extern void arrayalloc_freez(ARAL *ar, void *ptr); + +#endif // ARRAYALLOC_H diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 880417551..8a32184f6 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -65,9 +65,9 @@ void buffer_char_replace(BUFFER *wb, char from, char to) } // This trick seems to give an 80% speed increase in 32bit systems -// print_calculated_number_llu_r() will just print the digits up to the +// print_number_llu_r() will just print the digits up to the // point the remaining value fits in 32 bits, and then calls -// print_calculated_number_lu_r() to print the rest with 32 bit arithmetic. +// print_number_lu_r() to print the rest with 32 bit arithmetic. inline char *print_number_lu_r(char *str, unsigned long uvalue) { char *wstr = str; @@ -299,7 +299,7 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...) } -void buffer_rrd_value(BUFFER *wb, calculated_number value) +void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value) { buffer_need_bytes(wb, 50); @@ -308,7 +308,7 @@ void buffer_rrd_value(BUFFER *wb, calculated_number value) return; } else - wb->len += print_calculated_number(&wb->buffer[wb->len], value); + wb->len += print_netdata_double(&wb->buffer[wb->len], value); // terminate it buffer_need_bytes(wb, 1); diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index ceaeadd9b..42425b4cb 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -56,7 +56,7 @@ extern void buffer_reset(BUFFER *wb); extern void buffer_strcat(BUFFER *wb, const char *txt); extern void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len); -extern void buffer_rrd_value(BUFFER *wb, calculated_number value); +extern void buffer_rrd_value(BUFFER *wb, NETDATA_DOUBLE value); extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); extern void buffer_jsdate(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c index f0e17b232..e4dca6c8b 100644 --- a/libnetdata/clocks/clocks.c +++ b/libnetdata/clocks/clocks.c @@ -393,7 +393,7 @@ static inline collected_number read_proc_uptime(char *filename) { return 0; } - return (collected_number)(strtold(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0); + return (collected_number)(strtondd(procfile_lineword(read_proc_uptime_ff, 0, 0), NULL) * 1000.0); } inline collected_number uptime_msec(char *filename){ diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c index 0272877bf..1288366da 100644 --- a/libnetdata/config/appconfig.c +++ b/libnetdata/config/appconfig.c @@ -462,15 +462,15 @@ long long appconfig_get_number(struct config *root, const char *section, const c return strtoll(s, NULL, 0); } -LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) +NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) { char buffer[100], *s; - sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); + sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); s = appconfig_get(root, section, name, buffer); if(!s) return value; - return str2ld(s, NULL); + return str2ndd(s, NULL); } static inline int appconfig_test_boolean_value(char *s) { @@ -588,10 +588,10 @@ long long appconfig_set_number(struct config *root, const char *section, const c return value; } -LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value) +NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value) { char buffer[100]; - sprintf(buffer, "%0.5" LONG_DOUBLE_MODIFIER, value); + sprintf(buffer, "%0.5" NETDATA_DOUBLE_MODIFIER, value); appconfig_set(root, section, name, buffer); @@ -805,6 +805,18 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) struct section *co; struct config_option *cv; + { + int found_host_labels = 0; + for (co = root->first_section; co; co = co->next) + if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) + found_host_labels = 1; + + if(!found_host_labels) { + appconfig_section_create(root, CONFIG_SECTION_HOST_LABEL); + appconfig_get(root, CONFIG_SECTION_HOST_LABEL, "name", "value"); + } + } + buffer_strcat(wb, "# netdata configuration\n" "#\n" @@ -819,26 +831,27 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) "#\n" "\n# global netdata configuration\n"); - for(i = 0; i <= 15 ;i++) { + for(i = 0; i <= 16 ;i++) { appconfig_wrlock(root); for(co = root->first_section; co ; co = co->next) { if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)) pri = 0; - else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 1; - else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 2; - else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 3; - else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 4; - else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 5; - else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 6; - else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 7; - else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 8; - else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 9; - // by default, new sections will get pri = 10 (set at the end, below) - else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 11; - else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 12; - else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 13; - else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 14; - else if(!strncmp(co->name, "plugin:", 7)) pri = 15; // << change the loop too if you change this - else pri = 10; // this is used for any new (currently unknown) sections + else if(!strcmp(co->name, CONFIG_SECTION_DB)) pri = 1; + else if(!strcmp(co->name, CONFIG_SECTION_DIRECTORIES)) pri = 2; + else if(!strcmp(co->name, CONFIG_SECTION_LOGS)) pri = 3; + else if(!strcmp(co->name, CONFIG_SECTION_ENV_VARS)) pri = 4; + else if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL)) pri = 5; + else if(!strcmp(co->name, CONFIG_SECTION_SQLITE)) pri = 6; + else if(!strcmp(co->name, CONFIG_SECTION_CLOUD)) pri = 7; + else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 8; + else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 9; + else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 10; + // by default, new sections will get pri = 11 (set at the end, below) + else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 12; + else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 13; + else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 14; + else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 15; + else if(!strncmp(co->name, "plugin:", 7)) pri = 16; // << change the loop too if you change this + else pri = 11; // this is used for any new (currently unknown) sections if(i == pri) { int loaded = 0; @@ -904,7 +917,7 @@ int config_parse_duration(const char* string, int* result) { if(!(isdigit(*string) || *string == '+' || *string == '-')) goto fallback; char *e = NULL; - calculated_number n = str2ld(string, &e); + NETDATA_DOUBLE n = str2ndd(string, &e); if(e && *e) { switch (*e) { case 'Y': diff --git a/libnetdata/config/appconfig.h b/libnetdata/config/appconfig.h index f1f61e31d..d72a3140e 100644 --- a/libnetdata/config/appconfig.h +++ b/libnetdata/config/appconfig.h @@ -100,6 +100,8 @@ #define CONFIG_SECTION_HOST_LABEL "host labels" #define EXPORTING_CONF "exporting.conf" #define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics" +#define CONFIG_SECTION_DB "db" + // these are used to limit the configuration names and values lengths // they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) @@ -168,7 +170,7 @@ extern void config_section_unlock(struct section *co); extern char *appconfig_get_by_section(struct section *co, const char *name, const char *default_value); extern char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value); extern long long appconfig_get_number(struct config *root, const char *section, const char *name, long long value); -extern LONG_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); +extern NETDATA_DOUBLE appconfig_get_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); extern int appconfig_get_boolean_by_section(struct section *co, const char *name, int value); extern int appconfig_get_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_get_boolean_ondemand(struct config *root, const char *section, const char *name, int value); @@ -177,7 +179,7 @@ extern int appconfig_get_duration(struct config *root, const char *section, cons extern const char *appconfig_set(struct config *root, const char *section, const char *name, const char *value); extern const char *appconfig_set_default(struct config *root, const char *section, const char *name, const char *value); extern long long appconfig_set_number(struct config *root, const char *section, const char *name, long long value); -extern LONG_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, LONG_DOUBLE value); +extern NETDATA_DOUBLE appconfig_set_float(struct config *root, const char *section, const char *name, NETDATA_DOUBLE value); extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_exists(struct config *root, const char *section, const char *name); diff --git a/libnetdata/dictionary/README.md b/libnetdata/dictionary/README.md index 28d0cfbbd..879c1bcc1 100644 --- a/libnetdata/dictionary/README.md +++ b/libnetdata/dictionary/README.md @@ -22,7 +22,7 @@ Dictionaries are **ordered**, meaning that the order they have been added is pre Dictionaries guarantee **uniqueness** of all items added to them, meaning that only one item with a given name can exist in the dictionary at any given time. -Dictionaries are extremely fast in all operations. They are indexing the keys with `JudyHS` and they utilize a double-linked-list for the traversal operations. Deletion is the most expensive operation, usually somewhat slower than insertion. +Dictionaries are extremely fast in all operations. They are indexing the keys with `JudyHS` (or `AVL` when `libJudy` is not available) and they utilize a double-linked-list for the traversal operations. Deletion is the most expensive operation, usually somewhat slower than insertion. ## Memory management @@ -31,13 +31,13 @@ Dictionaries come with 2 memory management options: - **Clone** (copy) the name and/or the value to memory allocated by the dictionary. - **Link** the name and/or the value, without allocating any memory about them. -In **clone** mode, the dictionary guarantees that all operations on the dictionary items will automatically take care of the memory used by the name and/or the value. In case the value is an object needs to have user allocated memory, the following callback functions can be registered: +In **clone** mode, the dictionary guarantees that all operations on the dictionary items will automatically take care of the memory used by the name and/or the value. In case the value is an object that needs to have user allocated memory, the following callback functions can be registered: 1.`dictionary_register_insert_callback()` that will be called just after the insertion of an item to the dictionary, or after the replacement of the value of a dictionary item (but while the dictionary is write-locked - if locking is enabled). 2. `dictionary_register_delete_callback()` that will be called just prior to the deletion of an item from the dictionary, or prior to the replacement of the value of a dictionary item (but while the dictionary is write-locked - if locking is enabled). 3. `dictionary_register_conflict_callback()` that will be called when `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` is set and another value is attempted to be inserted for the same key. -In **link** mode, the name and/or the value are just linked to the dictionary item, and it is the user's responsibility to free the memory used after an item is deleted from the dictionary. +In **link** mode, the name and/or the value are just linked to the dictionary item, and it is the user's responsibility to free the memory they use after an item is deleted from the dictionary or when the dictionary is destroyed. By default, **clone** mode is used for both the name and the value. @@ -63,7 +63,7 @@ The dictionary supports the following operations supported by the hash table: Use `dictionary_create()` to create a dictionary. -Use `dictionary_destroy()` to destroy a dictionary. When destroyed, a dictionary frees all the memory it has allocated on its own. The exception is the registration of a deletion callback function that can be called on deletion of an item, which may free additional resources. +Use `dictionary_destroy()` to destroy a dictionary. When destroyed, a dictionary frees all the memory it has allocated on its own. This can be complemented by the registration of a deletion callback function that can be called upon deletion of each item in the dictionary, which may free additional resources. ### dictionary_set() @@ -72,7 +72,7 @@ This call is used to: - **add** an item to the dictionary. - **reset** the value of an existing item in the dictionary. -If **resetting** is not desired, add `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored. +If **resetting** is not desired, add `DICTIONARY_FLAG_DONT_OVERWRITE_VALUE` to the flags when creating the dictionary. In this case, `dictionary_set()` will return the value of the original item found in the dictionary instead of resetting it and the value passed to the call will be ignored. Optionally a conflict callback function can be registered, to manipulate (probably merge or extend) the original value, based on the new value attempted to be added to the dictionary. For **multi-threaded** operation, the `dictionary_set()` calls get an exclusive write lock on the dictionary. @@ -89,14 +89,16 @@ Where: * `value` is a pointer to the value associated with this item. In **clone** mode, if `value` is `NULL`, a new memory allocation will be made of `value_len` size and will be initialized to zero. * `value_len` is the size of the `value` data. If `value_len` is zero, no allocation will be done and the dictionary item will permanently have the `NULL` value. -> **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary. It should never be called without an active lock on the dictionary, which can only be acquired while traversing. +> **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary in write mode. It should never be called without an active lock on the dictionary, which can only be acquired while traversing. ### dictionary_get() -This call is used to get the value of an item, given its name. It utilizes the JudyHS hash table for making the lookup. +This call is used to get the value of an item, given its name. It utilizes the `JudyHS` hash table for making the lookup. For **multi-threaded** operation, the `dictionary_get()` call gets a shared read lock on the dictionary. +In clone mode, the value returned is not guaranteed to be valid, as any other thread may delete the item from the dictionary at any time. To ensure the value will be available, use `dictionary_get_and_acquire_item()`, which uses a reference counter to defer deletes until the item is released. + The format is: ```c @@ -114,7 +116,7 @@ Where: This call is used to delete an item from the dictionary, given its name. -If there is a delete callback registered to the dictionary (`dictionary_register_delete_callback()`), it is called prior to the actual deletion of the item. +If there is a deletion callback registered to the dictionary (`dictionary_register_delete_callback()`), it is called prior to the actual deletion of the item. For **multi-threaded** operation, the `dictionary_del()` calls get an exclusive write lock on the dictionary. @@ -131,29 +133,65 @@ Where: > **IMPORTANT**<br/>There is also an **unsafe** version (without locks) of this call. This is to be used when traversing the dictionary, to delete the current item. It should never be called without an active lock on the dictionary, which can only be acquired while traversing. +### dictionary_get_and_acquire_item() + +This call can be used the search and get a dictionary item, while ensuring that it will be available for use, until `dictionary_acquired_item_release()` is called. + +This call **does not return the value** of the dictionary item. It returns an internal pointer to a structure that maintains the reference counter used to protect the actual value. To get the value of the item (the same value as returned by `dictionary_get()`), the function `dictionary_acquired_item_value()` has to be called. + +Example: + +```c +// create the dictionary +DICTIONARY *dict = dictionary_create(DICTIONARY_FLAGS_NONE); + +// add an item to it +dictionary_set(dict, "name", "value", 6); + +// find the item we added and acquire it +void *item = dictionary_get_and_acquire_item(dict, "name"); + +// extract its value +char *value = (char *)dictionary_acquired_item_value(dict, item); + +// now value points to the string "value" +printf("I got value = '%s'\n", value); + +// release the item, so that it can deleted +dictionary_acquired_item_release(dict, item); + +// destroy the dictionary +dictionary_destroy(dict); +``` + +When items are acquired, a reference counter is maintained to keep track of how many users exist for it. If an item with a non-zero number of users is deleted, it is removed from the index, it can be added again to the index (without conflict), and although it exists in the linked-list, it is not offered during traversal. Garbage collection to actually delete the item happens every time a write-locked dictionary is unlocked (just before the unlock) and items are deleted only if no users are using them. + +If any item is still acquired when the dictionary is destroyed, the destruction of the dictionary is also deferred until all the acquired items are released. When the dictionary is destroyed like that, all operations on the dictionary fail (traversals do not traverse, insertions do not insert, deletions do not delete, searches do not find any items, etc). Once the last item in the dictionary is released, the dictionary is automatically destroyed too. + ## Traversal -Dictionaries offer 2 ways to traverse the entire dictionary: +Dictionaries offer 3 ways to traverse the entire dictionary: - **walkthrough**, implemented by setting a callback function to be called for every item. +- **sorted walkthrough**, which first sorts the dictionary and then call a callback function for every item. - **foreach**, a way to traverse the dictionary with a for-next loop. -Both of these methods are available in **read** or **write** mode. In **read** mode only lookups are allowed to the dictionary. In **write** both lookups but also deletion of the currently working item is also allowed. +All these methods are available in **read** or **write** mode. In **read** mode only lookups are allowed to the dictionary. In **write** lookups but also insertions and deletions are allowed. -While traversing the dictionary with any of these methods, all calls to the dictionary have to use the `_unsafe` versions of the function calls, otherwise deadlock may arise. +While traversing the dictionary with any of these methods, all calls to the dictionary have to use the `_unsafe` versions of the function calls, otherwise deadlocks may arise. > **IMPORTANT**<br/>The dictionary itself does not check to ensure that a user is actually using the right lock mode (read or write) while traversing the dictionary for each of the unsafe calls. ### walkthrough (callback) -There are 2 calls: +There are 4 calls: -- `dictionary_walkthrough_read()` that acquires a shared read lock, and it calls a callback function for every item of the dictionary. The callback function may use the unsafe versions of the `dictionary_get()` calls to lookup other items in the dictionary, but it should not add or remove item from the dictionary. -- `dictionary_walkthrough_write()` that acquires an exclusive write lock, and it calls a callback function for every item of the dictionary. This is to be used when items need to be added to the dictionary, or when the current item may need to be deleted. If the callback function deletes any other items, the behavior may be undefined (actually, the item next to the one currently working should not be deleted - a pointer to it is held by the traversal function to move on traversing the dictionary). +- `dictionary_walkthrough_read()` and `dictionary_sorted_walkthrough_read()` that acquire a shared read lock, and they call a callback function for every item of the dictionary. The callback function may use the unsafe versions of the `dictionary_get()` calls to lookup other items in the dictionary, but it should not attempt to add or remove items to/from the dictionary. +- `dictionary_walkthrough_write()` and `dictionary_sorted_walkthrough_write()` that acquire an exclusive write lock, and they call a callback function for every item of the dictionary. This is to be used when items need to be added to or removed from the dictionary. The `write` versions can be used to delete any or all the items from the dictionary, including the currently working one. For the `sorted` version, all items in the dictionary maintain a reference counter, so all deletions are deferred until the sorted walkthrough finishes.** -The items are traversed in the same order they have been added to the dictionary (or the reverse order if the flag `DICTIONARY_FLAG_ADD_IN_FRONT` is set during dictionary creation). +The non sorted versions traverse the items in the same order they have been added to the dictionary (or the reverse order if the flag `DICTIONARY_FLAG_ADD_IN_FRONT` is set during dictionary creation). The sorted versions sort alphabetically the items based on their name, and then they traverse them in the sorted order. -The callback function returns an `int`. If this value is negative, traversal of the dictionary is stopped immediately and the negative value is returned to the caller. If the returned value of all callbacks is zero or positive, the walkthrough functions return the sum of the return values of all callbacks. So, if you are just interested to know how many items fall into some condition, write a callback function that returns 1 when the item satisfies that condition and 0 when it does not and the walkthrough function will return how many tested positive. +The callback function returns an `int`. If this value is negative, traversal of the dictionary is stopped immediately and the negative value is returned to the caller. If the returned value of all callback calls is zero or positive, the walkthrough functions return the sum of the return values of all callbacks. So, if you are just interested to know how many items fall into some condition, write a callback function that returns 1 when the item satisfies that condition and 0 when it does not and the walkthrough function will return how many tested positive. ### foreach (for-next loop) @@ -186,14 +224,14 @@ else something else; ``` -In the above, the `if(x)` condition will work as expected. It will do the foreach loop when x is 1, otherwise it will run `something else`. +In the above, the `if(x == 1)` condition will work as expected. It will do the foreach loop when x is 1, otherwise it will run `something else`. There are 2 versions of `dfe_start`: - `dfe_start_read()` that acquires a shared read lock to the dictionary. - `dfe_start_write()` that acquires an exclusive write lock to the dictionary. -While in the loop, depending on the read or write versions of `dfe_start`, the caller may lookup or manipulate the dictionary. The rules are the same with the walkthrough callback functions. +While in the loop, depending on the read or write versions of `dfe_start`, the caller may lookup or manipulate the dictionary using the unsafe functions. The rules are the same with the unsorted walkthrough callback functions. PS: DFE is Dictionary For Each. diff --git a/libnetdata/dictionary/dictionary.c b/libnetdata/dictionary/dictionary.c index 42285037d..c1325ecb5 100644 --- a/libnetdata/dictionary/dictionary.c +++ b/libnetdata/dictionary/dictionary.c @@ -1,7 +1,12 @@ // SPDX-License-Identifier: GPL-3.0-or-later -// NOT TO BE USED BY USERS YET -#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 6) // maintain reference counter in walkthrough and foreach +// NOT TO BE USED BY USERS +#define DICTIONARY_FLAG_EXCLUSIVE_ACCESS (1 << 29) // there is only one thread accessing the dictionary +#define DICTIONARY_FLAG_DESTROYED (1 << 30) // this dictionary has been destroyed +#define DICTIONARY_FLAG_DEFER_ALL_DELETIONS (1 << 31) // defer all deletions of items in the dictionary + +// our reserved flags that cannot be set by users +#define DICTIONARY_FLAGS_RESERVED (DICTIONARY_FLAG_EXCLUSIVE_ACCESS|DICTIONARY_FLAG_DESTROYED|DICTIONARY_FLAG_DEFER_ALL_DELETIONS) typedef struct dictionary DICTIONARY; #define DICTIONARY_INTERNALS @@ -19,99 +24,49 @@ typedef struct dictionary DICTIONARY; #include <Judy.h> #endif -/* - * This version uses JudyHS arrays to index the dictionary - * - * The following output is from the unit test, at the end of this file: - * - * This is the JudyHS version: - * - * 1000000 x dictionary_set() (dictionary size 0 entries, 0 KB)... - * 1000000 x dictionary_get(existing) (dictionary size 1000000 entries, 74001 KB)... - * 1000000 x dictionary_get(non-existing) (dictionary size 1000000 entries, 74001 KB)... - * Walking through the dictionary (dictionary size 1000000 entries, 74001 KB)... - * 1000000 x dictionary_del(existing) (dictionary size 1000000 entries, 74001 KB)... - * 1000000 x dictionary_set() (dictionary size 0 entries, 0 KB)... - * Destroying dictionary (dictionary size 1000000 entries, 74001 KB)... - * - * TIMINGS: - * adding 316027 usec, positive search 156740 usec, negative search 84524, walk through 15036 usec, deleting 361444, destroy 107394 usec - * - * This is from the JudySL version: - * - * Creating dictionary of 1000000 entries... - * Checking index of 1000000 entries... - * Walking 1000000 entries and checking name-value pairs... - * Created and checked 1000000 entries, found 0 errors - used 58376 KB of memory - * Destroying dictionary of 1000000 entries... - * Deleted 1000000 entries - * create 338975 usec, check 156080 usec, walk 80764 usec, destroy 444569 usec - * - * This is the AVL version: - * - * Creating dictionary of 1000000 entries... - * Checking index of 1000000 entries... - * Walking 1000000 entries and checking name-value pairs... - * Created and checked 1000000 entries, found 0 errors - used 89626 KB of memory - * Destroying dictionary of 1000000 entries... - * create 413892 usec, check 220006 usec, walk 34247 usec, destroy 98062 usec - * - * So, the JudySL is a lot slower to WALK and DESTROY (DESTROY does a WALK) - * It is slower, because for every item, JudySL copies the KEY/NAME to a - * caller supplied buffer (Index). So, by just walking over 1 million items, - * JudySL does 1 million strcpy() !!! - * - * It also seems that somehow JudySLDel() is unbelievably slow too! - * - */ +typedef enum name_value_flags { + NAME_VALUE_FLAG_NONE = 0, + NAME_VALUE_FLAG_NAME_IS_ALLOCATED = (1 << 0), // the name pointer is a STRING + NAME_VALUE_FLAG_DELETED = (1 << 1), // this item is deleted, so it is not available for traversal + NAME_VALUE_FLAG_NEW_OR_UPDATED = (1 << 2), // this item is new or just updated (used by the react callback) + // IMPORTANT: IF YOU ADD ANOTHER FLAG, YOU NEED TO ALLOCATE ANOTHER BIT TO FLAGS IN NAME_VALUE !!! +} NAME_VALUE_FLAGS; /* * Every item in the dictionary has the following structure. */ + typedef struct name_value { #ifdef DICTIONARY_WITH_AVL avl_t avl_node; #endif +#ifdef NETDATA_INTERNAL_CHECKS + DICTIONARY *dict; +#endif + struct name_value *next; // a double linked list to allow fast insertions and deletions struct name_value *prev; - char *name; // the name of the dictionary item + uint32_t refcount; // the reference counter + uint32_t value_len:29; // the size of the value (assumed binary) + uint8_t flags:3; // the flags for this item + void *value; // the value of the dictionary item + union { + STRING *string_name; // the name of the dictionary item + char *caller_name; // the user supplied string pointer + }; } NAME_VALUE; -/* - * When DICTIONARY_FLAG_WITH_STATISTICS is set, we need to keep track of all the memory - * we allocate and free. So, we need to keep track of the sizes of all names and values. - * We do this by overloading NAME_VALUE with the following additional fields. - */ - -typedef enum name_value_flags { - NAME_VALUE_FLAG_NONE = 0, - NAME_VALUE_FLAG_DELETED = (1 << 0), // this item is deleted -} NAME_VALUE_FLAGS; - -typedef struct name_value_with_stats { - NAME_VALUE name_value_data_here; // never used - just to put the lengths at the right position - - size_t name_len; // the size of the name, including the terminating zero - size_t value_len; // the size of the value (assumed binary) - - size_t refcount; // the reference counter - NAME_VALUE_FLAGS flags; // the flags for this item -} NAME_VALUE_WITH_STATS; - -struct dictionary_stats { - size_t inserts; - size_t deletes; - size_t searches; - size_t resets; - size_t entries; - size_t memory; -}; - struct dictionary { +#ifdef NETDATA_INTERNAL_CHECKS + const char *creation_function; + const char *creation_file; + size_t creation_line; +#endif + DICTIONARY_FLAGS flags; // the flags of the dictionary NAME_VALUE *first_item; // the double linked list base pointers @@ -120,26 +75,51 @@ struct dictionary { #ifdef DICTIONARY_WITH_AVL avl_tree_type values_index; NAME_VALUE *hash_base; + void *(*get_thread_static_name_value)(const char *name); #endif #ifdef DICTIONARY_WITH_JUDYHS Pvoid_t JudyHSArray; // the hash table #endif - netdata_rwlock_t *rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set + netdata_rwlock_t rwlock; // the r/w lock when DICTIONARY_FLAG_SINGLE_THREADED is not set void (*ins_callback)(const char *name, void *value, void *data); void *ins_callback_data; + void (*react_callback)(const char *name, void *value, void *data); + void *react_callback_data; + void (*del_callback)(const char *name, void *value, void *data); void *del_callback_data; void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data); void *conflict_callback_data; - struct dictionary_stats *stats; // the statistics when DICTIONARY_FLAG_WITH_STATISTICS is set + size_t version; // the current version of the dictionary + size_t inserts; // how many index insertions have been performed + size_t deletes; // how many index deletions have been performed + size_t searches; // how many index searches have been performed + size_t resets; // how many times items have reset their values + size_t walkthroughs; // how many walkthroughs have been done + long int memory; // how much memory the dictionary has currently allocated + long int entries; // how many items are currently in the index (the linked list may have more) + long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties + long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet + int readers; // how many readers are currently using the dictionary + int writers; // how many writers are currently using the dictionary + + size_t scratchpad_size; // the size of the scratchpad in bytes + uint8_t scratchpad[]; // variable size scratchpad requested by the caller }; +static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VALUE *nv); +static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv); +static inline const char *namevalue_get_name(NAME_VALUE *nv); + +// ---------------------------------------------------------------------------- +// callbacks registration + void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data) { dict->ins_callback = ins_callback; dict->ins_callback_data = data; @@ -155,63 +135,156 @@ void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_cal dict->conflict_callback_data = data; } +void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const char *name, void *value, void *data), void *data) { + dict->react_callback = react_callback; + dict->react_callback_data = data; +} + // ---------------------------------------------------------------------------- // dictionary statistics maintenance -size_t dictionary_stats_allocated_memory(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->memory; - return 0; +long int dictionary_stats_allocated_memory(DICTIONARY *dict) { + return dict->memory; } -size_t dictionary_stats_entries(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->entries; - return 0; +long int dictionary_stats_entries(DICTIONARY *dict) { + return dict->entries; +} +size_t dictionary_stats_version(DICTIONARY *dict) { + return dict->version; } size_t dictionary_stats_searches(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->searches; - return 0; + return dict->searches; } size_t dictionary_stats_inserts(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->inserts; - return 0; + return dict->inserts; } size_t dictionary_stats_deletes(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->deletes; - return 0; + return dict->deletes; } size_t dictionary_stats_resets(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - return dict->stats->resets; - return 0; + return dict->resets; +} +size_t dictionary_stats_walkthroughs(DICTIONARY *dict) { + return dict->walkthroughs; +} +size_t dictionary_stats_referenced_items(DICTIONARY *dict) { + return __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST); } static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) - dict->stats->searches++; + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->searches++; + } + else { + __atomic_fetch_add(&dict->searches, 1, __ATOMIC_RELAXED); + } } static inline void DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict, size_t size) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - dict->stats->inserts++; - dict->stats->entries++; - dict->stats->memory += size; + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->version++; + dict->inserts++; + dict->entries++; + dict->memory += (long)size; + } + else { + __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&dict->inserts, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->entries, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->memory, (long)size, __ATOMIC_RELAXED); } } -static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict, size_t size) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - dict->stats->deletes++; - dict->stats->entries--; - dict->stats->memory -= size; +static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict) { + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->version++; + dict->deletes++; + dict->entries--; + } + else { + __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&dict->deletes, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_RELAXED); + } +} +static inline void DICTIONARY_STATS_ENTRIES_MINUS_MEMORY(DICTIONARY *dict, size_t size) { + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->memory -= (long)size; + } + else { + __atomic_fetch_sub(&dict->memory, (long)size, __ATOMIC_RELAXED); } } static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t oldsize, size_t newsize) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - dict->stats->resets++; - dict->stats->memory += newsize; - dict->stats->memory -= oldsize; + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->version++; + dict->resets++; + dict->memory += (long)newsize; + dict->memory -= (long)oldsize; + } + else { + __atomic_fetch_add(&dict->version, 1, __ATOMIC_SEQ_CST); + __atomic_fetch_add(&dict->resets, 1, __ATOMIC_RELAXED); + __atomic_fetch_add(&dict->memory, (long)newsize, __ATOMIC_RELAXED); + __atomic_fetch_sub(&dict->memory, (long)oldsize, __ATOMIC_RELAXED); + } +} + +static inline void DICTIONARY_STATS_WALKTHROUGHS_PLUS1(DICTIONARY *dict) { + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + dict->walkthroughs++; + } + else { + __atomic_fetch_add(&dict->walkthroughs, 1, __ATOMIC_RELAXED); + } +} + +static inline size_t DICTIONARY_STATS_REFERENCED_ITEMS_PLUS1(DICTIONARY *dict) { + return __atomic_add_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); +} + +static inline size_t DICTIONARY_STATS_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) { + return __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST); +} + +static inline size_t DICTIONARY_STATS_PENDING_DELETES_PLUS1(DICTIONARY *dict) { + return __atomic_add_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); +} + +static inline size_t DICTIONARY_STATS_PENDING_DELETES_MINUS1(DICTIONARY *dict) { + return __atomic_sub_fetch(&dict->pending_deletion_items, 1, __ATOMIC_SEQ_CST); +} + +static inline size_t DICTIONARY_STATS_PENDING_DELETES_GET(DICTIONARY *dict) { + return __atomic_load_n(&dict->pending_deletion_items, __ATOMIC_SEQ_CST); +} + +static inline int DICTIONARY_NAME_VALUE_REFCOUNT_GET(NAME_VALUE *nv) { + return __atomic_load_n(&nv->refcount, __ATOMIC_SEQ_CST); +} + +// ---------------------------------------------------------------------------- +// garbage collector +// it is called every time someone gets a write lock to the dictionary + +static void garbage_collect_pending_deletes_unsafe(DICTIONARY *dict) { + if(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS)) return; + + if(likely(!DICTIONARY_STATS_PENDING_DELETES_GET(dict))) return; + + NAME_VALUE *nv = dict->first_item; + while(nv) { + if((nv->flags & NAME_VALUE_FLAG_DELETED) && DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv) == 0) { + NAME_VALUE *nv_next = nv->next; + + linkedlist_namevalue_unlink_unsafe(dict, nv); + namevalue_destroy_unsafe(dict, nv); + + size_t pending = DICTIONARY_STATS_PENDING_DELETES_MINUS1(dict); + if(!pending) break; + + nv = nv_next; + } + else + nv = nv->next; } } @@ -220,41 +293,104 @@ static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t static inline size_t dictionary_lock_init(DICTIONARY *dict) { if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - dict->rwlock = mallocz(sizeof(netdata_rwlock_t)); - netdata_rwlock_init(dict->rwlock); - return sizeof(netdata_rwlock_t); + netdata_rwlock_init(&dict->rwlock); + + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) + dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; + + return 0; } - dict->rwlock = NULL; + + // we are single threaded + dict->flags |= DICTIONARY_FLAG_EXCLUSIVE_ACCESS; return 0; } static inline size_t dictionary_lock_free(DICTIONARY *dict) { if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - netdata_rwlock_destroy(dict->rwlock); - freez(dict->rwlock); - return sizeof(netdata_rwlock_t); + netdata_rwlock_destroy(&dict->rwlock); + return 0; } return 0; } -static inline void dictionary_lock_rlock(DICTIONARY *dict) { - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - // debug(D_DICTIONARY, "Dictionary READ lock"); - netdata_rwlock_rdlock(dict->rwlock); +static void dictionary_lock(DICTIONARY *dict, char rw) { + if(rw == 'u' || rw == 'U') return; + + if(rw == 'r' || rw == 'R') { + // read lock + __atomic_add_fetch(&dict->readers, 1, __ATOMIC_RELAXED); + } + else { + // write lock + __atomic_add_fetch(&dict->writers, 1, __ATOMIC_RELAXED); + } + + if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) + return; + + if(rw == 'r' || rw == 'R') { + // read lock + netdata_rwlock_rdlock(&dict->rwlock); + + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) { + internal_error(true, "DICTIONARY: left-over exclusive access to dictionary created by %s (%zu@%s) found", dict->creation_function, dict->creation_line, dict->creation_file); + dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; + } + } + else { + // write lock + netdata_rwlock_wrlock(&dict->rwlock); + + dict->flags |= DICTIONARY_FLAG_EXCLUSIVE_ACCESS; } } -static inline void dictionary_lock_wrlock(DICTIONARY *dict) { - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - // debug(D_DICTIONARY, "Dictionary WRITE lock"); - netdata_rwlock_wrlock(dict->rwlock); +static void dictionary_unlock(DICTIONARY *dict, char rw) { + if(rw == 'u' || rw == 'U') return; + + if(rw == 'r' || rw == 'R') { + // read unlock + __atomic_sub_fetch(&dict->readers, 1, __ATOMIC_RELAXED); + } + else { + // write unlock + garbage_collect_pending_deletes_unsafe(dict); + __atomic_sub_fetch(&dict->writers, 1, __ATOMIC_RELAXED); } + + if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) + return; + + if(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS) + dict->flags &= ~DICTIONARY_FLAG_EXCLUSIVE_ACCESS; + + netdata_rwlock_unlock(&dict->rwlock); } -static inline void dictionary_unlock(DICTIONARY *dict) { - if(likely(!(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED))) { - // debug(D_DICTIONARY, "Dictionary UNLOCK lock"); - netdata_rwlock_unlock(dict->rwlock); +// ---------------------------------------------------------------------------- +// deferred deletions + +void dictionary_defer_all_deletions_unsafe(DICTIONARY *dict, char rw) { + if(rw == 'r' || rw == 'R') { + // read locked - no need to defer deletions + ; + } + else { + // write locked - defer deletions + dict->flags |= DICTIONARY_FLAG_DEFER_ALL_DELETIONS; + } +} + +void dictionary_restore_all_deletions_unsafe(DICTIONARY *dict, char rw) { + if(rw == 'r' || rw == 'R') { + // read locked - no need to defer deletions + internal_error(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS, "DICTIONARY: deletions are deferred on a read lock"); + } + else { + // write locked - defer deletions + if(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS) + dict->flags &= ~DICTIONARY_FLAG_DEFER_ALL_DELETIONS; } } @@ -277,39 +413,90 @@ static inline size_t reference_counter_free(DICTIONARY *dict) { return 0; } -static void reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - __atomic_fetch_add(&nvs->refcount, 1, __ATOMIC_SEQ_CST); - } +static int reference_counter_increase(NAME_VALUE *nv) { + int refcount = __atomic_add_fetch(&nv->refcount, 1, __ATOMIC_SEQ_CST); + if(refcount == 1) + fatal("DICTIONARY: request to dup item '%s' but its reference counter was zero", namevalue_get_name(nv)); + return refcount; } -static void reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - __atomic_fetch_sub(&nvs->refcount, 1, __ATOMIC_SEQ_CST); +static int reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) { + int refcount; + if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) + refcount = ++nv->refcount; + else + refcount = __atomic_add_fetch(&nv->refcount, 1, __ATOMIC_SEQ_CST); + + if(refcount == 1) { + // referenced items counts number of unique items referenced + // so, we increase it only when refcount == 1 + DICTIONARY_STATS_REFERENCED_ITEMS_PLUS1(dict); + + // if this is a deleted item, but the counter increased to 1 + // we need to remove it from the pending items to delete + if (nv->flags & NAME_VALUE_FLAG_DELETED) + DICTIONARY_STATS_PENDING_DELETES_MINUS1(dict); } + + return refcount; } -static int reference_counter_mark_deleted(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - nvs->flags |= NAME_VALUE_FLAG_DELETED; - return 1; +static uint32_t reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv, bool can_get_write_lock) { + // this function may be called without any lock on the dictionary + // or even when someone else has a write lock on the dictionary + // so, we cannot check for EXCLUSIVE ACCESS + + uint32_t refcount; + if(likely(dict->flags & DICTIONARY_FLAG_SINGLE_THREADED)) + refcount = nv->refcount--; + else + refcount = __atomic_fetch_sub(&nv->refcount, 1, __ATOMIC_SEQ_CST); + + if(refcount == 0) { + internal_error(true, "DICTIONARY: attempted to release item without references: '%s' on dictionary created by %s() (%zu@%s)", namevalue_get_name(nv), dict->creation_function, dict->creation_line, dict->creation_file); + fatal("DICTIONARY: attempted to release item without references: '%s'", namevalue_get_name(nv)); } - return 0; + + if(refcount == 1) { + if((nv->flags & NAME_VALUE_FLAG_DELETED)) + DICTIONARY_STATS_PENDING_DELETES_PLUS1(dict); + + // referenced items counts number of unique items referenced + // so, we decrease it only when refcount == 0 + DICTIONARY_STATS_REFERENCED_ITEMS_MINUS1(dict); + } + + if(can_get_write_lock && DICTIONARY_STATS_PENDING_DELETES_GET(dict)) { + // we can garbage collect now + + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); + garbage_collect_pending_deletes_unsafe(dict); + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); + } + + return refcount; } // ---------------------------------------------------------------------------- // hash table #ifdef DICTIONARY_WITH_AVL +static inline const char *namevalue_get_name(NAME_VALUE *nv); + static int name_value_compare(void* a, void* b) { - return strcmp(((NAME_VALUE *)a)->name, ((NAME_VALUE *)b)->name); + return strcmp(namevalue_get_name((NAME_VALUE *)a), namevalue_get_name((NAME_VALUE *)b)); +} + +static void *get_thread_static_name_value(const char *name) { + static __thread NAME_VALUE tmp = { 0 }; + tmp.flags = NAME_VALUE_FLAG_NONE; + tmp.caller_name = (char *)name; + return &tmp; } static void hashtable_init_unsafe(DICTIONARY *dict) { avl_init(&dict->values_index, name_value_compare); + dict->get_thread_static_name_value = get_thread_static_name_value; } static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { @@ -317,7 +504,7 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { return 0; } -static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) { +static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) { (void)name; (void)name_len; @@ -330,9 +517,8 @@ static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, si static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { (void)name_len; - NAME_VALUE tmp; - tmp.name = (char *)name; - return (NAME_VALUE *)avl_search(&(dict->values_index), (avl_t *) &tmp); + void *tmp = dict->get_thread_static_name_value(name); + return (NAME_VALUE *)avl_search(&(dict->values_index), (avl_t *)tmp); } static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { @@ -346,13 +532,10 @@ static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char return &dict->hash_base; } -static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) { +static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *nv) { // we have our new NAME_VALUE object. // Let's index it. - (void)name; - (void)name_len; - if(unlikely(avl_insert(&((dict)->values_index), (avl_t *)(nv)) != (avl_t *)nv)) error("dictionary: INTERNAL ERROR: duplicate insertion to dictionary."); } @@ -380,6 +563,8 @@ static size_t hashtable_destroy_unsafe(DICTIONARY *dict) { } static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char *name, size_t name_len) { + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); + JError_t J_Error; Pvoid_t *Rc = JudyHSIns(&dict->JudyHSArray, (void *)name, name_len, &J_Error); if (unlikely(Rc == PJERR)) { @@ -397,9 +582,10 @@ static inline NAME_VALUE **hashtable_insert_unsafe(DICTIONARY *dict, const char return (NAME_VALUE **)Rc; } -static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) { - (void)nv; +static inline int hashtable_delete_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *nv) { + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: deleting item from the index without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); + (void)nv; if(unlikely(!dict->JudyHSArray)) return 0; JError_t J_Error; @@ -440,10 +626,8 @@ static inline NAME_VALUE *hashtable_get_unsafe(DICTIONARY *dict, const char *nam } } -static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, const char *name, size_t name_len, NAME_VALUE *nv) { +static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, void *nv) { (void)dict; - (void)name; - (void)name_len; (void)nv; ; } @@ -454,6 +638,8 @@ static inline void hashtable_inserted_name_value_unsafe(DICTIONARY *dict, const // linked list management static inline void linkedlist_namevalue_link_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: adding item to the linked-list without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); + if (unlikely(!dict->first_item)) { // we are the only ones here nv->next = NULL; @@ -481,6 +667,8 @@ static inline void linkedlist_namevalue_link_unsafe(DICTIONARY *dict, NAME_VALUE } static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: removing item from the linked-list without exclusive access to the dictionary created by %s() (%zu@%s)", dict->creation_function, dict->creation_line, dict->creation_file); + if(nv->next) nv->next->prev = nv->prev; if(nv->prev) nv->prev->next = nv->next; if(dict->first_item == nv) dict->first_item = nv->next; @@ -490,54 +678,47 @@ static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VAL // ---------------------------------------------------------------------------- // NAME_VALUE methods -static inline size_t namevalue_alloc_size(DICTIONARY *dict) { - return (dict->flags & DICTIONARY_FLAG_WITH_STATISTICS) ? sizeof(NAME_VALUE_WITH_STATS) : sizeof(NAME_VALUE); -} - -static inline size_t namevalue_get_namelen(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - return nvs->name_len; +static inline size_t namevalue_set_name(DICTIONARY *dict, NAME_VALUE *nv, const char *name, size_t name_len) { + if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) { + nv->caller_name = (char *)name; + return 0; } - return 0; + + nv->string_name = string_strdupz(name); + nv->flags |= NAME_VALUE_FLAG_NAME_IS_ALLOCATED; + return name_len; } -static inline size_t namevalue_get_valuelen(DICTIONARY *dict, NAME_VALUE *nv) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - return nvs->value_len; - } + +static inline size_t namevalue_free_name(DICTIONARY *dict, NAME_VALUE *nv) { + if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) + string_freez(nv->string_name); + return 0; } -static inline void namevalue_set_valuelen(DICTIONARY *dict, NAME_VALUE *nv, size_t value_len) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - nvs->value_len = value_len; - } -} -static inline void namevalue_set_namevaluelen(DICTIONARY *dict, NAME_VALUE *nv, size_t name_len, size_t value_len) { - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv; - nvs->name_len = name_len; - nvs->value_len = value_len; - } + +static inline const char *namevalue_get_name(NAME_VALUE *nv) { + if(nv->flags & NAME_VALUE_FLAG_NAME_IS_ALLOCATED) + return string2str(nv->string_name); + else + return nv->caller_name; } static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len) { debug(D_DICTIONARY, "Creating name value entry for name '%s'.", name); - size_t size = namevalue_alloc_size(dict); + size_t size = sizeof(NAME_VALUE); NAME_VALUE *nv = mallocz(size); size_t allocated = size; - namevalue_set_namevaluelen(dict, nv, name_len, value_len); +#ifdef NETDATA_INTERNAL_CHECKS + nv->dict = dict; +#endif - if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE)) - nv->name = (char *)name; - else { - nv->name = mallocz(name_len); - memcpy(nv->name, name, name_len); - allocated += name_len; - } + nv->refcount = 0; + nv->flags = NAME_VALUE_FLAG_NONE; + nv->value_len = value_len; + + allocated += namevalue_set_name(dict, nv, name, name_len); if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) nv->value = value; @@ -556,7 +737,7 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s } } else { - // the caller want an item without any value + // the caller wants an item without any value nv->value = NULL; } @@ -565,107 +746,189 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s DICTIONARY_STATS_ENTRIES_PLUS1(dict, allocated); + if(dict->ins_callback) + dict->ins_callback(namevalue_get_name(nv), nv->value, dict->ins_callback_data); + return nv; } static void namevalue_reset_unsafe(DICTIONARY *dict, NAME_VALUE *nv, void *value, size_t value_len) { - debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", nv->name); + debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", namevalue_get_name(nv)); + + DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, nv->value_len, value_len); + + if(dict->del_callback) + dict->del_callback(namevalue_get_name(nv), nv->value, dict->del_callback_data); if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) { - debug(D_DICTIONARY, "Dictionary: linking value to '%s'", nv->name); + debug(D_DICTIONARY, "Dictionary: linking value to '%s'", namevalue_get_name(nv)); nv->value = value; - namevalue_set_valuelen(dict, nv, value_len); + nv->value_len = value_len; } else { - debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", nv->name); - DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, namevalue_get_valuelen(dict, nv), value_len); - - void *old = nv->value; - void *new = mallocz(value_len); - memcpy(new, value, value_len); - nv->value = new; - namevalue_set_valuelen(dict, nv, value_len); + debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", namevalue_get_name(nv)); + + void *oldvalue = nv->value; + void *newvalue = NULL; + if(value_len) { + newvalue = mallocz(value_len); + if(value) memcpy(newvalue, value, value_len); + else memset(newvalue, 0, value_len); + } + nv->value = newvalue; + nv->value_len = value_len; - debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", nv->name); - freez(old); + debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", namevalue_get_name(nv)); + freez(oldvalue); } + + if(dict->ins_callback) + dict->ins_callback(namevalue_get_name(nv), nv->value, dict->ins_callback_data); } static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv) { - debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name); + debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", namevalue_get_name(nv)); + + if(dict->del_callback) + dict->del_callback(namevalue_get_name(nv), nv->value, dict->del_callback_data); size_t freed = 0; if(unlikely(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))) { - debug(D_DICTIONARY, "Dictionary freeing value of '%s'", nv->name); + debug(D_DICTIONARY, "Dictionary freeing value of '%s'", namevalue_get_name(nv)); freez(nv->value); - freed += namevalue_get_valuelen(dict, nv); + freed += nv->value_len; } if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) { - debug(D_DICTIONARY, "Dictionary freeing name '%s'", nv->name); - freez(nv->name); - freed += namevalue_get_namelen(dict, nv); + debug(D_DICTIONARY, "Dictionary freeing name '%s'", namevalue_get_name(nv)); + freed += namevalue_free_name(dict, nv); } freez(nv); - freed += namevalue_alloc_size(dict); + freed += sizeof(NAME_VALUE); - DICTIONARY_STATS_ENTRIES_MINUS1(dict, freed); + DICTIONARY_STATS_ENTRIES_MINUS_MEMORY(dict, freed); return freed; } +// if a dictionary item can be deleted, return true, otherwise return false +static bool name_value_can_be_deleted(DICTIONARY *dict, NAME_VALUE *nv) { + if(unlikely(dict->flags & DICTIONARY_FLAG_DEFER_ALL_DELETIONS)) + return false; + + if(unlikely(DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv) > 0)) + return false; + + return true; +} + // ---------------------------------------------------------------------------- // API - dictionary management - -DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) { +#ifdef NETDATA_INTERNAL_CHECKS +DICTIONARY *dictionary_create_advanced_with_trace(DICTIONARY_FLAGS flags, size_t scratchpad_size, const char *function, size_t line, const char *file) { +#else +DICTIONARY *dictionary_create_advanced(DICTIONARY_FLAGS flags, size_t scratchpad_size) { +#endif debug(D_DICTIONARY, "Creating dictionary."); - if((flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) && (flags & DICTIONARY_FLAG_SINGLE_THREADED)) { - error("DICTIONARY: requested reference counters on single threaded dictionary. Not adding reference counters."); - flags &= ~DICTIONARY_FLAG_REFERENCE_COUNTERS; - } + if(unlikely(flags & DICTIONARY_FLAGS_RESERVED)) + flags &= ~DICTIONARY_FLAGS_RESERVED; - if(flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) { - // we need statistics to allocate the extra NAME_VALUE attributes - flags |= DICTIONARY_FLAG_WITH_STATISTICS; - } - - DICTIONARY *dict = callocz(1, sizeof(DICTIONARY)); - size_t allocated = sizeof(DICTIONARY); + DICTIONARY *dict = callocz(1, sizeof(DICTIONARY) + scratchpad_size); + size_t allocated = sizeof(DICTIONARY) + scratchpad_size; + dict->scratchpad_size = scratchpad_size; dict->flags = flags; dict->first_item = dict->last_item = NULL; allocated += dictionary_lock_init(dict); allocated += reference_counter_init(dict); - - if(flags & DICTIONARY_FLAG_WITH_STATISTICS) { - dict->stats = callocz(1, sizeof(struct dictionary_stats)); - allocated += sizeof(struct dictionary_stats); - dict->stats->memory = allocated; - } - else - dict->stats = NULL; + dict->memory = (long)allocated; hashtable_init_unsafe(dict); + +#ifdef NETDATA_INTERNAL_CHECKS + dict->creation_function = function; + dict->creation_file = file; + dict->creation_line = line; +#endif + return (DICTIONARY *)dict; } +void *dictionary_scratchpad(DICTIONARY *dict) { + return &dict->scratchpad; +} + size_t dictionary_destroy(DICTIONARY *dict) { + if(!dict) return 0; + + NAME_VALUE *nv; + debug(D_DICTIONARY, "Destroying dictionary."); - dictionary_lock_wrlock(dict); + long referenced_items = 0; + size_t retries = 0; + do { + referenced_items = __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST); + if (referenced_items) { + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); + + // there are referenced items + // delete all items individually, so that only the referenced will remain + NAME_VALUE *nv_next; + for (nv = dict->first_item; nv; nv = nv_next) { + nv_next = nv->next; + size_t refcount = DICTIONARY_NAME_VALUE_REFCOUNT_GET(nv); + if (!refcount && !(nv->flags & NAME_VALUE_FLAG_DELETED)) + dictionary_del_unsafe(dict, namevalue_get_name(nv)); + } + + internal_error( + retries == 0, + "DICTIONARY: waiting (try %zu) for destruction of dictionary created from %s() %zu@%s, because it has %ld referenced items in it (%ld total).", + retries + 1, + dict->creation_function, + dict->creation_line, + dict->creation_file, + referenced_items, + dict->entries); + + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); + sleep_usec(10000); + } + } while(referenced_items > 0 && ++retries < 10); + + if(referenced_items) { + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); + + dict->flags |= DICTIONARY_FLAG_DESTROYED; + internal_error( + true, + "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s after %zu retries, because it has %ld referenced items in it (%ld total).", + dict->creation_function, + dict->creation_line, + dict->creation_file, + retries, + referenced_items, + dict->entries); + + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); + return 0; + } + + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); size_t freed = 0; - NAME_VALUE *nv = dict->first_item; + nv = dict->first_item; while (nv) { // cache nv->next // because we are going to free nv - NAME_VALUE *nvnext = nv->next; + NAME_VALUE *nv_next = nv->next; freed += namevalue_destroy_unsafe(dict, nv); - nv = nvnext; + nv = nv_next; // to speed up destruction, we don't // unlink nv from the linked-list here } @@ -676,31 +939,31 @@ size_t dictionary_destroy(DICTIONARY *dict) { // destroy the dictionary freed += hashtable_destroy_unsafe(dict); - dictionary_unlock(dict); + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); freed += dictionary_lock_free(dict); freed += reference_counter_free(dict); - - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) { - freez(dict->stats); - dict->stats = NULL; - freed += sizeof(struct dictionary_stats); - } - + freed += sizeof(DICTIONARY) + dict->scratchpad_size; freez(dict); - freed += sizeof(DICTIONARY); return freed; } // ---------------------------------------------------------------------------- -// API - items management +// helpers -void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - if(unlikely(!name || !*name)) { - error("Attempted to dictionary_set() a dictionary item without a name"); +static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + if(unlikely(!name)) { + internal_error(true, "DICTIONARY: attempted to dictionary_set() a dictionary item without a name"); return NULL; } + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_set() on a destroyed dictionary"); + return NULL; + } + + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: inserting dictionary item '%s' without exclusive access to dictionary", name); + size_t name_len = strlen(name) + 1; // we need the terminating null too debug(D_DICTIONARY, "SET dictionary entry with name '%s'.", name); @@ -720,11 +983,9 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz if(likely(*pnv == 0)) { // a new item added to the index nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len); - hashtable_inserted_name_value_unsafe(dict, name, name_len, nv); + hashtable_inserted_name_value_unsafe(dict, nv); linkedlist_namevalue_link_unsafe(dict, nv); - - if(dict->ins_callback) - dict->ins_callback(nv->name, nv->value, dict->ins_callback_data); + nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; } else { // the item is already in the index @@ -732,33 +993,34 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz // or overwrite the value, depending on dictionary flags nv = *pnv; - if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) { - - if(dict->del_callback) - dict->del_callback(nv->name, nv->value, dict->del_callback_data); + if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) { namevalue_reset_unsafe(dict, nv, value, value_len); + nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; + } - if(dict->ins_callback) - dict->ins_callback(nv->name, nv->value, dict->ins_callback_data); + else if(dict->conflict_callback) { + dict->conflict_callback(namevalue_get_name(nv), nv->value, value, dict->conflict_callback_data); + nv->flags |= NAME_VALUE_FLAG_NEW_OR_UPDATED; + } + + else { + // make sure this flag is not set + nv->flags &= ~NAME_VALUE_FLAG_NEW_OR_UPDATED; } - else if(dict->conflict_callback) - dict->conflict_callback(nv->name, nv->value, value, dict->conflict_callback_data); } - return nv->value; + return nv; } -void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) { - dictionary_lock_wrlock(dict); - void *ret = dictionary_set_unsafe(dict, name, value, value_len); - dictionary_unlock(dict); - return ret; -} +static NAME_VALUE *dictionary_get_name_value_unsafe(DICTIONARY *dict, const char *name) { + if(unlikely(!name)) { + internal_error(true, "attempted to dictionary_get() without a name"); + return NULL; + } -void *dictionary_get_unsafe(DICTIONARY *dict, const char *name) { - if(unlikely(!name || !*name)) { - error("Attempted to dictionary_get() without a name"); + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_get() on a destroyed dictionary"); return NULL; } @@ -773,22 +1035,168 @@ void *dictionary_get_unsafe(DICTIONARY *dict, const char *name) { } debug(D_DICTIONARY, "Found dictionary entry with name '%s'.", name); + return nv; +} + +// ---------------------------------------------------------------------------- +// API - items management + +void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); + + if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { + // we need to call the react callback with a reference counter on nv + reference_counter_acquire(dict, nv); + dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); + reference_counter_release(dict, nv, false); + } + + return nv ? nv->value : NULL; +} + +void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); + NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); + + // we need to get a reference counter for the react callback + // before we unlock the dictionary + if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) + reference_counter_acquire(dict, nv); + + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); + + if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { + // we got the reference counter we need, above + dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); + reference_counter_release(dict, nv, false); + } + + return nv ? nv->value : NULL; +} + +DICTIONARY_ITEM *dictionary_set_and_acquire_item_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); + + if(unlikely(!nv)) + return NULL; + + reference_counter_acquire(dict, nv); + + if(unlikely(dict->react_callback && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { + dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); + } + + return (DICTIONARY_ITEM *)nv; +} + +DICTIONARY_ITEM *dictionary_set_and_acquire_item(DICTIONARY *dict, const char *name, void *value, size_t value_len) { + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); + NAME_VALUE *nv = dictionary_set_name_value_unsafe(dict, name, value, value_len); + + // we need to get the reference counter before we unlock + if(nv) reference_counter_acquire(dict, nv); + + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); + + if(unlikely(dict->react_callback && nv && (nv->flags & NAME_VALUE_FLAG_NEW_OR_UPDATED))) { + // we already have a reference counter, for the caller, no need for another one + dict->react_callback(namevalue_get_name(nv), nv->value, dict->react_callback_data); + } + + return (DICTIONARY_ITEM *)nv; +} + +void *dictionary_get_unsafe(DICTIONARY *dict, const char *name) { + NAME_VALUE *nv = dictionary_get_name_value_unsafe(dict, name); + + if(unlikely(!nv)) + return NULL; + return nv->value; } void *dictionary_get(DICTIONARY *dict, const char *name) { - dictionary_lock_rlock(dict); + dictionary_lock(dict, DICTIONARY_LOCK_READ); void *ret = dictionary_get_unsafe(dict, name); - dictionary_unlock(dict); + dictionary_unlock(dict, DICTIONARY_LOCK_READ); return ret; } +DICTIONARY_ITEM *dictionary_get_and_acquire_item_unsafe(DICTIONARY *dict, const char *name) { + NAME_VALUE *nv = dictionary_get_name_value_unsafe(dict, name); + + if(unlikely(!nv)) + return NULL; + + reference_counter_acquire(dict, nv); + return (DICTIONARY_ITEM *)nv; +} + +DICTIONARY_ITEM *dictionary_get_and_acquire_item(DICTIONARY *dict, const char *name) { + dictionary_lock(dict, DICTIONARY_LOCK_READ); + void *ret = dictionary_get_and_acquire_item_unsafe(dict, name); + dictionary_unlock(dict, DICTIONARY_LOCK_READ); + return ret; +} + +DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY_ITEM *item) { + if(unlikely(!item)) return NULL; + reference_counter_increase((NAME_VALUE *)item); + return item; +} + +const char *dictionary_acquired_item_name(DICTIONARY_ITEM *item) { + if(unlikely(!item)) return NULL; + return namevalue_get_name((NAME_VALUE *)item); +} + +void *dictionary_acquired_item_value(DICTIONARY_ITEM *item) { + if(unlikely(!item)) return NULL; + return ((NAME_VALUE *)item)->value; +} + +void dictionary_acquired_item_release_unsafe(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(unlikely(!item)) return; + +#ifdef NETDATA_INTERNAL_CHECKS + if(((NAME_VALUE *)item)->dict != dict) + fatal("DICTIONARY: %s(): name_value item with name '%s' does not belong to this dictionary", __FUNCTION__, namevalue_get_name((NAME_VALUE *)item)); +#endif + + reference_counter_release(dict, (NAME_VALUE *)item, false); +} + +void dictionary_acquired_item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) { + if(unlikely(!item)) return; + +#ifdef NETDATA_INTERNAL_CHECKS + if(((NAME_VALUE *)item)->dict != dict) + fatal("DICTIONARY: %s(): name_value item with name '%s' does not belong to this dictionary", __FUNCTION__, namevalue_get_name((NAME_VALUE *)item)); +#endif + + // no need to get a lock here + // we pass the last parameter to reference_counter_release() as true + // so that the release may get a write-lock if required to clean up + + reference_counter_release(dict, (NAME_VALUE *)item, true); + + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) + dictionary_destroy(dict); +} + int dictionary_del_unsafe(DICTIONARY *dict, const char *name) { + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_del() on a destroyed dictionary"); + return -1; + } + if(unlikely(!name || !*name)) { - error("Attempted to dictionary_det() without a name"); + internal_error(true, "DICTIONARY: attempted to dictionary_del() without a name"); return -1; } + internal_error(!(dict->flags & DICTIONARY_FLAG_EXCLUSIVE_ACCESS), "DICTIONARY: INTERNAL ERROR: deleting dictionary item '%s' without exclusive access to dictionary", name); + size_t name_len = strlen(name) + 1; // we need the terminating null too debug(D_DICTIONARY, "DEL dictionary entry with name '%s'.", name); @@ -809,23 +1217,25 @@ int dictionary_del_unsafe(DICTIONARY *dict, const char *name) { if(hashtable_delete_unsafe(dict, name, name_len, nv) == 0) error("DICTIONARY: INTERNAL ERROR: tried to delete item with name '%s' that is not in the index", name); - if(!reference_counter_mark_deleted(dict, nv)) { + if(name_value_can_be_deleted(dict, nv)) { linkedlist_namevalue_unlink_unsafe(dict, nv); - - if(dict->del_callback) - dict->del_callback(nv->name, nv->value, dict->del_callback_data); - namevalue_destroy_unsafe(dict, nv); } + else + nv->flags |= NAME_VALUE_FLAG_DELETED; + ret = 0; + + DICTIONARY_STATS_ENTRIES_MINUS1(dict); + } return ret; } int dictionary_del(DICTIONARY *dict, const char *name) { - dictionary_lock_wrlock(dict); + dictionary_lock(dict, DICTIONARY_LOCK_WRITE); int ret = dictionary_del_unsafe(dict, name); - dictionary_unlock(dict); + dictionary_unlock(dict, DICTIONARY_LOCK_WRITE); return ret; } @@ -835,25 +1245,37 @@ int dictionary_del(DICTIONARY *dict, const char *name) { void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { if(unlikely(!dfe || !dict)) return NULL; + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_foreach_start_rw() on a destroyed dictionary"); + dfe->last_item = NULL; + dfe->name = NULL; + dfe->value = NULL; + return NULL; + } + dfe->dict = dict; + dfe->rw = rw; dfe->started_ut = now_realtime_usec(); - if(rw == 'r' || rw == 'R') - dictionary_lock_rlock(dict); - else - dictionary_lock_wrlock(dict); + dictionary_lock(dict, dfe->rw); + + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); + // get the first item from the list NAME_VALUE *nv = dict->first_item; - dfe->last_position_index = (void *)nv; + + // skip all the deleted items + while(nv && (nv->flags & NAME_VALUE_FLAG_DELETED)) + nv = nv->next; if(likely(nv)) { - dfe->next_position_index = (void *)nv->next; - dfe->name = nv->name; - dfe->value = (void *)nv->value; + dfe->last_item = nv; + dfe->name = (char *)namevalue_get_name(nv); + dfe->value = nv->value; reference_counter_acquire(dict, nv); } else { - dfe->next_position_index = NULL; + dfe->last_item = NULL; dfe->name = NULL; dfe->value = NULL; } @@ -864,21 +1286,36 @@ void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) { void *dictionary_foreach_next(DICTFE *dfe) { if(unlikely(!dfe || !dfe->dict)) return NULL; - NAME_VALUE *nv = (NAME_VALUE *)dfe->last_position_index; - if(likely(nv)) - reference_counter_release(dfe->dict, nv); + if(unlikely(dfe->dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary"); + dfe->last_item = NULL; + dfe->name = NULL; + dfe->value = NULL; + return NULL; + } - nv = dfe->last_position_index = dfe->next_position_index; + // the item we just did + NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item; - if(likely(nv)) { - dfe->next_position_index = (void *)nv->next; - dfe->name = nv->name; - dfe->value = (void *)nv->value; + // get the next item from the list + NAME_VALUE *nv_next = (nv) ? nv->next : NULL; + + // skip all the deleted items + while(nv_next && (nv_next->flags & NAME_VALUE_FLAG_DELETED)) + nv_next = nv_next->next; + // release the old, so that it can possibly be deleted + if(likely(nv)) + reference_counter_release(dfe->dict, nv, false); + + if(likely(nv = nv_next)) { + dfe->last_item = nv; + dfe->name = (char *)namevalue_get_name(nv); + dfe->value = nv->value; reference_counter_acquire(dfe->dict, nv); } else { - dfe->next_position_index = NULL; + dfe->last_item = NULL; dfe->name = NULL; dfe->value = NULL; } @@ -889,14 +1326,21 @@ void *dictionary_foreach_next(DICTFE *dfe) { usec_t dictionary_foreach_done(DICTFE *dfe) { if(unlikely(!dfe || !dfe->dict)) return 0; - NAME_VALUE *nv = (NAME_VALUE *)dfe->last_position_index; - if(nv) - reference_counter_release(dfe->dict, nv); + if(unlikely(dfe->dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_foreach_next() on a destroyed dictionary"); + return 0; + } + + // the item we just did + NAME_VALUE *nv = (NAME_VALUE *)dfe->last_item; + + // release it, so that it can possibly be deleted + if(likely(nv)) + reference_counter_release(dfe->dict, nv, false); - dictionary_unlock((DICTIONARY *)dfe->dict); + dictionary_unlock(dfe->dict, dfe->rw); dfe->dict = NULL; - dfe->last_position_index = NULL; - dfe->next_position_index = NULL; + dfe->last_item = NULL; dfe->name = NULL; dfe->value = NULL; @@ -912,21 +1356,40 @@ usec_t dictionary_foreach_done(DICTFE *dfe) { // do not use other dictionary calls while walking the dictionary - deadlock! int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) { - if(rw == 'r' || rw == 'R') - dictionary_lock_rlock(dict); - else - dictionary_lock_wrlock(dict); + if(unlikely(!dict)) return 0; + + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_walkthrough_rw() on a destroyed dictionary"); + return 0; + } + + dictionary_lock(dict, rw); + + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); // written in such a way, that the callback can delete the active element int ret = 0; NAME_VALUE *nv = dict->first_item, *nv_next; while(nv) { - nv_next = nv->next; + // skip the deleted items + if(unlikely(nv->flags & NAME_VALUE_FLAG_DELETED)) { + nv = nv->next; + continue; + } + + // get a reference counter, so that our item will not be deleted + // while we are using it reference_counter_acquire(dict, nv); - int r = callback(nv->name, nv->value, data); - reference_counter_release(dict, nv); + + int r = callback(namevalue_get_name(nv), nv->value, data); + + // since we have a reference counter, this item cannot be deleted + // until we release the reference counter, so the pointers are there + nv_next = nv->next; + reference_counter_release(dict, nv, false); + if(unlikely(r < 0)) { ret = r; break; @@ -937,12 +1400,249 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c nv = nv_next; } - dictionary_unlock(dict); + dictionary_unlock(dict, rw); return ret; } // ---------------------------------------------------------------------------- +// sorted walkthrough + +static int dictionary_sort_compar(const void *nv1, const void *nv2) { + return strcmp(namevalue_get_name((*(NAME_VALUE **)nv1)), namevalue_get_name((*(NAME_VALUE **)nv2))); +} + +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) { + if(unlikely(!dict || !dict->entries)) return 0; + + if(unlikely(dict->flags & DICTIONARY_FLAG_DESTROYED)) { + internal_error(true, "DICTIONARY: attempted to dictionary_sorted_walkthrough_rw() on a destroyed dictionary"); + return 0; + } + + dictionary_lock(dict, rw); + dictionary_defer_all_deletions_unsafe(dict, rw); + + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict); + + size_t count = dict->entries; + NAME_VALUE **array = mallocz(sizeof(NAME_VALUE *) * count); + + size_t i; + NAME_VALUE *nv; + for(nv = dict->first_item, i = 0; nv && i < count ;nv = nv->next) { + if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) + array[i++] = nv; + } + + internal_error(nv != NULL, "DICTIONARY: during sorting expected to have %zu items in dictionary, but there are more. Sorted results may be incomplete. Dictionary fails to maintain an accurate number of the number of entries it has.", count); + + if(unlikely(i != count)) { + internal_error(true, "DICTIONARY: during sorting expected to have %zu items in dictionary, but there are %zu. Sorted results may be incomplete. Dictionary fails to maintain an accurate number of the number of entries it has.", count, i); + count = i; + } + + qsort(array, count, sizeof(NAME_VALUE *), dictionary_sort_compar); + + int ret = 0; + for(i = 0; i < count ;i++) { + nv = array[i]; + if(likely(!(nv->flags & NAME_VALUE_FLAG_DELETED))) { + reference_counter_acquire(dict, nv); + int r = callback(namevalue_get_name(nv), nv->value, data); + reference_counter_release(dict, nv, false); + if (r < 0) { + ret = r; + break; + } + ret += r; + } + } + + dictionary_restore_all_deletions_unsafe(dict, rw); + dictionary_unlock(dict, rw); + freez(array); + + return ret; +} + +// ---------------------------------------------------------------------------- +// STRING implementation - dedup all STRINGs + +typedef struct string_entry { +#ifdef DICTIONARY_WITH_AVL + avl_t avl_node; +#endif + uint32_t length; // the string length with the terminating '\0' + uint32_t refcount; // how many times this string is used + const char str[]; // the string itself +} STRING_ENTRY; + +#ifdef DICTIONARY_WITH_AVL +static int string_entry_compare(void* a, void* b) { + return strcmp(((STRING_ENTRY *)a)->str, ((STRING_ENTRY *)b)->str); +} + +static void *get_thread_static_string_entry(const char *name) { + static __thread size_t _length = 0; + static __thread STRING_ENTRY *_tmp = NULL; + + size_t size = sizeof(STRING_ENTRY) + strlen(name) + 1; + if(likely(_tmp && _length < size)) { + freez(_tmp); + _tmp = NULL; + _length = 0; + } + + if(unlikely(!_tmp)) { + _tmp = callocz(1, size); + _length = size; + } + + strcpy((char *)&_tmp->str[0], name); + return _tmp; +} +#endif + +DICTIONARY string_dictionary = { +#ifdef DICTIONARY_WITH_AVL + .values_index = { + .root = NULL, + .compar = string_entry_compare + }, + .get_thread_static_name_value = get_thread_static_string_entry, +#endif + + .flags = DICTIONARY_FLAG_EXCLUSIVE_ACCESS, + .rwlock = NETDATA_RWLOCK_INITIALIZER +}; + +static netdata_mutex_t string_mutex = NETDATA_MUTEX_INITIALIZER; + +STRING *string_dup(STRING *string) { + if(unlikely(!string)) return NULL; + + STRING_ENTRY *se = (STRING_ENTRY *)string; + netdata_mutex_lock(&string_mutex); + se->refcount++; + netdata_mutex_unlock(&string_mutex); + return string; +} + +STRING *string_strdupz(const char *str) { + if(unlikely(!str || !*str)) return NULL; + + netdata_mutex_lock(&string_mutex); + + size_t length = strlen(str) + 1; + STRING_ENTRY *se; + STRING_ENTRY **ptr = (STRING_ENTRY **)hashtable_insert_unsafe(&string_dictionary, str, length); + if(unlikely(*ptr == 0)) { + // a new item added to the index + size_t mem_size = sizeof(STRING_ENTRY) + length; + se = mallocz(mem_size); + strcpy((char *)se->str, str); + se->length = length; + se->refcount = 1; + *ptr = se; + hashtable_inserted_name_value_unsafe(&string_dictionary, se); + string_dictionary.version++; + string_dictionary.inserts++; + string_dictionary.entries++; + string_dictionary.memory += (long)mem_size; + } + else { + // the item is already in the index + se = *ptr; + se->refcount++; + string_dictionary.searches++; + } + + netdata_mutex_unlock(&string_mutex); + return (STRING *)se; +} + +void string_freez(STRING *string) { + if(unlikely(!string)) return; + netdata_mutex_lock(&string_mutex); + + STRING_ENTRY *se = (STRING_ENTRY *)string; + + if(se->refcount == 0) + fatal("STRING: tried to free string that has zero references."); + + se->refcount--; + if(unlikely(se->refcount == 0)) { + if(hashtable_delete_unsafe(&string_dictionary, se->str, se->length, se) == 0) + error("STRING: INTERNAL ERROR: tried to delete '%s' that is not in the index", se->str); + + size_t mem_size = sizeof(STRING_ENTRY) + se->length; + freez(se); + string_dictionary.version++; + string_dictionary.deletes++; + string_dictionary.entries--; + string_dictionary.memory -= (long)mem_size; + } + + netdata_mutex_unlock(&string_mutex); +} + +size_t string_length(STRING *string) { + if(unlikely(!string)) return 0; + return ((STRING_ENTRY *)string)->length - 1; +} + +const char *string2str(STRING *string) { + if(unlikely(!string)) return ""; + return ((STRING_ENTRY *)string)->str; +} + +STRING *string_2way_merge(STRING *a, STRING *b) { + static STRING *X = NULL; + + if(unlikely(!X)) { + X = string_strdupz("[x]"); + } + + if(unlikely(a == b)) return string_dup(a); + if(unlikely(a == X)) return string_dup(a); + if(unlikely(b == X)) return string_dup(b); + if(unlikely(!a)) return string_dup(X); + if(unlikely(!b)) return string_dup(X); + + size_t alen = string_length(a); + size_t blen = string_length(b); + size_t length = alen + blen + string_length(X) + 1; + char buf1[length + 1], buf2[length + 1], *dst1; + const char *s1, *s2; + + s1 = string2str(a); + s2 = string2str(b); + dst1 = buf1; + for( ; *s1 && *s2 && *s1 == *s2 ;s1++, s2++) + *dst1++ = *s1; + + *dst1 = '\0'; + + if(*s1 != '\0' || *s2 != '\0') { + *dst1++ = '['; + *dst1++ = 'x'; + *dst1++ = ']'; + + s1 = &(string2str(a))[alen - 1]; + s2 = &(string2str(b))[blen - 1]; + char *dst2 = &buf2[length]; + *dst2 = '\0'; + for (; *s1 && *s2 && *s1 == *s2; s1--, s2--) + *(--dst2) = *s1; + + strcpy(dst1, dst2); + } + + return string_strdupz(buf1); +} + +// ---------------------------------------------------------------------------- // unit test static void dictionary_unittest_free_char_pp(char **pp, size_t entries) { @@ -983,6 +1683,22 @@ static size_t dictionary_unittest_set_clone(DICTIONARY *dict, char **names, char return errors; } +static size_t dictionary_unittest_set_null(DICTIONARY *dict, char **names, char **values, size_t entries) { + (void)values; + size_t errors = 0; + long i = 0; + for(; i < (long)entries ;i++) { + void *val = dictionary_set(dict, names[i], NULL, 0); + if(val != NULL) { fprintf(stderr, ">>> %s() returns a non NULL value\n", __FUNCTION__); errors++; } + } + if(dictionary_stats_entries(dict) != i) { + fprintf(stderr, ">>> %s() dictionary items do not match\n", __FUNCTION__); + errors++; + } + return errors; +} + + static size_t dictionary_unittest_set_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries) { size_t errors = 0; for(size_t i = 0; i < entries ;i++) { @@ -1182,7 +1898,7 @@ static size_t dictionary_unittest_destroy(DICTIONARY *dict, char **names, char * } static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *message, char **names, char **values, size_t entries, size_t *errors, size_t (*callback)(DICTIONARY *dict, char **names, char **values, size_t entries)) { - fprintf(stderr, "%-40s... ", message); + fprintf(stderr, "%40s ... ", message); usec_t started = now_realtime_usec(); size_t errs = callback(dict, names, values, entries); @@ -1191,12 +1907,12 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m if(callback == dictionary_unittest_destroy) dict = NULL; - fprintf(stderr, " %zu errors, %zu items in dictionary, %llu usec \n", errs, dict? dictionary_stats_entries(dict):0, dt); + fprintf(stderr, " %zu errors, %ld items in dictionary, %llu usec \n", errs, dict? dictionary_stats_entries(dict):0, dt); *errors += errs; return dt; } -void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { +static void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone); dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_clone); dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting); @@ -1211,7 +1927,7 @@ void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, si dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy); } -void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { +static void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_nonclone); dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting); @@ -1226,6 +1942,217 @@ void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy); } +struct dictionary_unittest_sorting { + const char *oldname; + const char *oldvalue; + size_t count; +}; + +static int dictionary_unittest_sorting_callback(const char *name, void *value, void *data) { + struct dictionary_unittest_sorting *t = (struct dictionary_unittest_sorting *)data; + const char *v = (const char *)value; + + int ret = 0; + if(t->oldname && strcmp(t->oldname, name) > 0) { + fprintf(stderr, "name '%s' should be after '%s'\n", t->oldname, name); + ret = 1; + } + t->count++; + t->oldname = name; + t->oldvalue = v; + + return ret; +} + +static size_t dictionary_unittest_sorted_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) { + (void)names; + (void)values; + struct dictionary_unittest_sorting tmp = { .oldname = NULL, .oldvalue = NULL, .count = 0 }; + size_t errors; + errors = dictionary_sorted_walkthrough_read(dict, dictionary_unittest_sorting_callback, &tmp); + + if(tmp.count != entries) { + fprintf(stderr, "Expected %zu entries, counted %zu\n", entries, tmp.count); + errors++; + } + return errors; +} + +static void dictionary_unittest_sorting(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { + dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone); + dictionary_unittest_run_and_measure_time(dict, "sorted walkthrough", names, values, entries, errors, dictionary_unittest_sorted_walkthrough); +} + +static void dictionary_unittest_null_dfe(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) { + dictionary_unittest_run_and_measure_time(dict, "adding null value entries", names, values, entries, errors, dictionary_unittest_set_null); + dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, errors, dictionary_unittest_foreach); +} + + +static int check_dictionary_callback(const char *name, void *value, void *data) { + (void)name; + (void)value; + (void)data; + return 1; +} + +static size_t check_dictionary(DICTIONARY *dict, size_t entries, size_t linked_list_members) { + size_t errors = 0; + + fprintf(stderr, "dictionary entries %ld, expected %zu...\t\t\t\t\t", dictionary_stats_entries(dict), entries); + if (dictionary_stats_entries(dict) != (long)entries) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + size_t ll = 0; + void *t; + dfe_start_read(dict, t) + ll++; + dfe_done(t); + + fprintf(stderr, "dictionary foreach entries %zu, expected %zu...\t\t\t\t", ll, entries); + if(ll != entries) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + ll = dictionary_walkthrough_read(dict, check_dictionary_callback, NULL); + fprintf(stderr, "dictionary walkthrough entries %zu, expected %zu...\t\t\t\t", ll, entries); + if(ll != entries) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + ll = dictionary_sorted_walkthrough_read(dict, check_dictionary_callback, NULL); + fprintf(stderr, "dictionary sorted walkthrough entries %zu, expected %zu...\t\t\t", ll, entries); + if(ll != entries) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + NAME_VALUE *nv; + for(ll = 0, nv = dict->first_item; nv ;nv = nv->next) + ll++; + + fprintf(stderr, "dictionary linked list entries %zu, expected %zu...\t\t\t\t", ll, linked_list_members); + if(ll != linked_list_members) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + return errors; +} + +static int check_name_value_callback(const char *name, void *value, void *data) { + (void)name; + return value == data; +} + +static size_t check_name_value_deleted_flag(DICTIONARY *dict, NAME_VALUE *nv, const char *name, const char *value, unsigned refcount, NAME_VALUE_FLAGS deleted_flags, bool searchable, bool browsable, bool linked) { + size_t errors = 0; + + fprintf(stderr, "NAME_VALUE name is '%s', expected '%s'...\t\t\t\t", namevalue_get_name(nv), name); + if(strcmp(namevalue_get_name(nv), name) != 0) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "NAME_VALUE value is '%s', expected '%s'...\t\t\t", (const char *)nv->value, value); + if(strcmp((const char *)nv->value, value) != 0) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "NAME_VALUE refcount is %u, expected %u...\t\t\t\t\t", nv->refcount, refcount); + if (nv->refcount != refcount) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + fprintf(stderr, "NAME_VALUE deleted flag is %s, expected %s...\t\t\t", (nv->flags & NAME_VALUE_FLAG_DELETED)?"TRUE":"FALSE", (deleted_flags & NAME_VALUE_FLAG_DELETED)?"TRUE":"FALSE"); + if ((nv->flags & NAME_VALUE_FLAG_DELETED) != (deleted_flags & NAME_VALUE_FLAG_DELETED)) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + void *v = dictionary_get(dict, name); + bool found = v == nv->value; + fprintf(stderr, "NAME_VALUE searchable %5s, expected %5s...\t\t\t\t", found?"true":"false", searchable?"true":"false"); + if(found != searchable) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + found = false; + void *t; + dfe_start_read(dict, t) { + if(t == nv->value) found = true; + } + dfe_done(t); + + fprintf(stderr, "NAME_VALUE dfe browsable %5s, expected %5s...\t\t\t", found?"true":"false", browsable?"true":"false"); + if(found != browsable) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + found = dictionary_walkthrough_read(dict, check_name_value_callback, nv->value); + fprintf(stderr, "NAME_VALUE walkthrough browsable %5s, expected %5s...\t\t", found?"true":"false", browsable?"true":"false"); + if(found != browsable) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + found = dictionary_sorted_walkthrough_read(dict, check_name_value_callback, nv->value); + fprintf(stderr, "NAME_VALUE sorted walkthrough browsable %5s, expected %5s...\t", found?"true":"false", browsable?"true":"false"); + if(found != browsable) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + found = false; + NAME_VALUE *n; + for(n = dict->first_item; n ;n = n->next) + if(n == nv) found = true; + + fprintf(stderr, "NAME_VALUE linked %5s, expected %5s...\t\t\t\t", found?"true":"false", linked?"true":"false"); + if(found != linked) { + fprintf(stderr, "FAILED\n"); + errors++; + } + else + fprintf(stderr, "OK\n"); + + return errors; +} + int dictionary_unittest(size_t entries) { if(entries < 10) entries = 10; @@ -1237,23 +2164,23 @@ int dictionary_unittest(size_t entries) { char **values = dictionary_unittest_generate_values(entries); fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); dictionary_unittest_clone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS); + dict = dictionary_create(DICTIONARY_FLAG_NONE); dictionary_unittest_clone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); dictionary_unittest_nonclone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT); dictionary_unittest_nonclone(dict, names, values, entries, &errors); fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone); dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach); @@ -1262,22 +2189,222 @@ int dictionary_unittest(size_t entries) { dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this); dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy); fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries); - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE); dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone); dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this); dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop empty", names, values, 0, &errors, dictionary_unittest_foreach); dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback empty", names, values, 0, &errors, dictionary_unittest_walkthrough); dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy); + fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dictionary_unittest_sorting(dict, names, values, entries, &errors); + dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); + + fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED); + dictionary_unittest_null_dfe(dict, names, values, entries, &errors); + dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); + + fprintf(stderr, "\nCreating dictionary single threaded, noclone, %zu items\n", entries); + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE); + dictionary_unittest_null_dfe(dict, names, values, entries, &errors); + dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy); + + // check reference counters + { + fprintf(stderr, "\nTesting reference counters:\n"); + dict = dictionary_create(DICTIONARY_FLAG_NONE|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE); + errors += check_dictionary(dict, 0, 0); + + fprintf(stderr, "\nAdding test item to dictionary and acquiring it\n"); + dictionary_set(dict, "test", "ITEM1", 6); + NAME_VALUE *nv = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); + + errors += check_dictionary(dict, 1, 1); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nChecking that reference counters are increased:\n"); + void *t; + dfe_start_read(dict, t) { + errors += check_dictionary(dict, 1, 1); + errors += + check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 2, NAME_VALUE_FLAG_NONE, true, true, true); + } + dfe_done(t); + + fprintf(stderr, "\nChecking that reference counters are decreased:\n"); + errors += check_dictionary(dict, 1, 1); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nDeleting the item we have acquired:\n"); + dictionary_del(dict, "test"); + + errors += check_dictionary(dict, 0, 1); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + + fprintf(stderr, "\nAdding another item with the same name of the item we deleted, while being acquired:\n"); + dictionary_set(dict, "test", "ITEM2", 6); + errors += check_dictionary(dict, 1, 2); + + fprintf(stderr, "\nAcquiring the second item:\n"); + NAME_VALUE *nv2 = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + errors += check_name_value_deleted_flag(dict, nv2, "test", "ITEM2", 1, NAME_VALUE_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nReleasing the second item (the first is still acquired):\n"); + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv2); + errors += check_dictionary(dict, 1, 2); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + errors += check_name_value_deleted_flag(dict, nv2, "test", "ITEM2", 0, NAME_VALUE_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nDeleting the second item (the first is still acquired):\n"); + dictionary_del(dict, "test"); + errors += check_dictionary(dict, 0, 1); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_DELETED, false, false, true); + + fprintf(stderr, "\nReleasing the first item (which we have already deleted):\n"); + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv); + errors += check_dictionary(dict, 0, 0); + + fprintf(stderr, "\nAdding again the test item to dictionary and acquiring it\n"); + dictionary_set(dict, "test", "ITEM1", 6); + nv = (NAME_VALUE *)dictionary_get_and_acquire_item(dict, "test"); + + errors += check_dictionary(dict, 1, 1); + errors += check_name_value_deleted_flag(dict, nv, "test", "ITEM1", 1, NAME_VALUE_FLAG_NONE, true, true, true); + + fprintf(stderr, "\nDestroying the dictionary while we have acquired an item\n"); + dictionary_destroy(dict); + + fprintf(stderr, "Releasing the item (on a destroyed dictionary)\n"); + dictionary_acquired_item_release(dict, (DICTIONARY_ITEM *)nv); + nv = NULL; + dict = NULL; + } + + // check string + { + long string_entries_starting = dictionary_stats_entries(&string_dictionary); + + fprintf(stderr, "\nChecking strings...\n"); + + STRING *s1 = string_strdupz("hello unittest"); + STRING *s2 = string_strdupz("hello unittest"); + if(s1 != s2) { + errors++; + fprintf(stderr, "ERROR: duplicating strings are not deduplicated\n"); + } + else + fprintf(stderr, "OK: duplicating string are deduplicated\n"); + + STRING *s3 = string_dup(s1); + if(s3 != s1) { + errors++; + fprintf(stderr, "ERROR: cloning strings are not deduplicated\n"); + } + else + fprintf(stderr, "OK: cloning string are deduplicated\n"); + + STRING_ENTRY *se = (STRING_ENTRY *)s1; + if(se->refcount != 3) { + errors++; + fprintf(stderr, "ERROR: string refcount is not 3\n"); + } + else + fprintf(stderr, "OK: string refcount is 3\n"); + + STRING *s4 = string_strdupz("world unittest"); + if(s4 == s1) { + errors++; + fprintf(stderr, "ERROR: string is sharing pointers on different strings\n"); + } + else + fprintf(stderr, "OK: string is properly handling different strings\n"); + + usec_t start_ut, end_ut; + STRING **strings = mallocz(entries * sizeof(STRING *)); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + strings[i] = string_strdupz(names[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Created %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + strings[i] = string_dup(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Cloned %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + start_ut = now_realtime_usec(); + for(size_t i = 0; i < entries ;i++) { + string_freez(strings[i]); + string_freez(strings[i]); + } + end_ut = now_realtime_usec(); + fprintf(stderr, "Freed %zu strings in %llu usecs\n", entries, end_ut - start_ut); + + freez(strings); + + if(dictionary_stats_entries(&string_dictionary) != string_entries_starting + 2) { + errors++; + fprintf(stderr, "ERROR: strings dictionary should have %ld items but it has %ld\n", string_entries_starting + 2, dictionary_stats_entries(&string_dictionary)); + } + else + fprintf(stderr, "OK: strings dictionary has 2 items\n"); + } + + // check 2-way merge + { + struct testcase { + char *src1; char *src2; char *expected; + } tests[] = { + { "", "", ""}, + { "a", "", "[x]"}, + { "", "a", "[x]"}, + { "a", "a", "a"}, + { "abcd", "abcd", "abcd"}, + { "foo_cs", "bar_cs", "[x]_cs"}, + { "cp_UNIQUE_INFIX_cs", "cp_unique_infix_cs", "cp_[x]_cs"}, + { "cp_UNIQUE_INFIX_ci_unique_infix_cs", "cp_unique_infix_ci_UNIQUE_INFIX_cs", "cp_[x]_cs"}, + { "foo[1234]", "foo[4321]", "foo[[x]]"}, + { NULL, NULL, NULL }, + }; + + for (struct testcase *tc = &tests[0]; tc->expected != NULL; tc++) { + STRING *src1 = string_strdupz(tc->src1); + STRING *src2 = string_strdupz(tc->src2); + STRING *expected = string_strdupz(tc->expected); + + STRING *result = string_2way_merge(src1, src2); + if (string_cmp(result, expected) != 0) { + fprintf(stderr, "string_2way_merge(\"%s\", \"%s\") -> \"%s\" (expected=\"%s\")\n", + string2str(src1), + string2str(src2), + string2str(result), + string2str(expected)); + errors++; + } + + string_freez(src1); + string_freez(src2); + string_freez(expected); + string_freez(result); + } + } + dictionary_unittest_free_char_pp(names, entries); dictionary_unittest_free_char_pp(values, entries); fprintf(stderr, "\n%zu errors found\n", errors); - return (int)errors; + return errors ? 1 : 0; } diff --git a/libnetdata/dictionary/dictionary.h b/libnetdata/dictionary/dictionary.h index 356bf1895..fdb2088c0 100644 --- a/libnetdata/dictionary/dictionary.h +++ b/libnetdata/dictionary/dictionary.h @@ -35,23 +35,34 @@ * */ -#ifndef DICTIONARY_INTERNALS -typedef void DICTIONARY; -#endif +typedef struct dictionary DICTIONARY; +typedef struct dictionary_item DICTIONARY_ITEM; typedef enum dictionary_flags { - DICTIONARY_FLAG_NONE = 0, // the default is the opposite of all below - DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks) - DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy) - DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy) - DICTIONARY_FLAG_WITH_STATISTICS = (1 << 3), // maintain statistics about dictionary operations (default: disabled) - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 4), // don't overwrite values of dictionary items (default: overwrite) - DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 5), // add dictionary items at the front of the linked list (default: at the end) - DICTIONARY_FLAG_RESERVED1 = (1 << 6), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS + DICTIONARY_FLAG_NONE = 0, // the default is the opposite of all below + DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks) + DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy) + DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy) + DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite) + DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end) + + // to change the value of the following, you also need to change the corresponding #defines in dictionary.c + DICTIONARY_FLAG_RESERVED1 = (1 << 29), // reserved for DICTIONARY_FLAG_EXCLUSIVE_ACCESS + DICTIONARY_FLAG_RESERVED2 = (1 << 30), // reserved for DICTIONARY_FLAG_DESTROYED + DICTIONARY_FLAG_RESERVED3 = (1 << 31), // reserved for DICTIONARY_FLAG_DEFER_ALL_DELETIONS } DICTIONARY_FLAGS; // Create a dictionary -extern DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags); +#ifdef NETDATA_INTERNAL_CHECKS +#define dictionary_create(flags) dictionary_create_advanced_with_trace(flags, 0, __FUNCTION__, __LINE__, __FILE__); +#define dictionary_create_advanced(flags) dictionary_create_advanced_with_trace(flags, 0, __FUNCTION__, __LINE__, __FILE__); +extern DICTIONARY *dictionary_create_advanced_with_trace(DICTIONARY_FLAGS flags, size_t scratchpad_size, const char *function, size_t line, const char *file); +#else +#define dictionary_create(flags) dictionary_create_advanced(flags, 0); +extern DICTIONARY *dictionary_create_advanced(DICTIONARY_FLAGS flags, size_t scratchpad_size); +#endif + +extern void *dictionary_scratchpad(DICTIONARY *dict); // an insert callback to be called just after an item is added to the dictionary // this callback is called while the dictionary is write locked! @@ -66,6 +77,11 @@ extern void dictionary_register_delete_callback(DICTIONARY *dict, void (*del_cal // the old_value will remain in the dictionary - the new_value is ignored extern void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data), void *data); +// a reaction callback to be called after every item insertion or conflict +// after the constructors have finished and the items are fully available for use +// and the dictionary is not write locked anymore +extern void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)(const char *name, void *value, void *data), void *data); + // Destroy a dictionary // returns the number of bytes freed // the returned value will not include name and value sizes if DICTIONARY_FLAG_WITH_STATISTICS is not set @@ -85,7 +101,7 @@ extern size_t dictionary_destroy(DICTIONARY *dict); // // Passing NULL as value, the dictionary will callocz() the newly allocated value, otherwise it will copy it. // Passing 0 as value_len, the dictionary will set the value to NULL (no allocations for value will be made). -extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len) NEVERNULL; +extern void *dictionary_set(DICTIONARY *dict, const char *name, void *value, size_t value_len); // Get an item from the dictionary // If it returns NULL, the item is not found @@ -96,6 +112,19 @@ extern void *dictionary_get(DICTIONARY *dict, const char *name); // returns -1 if the item was not found in the index extern int dictionary_del(DICTIONARY *dict, const char *name); +extern DICTIONARY_ITEM *dictionary_get_and_acquire_item_unsafe(DICTIONARY *dict, const char *name); +extern DICTIONARY_ITEM *dictionary_get_and_acquire_item(DICTIONARY *dict, const char *name); + +extern DICTIONARY_ITEM *dictionary_set_and_acquire_item_unsafe(DICTIONARY *dict, const char *name, void *value, size_t value_len); +extern DICTIONARY_ITEM *dictionary_set_and_acquire_item(DICTIONARY *dict, const char *name, void *value, size_t value_len); + +extern void dictionary_acquired_item_release_unsafe(DICTIONARY *dict, DICTIONARY_ITEM *item); +extern void dictionary_acquired_item_release(DICTIONARY *dict, DICTIONARY_ITEM *item); + +extern DICTIONARY_ITEM *dictionary_acquired_item_dup(DICTIONARY_ITEM *item); +extern const char *dictionary_acquired_item_name(DICTIONARY_ITEM *item); +extern void *dictionary_acquired_item_value(DICTIONARY_ITEM *item); + // UNSAFE functions, without locks // to be used when the user is traversing with the right lock type // Read lock is acquired by dictionary_walktrhough_read() and dfe_start_read() @@ -126,6 +155,10 @@ extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name); #define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data) extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *value, void *data), void *data); +#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data) +#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data) +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data); + // Traverse with foreach // // Use like this: @@ -146,29 +179,35 @@ extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)( #define DICTFE_CONST const #endif +#define DICTIONARY_LOCK_READ 'r' +#define DICTIONARY_LOCK_WRITE 'w' +#define DICTIONARY_LOCK_NONE 'u' + typedef DICTFE_CONST struct dictionary_foreach { DICTFE_CONST char *name; // the dictionary name of the last item used void *value; // the dictionary value of the last item used // same as the return value of dictfe_start() and dictfe_next() // the following are for internal use only - to keep track of the point we are + char rw; // the lock mode 'r' or 'w' usec_t started_ut; // the time the caller started iterating (now_realtime_usec()) DICTIONARY *dict; // the dictionary upon we work - void *last_position_index; // the internal position index, to remember the position we are at - void *next_position_index; // the internal position index, of the next item + void *last_item; // the item we work on, to remember the position we are at } DICTFE; -#define dfe_start_read(dict, value) dfe_start_rw(dict, value, 'r') -#define dfe_start_write(dict, value) dfe_start_rw(dict, value, 'w') +#define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ) +#define dfe_start_write(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_WRITE) #define dfe_start_rw(dict, value, mode) \ do { \ DICTFE value ## _dfe = {}; \ - const char *value ## _name; (void)(value ## _name); \ + const char *value ## _name; (void)(value ## _name); (void)value; \ for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)), ( value ## _name ) = value ## _dfe.name; \ - (value) ;\ - (value) = dictionary_foreach_next(&value ## _dfe), ( value ## _name ) = value ## _dfe.name) + (value ## _dfe.name) ;\ + (value) = dictionary_foreach_next(&value ## _dfe), ( value ## _name ) = value ## _dfe.name) \ + { #define dfe_done(value) \ + } \ dictionary_foreach_done(&value ## _dfe); \ } while(0) @@ -177,14 +216,37 @@ extern void * dictionary_foreach_next(DICTFE *dfe); extern usec_t dictionary_foreach_done(DICTFE *dfe); // Get statistics about the dictionary -// If DICTIONARY_FLAG_WITH_STATISTICS is not set, these return zero -extern size_t dictionary_stats_allocated_memory(DICTIONARY *dict); -extern size_t dictionary_stats_entries(DICTIONARY *dict); +extern long int dictionary_stats_allocated_memory(DICTIONARY *dict); +extern long int dictionary_stats_entries(DICTIONARY *dict); +extern size_t dictionary_stats_version(DICTIONARY *dict); extern size_t dictionary_stats_inserts(DICTIONARY *dict); extern size_t dictionary_stats_searches(DICTIONARY *dict); extern size_t dictionary_stats_deletes(DICTIONARY *dict); extern size_t dictionary_stats_resets(DICTIONARY *dict); +extern size_t dictionary_stats_walkthroughs(DICTIONARY *dict); +extern size_t dictionary_stats_referenced_items(DICTIONARY *dict); extern int dictionary_unittest(size_t entries); +// ---------------------------------------------------------------------------- +// STRING implementation + +typedef struct netdata_string STRING; +extern STRING *string_strdupz(const char *str); +extern STRING *string_dup(STRING *string); +extern void string_freez(STRING *string); +extern size_t string_length(STRING *string); +extern const char *string2str(STRING *string) NEVERNULL; + +// keep common prefix/suffix and replace everything else with [x] +extern STRING *string_2way_merge(STRING *a, STRING *b); + +static inline int string_cmp(STRING *s1, STRING *s2) { + // STRINGs are deduplicated, so the same strings have the same pointer + if(unlikely(s1 == s2)) return 0; + + // they differ, do the typical comparison + return strcmp(string2str(s1), string2str(s2)); +} + #endif /* NETDATA_DICTIONARY_H */ diff --git a/libnetdata/ebpf/ebpf.c b/libnetdata/ebpf/ebpf.c index ffb602307..c48f2f24e 100644 --- a/libnetdata/ebpf/ebpf.c +++ b/libnetdata/ebpf/ebpf.c @@ -608,7 +608,7 @@ static void ebpf_update_maps(ebpf_module_t *em, struct bpf_object *obj) void ebpf_update_controller(int fd, ebpf_module_t *em) { uint32_t key = NETDATA_CONTROLLER_APPS_ENABLED; - uint32_t value = em->apps_charts | em->cgroup_charts; + uint32_t value = (em->apps_charts & NETDATA_EBPF_APPS_FLAG_YES) | em->cgroup_charts; int ret = bpf_map_update_elem(fd, &key, &value, 0); if (ret) error("Add key(%u) for controller table failed.", key); @@ -969,7 +969,7 @@ void ebpf_update_module_using_config(ebpf_module_t *modules) EBPF_CFG_UPDATE_EVERY, modules->update_every); modules->apps_charts = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_APPLICATION, - modules->apps_charts); + (int) (modules->apps_charts & NETDATA_EBPF_APPS_FLAG_YES)); modules->cgroup_charts = appconfig_get_boolean(modules->cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_CGROUP, modules->cgroup_charts); diff --git a/libnetdata/ebpf/ebpf.h b/libnetdata/ebpf/ebpf.h index ec486b59a..55b68a51e 100644 --- a/libnetdata/ebpf/ebpf.h +++ b/libnetdata/ebpf/ebpf.h @@ -34,6 +34,7 @@ #define EBPF_CFG_PROGRAM_PATH "btf path" #define EBPF_CFG_UPDATE_EVERY "update every" +#define EBPF_CFG_UPDATE_APPS_EVERY_DEFAULT 10 #define EBPF_CFG_PID_SIZE "pid table size" #define EBPF_CFG_APPLICATION "apps" #define EBPF_CFG_CGROUP "cgroups" @@ -212,6 +213,12 @@ typedef struct ebpf_plugin_stats { uint32_t trampolines; // Number of trampolines used } ebpf_plugin_stats_t; +typedef enum netdata_apps_integration_flags { + NETDATA_EBPF_APPS_FLAG_NO, + NETDATA_EBPF_APPS_FLAG_YES, + NETDATA_EBPF_APPS_FLAG_CHART_CREATED +} netdata_apps_integration_flags_t; + typedef struct ebpf_module { const char *thread_name; const char *config_name; @@ -219,7 +226,7 @@ typedef struct ebpf_module { void *(*start_routine)(void *); int update_every; int global_charts; - int apps_charts; + netdata_apps_integration_flags_t apps_charts; int cgroup_charts; netdata_run_mode_t mode; uint32_t thread_id; @@ -233,6 +240,8 @@ typedef struct ebpf_module { uint64_t kernels; netdata_ebpf_load_mode_t load; netdata_ebpf_targets_t *targets; + struct bpf_link **probe_links; + struct bpf_object *objects; } ebpf_module_t; extern int ebpf_get_kernel_version(); @@ -266,6 +275,43 @@ typedef struct netdata_ebpf_histogram { uint64_t histogram[NETDATA_EBPF_HIST_MAX_BINS]; } netdata_ebpf_histogram_t; +typedef struct ebpf_filesystem_partitions { + char *filesystem; + char *optional_filesystem; + char *family; + char *family_name; + struct bpf_object *objects; + struct bpf_link **probe_links; + + netdata_ebpf_histogram_t hread; + netdata_ebpf_histogram_t hwrite; + netdata_ebpf_histogram_t hopen; + netdata_ebpf_histogram_t hadditional; + + uint32_t flags; + uint32_t enabled; + + ebpf_addresses_t addresses; + uint64_t kernels; +} ebpf_filesystem_partitions_t; + +typedef struct ebpf_sync_syscalls { + char *syscall; + int enabled; + uint32_t flags; + + // BTF structure + struct bpf_object *objects; + struct bpf_link **probe_links; + + // BPF structure +#ifdef LIBBPF_MAJOR_VERSION + struct sync_bpf *sync_obj; +#else + void *sync_obj; +#endif +} ebpf_sync_syscalls_t; + extern void ebpf_histogram_dimension_cleanup(char **ptr, size_t length); // Tracepoint helpers diff --git a/libnetdata/eval/eval.c b/libnetdata/eval/eval.c index 7ca45882f..e86cbd587 100644 --- a/libnetdata/eval/eval.c +++ b/libnetdata/eval/eval.c @@ -9,7 +9,7 @@ typedef struct eval_value { int type; union { - calculated_number number; + NETDATA_DOUBLE number; EVAL_VARIABLE *variable; struct eval_node *expression; }; @@ -54,16 +54,16 @@ typedef struct eval_node { static inline void eval_node_free(EVAL_NODE *op); static inline EVAL_NODE *parse_full_expression(const char **string, int *error); static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error); -static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); +static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); static inline void print_parsed_as_node(BUFFER *out, EVAL_NODE *op, int *error); -static inline void print_parsed_as_constant(BUFFER *out, calculated_number n); +static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n); // ---------------------------------------------------------------------------- // evaluation of expressions -static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { +static inline NETDATA_DOUBLE eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABLE *v, int *error) { static uint32_t this_hash = 0, now_hash = 0, after_hash = 0, before_hash = 0, status_hash = 0, removed_hash = 0, uninitialized_hash = 0, undefined_hash = 0, clear_hash = 0, warning_hash = 0, critical_hash = 0; - calculated_number n; + NETDATA_DOUBLE n; if(unlikely(this_hash == 0)) { this_hash = simple_hash("this"); @@ -179,8 +179,8 @@ static inline calculated_number eval_variable(EVAL_EXPRESSION *exp, EVAL_VARIABL return NAN; } -static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) { - calculated_number n; +static inline NETDATA_DOUBLE eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, int *error) { + NETDATA_DOUBLE n; switch(v->type) { case EVAL_VALUE_EXPRESSION: @@ -204,101 +204,101 @@ static inline calculated_number eval_value(EVAL_EXPRESSION *exp, EVAL_VALUE *v, return n; } -static inline int is_true(calculated_number n) { +static inline int is_true(NETDATA_DOUBLE n) { if(isnan(n)) return 0; if(isinf(n)) return 1; if(n == 0) return 0; return 1; } -calculated_number eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_and(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return is_true(eval_value(exp, &op->ops[0], error)) && is_true(eval_value(exp, &op->ops[1], error)); } -calculated_number eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_or(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return is_true(eval_value(exp, &op->ops[0], error)) || is_true(eval_value(exp, &op->ops[1], error)); } -calculated_number eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_greater_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); return isgreaterequal(n1, n2); } -calculated_number eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_less_than_or_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); return islessequal(n1, n2); } -calculated_number eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); if(isnan(n1) && isnan(n2)) return 1; if(isinf(n1) && isinf(n2)) return 1; if(isnan(n1) || isnan(n2)) return 0; if(isinf(n1) || isinf(n2)) return 0; - return calculated_number_equal(n1, n2); + return considered_equal_ndd(n1, n2); } -calculated_number eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_not_equal(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return !eval_equal(exp, op, error); } -calculated_number eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_less(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); return isless(n1, n2); } -calculated_number eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_greater(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); return isgreater(n1, n2); } -calculated_number eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); if(isnan(n1) || isnan(n2)) return NAN; if(isinf(n1) || isinf(n2)) return INFINITY; return n1 + n2; } -calculated_number eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); if(isnan(n1) || isnan(n2)) return NAN; if(isinf(n1) || isinf(n2)) return INFINITY; return n1 - n2; } -calculated_number eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_multiply(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); if(isnan(n1) || isnan(n2)) return NAN; if(isinf(n1) || isinf(n2)) return INFINITY; return n1 * n2; } -calculated_number eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); - calculated_number n2 = eval_value(exp, &op->ops[1], error); +NETDATA_DOUBLE eval_divide(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); + NETDATA_DOUBLE n2 = eval_value(exp, &op->ops[1], error); if(isnan(n1) || isnan(n2)) return NAN; if(isinf(n1) || isinf(n2)) return INFINITY; return n1 / n2; } -calculated_number eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_nop(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return eval_value(exp, &op->ops[0], error); } -calculated_number eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_not(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return !is_true(eval_value(exp, &op->ops[0], error)); } -calculated_number eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_sign_plus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { return eval_value(exp, &op->ops[0], error); } -calculated_number eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); +NETDATA_DOUBLE eval_sign_minus(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); if(isnan(n1)) return NAN; if(isinf(n1)) return INFINITY; return -n1; } -calculated_number eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { - calculated_number n1 = eval_value(exp, &op->ops[0], error); +NETDATA_DOUBLE eval_abs(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { + NETDATA_DOUBLE n1 = eval_value(exp, &op->ops[0], error); if(isnan(n1)) return NAN; if(isinf(n1)) return INFINITY; return ABS(n1); } -calculated_number eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +NETDATA_DOUBLE eval_if_then_else(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { if(is_true(eval_value(exp, &op->ops[0], error))) return eval_value(exp, &op->ops[1], error); else @@ -310,7 +310,7 @@ static struct operator { char precedence; char parameters; char isfunction; - calculated_number (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); + NETDATA_DOUBLE (*eval)(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error); } operators[256] = { // this is a random access array // we always access it with a known EVAL_OPERATOR_X @@ -341,13 +341,13 @@ static struct operator { #define eval_precedence(operator) (operators[(unsigned char)(operator)].precedence) -static inline calculated_number eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { +static inline NETDATA_DOUBLE eval_node(EVAL_EXPRESSION *exp, EVAL_NODE *op, int *error) { if(unlikely(op->count != operators[op->operator].parameters)) { *error = EVAL_ERROR_INVALID_NUMBER_OF_OPERANDS; return 0; } - calculated_number n = operators[op->operator].eval(exp, op, error); + NETDATA_DOUBLE n = operators[op->operator].eval(exp, op, error); return n; } @@ -360,7 +360,7 @@ static inline void print_parsed_as_variable(BUFFER *out, EVAL_VARIABLE *v, int * buffer_sprintf(out, "${%s}", v->name); } -static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) { +static inline void print_parsed_as_constant(BUFFER *out, NETDATA_DOUBLE n) { if(unlikely(isnan(n))) { buffer_strcat(out, "nan"); return; @@ -372,7 +372,7 @@ static inline void print_parsed_as_constant(BUFFER *out, calculated_number n) { } char b[100+1], *s; - snprintfz(b, 100, CALCULATED_NUMBER_FORMAT, n); + snprintfz(b, 100, NETDATA_DOUBLE_FORMAT, n); s = &b[strlen(b) - 1]; while(s > b && *s == '0') { @@ -737,9 +737,9 @@ static inline int parse_variable(const char **string, char *buffer, size_t len) return 0; } -static inline int parse_constant(const char **string, calculated_number *number) { +static inline int parse_constant(const char **string, NETDATA_DOUBLE *number) { char *end = NULL; - calculated_number n = str2ld(*string, &end); + NETDATA_DOUBLE n = str2ndd(*string, &end); if(unlikely(!end || *string == end)) { *number = 0; return 0; @@ -845,7 +845,7 @@ static inline void eval_node_set_value_to_node(EVAL_NODE *op, int pos, EVAL_NODE op->ops[pos].expression = value; } -static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, calculated_number value) { +static inline void eval_node_set_value_to_constant(EVAL_NODE *op, int pos, NETDATA_DOUBLE value) { if(pos >= op->count) fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1); @@ -911,7 +911,7 @@ static inline EVAL_NODE *parse_next_operand_given_its_operator(const char **stri static inline EVAL_NODE *parse_one_full_operand(const char **string, int *error) { char variable_buffer[EVAL_MAX_VARIABLE_NAME_LENGTH + 1]; EVAL_NODE *op1 = NULL; - calculated_number number; + NETDATA_DOUBLE number; *error = EVAL_ERROR_OK; diff --git a/libnetdata/eval/eval.h b/libnetdata/eval/eval.h index fc03d7c1f..086d171aa 100644 --- a/libnetdata/eval/eval.h +++ b/libnetdata/eval/eval.h @@ -28,11 +28,11 @@ typedef struct eval_expression { const char *parsed_as; RRDCALC_STATUS *status; - calculated_number *myself; + NETDATA_DOUBLE *myself; time_t *after; time_t *before; - calculated_number result; + NETDATA_DOUBLE result; int error; BUFFER *error_msg; @@ -83,6 +83,6 @@ extern const char *expression_strerror(int error); // 2 = FAILED, the error message is in: buffer_tostring(expression->error_msg) extern int expression_evaluate(EVAL_EXPRESSION *expression); -extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result); +extern int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result); #endif //NETDATA_EVAL_H diff --git a/libnetdata/inlined.h b/libnetdata/inlined.h index 7e7d8ebed..5c265fc01 100644 --- a/libnetdata/inlined.h +++ b/libnetdata/inlined.h @@ -151,77 +151,6 @@ static inline long long str2ll(const char *s, char **endptr) { return n; } -static inline long double str2ld(const char *s, char **endptr) { - int negative = 0; - const char *start = s; - unsigned long long integer_part = 0; - unsigned long decimal_part = 0; - size_t decimal_digits = 0; - - switch(*s) { - case '-': - s++; - negative = 1; - break; - - case '+': - s++; - break; - - case 'n': - if(s[1] == 'a' && s[2] == 'n') { - if(endptr) *endptr = (char *)&s[3]; - return NAN; - } - break; - - case 'i': - if(s[1] == 'n' && s[2] == 'f') { - if(endptr) *endptr = (char *)&s[3]; - return INFINITY; - } - break; - - default: - break; - } - - while (*s >= '0' && *s <= '9') { - integer_part = (integer_part * 10) + (*s - '0'); - s++; - } - - if(unlikely(*s == '.')) { - decimal_part = 0; - s++; - - while (*s >= '0' && *s <= '9') { - decimal_part = (decimal_part * 10) + (*s - '0'); - s++; - decimal_digits++; - } - } - - if(unlikely(*s == 'e' || *s == 'E')) - return strtold(start, endptr); - - if(unlikely(endptr)) - *endptr = (char *)s; - - if(unlikely(negative)) { - if(unlikely(decimal_digits)) - return -((long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits)); - else - return -((long double)integer_part); - } - else { - if(unlikely(decimal_digits)) - return (long double)integer_part + (long double)decimal_part / powl(10.0, decimal_digits); - else - return (long double)integer_part; - } -} - static inline char *strncpyz(char *dst, const char *src, size_t n) { char *p = dst; @@ -233,21 +162,21 @@ static inline char *strncpyz(char *dst, const char *src, size_t n) { return p; } -static inline void sanitize_json_string(char *dst, char *src, size_t len) { - while (*src != '\0' && len > 1) { - if (*src == '\\' || *src == '\"' || *src < 0x1F) { - if (*src < 0x1F) { - *dst++ = '_'; - src++; - len--; - } else { - *dst++ = '\\'; - *dst++ = *src++; - len -= 2; - } - } else { +static inline void sanitize_json_string(char *dst, const char *src, size_t dst_size) { + while (*src != '\0' && dst_size > 1) { + if (*src < 0x1F) { + *dst++ = '_'; + src++; + dst_size--; + } + else if (*src == '\\' || *src == '\"') { + *dst++ = '\\'; + *dst++ = *src++; + dst_size -= 2; + } + else { *dst++ = *src++; - len--; + dst_size--; } } *dst = '\0'; diff --git a/libnetdata/json/json.c b/libnetdata/json/json.c index 1f391eeaa..e03556b50 100644 --- a/libnetdata/json/json.c +++ b/libnetdata/json/json.c @@ -111,7 +111,7 @@ int json_callback_print(JSON_ENTRY *e) break; case JSON_NUMBER: - sprintf(txt,"%Lf", e->data.number); + sprintf(txt, NETDATA_DOUBLE_FORMAT_AUTO, e->data.number); buffer_strcat(wb,txt); break; @@ -168,7 +168,7 @@ static inline void json_jsonc_set_integer(JSON_ENTRY *e, char *key, int64_t valu e->type = JSON_NUMBER; memcpy(e->name, key, len); e->name[len] = 0; - e->data.number = value; + e->data.number = (NETDATA_DOUBLE)value; } /** diff --git a/libnetdata/json/json.h b/libnetdata/json/json.h index 79b58b170..b43f06b50 100644 --- a/libnetdata/json/json.h +++ b/libnetdata/json/json.h @@ -3,8 +3,13 @@ #if ENABLE_JSONC -# include <json-c/json.h> -#endif +#include <json-c/json.h> +// fix an older json-c bug +// https://github.com/json-c/json-c/issues/135 +#ifdef error_description +#undef error_description +#endif // error_description +#endif // ENABLE_JSONC #include "jsmn.h" @@ -26,12 +31,12 @@ typedef struct json_entry { char name[JSON_NAME_LEN + 1]; char fullname[JSON_FULLNAME_LEN + 1]; union { - char *string; // type == JSON_STRING - long double number; // type == JSON_NUMBER - int boolean; // type == JSON_BOOLEAN - size_t items; // type == JSON_ARRAY + char *string; // type == JSON_STRING + NETDATA_DOUBLE number; // type == JSON_NUMBER + int boolean; // type == JSON_BOOLEAN + size_t items; // type == JSON_ARRAY } data; - size_t pos; // the position of this item in its parent + size_t pos; // the position of this item in its parent char *original_string; diff --git a/libnetdata/libjudy/src/Judy.h b/libnetdata/libjudy/src/Judy.h new file mode 100644 index 000000000..adfb5b53b --- /dev/null +++ b/libnetdata/libjudy/src/Judy.h @@ -0,0 +1,622 @@ +#ifndef _JUDY_INCLUDED +#define _JUDY_INCLUDED +// _________________ +// +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.52 $ $Source: /judy/src/Judy.h $ +// +// HEADER FILE FOR EXPORTED FEATURES IN JUDY LIBRARY, libJudy.* +// +// See the manual entries for details. +// +// Note: This header file uses old-style comments on #-directive lines and +// avoids "()" on macro names in comments for compatibility with older cc -Aa +// and some tools on some platforms. + + +// PLATFORM-SPECIFIC + +#ifdef JU_WIN /* =============================================== */ + +typedef __int8 int8_t; +typedef __int16 int16_t; +typedef __int32 int32_t; +typedef __int64 int64_t; + +typedef unsigned __int8 uint8_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; + +#else /* ================ ! JU_WIN ============================= */ + +// ISO C99: 7.8 Format conversion of integer types <inttypes.h> +#include <inttypes.h> /* if this FAILS, try #include <stdint.h> */ + +// ISO C99: 7.18 Integer types uint*_t +//#include <stdint.h> + +#endif /* ================ ! JU_WIN ============================= */ + +// ISO C99 Standard: 7.20 General utilities +#include <stdlib.h> + +// ISO C99 Standard: 7.10/5.2.4.2.1 Sizes of integer types +#include <limits.h> + +#ifdef __cplusplus /* support use by C++ code */ +extern "C" { +#endif + + +// **************************************************************************** +// DECLARE SOME BASE TYPES IN CASE THEY ARE MISSING: +// +// These base types include "const" where appropriate, but only where of +// interest to the caller. For example, a caller cares that a variable passed +// by reference will not be modified, such as, "const void * Pindex", but not +// that the called function internally does not modify the pointer itself, such +// as, "void * const Pindex". +// +// Note that its OK to pass a Pvoid_t to a Pcvoid_t; the latter is the same, +// only constant. Callers need to do this so they can also pass & Pvoid_t to +// PPvoid_t (non-constant). + +#ifndef _PCVOID_T +#define _PCVOID_T +typedef const void * Pcvoid_t; +#endif + +#ifndef _PVOID_T +#define _PVOID_T +typedef void * Pvoid_t; +typedef void ** PPvoid_t; +#endif + +#ifndef _WORD_T +#define _WORD_T +typedef unsigned long Word_t, * PWord_t; // expect 32-bit or 64-bit words. +#endif + +#ifndef NULL +#define NULL 0 +#endif + + +// **************************************************************************** +// SUPPORT FOR ERROR HANDLING: +// +// Judy error numbers: +// +// Note: These are an enum so theres a related typedef, but the numbers are +// spelled out so you can map a number back to its name. + +typedef enum // uint8_t -- but C does not support this type of enum. +{ + +// Note: JU_ERRNO_NONE and JU_ERRNO_FULL are not real errors. They specify +// conditions which are otherwise impossible return values from 32-bit +// Judy1Count, which has 2^32 + 1 valid returns (0..2^32) plus one error +// return. These pseudo-errors support the return values that cannot otherwise +// be unambiguously represented in a 32-bit word, and will never occur on a +// 64-bit system. + + JU_ERRNO_NONE = 0, + JU_ERRNO_FULL = 1, + JU_ERRNO_NFMAX = JU_ERRNO_FULL, + +// JU_ERRNO_NOMEM comes from malloc(3C) when Judy cannot obtain needed memory. +// The system errno value is also set to ENOMEM. This error can be recoverable +// if the calling application frees other memory. +// +// TBD: Currently there is no guarantee the Judy array has no memory leaks +// upon JU_ERRNO_NOMEM. + + JU_ERRNO_NOMEM = 2, + +// Problems with parameters from the calling program: +// +// JU_ERRNO_NULLPPARRAY means PPArray was null; perhaps PArray was passed where +// &PArray was intended. Similarly, JU_ERRNO_NULLPINDEX means PIndex was null; +// perhaps &Index was intended. Also, JU_ERRNO_NONNULLPARRAY, +// JU_ERRNO_NULLPVALUE, and JU_ERRNO_UNSORTED, all added later (hence with +// higher numbers), mean: A non-null array was passed in where a null pointer +// was required; PValue was null; and unsorted indexes were detected. + + JU_ERRNO_NULLPPARRAY = 3, // see above. + JU_ERRNO_NONNULLPARRAY = 10, // see above. + JU_ERRNO_NULLPINDEX = 4, // see above. + JU_ERRNO_NULLPVALUE = 11, // see above. + JU_ERRNO_NOTJUDY1 = 5, // PArray is not to a Judy1 array. + JU_ERRNO_NOTJUDYL = 6, // PArray is not to a JudyL array. + JU_ERRNO_NOTJUDYSL = 7, // PArray is not to a JudySL array. + JU_ERRNO_UNSORTED = 12, // see above. + +// Errors below this point are not recoverable; further tries to access the +// Judy array might result in EFAULT and a core dump: +// +// JU_ERRNO_OVERRUN occurs when Judy detects, upon reallocation, that a block +// of memory in its own freelist was modified since being freed. + + JU_ERRNO_OVERRUN = 8, + +// JU_ERRNO_CORRUPT occurs when Judy detects an impossible value in a Judy data +// structure: +// +// Note: The Judy data structure contains some redundant elements that support +// this type of checking. + + JU_ERRNO_CORRUPT = 9 + +// Warning: At least some C or C++ compilers do not tolerate a trailing comma +// above here. At least we know of one case, in aCC; see JAGad58928. + +} JU_Errno_t; + + +// Judy errno structure: +// +// WARNING: For compatibility with possible future changes, the fields of this +// struct should not be referenced directly. Instead use the macros supplied +// below. + +// This structure should be declared on the stack in a threaded process. + +typedef struct J_UDY_ERROR_STRUCT +{ + JU_Errno_t je_Errno; // one of the enums above. + int je_ErrID; // often an internal source line number. + Word_t je_reserved[4]; // for future backward compatibility. + +} JError_t, * PJError_t; + + +// Related macros: +// +// Fields from error struct: + +#define JU_ERRNO(PJError) ((PJError)->je_Errno) +#define JU_ERRID(PJError) ((PJError)->je_ErrID) + +// For checking return values from various Judy functions: +// +// Note: Define JERR as -1, not as the seemingly more portable (Word_t) +// (~0UL), to avoid a compiler "overflow in implicit constant conversion" +// warning. + +#define JERR (-1) /* functions returning int or Word_t */ +#define PJERR ((Pvoid_t) (~0UL)) /* mainly for use here, see below */ +#define PPJERR ((PPvoid_t) (~0UL)) /* functions that return PPvoid_t */ + +// Convenience macro for when detailed error information (PJError_t) is not +// desired by the caller; a purposely short name: + +#define PJE0 ((PJError_t) NULL) + + +// **************************************************************************** +// JUDY FUNCTIONS: +// +// P_JE is a shorthand for use below: + +#define P_JE PJError_t PJError + +// **************************************************************************** +// JUDY1 FUNCTIONS: + +extern int Judy1Test( Pcvoid_t PArray, Word_t Index, P_JE); +extern int Judy1Set( PPvoid_t PPArray, Word_t Index, P_JE); +extern int Judy1SetArray( PPvoid_t PPArray, Word_t Count, + const Word_t * const PIndex, + P_JE); +extern int Judy1Unset( PPvoid_t PPArray, Word_t Index, P_JE); +extern Word_t Judy1Count( Pcvoid_t PArray, Word_t Index1, + Word_t Index2, P_JE); +extern int Judy1ByCount( Pcvoid_t PArray, Word_t Count, + Word_t * PIndex, P_JE); +extern Word_t Judy1FreeArray( PPvoid_t PPArray, P_JE); +extern Word_t Judy1MemUsed( Pcvoid_t PArray); +extern Word_t Judy1MemActive( Pcvoid_t PArray); +extern int Judy1First( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1Next( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1Last( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1Prev( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1FirstEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1NextEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1LastEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int Judy1PrevEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); + +extern PPvoid_t JudyLGet( Pcvoid_t PArray, Word_t Index, P_JE); +extern PPvoid_t JudyLIns( PPvoid_t PPArray, Word_t Index, P_JE); +extern int JudyLInsArray( PPvoid_t PPArray, Word_t Count, + const Word_t * const PIndex, + const Word_t * const PValue, + +// **************************************************************************** +// JUDYL FUNCTIONS: + P_JE); +extern int JudyLDel( PPvoid_t PPArray, Word_t Index, P_JE); +extern Word_t JudyLCount( Pcvoid_t PArray, Word_t Index1, + Word_t Index2, P_JE); +extern PPvoid_t JudyLByCount( Pcvoid_t PArray, Word_t Count, + Word_t * PIndex, P_JE); +extern Word_t JudyLFreeArray( PPvoid_t PPArray, P_JE); +extern Word_t JudyLMemUsed( Pcvoid_t PArray); +extern Word_t JudyLMemActive( Pcvoid_t PArray); +extern PPvoid_t JudyLFirst( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern PPvoid_t JudyLNext( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern PPvoid_t JudyLLast( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern PPvoid_t JudyLPrev( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int JudyLFirstEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int JudyLNextEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int JudyLLastEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); +extern int JudyLPrevEmpty( Pcvoid_t PArray, Word_t * PIndex, P_JE); + +// **************************************************************************** +// JUDYSL FUNCTIONS: + +extern PPvoid_t JudySLGet( Pcvoid_t, const uint8_t * Index, P_JE); +extern PPvoid_t JudySLIns( PPvoid_t, const uint8_t * Index, P_JE); +extern int JudySLDel( PPvoid_t, const uint8_t * Index, P_JE); +extern Word_t JudySLFreeArray( PPvoid_t, P_JE); +extern PPvoid_t JudySLFirst( Pcvoid_t, uint8_t * Index, P_JE); +extern PPvoid_t JudySLNext( Pcvoid_t, uint8_t * Index, P_JE); +extern PPvoid_t JudySLLast( Pcvoid_t, uint8_t * Index, P_JE); +extern PPvoid_t JudySLPrev( Pcvoid_t, uint8_t * Index, P_JE); + +// **************************************************************************** +// JUDYHSL FUNCTIONS: + +extern PPvoid_t JudyHSGet( Pcvoid_t, void *, Word_t); +extern PPvoid_t JudyHSIns( PPvoid_t, void *, Word_t, P_JE); +extern int JudyHSDel( PPvoid_t, void *, Word_t, P_JE); +extern Word_t JudyHSFreeArray( PPvoid_t, P_JE); + +extern const char *Judy1MallocSizes; +extern const char *JudyLMallocSizes; + +// **************************************************************************** +// JUDY memory interface to malloc() FUNCTIONS: + +extern Word_t JudyMalloc(Word_t); // words reqd => words allocd. +extern Word_t JudyMallocVirtual(Word_t); // words reqd => words allocd. +extern void JudyFree(Pvoid_t, Word_t); // free, size in words. +extern void JudyFreeVirtual(Pvoid_t, Word_t); // free, size in words. + +#define JLAP_INVALID 0x1 /* flag to mark pointer "not a Judy array" */ + +// **************************************************************************** +// MACRO EQUIVALENTS FOR JUDY FUNCTIONS: +// +// The following macros, such as J1T, are shorthands for calling Judy functions +// with parameter address-of and detailed error checking included. Since they +// are macros, the error checking code is replicated each time the macro is +// used, but it runs fast in the normal case of no error. +// +// If the caller does not like the way the default JUDYERROR macro handles +// errors (such as an exit(1) call when out of memory), they may define their +// own before the "#include <Judy.h>". A routine such as HandleJudyError +// could do checking on specific error numbers and print a different message +// dependent on the error. The following is one example: +// +// Note: the back-slashes are removed because some compilers will not accept +// them in comments. +// +// void HandleJudyError(uint8_t *, int, uint8_t *, int, int); +// #define JUDYERROR(CallerFile, CallerLine, JudyFunc, JudyErrno, JudyErrID) +// { +// HandleJudyError(CallerFile, CallerLine, JudyFunc, JudyErrno, JudyErrID); +// } +// +// The routine HandleJudyError could do checking on specific error numbers and +// print a different message dependent on the error. +// +// The macro receives five parameters that are: +// +// 1. CallerFile: Source filename where a Judy call returned a serious error. +// 2. CallerLine: Line number in that source file. +// 3. JudyFunc: Name of Judy function reporting the error. +// 4. JudyErrno: One of the JU_ERRNO* values enumerated above. +// 5. JudyErrID: The je_ErrID field described above. + +#ifndef JUDYERROR_NOTEST +#ifndef JUDYERROR /* supply a default error macro */ +#include <stdio.h> + +#define JUDYERROR(CallerFile, CallerLine, JudyFunc, JudyErrno, JudyErrID) \ + { \ + (void) fprintf(stderr, "File '%s', line %d: %s(), " \ + "JU_ERRNO_* == %d, ID == %d\n", \ + CallerFile, CallerLine, \ + JudyFunc, JudyErrno, JudyErrID); \ + exit(1); \ + } + +#endif /* JUDYERROR */ +#endif /* JUDYERROR_NOTEST */ + +// If the JUDYERROR macro is not desired at all, then the following eliminates +// it. However, the return code from each Judy function (that is, the first +// parameter of each macro) must be checked by the caller to assure that an +// error did not occur. +// +// Example: +// +// #define JUDYERROR_NOTEST 1 +// #include <Judy.h> +// +// or use this cc option at compile time: +// +// cc -DJUDYERROR_NOTEST ... +// +// Example code: +// +// J1S(Rc, PArray, Index); +// if (Rc == JERR) goto ...error +// +// or: +// +// JLI(PValue, PArray, Index); +// if (PValue == PJERR) goto ...error + + +// Internal shorthand macros for writing the J1S, etc. macros: + +#ifdef JUDYERROR_NOTEST /* ============================================ */ + +// "Judy Set Error": + +#define J_SE(FuncName,Errno) ((void) 0) + +// Note: In each J_*() case below, the digit is the number of key parameters +// to the Judy*() call. Just assign the Func result to the callers Rc value +// without a cast because none is required, and this keeps the API simpler. +// However, a family of different J_*() macros is needed to support the +// different numbers of key parameters (0,1,2) and the Func return type. +// +// In the names below, "I" = integer result; "P" = pointer result. Note, the +// Funcs for J_*P() return PPvoid_t, but cast this to a Pvoid_t for flexible, +// error-free assignment, and then compare to PJERR. + +#define J_0I(Rc,PArray,Func,FuncName) \ + { (Rc) = Func(PArray, PJE0); } + +#define J_1I(Rc,PArray,Index,Func,FuncName) \ + { (Rc) = Func(PArray, Index, PJE0); } + +#define J_1P(PV,PArray,Index,Func,FuncName) \ + { (PV) = (Pvoid_t) Func(PArray, Index, PJE0); } + +#define J_2I(Rc,PArray,Index,Arg2,Func,FuncName) \ + { (Rc) = Func(PArray, Index, Arg2, PJE0); } + +#define J_2C(Rc,PArray,Index1,Index2,Func,FuncName) \ + { (Rc) = Func(PArray, Index1, Index2, PJE0); } + +#define J_2P(PV,PArray,Index,Arg2,Func,FuncName) \ + { (PV) = (Pvoid_t) Func(PArray, Index, Arg2, PJE0); } + +// Variations for Judy*Set/InsArray functions: + +#define J_2AI(Rc,PArray,Count,PIndex,Func,FuncName) \ + { (Rc) = Func(PArray, Count, PIndex, PJE0); } +#define J_3AI(Rc,PArray,Count,PIndex,PValue,Func,FuncName) \ + { (Rc) = Func(PArray, Count, PIndex, PValue, PJE0); } + +#else /* ================ ! JUDYERROR_NOTEST ============================= */ + +#define J_E(FuncName,PJE) \ + JUDYERROR(__FILE__, __LINE__, FuncName, JU_ERRNO(PJE), JU_ERRID(PJE)) + +#define J_SE(FuncName,Errno) \ + { \ + JError_t J_Error; \ + JU_ERRNO(&J_Error) = (Errno); \ + JU_ERRID(&J_Error) = __LINE__; \ + J_E(FuncName, &J_Error); \ + } + +// Note: In each J_*() case below, the digit is the number of key parameters +// to the Judy*() call. Just assign the Func result to the callers Rc value +// without a cast because none is required, and this keeps the API simpler. +// However, a family of different J_*() macros is needed to support the +// different numbers of key parameters (0,1,2) and the Func return type. +// +// In the names below, "I" = integer result; "P" = pointer result. Note, the +// Funcs for J_*P() return PPvoid_t, but cast this to a Pvoid_t for flexible, +// error-free assignment, and then compare to PJERR. + +#define J_0I(Rc,PArray,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = Func(PArray, &J_Error)) == JERR) \ + J_E(FuncName, &J_Error); \ + } + +#define J_1I(Rc,PArray,Index,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = Func(PArray, Index, &J_Error)) == JERR) \ + J_E(FuncName, &J_Error); \ + } + +#define J_1P(Rc,PArray,Index,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = (Pvoid_t) Func(PArray, Index, &J_Error)) == PJERR) \ + J_E(FuncName, &J_Error); \ + } + +#define J_2I(Rc,PArray,Index,Arg2,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = Func(PArray, Index, Arg2, &J_Error)) == JERR) \ + J_E(FuncName, &J_Error); \ + } + +// Variation for Judy*Count functions, which return 0, not JERR, for error (and +// also for other non-error cases): +// +// Note: JU_ERRNO_NFMAX should only apply to 32-bit Judy1, but this header +// file lacks the necessary ifdefs to make it go away otherwise, so always +// check against it. + +#define J_2C(Rc,PArray,Index1,Index2,Func,FuncName) \ + { \ + JError_t J_Error; \ + if ((((Rc) = Func(PArray, Index1, Index2, &J_Error)) == 0) \ + && (JU_ERRNO(&J_Error) > JU_ERRNO_NFMAX)) \ + { \ + J_E(FuncName, &J_Error); \ + } \ + } + +#define J_2P(PV,PArray,Index,Arg2,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((PV) = (Pvoid_t) Func(PArray, Index, Arg2, &J_Error)) \ + == PJERR) J_E(FuncName, &J_Error); \ + } + +// Variations for Judy*Set/InsArray functions: + +#define J_2AI(Rc,PArray,Count,PIndex,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = Func(PArray, Count, PIndex, &J_Error)) == JERR) \ + J_E(FuncName, &J_Error); \ + } + +#define J_3AI(Rc,PArray,Count,PIndex,PValue,Func,FuncName) \ + { \ + JError_t J_Error; \ + if (((Rc) = Func(PArray, Count, PIndex, PValue, &J_Error)) \ + == JERR) J_E(FuncName, &J_Error); \ + } + +#endif /* ================ ! JUDYERROR_NOTEST ============================= */ + +// Some of the macros are special cases that use inlined shortcuts for speed +// with root-level leaves: + +// This is a slower version with current processors, but in the future... + +#define J1T(Rc,PArray,Index) \ + (Rc) = Judy1Test((Pvoid_t)(PArray), Index, PJE0) + +#define J1S( Rc, PArray, Index) \ + J_1I(Rc, (&(PArray)), Index, Judy1Set, "Judy1Set") +#define J1SA(Rc, PArray, Count, PIndex) \ + J_2AI(Rc,(&(PArray)), Count, PIndex, Judy1SetArray, "Judy1SetArray") +#define J1U( Rc, PArray, Index) \ + J_1I(Rc, (&(PArray)), Index, Judy1Unset, "Judy1Unset") +#define J1F( Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1First, "Judy1First") +#define J1N( Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1Next, "Judy1Next") +#define J1L( Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1Last, "Judy1Last") +#define J1P( Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1Prev, "Judy1Prev") +#define J1FE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1FirstEmpty, "Judy1FirstEmpty") +#define J1NE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1NextEmpty, "Judy1NextEmpty") +#define J1LE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1LastEmpty, "Judy1LastEmpty") +#define J1PE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), Judy1PrevEmpty, "Judy1PrevEmpty") +#define J1C( Rc, PArray, Index1, Index2) \ + J_2C(Rc, PArray, Index1, Index2, Judy1Count, "Judy1Count") +#define J1BC(Rc, PArray, Count, Index) \ + J_2I(Rc, PArray, Count, &(Index), Judy1ByCount, "Judy1ByCount") +#define J1FA(Rc, PArray) \ + J_0I(Rc, (&(PArray)), Judy1FreeArray, "Judy1FreeArray") +#define J1MU(Rc, PArray) \ + (Rc) = Judy1MemUsed(PArray) + +#define JLG(PV,PArray,Index) \ + (PV) = (Pvoid_t)JudyLGet((Pvoid_t)PArray, Index, PJE0) + +#define JLI( PV, PArray, Index) \ + J_1P(PV, (&(PArray)), Index, JudyLIns, "JudyLIns") + +#define JLIA(Rc, PArray, Count, PIndex, PValue) \ + J_3AI(Rc,(&(PArray)), Count, PIndex, PValue, JudyLInsArray, \ + "JudyLInsArray") +#define JLD( Rc, PArray, Index) \ + J_1I(Rc, (&(PArray)), Index, JudyLDel, "JudyLDel") + +#define JLF( PV, PArray, Index) \ + J_1P(PV, PArray, &(Index), JudyLFirst, "JudyLFirst") + +#define JLN( PV, PArray, Index) \ + J_1P(PV, PArray, &(Index), JudyLNext, "JudyLNext") + +#define JLL( PV, PArray, Index) \ + J_1P(PV, PArray, &(Index), JudyLLast, "JudyLLast") +#define JLP( PV, PArray, Index) \ + J_1P(PV, PArray, &(Index), JudyLPrev, "JudyLPrev") +#define JLFE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), JudyLFirstEmpty, "JudyLFirstEmpty") +#define JLNE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), JudyLNextEmpty, "JudyLNextEmpty") +#define JLLE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), JudyLLastEmpty, "JudyLLastEmpty") +#define JLPE(Rc, PArray, Index) \ + J_1I(Rc, PArray, &(Index), JudyLPrevEmpty, "JudyLPrevEmpty") +#define JLC( Rc, PArray, Index1, Index2) \ + J_2C(Rc, PArray, Index1, Index2, JudyLCount, "JudyLCount") +#define JLBC(PV, PArray, Count, Index) \ + J_2P(PV, PArray, Count, &(Index), JudyLByCount, "JudyLByCount") +#define JLFA(Rc, PArray) \ + J_0I(Rc, (&(PArray)), JudyLFreeArray, "JudyLFreeArray") +#define JLMU(Rc, PArray) \ + (Rc) = JudyLMemUsed(PArray) + +#define JHSI(PV, PArray, PIndex, Count) \ + J_2P(PV, (&(PArray)), PIndex, Count, JudyHSIns, "JudyHSIns") +#define JHSG(PV, PArray, PIndex, Count) \ + (PV) = (Pvoid_t) JudyHSGet(PArray, PIndex, Count) +#define JHSD(Rc, PArray, PIndex, Count) \ + J_2I(Rc, (&(PArray)), PIndex, Count, JudyHSDel, "JudyHSDel") +#define JHSFA(Rc, PArray) \ + J_0I(Rc, (&(PArray)), JudyHSFreeArray, "JudyHSFreeArray") + +#define JSLG( PV, PArray, Index) \ + J_1P( PV, PArray, Index, JudySLGet, "JudySLGet") +#define JSLI( PV, PArray, Index) \ + J_1P( PV, (&(PArray)), Index, JudySLIns, "JudySLIns") +#define JSLD( Rc, PArray, Index) \ + J_1I( Rc, (&(PArray)), Index, JudySLDel, "JudySLDel") +#define JSLF( PV, PArray, Index) \ + J_1P( PV, PArray, Index, JudySLFirst, "JudySLFirst") +#define JSLN( PV, PArray, Index) \ + J_1P( PV, PArray, Index, JudySLNext, "JudySLNext") +#define JSLL( PV, PArray, Index) \ + J_1P( PV, PArray, Index, JudySLLast, "JudySLLast") +#define JSLP( PV, PArray, Index) \ + J_1P( PV, PArray, Index, JudySLPrev, "JudySLPrev") +#define JSLFA(Rc, PArray) \ + J_0I( Rc, (&(PArray)), JudySLFreeArray, "JudySLFreeArray") + +#ifdef __cplusplus +} +#endif +#endif /* ! _JUDY_INCLUDED */ diff --git a/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c b/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c new file mode 100644 index 000000000..09a20e399 --- /dev/null +++ b/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c @@ -0,0 +1,87 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.33 $ $Source: /judy/src/JudyCommon/JudyMalloc.c $ +// ************************************************************************ // +// JUDY - Memory Allocater // +// -by- // +// Douglas L. Baskins // +// Hewlett Packard // +// Fort Collins, Co // +// (970) 229-2027 // +// // +// ************************************************************************ // + +// JUDY INCLUDE FILES +#include "Judy.h" + +// **************************************************************************** +// J U D Y M A L L O C +// +// Allocate RAM. This is the single location in Judy code that calls +// malloc(3C). Note: JPM accounting occurs at a higher level. + +Word_t JudyMalloc( + Word_t Words) +{ + Word_t Addr; + + Addr = (Word_t) malloc(Words * sizeof(Word_t)); + return(Addr); + +} // JudyMalloc() + + +// **************************************************************************** +// J U D Y F R E E + +void JudyFree( + void * PWord, + Word_t Words) +{ + (void) Words; + free(PWord); + +} // JudyFree() + + +// **************************************************************************** +// J U D Y M A L L O C +// +// Higher-level "wrapper" for allocating objects that need not be in RAM, +// although at this time they are in fact only in RAM. Later we hope that some +// entire subtrees (at a JPM or branch) can be "virtual", so their allocations +// and frees should go through this level. + +Word_t JudyMallocVirtual( + Word_t Words) +{ + return(JudyMalloc(Words)); + +} // JudyMallocVirtual() + + +// **************************************************************************** +// J U D Y F R E E + +void JudyFreeVirtual( + void * PWord, + Word_t Words) +{ + JudyFree(PWord, Words); + +} // JudyFreeVirtual() diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h b/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h new file mode 100644 index 000000000..350631f01 --- /dev/null +++ b/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h @@ -0,0 +1,1613 @@ +#ifndef _JUDYPRIVATE_INCLUDED +#define _JUDYPRIVATE_INCLUDED +// _________________ +// +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.77 $ $Source: /judy/src/JudyCommon/JudyPrivate.h $ +// +// Header file for all Judy sources, for global but private (non-exported) +// declarations. + +#include "Judy.h" + +// **************************************************************************** +// A VERY BRIEF EXPLANATION OF A JUDY ARRAY +// +// A Judy array is, effectively, a digital tree (or Trie) with 256 element +// branches (nodes), and with "compression tricks" applied to low-population +// branches or leaves to save a lot of memory at the cost of relatively little +// CPU time or cache fills. +// +// In the actual implementation, a Judy array is level-less, and traversing the +// "tree" actually means following the states in a state machine (SM) as +// directed by the Index. A Judy array is referred to here as an "SM", rather +// than as a "tree"; having "states", rather than "levels". +// +// Each branch or leaf in the SM decodes a portion ("digit") of the original +// Index; with 256-way branches there are 8 bits per digit. There are 3 kinds +// of branches, called: Linear, Bitmap and Uncompressed, of which the first 2 +// are compressed to contain no NULL entries. +// +// An Uncompressed branch has a 1.0 cache line fill cost to decode 8 bits of +// (digit, part of an Index), but it might contain many NULL entries, and is +// therefore inefficient with memory if lightly populated. +// +// A Linear branch has a ~1.75 cache line fill cost when at maximum population. +// A Bitmap branch has ~2.0 cache line fills. Linear and Bitmap branches are +// converted to Uncompressed branches when the additional memory can be +// amortized with larger populations. Higher-state branches have higher +// priority to be converted. +// +// Linear branches can hold 28 elements (based on detailed analysis) -- thus 28 +// expanses. A Linear branch is converted to a Bitmap branch when the 29th +// expanse is required. +// +// A Bitmap branch could hold 256 expanses, but is forced to convert to an +// Uncompressed branch when 185 expanses are required. Hopefully, it is +// converted before that because of population growth (again, based on detailed +// analysis and heuristics in the code). +// +// A path through the SM terminates to a leaf when the Index (or key) +// population in the expanse below a pointer will fit into 1 or 2 cache lines +// (~31..255 Indexes). A maximum-population Leaf has ~1.5 cache line fill +// cost. +// +// Leaves are sorted arrays of Indexes, where the Index Sizes (IS) are: 0, 1, +// 8, 16, 24, 32, [40, 48, 56, 64] bits. The IS depends on the "density" +// (population/expanse) of the values in the Leaf. Zero bits are possible if +// population == expanse in the SM (that is, a full small expanse). +// +// Elements of a branches are called Judy Pointers (JPs). Each JP object +// points to the next object in the SM, plus, a JP can decode an additional +// 2[6] bytes of an Index, but at the cost of "narrowing" the expanse +// represented by the next object in the SM. A "narrow" JP (one which has +// decode bytes/digits) is a way of skipping states in the SM. +// +// Although counterintuitive, we think a Judy SM is optimal when the Leaves are +// stored at MINIMUM compression (narrowing, or use of Decode bytes). If more +// aggressive compression was used, decompression of a leaf be required to +// insert an index. Additional compression would save a little memory but not +// help performance significantly. + + +#ifdef A_PICTURE_IS_WORTH_1000_WORDS +******************************************************************************* + +JUDY 32-BIT STATE MACHINE (SM) EXAMPLE, FOR INDEX = 0x02040103 + +The Index used in this example is purposely chosen to allow small, simple +examples below; each 1-byte "digit" from the Index has a small numeric value +that fits in one column. In the drawing below: + + JRP == Judy Root Pointer; + + C == 1 byte of a 1..3 byte Population (count of Indexes) below this + pointer. Since this is shared with the Decode field, the combined + sizes must be 3[7], that is, 1 word less 1 byte for the JP Type. + + The 1-byte field jp_Type is represented as: + + 1..3 == Number of bytes in the population (Pop0) word of the Branch or Leaf + below the pointer (note: 1..7 on 64-bit); indicates: + - number of bytes in Decode field == 3 - this number; + - number of bytes remaining to decode. + Note: The maximum is 3, not 4, because the 1st byte of the Index is + always decoded digitally in the top branch. + -B- == JP points to a Branch (there are many kinds of Branches). + -L- == JP points to a Leaf (there are many kinds of Leaves). + + (2) == Digit of Index decoded by position offset in branch (really + 0..0xff). + + 4* == Digit of Index necessary for decoding a "narrow" pointer, in a + Decode field; replaces 1 missing branch (really 0..0xff). + + 4+ == Digit of Index NOT necessary for decoding a "narrow" pointer, but + used for fast traversal of the SM by Judy1Test() and JudyLGet() + (see the code) (really 0..0xff). + + 0 == Byte in a JPs Pop0 field that is always ignored, because a leaf + can never contain more than 256 Indexes (Pop0 <= 255). + + +----- == A Branch or Leaf; drawn open-ended to remind you that it could + | have up to 256 columns. + +----- + + | + | == Pointer to next Branch or Leaf. + V + + | + O == A state is skipped by using a "narrow" pointer. + | + + < 1 > == Digit (Index) shown as an example is not necessarily in the + position shown; is sorted in order with neighbor Indexes. + (Really 0..0xff.) + +Note that this example shows every possibly topology to reach a leaf in a +32-bit Judy SM, although this is a very subtle point! + + STATE or` + LEVEL + +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ + |RJP| |RJP| |RJP| |RJP| |RJP| |RJP| |RJP| |RJP| + L---+ B---+ B---+ B---+ B---+ B---+ B---+ B---+ + | | | | | | | | + | | | | | | | | + V V (2) V (2) V (2) V (2) V (2) V (2) V (2) + +------ +------ +------ +------ +------ +------ +------ +------ +Four |< 2 > | 0 | 4* | C | 4* | 4* | C | C +byte |< 4 > | 0 | 0 | C | 1* | C | C | C 4 +Index|< 1 > | C | C | C | C | C | C | C +Leaf |< 3 > | 3 | 2 | 3 | 1 | 2 | 3 | 3 + +------ +--L--- +--L--- +--B--- +--L--- +--B--- +--B--- +--B--- + | | | | | | | + / | / | | / / + / | / | | / / + | | | | | | | + V | V (4) | | V (4) V (4) + +------ | +------ | | +------ +------ + Three |< 4 > | | 4+ | | | 4+ | 4+ + byte Index|< 1 > O | 0 O O | 1* | C 3 + Leaf |< 3 > | | C | | | C | C + +------ | | 2 | | | 1 | 2 + / +----L- | | +----L- +----B- + / | | | | | + | / | / / / + | / | / / / + | / | | / / + | / | | / / + | | | | | | + V V | V(1) | V(1) + +------ +------ | +------ | +------ + Two byte |< 1 > |< 1 > | | 4+ | | 4+ + Index Leaf |< 3 > |< 3 > O | 1+ O | 1+ 2 + +------ +------ / | C | | C + / | 1 | | 1 + | +-L---- | +-L---- + | | | | + | / | / + | | | | + V V V V + +------ +------ +------ +------ + One byte Index Leaf |< 3 > |< 3 > |< 3 > |< 3 > 1 + +------ +------ +------ +------ + + +#endif // A_PICTURE_IS_WORTH_1000_WORDS + + +// **************************************************************************** +// MISCELLANEOUS GLOBALS: +// +// PLATFORM-SPECIFIC CONVENIENCE MACROS: +// +// These are derived from context (set by cc or in system header files) or +// based on JU_<PLATFORM> macros from make_includes/platform.*.mk. We decided +// on 011018 that any macro reliably derivable from context (cc or headers) for +// ALL platforms supported by Judy is based on that derivation, but ANY +// exception means to stop using the external macro completely and derive from +// JU_<PLATFORM> instead. + +// Other miscellaneous stuff: + +#ifndef _BOOL_T +#define _BOOL_T +typedef int bool_t; +#endif + +#define FUNCTION // null; easy to find functions. + +#ifndef TRUE +#define TRUE 1 +#endif + +#ifndef FALSE +#define FALSE 0 +#endif + +#ifdef TRACE // turn on all other tracing in the code: +#define TRACEJP 1 // JP traversals in JudyIns.c and JudyDel.c. +#define TRACEJPR 1 // JP traversals in retrieval code, JudyGet.c. +#define TRACECF 1 // cache fills in JudyGet.c. +#define TRACEMI 1 // malloc calls in JudyMallocIF.c. +#define TRACEMF 1 // malloc calls at a lower level in JudyMalloc.c. +#endif + + +// SUPPORT FOR DEBUG-ONLY CODE: +// +// By convention, use -DDEBUG to enable both debug-only code AND assertions in +// the Judy sources. +// +// Invert the sense of assertions, so they are off unless explicitly requested, +// in a uniform way. +// +// Note: It is NOT appropriate to put this in Judy.h; it would mess up +// application code. + +#ifndef DEBUG +#define NDEBUG 1 // must be 1 for "#if". +#endif + +// Shorthand notations to avoid #ifdefs for single-line conditional statements: +// +// Warning: These cannot be used around compiler directives, such as +// "#include", nor in the case where Code contains a comma other than nested +// within parentheses or quotes. + +#ifndef DEBUG +#define DBGCODE(Code) // null. +#else +#define DBGCODE(Code) Code +#endif + +#ifdef JUDY1 +#define JUDY1CODE(Code) Code +#define JUDYLCODE(Code) // null. +#endif + +#ifdef JUDYL +#define JUDYLCODE(Code) Code +#define JUDY1CODE(Code) // null. +#endif + +#include <assert.h> + +// **************************************************************************** +// FUNDAMENTAL CONSTANTS FOR MACHINE +// **************************************************************************** + +// Machine (CPU) cache line size: +// +// NOTE: A leaf size of 2 cache lines maximum is the target (optimal) for +// Judy. Its hard to obtain a machines cache line size at compile time, but +// if the machine has an unexpected cache line size, its not devastating if +// the following constants end up causing leaves that are 1 cache line in size, +// or even 4 cache lines in size. The assumed 32-bit system has 16-word = +// 64-byte cache lines, and the assumed 64-bit system has 16-word = 128-byte +// cache lines. + +#ifdef JU_64BIT +#define cJU_BYTESPERCL 128 // cache line size in bytes. +#else +#define cJU_BYTESPERCL 64 // cache line size in bytes. +#endif + +// Bits Per Byte: + +#define cJU_BITSPERBYTE 0x8 + +// Bytes Per Word and Bits Per Word, latter assuming sizeof(byte) is 8 bits: +// +// Expect 32 [64] bits per word. + +#define cJU_BYTESPERWORD (sizeof(Word_t)) +#define cJU_BITSPERWORD (sizeof(Word_t) * cJU_BITSPERBYTE) + +#define JU_BYTESTOWORDS(BYTES) \ + (((BYTES) + cJU_BYTESPERWORD - 1) / cJU_BYTESPERWORD) + +// A word that is all-ones, normally equal to -1UL, but safer with ~0: + +#define cJU_ALLONES (~0UL) + +// Note, these are forward references, but thats OK: + +#define cJU_FULLBITMAPB ((BITMAPB_t) cJU_ALLONES) +#define cJU_FULLBITMAPL ((BITMAPL_t) cJU_ALLONES) + + +// **************************************************************************** +// MISCELLANEOUS JUDY-SPECIFIC DECLARATIONS +// **************************************************************************** + +// ROOT STATE: +// +// State at the start of the Judy SM, based on 1 byte decoded per state; equal +// to the number of bytes per Index to decode. + +#define cJU_ROOTSTATE (sizeof(Word_t)) + + +// SUBEXPANSES PER STATE: +// +// Number of subexpanses per state traversed, which is the number of JPs in a +// branch (actual or theoretical) and the number of bits in a bitmap. + +#define cJU_SUBEXPPERSTATE 256 + + +// LEAF AND VALUE POINTERS: +// +// Some other basic object types are in declared in JudyPrivateBranch.h +// (Pjbl_t, Pjbb_t, Pjbu_t, Pjp_t) or are Judy1/L-specific (Pjlb_t). The +// few remaining types are declared below. +// +// Note: Leaf pointers are cast to different-sized objects depending on the +// leafs level, but are at least addresses (not just numbers), so use void * +// (Pvoid_t), not PWord_t or Word_t for them, except use Pjlw_t for whole-word +// (top-level, root-level) leaves. Value areas, however, are always whole +// words. +// +// Furthermore, use Pjll_t only for generic leaf pointers (for various size +// LeafLs). Use Pjlw_t for LeafWs. Use Pleaf (with type uint8_t *, uint16_t +// *, etc) when the leaf index size is known. + +typedef PWord_t Pjlw_t; // pointer to root-level leaf (whole-word indexes). +typedef Pvoid_t Pjll_t; // pointer to lower-level linear leaf. + +#ifdef JUDYL +typedef PWord_t Pjv_t; // pointer to JudyL value area. +#endif + + +// POINTER PREPARATION MACROS: +// +// These macros are used to strip malloc-namespace-type bits from a pointer + +// malloc-type word (which references any Judy mallocd object that might be +// obtained from other than a direct call of malloc()), prior to dereferencing +// the pointer as an address. The malloc-type bits allow Judy mallocd objects +// to come from different "malloc() namespaces". +// +// (root pointer) (JRP, see above) +// jp.jp_Addr generic pointer to next-level node, except when used +// as a JudyL Immed01 value area +// JU_JBB_PJP macro hides jbbs_Pjp (pointer to JP subarray) +// JL_JLB_PVALUE macro hides jLlbs_PValue (pointer to value subarray) +// +// When setting one of these fields or passing an address to j__udyFree*(), the +// "raw" memory address is used; otherwise the memory address must be passed +// through one of the macros below before its dereferenced. +// +// Note: After much study, the typecasts below appear in the macros rather +// than at the point of use, which is both simpler and allows the compiler to +// do type-checking. + + +#define P_JLW( ADDR) ((Pjlw_t) (ADDR)) // root leaf. +#define P_JPM( ADDR) ((Pjpm_t) (ADDR)) // root JPM. +#define P_JBL( ADDR) ((Pjbl_t) (ADDR)) // BranchL. +#define P_JBB( ADDR) ((Pjbb_t) (ADDR)) // BranchB. +#define P_JBU( ADDR) ((Pjbu_t) (ADDR)) // BranchU. +#define P_JLL( ADDR) ((Pjll_t) (ADDR)) // LeafL. +#define P_JLB( ADDR) ((Pjlb_t) (ADDR)) // LeafB1. +#define P_JP( ADDR) ((Pjp_t) (ADDR)) // JP. + +#ifdef JUDYL +#define P_JV( ADDR) ((Pjv_t) (ADDR)) // &value. +#endif + + +// LEAST BYTES: +// +// Mask for least bytes of a word, and a macro to perform this mask on an +// Index. +// +// Note: This macro has been problematic in the past to get right and to make +// portable. Its not OK on all systems to shift by the full word size. This +// macro should allow shifting by 1..N bytes, where N is the word size, but +// should produce a compiler warning if the macro is called with Bytes == 0. +// +// Warning: JU_LEASTBYTESMASK() is not a constant macro unless Bytes is a +// constant; otherwise it is a variable shift, which is expensive on some +// processors. + +#define JU_LEASTBYTESMASK(BYTES) \ + ((0x100UL << (cJU_BITSPERBYTE * ((BYTES) - 1))) - 1) + +#define JU_LEASTBYTES(INDEX,BYTES) ((INDEX) & JU_LEASTBYTESMASK(BYTES)) + + +// BITS IN EACH BITMAP SUBEXPANSE FOR BITMAP BRANCH AND LEAF: +// +// The bits per bitmap subexpanse times the number of subexpanses equals a +// constant (cJU_SUBEXPPERSTATE). You can also think of this as a compile-time +// choice of "aspect ratio" for bitmap branches and leaves (which can be set +// independently for each). +// +// A default aspect ratio is hardwired here if not overridden at compile time, +// such as by "EXTCCOPTS=-DBITMAP_BRANCH16x16 make". + +#if (! (defined(BITMAP_BRANCH8x32) || defined(BITMAP_BRANCH16x16) || defined(BITMAP_BRANCH32x8))) +#define BITMAP_BRANCH32x8 1 // 32 bits per subexpanse, 8 subexpanses. +#endif + +#ifdef BITMAP_BRANCH8x32 +#define BITMAPB_t uint8_t +#endif + +#ifdef BITMAP_BRANCH16x16 +#define BITMAPB_t uint16_t +#endif + +#ifdef BITMAP_BRANCH32x8 +#define BITMAPB_t uint32_t +#endif + +// Note: For bitmap leaves, BITMAP_LEAF64x4 is only valid for 64 bit: +// +// Note: Choice of aspect ratio mostly matters for JudyL bitmap leaves. For +// Judy1 the choice doesnt matter much -- the code generated for different +// BITMAP_LEAF* values choices varies, but correctness and performance are the +// same. + +#ifndef JU_64BIT + +#if (! (defined(BITMAP_LEAF8x32) || defined(BITMAP_LEAF16x16) || defined(BITMAP_LEAF32x8))) +#define BITMAP_LEAF32x8 // 32 bits per subexpanse, 8 subexpanses. +#endif + +#else // 32BIT + +#if (! (defined(BITMAP_LEAF8x32) || defined(BITMAP_LEAF16x16) || defined(BITMAP_LEAF32x8) || defined(BITMAP_LEAF64x4))) +#define BITMAP_LEAF64x4 // 64 bits per subexpanse, 4 subexpanses. + +#endif +#endif // JU_64BIT + +#ifdef BITMAP_LEAF8x32 +#define BITMAPL_t uint8_t +#endif + +#ifdef BITMAP_LEAF16x16 +#define BITMAPL_t uint16_t +#endif + +#ifdef BITMAP_LEAF32x8 +#define BITMAPL_t uint32_t +#endif + +#ifdef BITMAP_LEAF64x4 +#define BITMAPL_t uint64_t +#endif + + +// EXPORTED DATA AND FUNCTIONS: + +#ifdef JUDY1 +extern const uint8_t j__1_BranchBJPPopToWords[]; +#endif + +#ifdef JUDYL +extern const uint8_t j__L_BranchBJPPopToWords[]; +#endif + +// Fast LeafL search routine used for inlined code: + +#if (! defined(SEARCH_BINARY)) || (! defined(SEARCH_LINEAR)) +// default a binary search leaf method +#define SEARCH_BINARY 1 +//#define SEARCH_LINEAR 1 +#endif + +#ifdef SEARCH_LINEAR + +#define SEARCHLEAFNATIVE(LEAFTYPE,ADDR,POP1,INDEX) \ + LEAFTYPE *P_leaf = (LEAFTYPE *)(ADDR); \ + LEAFTYPE I_ndex = (INDEX); /* with masking */ \ + if (I_ndex > P_leaf[(POP1) - 1]) return(~(POP1)); \ + while(I_ndex > *P_leaf) P_leaf++; \ + if (I_ndex == *P_leaf) return(P_leaf - (LEAFTYPE *)(ADDR)); \ + return(~(P_leaf - (LEAFTYPE *)(ADDR))); + + +#define SEARCHLEAFNONNAT(ADDR,POP1,INDEX,LFBTS,COPYINDEX) \ +{ \ + uint8_t *P_leaf, *P_leafEnd; \ + Word_t i_ndex; \ + Word_t I_ndex = JU_LEASTBYTES((INDEX), (LFBTS)); \ + Word_t p_op1; \ + \ + P_leaf = (uint8_t *)(ADDR); \ + P_leafEnd = P_leaf + ((POP1) * (LFBTS)); \ + \ + do { \ + JU_COPY3_PINDEX_TO_LONG(i_ndex, P_leaf); \ + if (I_ndex <= i_ndex) break; \ + P_leaf += (LFBTS); \ + } while (P_leaf < P_leafEnd); \ + \ + p_op1 = (P_leaf - (uint8_t *) (ADDR)) / (LFBTS); \ + if (I_ndex == i_ndex) return(p_op1); \ + return(~p_op1); \ +} +#endif // SEARCH_LINEAR + +#ifdef SEARCH_BINARY + +#define SEARCHLEAFNATIVE(LEAFTYPE,ADDR,POP1,INDEX) \ + LEAFTYPE *P_leaf = (LEAFTYPE *)(ADDR); \ + LEAFTYPE I_ndex = (LEAFTYPE)INDEX; /* truncate hi bits */ \ + Word_t l_ow = cJU_ALLONES; \ + Word_t m_id; \ + Word_t h_igh = POP1; \ + \ + while ((h_igh - l_ow) > 1UL) \ + { \ + m_id = (h_igh + l_ow) / 2; \ + if (P_leaf[m_id] > I_ndex) \ + h_igh = m_id; \ + else \ + l_ow = m_id; \ + } \ + if (l_ow == cJU_ALLONES || P_leaf[l_ow] != I_ndex) \ + return(~h_igh); \ + return(l_ow) + + +#define SEARCHLEAFNONNAT(ADDR,POP1,INDEX,LFBTS,COPYINDEX) \ + uint8_t *P_leaf = (uint8_t *)(ADDR); \ + Word_t l_ow = cJU_ALLONES; \ + Word_t m_id; \ + Word_t h_igh = POP1; \ + Word_t I_ndex = JU_LEASTBYTES((INDEX), (LFBTS)); \ + Word_t i_ndex; \ + \ + I_ndex = JU_LEASTBYTES((INDEX), (LFBTS)); \ + \ + while ((h_igh - l_ow) > 1UL) \ + { \ + m_id = (h_igh + l_ow) / 2; \ + COPYINDEX(i_ndex, &P_leaf[m_id * (LFBTS)]); \ + if (i_ndex > I_ndex) \ + h_igh = m_id; \ + else \ + l_ow = m_id; \ + } \ + if (l_ow == cJU_ALLONES) return(~h_igh); \ + \ + COPYINDEX(i_ndex, &P_leaf[l_ow * (LFBTS)]); \ + if (i_ndex != I_ndex) return(~h_igh); \ + return(l_ow) + +#endif // SEARCH_BINARY + +// Fast way to count bits set in 8..32[64]-bit int: +// +// For performance, j__udyCountBits*() are written to take advantage of +// platform-specific features where available. +// + +#ifdef JU_NOINLINE + +extern BITMAPB_t j__udyCountBitsB(BITMAPB_t word); +extern BITMAPL_t j__udyCountBitsL(BITMAPL_t word); + +// Compiler supports inline + +#elif defined(JU_HPUX_IPF) + +#define j__udyCountBitsB(WORD) _Asm_popcnt(WORD) +#define j__udyCountBitsL(WORD) _Asm_popcnt(WORD) + +#elif defined(JU_LINUX_IPF) + +static inline BITMAPB_t j__udyCountBitsB(BITMAPB_t word) +{ + BITMAPB_t result; + __asm__ ("popcnt %0=%1" : "=r" (result) : "r" (word)); + return(result); +} + +static inline BITMAPL_t j__udyCountBitsL(BITMAPL_t word) +{ + BITMAPL_t result; + __asm__ ("popcnt %0=%1" : "=r" (result) : "r" (word)); + return(result); +} + + +#else // No instructions available, use inline code + +// **************************************************************************** +// __ J U D Y C O U N T B I T S B +// +// Return the number of bits set in "Word", for a bitmap branch. +// +// Note: Bitmap branches have maximum bitmap size = 32 bits. + +#ifdef JU_WIN +static __inline BITMAPB_t j__udyCountBitsB(BITMAPB_t word) +#else +static inline BITMAPB_t j__udyCountBitsB(BITMAPB_t word) +#endif +{ + word = (word & 0x55555555) + ((word & 0xAAAAAAAA) >> 1); + word = (word & 0x33333333) + ((word & 0xCCCCCCCC) >> 2); + word = (word & 0x0F0F0F0F) + ((word & 0xF0F0F0F0) >> 4); // >= 8 bits. +#if defined(BITMAP_BRANCH16x16) || defined(BITMAP_BRANCH32x8) + word = (word & 0x00FF00FF) + ((word & 0xFF00FF00) >> 8); // >= 16 bits. +#endif + +#ifdef BITMAP_BRANCH32x8 + word = (word & 0x0000FFFF) + ((word & 0xFFFF0000) >> 16); // >= 32 bits. +#endif + return(word); + +} // j__udyCountBitsB() + + +// **************************************************************************** +// __ J U D Y C O U N T B I T S L +// +// Return the number of bits set in "Word", for a bitmap leaf. +// +// Note: Bitmap branches have maximum bitmap size = 32 bits. + +// Note: Need both 32-bit and 64-bit versions of j__udyCountBitsL() because +// bitmap leaves can have 64-bit bitmaps. + +#ifdef JU_WIN +static __inline BITMAPL_t j__udyCountBitsL(BITMAPL_t word) +#else +static inline BITMAPL_t j__udyCountBitsL(BITMAPL_t word) +#endif +{ +#ifndef JU_64BIT + + word = (word & 0x55555555) + ((word & 0xAAAAAAAA) >> 1); + word = (word & 0x33333333) + ((word & 0xCCCCCCCC) >> 2); + word = (word & 0x0F0F0F0F) + ((word & 0xF0F0F0F0) >> 4); // >= 8 bits. +#if defined(BITMAP_LEAF16x16) || defined(BITMAP_LEAF32x8) + word = (word & 0x00FF00FF) + ((word & 0xFF00FF00) >> 8); // >= 16 bits. +#endif +#ifdef BITMAP_LEAF32x8 + word = (word & 0x0000FFFF) + ((word & 0xFFFF0000) >> 16); // >= 32 bits. +#endif + +#else // JU_64BIT + + word = (word & 0x5555555555555555) + ((word & 0xAAAAAAAAAAAAAAAA) >> 1); + word = (word & 0x3333333333333333) + ((word & 0xCCCCCCCCCCCCCCCC) >> 2); + word = (word & 0x0F0F0F0F0F0F0F0F) + ((word & 0xF0F0F0F0F0F0F0F0) >> 4); +#if defined(BITMAP_LEAF16x16) || defined(BITMAP_LEAF32x8) || defined(BITMAP_LEAF64x4) + word = (word & 0x00FF00FF00FF00FF) + ((word & 0xFF00FF00FF00FF00) >> 8); +#endif +#if defined(BITMAP_LEAF32x8) || defined(BITMAP_LEAF64x4) + word = (word & 0x0000FFFF0000FFFF) + ((word & 0xFFFF0000FFFF0000) >>16); +#endif +#ifdef BITMAP_LEAF64x4 + word = (word & 0x00000000FFFFFFFF) + ((word & 0xFFFFFFFF00000000) >>32); +#endif +#endif // JU_64BIT + + return(word); + +} // j__udyCountBitsL() + +#endif // Compiler supports inline + +// GET POP0: +// +// Get from jp_DcdPopO the Pop0 for various JP Types. +// +// Notes: +// +// - Different macros require different parameters... +// +// - There are no simple macros for cJU_BRANCH* Types because their +// populations must be added up and dont reside in an already-calculated +// place. (TBD: This is no longer true, now its in the JPM.) +// +// - cJU_JPIMM_POP0() is not defined because it would be redundant because the +// Pop1 is already encoded in each enum name. +// +// - A linear or bitmap leaf Pop0 cannot exceed cJU_SUBEXPPERSTATE - 1 (Pop0 = +// 0..255), so use a simpler, faster macro for it than for other JP Types. +// +// - Avoid any complex calculations that would slow down the compiled code. +// Assume these macros are only called for the appropriate JP Types. +// Unfortunately theres no way to trigger an assertion here if the JP type +// is incorrect for the macro, because these are merely expressions, not +// statements. + +#define JU_LEAFW_POP0(JRP) (*P_JLW(JRP)) +#define cJU_JPFULLPOPU1_POP0 (cJU_SUBEXPPERSTATE - 1) + +// GET JP Type: +// Since bit fields greater than 32 bits are not supported in some compilers +// the jp_DcdPopO field is expanded to include the jp_Type in the high 8 bits +// of the Word_t. +// First the read macro: + +#define JU_JPTYPE(PJP) ((PJP)->jp_Type) + +#define JU_JPLEAF_POP0(PJP) ((PJP)->jp_DcdP0[sizeof(Word_t) - 2]) + +#ifdef JU_64BIT + +#define JU_JPDCDPOP0(PJP) \ + ((Word_t)(PJP)->jp_DcdP0[0] << 48 | \ + (Word_t)(PJP)->jp_DcdP0[1] << 40 | \ + (Word_t)(PJP)->jp_DcdP0[2] << 32 | \ + (Word_t)(PJP)->jp_DcdP0[3] << 24 | \ + (Word_t)(PJP)->jp_DcdP0[4] << 16 | \ + (Word_t)(PJP)->jp_DcdP0[5] << 8 | \ + (Word_t)(PJP)->jp_DcdP0[6]) + + +#define JU_JPSETADT(PJP,ADDR,DCDPOP0,TYPE) \ +{ \ + (PJP)->jp_Addr = (ADDR); \ + (PJP)->jp_DcdP0[0] = (uint8_t)((Word_t)(DCDPOP0) >> 48); \ + (PJP)->jp_DcdP0[1] = (uint8_t)((Word_t)(DCDPOP0) >> 40); \ + (PJP)->jp_DcdP0[2] = (uint8_t)((Word_t)(DCDPOP0) >> 32); \ + (PJP)->jp_DcdP0[3] = (uint8_t)((Word_t)(DCDPOP0) >> 24); \ + (PJP)->jp_DcdP0[4] = (uint8_t)((Word_t)(DCDPOP0) >> 16); \ + (PJP)->jp_DcdP0[5] = (uint8_t)((Word_t)(DCDPOP0) >> 8); \ + (PJP)->jp_DcdP0[6] = (uint8_t)((Word_t)(DCDPOP0)); \ + (PJP)->jp_Type = (TYPE); \ +} + +#else // 32 Bit + +#define JU_JPDCDPOP0(PJP) \ + ((Word_t)(PJP)->jp_DcdP0[0] << 16 | \ + (Word_t)(PJP)->jp_DcdP0[1] << 8 | \ + (Word_t)(PJP)->jp_DcdP0[2]) + + +#define JU_JPSETADT(PJP,ADDR,DCDPOP0,TYPE) \ +{ \ + (PJP)->jp_Addr = (ADDR); \ + (PJP)->jp_DcdP0[0] = (uint8_t)((Word_t)(DCDPOP0) >> 16); \ + (PJP)->jp_DcdP0[1] = (uint8_t)((Word_t)(DCDPOP0) >> 8); \ + (PJP)->jp_DcdP0[2] = (uint8_t)((Word_t)(DCDPOP0)); \ + (PJP)->jp_Type = (TYPE); \ +} + +#endif // 32 Bit + +// NUMBER OF BITS IN A BRANCH OR LEAF BITMAP AND SUBEXPANSE: +// +// Note: cJU_BITSPERBITMAP must be the same as the number of JPs in a branch. + +#define cJU_BITSPERBITMAP cJU_SUBEXPPERSTATE + +// Bitmaps are accessed in units of "subexpanses": + +#define cJU_BITSPERSUBEXPB (sizeof(BITMAPB_t) * cJU_BITSPERBYTE) +#define cJU_NUMSUBEXPB (cJU_BITSPERBITMAP / cJU_BITSPERSUBEXPB) + +#define cJU_BITSPERSUBEXPL (sizeof(BITMAPL_t) * cJU_BITSPERBYTE) +#define cJU_NUMSUBEXPL (cJU_BITSPERBITMAP / cJU_BITSPERSUBEXPL) + + +// MASK FOR A SPECIFIED BIT IN A BITMAP: +// +// Warning: If BitNum is a variable, this results in a variable shift that is +// expensive, at least on some processors. Use with caution. +// +// Warning: BitNum must be less than cJU_BITSPERWORD, that is, 0 .. +// cJU_BITSPERWORD - 1, to avoid a truncated shift on some machines. +// +// TBD: Perhaps use an array[32] of masks instead of calculating them. + +#define JU_BITPOSMASKB(BITNUM) (1L << ((BITNUM) % cJU_BITSPERSUBEXPB)) +#define JU_BITPOSMASKL(BITNUM) (1L << ((BITNUM) % cJU_BITSPERSUBEXPL)) + + +// TEST/SET/CLEAR A BIT IN A BITMAP LEAF: +// +// Test if a byte-sized Digit (portion of Index) has a corresponding bit set in +// a bitmap, or set a byte-sized Digits bit into a bitmap, by looking up the +// correct subexpanse and then checking/setting the correct bit. +// +// Note: Mask higher bits, if any, for the convenience of the user of this +// macro, in case they pass a full Index, not just a digit. If the caller has +// a true 8-bit digit, make it of type uint8_t and the compiler should skip the +// unnecessary mask step. + +#define JU_SUBEXPL(DIGIT) (((DIGIT) / cJU_BITSPERSUBEXPL) & (cJU_NUMSUBEXPL-1)) + +#define JU_BITMAPTESTL(PJLB, INDEX) \ + (JU_JLB_BITMAP(PJLB, JU_SUBEXPL(INDEX)) & JU_BITPOSMASKL(INDEX)) + +#define JU_BITMAPSETL(PJLB, INDEX) \ + (JU_JLB_BITMAP(PJLB, JU_SUBEXPL(INDEX)) |= JU_BITPOSMASKL(INDEX)) + +#define JU_BITMAPCLEARL(PJLB, INDEX) \ + (JU_JLB_BITMAP(PJLB, JU_SUBEXPL(INDEX)) ^= JU_BITPOSMASKL(INDEX)) + + +// MAP BITMAP BIT OFFSET TO DIGIT: +// +// Given a digit variable to set, a bitmap branch or leaf subexpanse (base 0), +// the bitmap (BITMAP*_t) for that subexpanse, and an offset (Nth set bit in +// the bitmap, base 0), compute the digit (also base 0) corresponding to the +// subexpanse and offset by counting all bits in the bitmap until offset+1 set +// bits are seen. Avoid expensive variable shifts. Offset should be less than +// the number of set bits in the bitmap; assert this. +// +// If theres a better way to do this, I dont know what it is. + +#define JU_BITMAPDIGITB(DIGIT,SUBEXP,BITMAP,OFFSET) \ + { \ + BITMAPB_t bitmap = (BITMAP); int remain = (OFFSET); \ + (DIGIT) = (SUBEXP) * cJU_BITSPERSUBEXPB; \ + \ + while ((remain -= (bitmap & 1)) >= 0) \ + { \ + bitmap >>= 1; ++(DIGIT); \ + assert((DIGIT) < ((SUBEXP) + 1) * cJU_BITSPERSUBEXPB); \ + } \ + } + +#define JU_BITMAPDIGITL(DIGIT,SUBEXP,BITMAP,OFFSET) \ + { \ + BITMAPL_t bitmap = (BITMAP); int remain = (OFFSET); \ + (DIGIT) = (SUBEXP) * cJU_BITSPERSUBEXPL; \ + \ + while ((remain -= (bitmap & 1)) >= 0) \ + { \ + bitmap >>= 1; ++(DIGIT); \ + assert((DIGIT) < ((SUBEXP) + 1) * cJU_BITSPERSUBEXPL); \ + } \ + } + + +// MASKS FOR PORTIONS OF 32-BIT WORDS: +// +// These are useful for bitmap subexpanses. +// +// "LOWER"/"HIGHER" means bits representing lower/higher-valued Indexes. The +// exact order of bits in the word is explicit here but is hidden from the +// caller. +// +// "EXC" means exclusive of the specified bit; "INC" means inclusive. +// +// In each case, BitPos is either "JU_BITPOSMASK*(BitNum)", or a variable saved +// from an earlier call of that macro; either way, it must be a 32-bit word +// with a single bit set. In the first case, assume the compiler is smart +// enough to optimize out common subexpressions. +// +// The expressions depend on unsigned decimal math that should be universal. + +#define JU_MASKLOWEREXC( BITPOS) ((BITPOS) - 1) +#define JU_MASKLOWERINC( BITPOS) (JU_MASKLOWEREXC(BITPOS) | (BITPOS)) +#define JU_MASKHIGHERINC(BITPOS) (-(BITPOS)) +#define JU_MASKHIGHEREXC(BITPOS) (JU_MASKHIGHERINC(BITPOS) ^ (BITPOS)) + + +// **************************************************************************** +// SUPPORT FOR NATIVE INDEX SIZES +// **************************************************************************** +// +// Copy a series of generic objects (uint8_t, uint16_t, uint32_t, Word_t) from +// one place to another. + +#define JU_COPYMEM(PDST,PSRC,POP1) \ + { \ + Word_t i_ndex = 0; \ + assert((POP1) > 0); \ + do { (PDST)[i_ndex] = (PSRC)[i_ndex]; } \ + while (++i_ndex < (POP1)); \ + } + + +// **************************************************************************** +// SUPPORT FOR NON-NATIVE INDEX SIZES +// **************************************************************************** +// +// Copy a 3-byte Index pointed by a uint8_t * to a Word_t: +// +#define JU_COPY3_PINDEX_TO_LONG(DESTLONG,PINDEX) \ + DESTLONG = (Word_t)(PINDEX)[0] << 16; \ + DESTLONG += (Word_t)(PINDEX)[1] << 8; \ + DESTLONG += (Word_t)(PINDEX)[2] + +// Copy a Word_t to a 3-byte Index pointed at by a uint8_t *: + +#define JU_COPY3_LONG_TO_PINDEX(PINDEX,SOURCELONG) \ + (PINDEX)[0] = (uint8_t)((SOURCELONG) >> 16); \ + (PINDEX)[1] = (uint8_t)((SOURCELONG) >> 8); \ + (PINDEX)[2] = (uint8_t)((SOURCELONG)) + +#ifdef JU_64BIT + +// Copy a 5-byte Index pointed by a uint8_t * to a Word_t: +// +#define JU_COPY5_PINDEX_TO_LONG(DESTLONG,PINDEX) \ + DESTLONG = (Word_t)(PINDEX)[0] << 32; \ + DESTLONG += (Word_t)(PINDEX)[1] << 24; \ + DESTLONG += (Word_t)(PINDEX)[2] << 16; \ + DESTLONG += (Word_t)(PINDEX)[3] << 8; \ + DESTLONG += (Word_t)(PINDEX)[4] + +// Copy a Word_t to a 5-byte Index pointed at by a uint8_t *: + +#define JU_COPY5_LONG_TO_PINDEX(PINDEX,SOURCELONG) \ + (PINDEX)[0] = (uint8_t)((SOURCELONG) >> 32); \ + (PINDEX)[1] = (uint8_t)((SOURCELONG) >> 24); \ + (PINDEX)[2] = (uint8_t)((SOURCELONG) >> 16); \ + (PINDEX)[3] = (uint8_t)((SOURCELONG) >> 8); \ + (PINDEX)[4] = (uint8_t)((SOURCELONG)) + +// Copy a 6-byte Index pointed by a uint8_t * to a Word_t: +// +#define JU_COPY6_PINDEX_TO_LONG(DESTLONG,PINDEX) \ + DESTLONG = (Word_t)(PINDEX)[0] << 40; \ + DESTLONG += (Word_t)(PINDEX)[1] << 32; \ + DESTLONG += (Word_t)(PINDEX)[2] << 24; \ + DESTLONG += (Word_t)(PINDEX)[3] << 16; \ + DESTLONG += (Word_t)(PINDEX)[4] << 8; \ + DESTLONG += (Word_t)(PINDEX)[5] + +// Copy a Word_t to a 6-byte Index pointed at by a uint8_t *: + +#define JU_COPY6_LONG_TO_PINDEX(PINDEX,SOURCELONG) \ + (PINDEX)[0] = (uint8_t)((SOURCELONG) >> 40); \ + (PINDEX)[1] = (uint8_t)((SOURCELONG) >> 32); \ + (PINDEX)[2] = (uint8_t)((SOURCELONG) >> 24); \ + (PINDEX)[3] = (uint8_t)((SOURCELONG) >> 16); \ + (PINDEX)[4] = (uint8_t)((SOURCELONG) >> 8); \ + (PINDEX)[5] = (uint8_t)((SOURCELONG)) + +// Copy a 7-byte Index pointed by a uint8_t * to a Word_t: +// +#define JU_COPY7_PINDEX_TO_LONG(DESTLONG,PINDEX) \ + DESTLONG = (Word_t)(PINDEX)[0] << 48; \ + DESTLONG += (Word_t)(PINDEX)[1] << 40; \ + DESTLONG += (Word_t)(PINDEX)[2] << 32; \ + DESTLONG += (Word_t)(PINDEX)[3] << 24; \ + DESTLONG += (Word_t)(PINDEX)[4] << 16; \ + DESTLONG += (Word_t)(PINDEX)[5] << 8; \ + DESTLONG += (Word_t)(PINDEX)[6] + +// Copy a Word_t to a 7-byte Index pointed at by a uint8_t *: + +#define JU_COPY7_LONG_TO_PINDEX(PINDEX,SOURCELONG) \ + (PINDEX)[0] = (uint8_t)((SOURCELONG) >> 48); \ + (PINDEX)[1] = (uint8_t)((SOURCELONG) >> 40); \ + (PINDEX)[2] = (uint8_t)((SOURCELONG) >> 32); \ + (PINDEX)[3] = (uint8_t)((SOURCELONG) >> 24); \ + (PINDEX)[4] = (uint8_t)((SOURCELONG) >> 16); \ + (PINDEX)[5] = (uint8_t)((SOURCELONG) >> 8); \ + (PINDEX)[6] = (uint8_t)((SOURCELONG)) + +#endif // JU_64BIT + +// **************************************************************************** +// COMMON CODE FRAGMENTS (MACROS) +// **************************************************************************** +// +// These code chunks are shared between various source files. + + +// SET (REPLACE) ONE DIGIT IN AN INDEX: +// +// To avoid endian issues, use masking and ORing, which operates in a +// big-endian register, rather than treating the Index as an array of bytes, +// though that would be simpler, but would operate in endian-specific memory. +// +// TBD: This contains two variable shifts, is that bad? + +#define JU_SETDIGIT(INDEX,DIGIT,STATE) \ + (INDEX) = ((INDEX) & (~cJU_MASKATSTATE(STATE))) \ + | (((Word_t) (DIGIT)) \ + << (((STATE) - 1) * cJU_BITSPERBYTE)) + +// Fast version for single LSB: + +#define JU_SETDIGIT1(INDEX,DIGIT) (INDEX) = ((INDEX) & ~0xff) | (DIGIT) + + +// SET (REPLACE) "N" LEAST DIGITS IN AN INDEX: + +#define JU_SETDIGITS(INDEX,INDEX2,cSTATE) \ + (INDEX) = ((INDEX ) & (~JU_LEASTBYTESMASK(cSTATE))) \ + | ((INDEX2) & ( JU_LEASTBYTESMASK(cSTATE))) + +// COPY DECODE BYTES FROM JP TO INDEX: +// +// Modify Index digit(s) to match the bytes in jp_DcdPopO in case one or more +// branches are skipped and the digits are significant. Its probably faster +// to just do this unconditionally than to check if its necessary. +// +// To avoid endian issues, use masking and ORing, which operates in a +// big-endian register, rather than treating the Index as an array of bytes, +// though that would be simpler, but would operate in endian-specific memory. +// +// WARNING: Must not call JU_LEASTBYTESMASK (via cJU_DCDMASK) with Bytes = +// cJU_ROOTSTATE or a bad mask is generated, but there are no Dcd bytes to copy +// in this case anyway. In fact there are no Dcd bytes unless State < +// cJU_ROOTSTATE - 1, so dont call this macro except in those cases. +// +// TBD: It would be nice to validate jp_DcdPopO against known digits to ensure +// no corruption, but this is non-trivial. + +#define JU_SETDCD(INDEX,PJP,cSTATE) \ + (INDEX) = ((INDEX) & ~cJU_DCDMASK(cSTATE)) \ + | (JU_JPDCDPOP0(PJP) & cJU_DCDMASK(cSTATE)) + +// INSERT/DELETE AN INDEX IN-PLACE IN MEMORY: +// +// Given a pointer to an array of "even" (native), same-sized objects +// (indexes), the current population of the array, an offset in the array, and +// a new Index to insert, "shift up" the array elements (Indexes) above the +// insertion point and insert the new Index. Assume there is sufficient memory +// to do this. +// +// In these macros, "i_offset" is an index offset, and "b_off" is a byte +// offset for odd Index sizes. +// +// Note: Endian issues only arise fro insertion, not deletion, and even for +// insertion, they are transparent when native (even) objects are used, and +// handled explicitly for odd (non-native) Index sizes. +// +// Note: The following macros are tricky enough that there is some test code +// for them appended to this file. + +#define JU_INSERTINPLACE(PARRAY,POP1,OFFSET,INDEX) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ + { \ + Word_t i_offset = (POP1); \ + \ + while (i_offset-- > (OFFSET)) \ + (PARRAY)[i_offset + 1] = (PARRAY)[i_offset]; \ + \ + (PARRAY)[OFFSET] = (INDEX); \ + } + + +// Variation for non-native Indexes, where cIS = Index Size +// and PByte must point to a uint8_t (byte); shift byte-by-byte: +// + +#define JU_INSERTINPLACE3(PBYTE,POP1,OFFSET,INDEX) \ +{ \ + Word_t i_off = POP1; \ + \ + while (i_off-- > (OFFSET)) \ + { \ + Word_t i_dx = i_off * 3; \ + (PBYTE)[i_dx + 0 + 3] = (PBYTE)[i_dx + 0]; \ + (PBYTE)[i_dx + 1 + 3] = (PBYTE)[i_dx + 1]; \ + (PBYTE)[i_dx + 2 + 3] = (PBYTE)[i_dx + 2]; \ + } \ + JU_COPY3_LONG_TO_PINDEX(&((PBYTE)[(OFFSET) * 3]), INDEX); \ +} + +#ifdef JU_64BIT + +#define JU_INSERTINPLACE5(PBYTE,POP1,OFFSET,INDEX) \ +{ \ + Word_t i_off = POP1; \ + \ + while (i_off-- > (OFFSET)) \ + { \ + Word_t i_dx = i_off * 5; \ + (PBYTE)[i_dx + 0 + 5] = (PBYTE)[i_dx + 0]; \ + (PBYTE)[i_dx + 1 + 5] = (PBYTE)[i_dx + 1]; \ + (PBYTE)[i_dx + 2 + 5] = (PBYTE)[i_dx + 2]; \ + (PBYTE)[i_dx + 3 + 5] = (PBYTE)[i_dx + 3]; \ + (PBYTE)[i_dx + 4 + 5] = (PBYTE)[i_dx + 4]; \ + } \ + JU_COPY5_LONG_TO_PINDEX(&((PBYTE)[(OFFSET) * 5]), INDEX); \ +} + +#define JU_INSERTINPLACE6(PBYTE,POP1,OFFSET,INDEX) \ +{ \ + Word_t i_off = POP1; \ + \ + while (i_off-- > (OFFSET)) \ + { \ + Word_t i_dx = i_off * 6; \ + (PBYTE)[i_dx + 0 + 6] = (PBYTE)[i_dx + 0]; \ + (PBYTE)[i_dx + 1 + 6] = (PBYTE)[i_dx + 1]; \ + (PBYTE)[i_dx + 2 + 6] = (PBYTE)[i_dx + 2]; \ + (PBYTE)[i_dx + 3 + 6] = (PBYTE)[i_dx + 3]; \ + (PBYTE)[i_dx + 4 + 6] = (PBYTE)[i_dx + 4]; \ + (PBYTE)[i_dx + 5 + 6] = (PBYTE)[i_dx + 5]; \ + } \ + JU_COPY6_LONG_TO_PINDEX(&((PBYTE)[(OFFSET) * 6]), INDEX); \ +} + +#define JU_INSERTINPLACE7(PBYTE,POP1,OFFSET,INDEX) \ +{ \ + Word_t i_off = POP1; \ + \ + while (i_off-- > (OFFSET)) \ + { \ + Word_t i_dx = i_off * 7; \ + (PBYTE)[i_dx + 0 + 7] = (PBYTE)[i_dx + 0]; \ + (PBYTE)[i_dx + 1 + 7] = (PBYTE)[i_dx + 1]; \ + (PBYTE)[i_dx + 2 + 7] = (PBYTE)[i_dx + 2]; \ + (PBYTE)[i_dx + 3 + 7] = (PBYTE)[i_dx + 3]; \ + (PBYTE)[i_dx + 4 + 7] = (PBYTE)[i_dx + 4]; \ + (PBYTE)[i_dx + 5 + 7] = (PBYTE)[i_dx + 5]; \ + (PBYTE)[i_dx + 6 + 7] = (PBYTE)[i_dx + 6]; \ + } \ + JU_COPY7_LONG_TO_PINDEX(&((PBYTE)[(OFFSET) * 7]), INDEX); \ +} +#endif // JU_64BIT + +// Counterparts to the above for deleting an Index: +// +// "Shift down" the array elements starting at the Index to be deleted. + +#define JU_DELETEINPLACE(PARRAY,POP1,OFFSET,IGNORE) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) < (Word_t) (POP1)); \ + { \ + Word_t i_offset = (OFFSET); \ + \ + while (++i_offset < (POP1)) \ + (PARRAY)[i_offset - 1] = (PARRAY)[i_offset]; \ + } + +// Variation for odd-byte-sized (non-native) Indexes, where cIS = Index Size +// and PByte must point to a uint8_t (byte); copy byte-by-byte: +// +// Note: If cIS == 1, JU_DELETEINPLACE_ODD == JU_DELETEINPLACE. +// +// Note: There are no endian issues here because bytes are just shifted as-is, +// not converted to/from an Index. + +#define JU_DELETEINPLACE_ODD(PBYTE,POP1,OFFSET,cIS) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) < (Word_t) (POP1)); \ + { \ + Word_t b_off = (((OFFSET) + 1) * (cIS)) - 1; \ + \ + while (++b_off < ((POP1) * (cIS))) \ + (PBYTE)[b_off - (cIS)] = (PBYTE)[b_off]; \ + } + + +// INSERT/DELETE AN INDEX WHILE COPYING OTHERS: +// +// Copy PSource[] to PDest[], where PSource[] has Pop1 elements (Indexes), +// inserting Index at PDest[Offset]. Unlike JU_*INPLACE*() above, these macros +// are used when moving Indexes from one memory object to another. + +#define JU_INSERTCOPY(PDEST,PSOURCE,POP1,OFFSET,INDEX) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ + { \ + Word_t i_offset; \ + \ + for (i_offset = 0; i_offset < (OFFSET); ++i_offset) \ + (PDEST)[i_offset] = (PSOURCE)[i_offset]; \ + \ + (PDEST)[i_offset] = (INDEX); \ + \ + for (/* null */; i_offset < (POP1); ++i_offset) \ + (PDEST)[i_offset + 1] = (PSOURCE)[i_offset]; \ + } + +#define JU_INSERTCOPY3(PDEST,PSOURCE,POP1,OFFSET,INDEX) \ +assert((long) (POP1) > 0); \ +assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ +{ \ + Word_t o_ff; \ + \ + for (o_ff = 0; o_ff < (OFFSET); o_ff++) \ + { \ + Word_t i_dx = o_ff * 3; \ + (PDEST)[i_dx + 0] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2] = (PSOURCE)[i_dx + 2]; \ + } \ + JU_COPY3_LONG_TO_PINDEX(&((PDEST)[(OFFSET) * 3]), INDEX); \ + \ + for (/* null */; o_ff < (POP1); o_ff++) \ + { \ + Word_t i_dx = o_ff * 3; \ + (PDEST)[i_dx + 0 + 3] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1 + 3] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2 + 3] = (PSOURCE)[i_dx + 2]; \ + } \ +} + +#ifdef JU_64BIT + +#define JU_INSERTCOPY5(PDEST,PSOURCE,POP1,OFFSET,INDEX) \ +assert((long) (POP1) > 0); \ +assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ +{ \ + Word_t o_ff; \ + \ + for (o_ff = 0; o_ff < (OFFSET); o_ff++) \ + { \ + Word_t i_dx = o_ff * 5; \ + (PDEST)[i_dx + 0] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4] = (PSOURCE)[i_dx + 4]; \ + } \ + JU_COPY5_LONG_TO_PINDEX(&((PDEST)[(OFFSET) * 5]), INDEX); \ + \ + for (/* null */; o_ff < (POP1); o_ff++) \ + { \ + Word_t i_dx = o_ff * 5; \ + (PDEST)[i_dx + 0 + 5] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1 + 5] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2 + 5] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3 + 5] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4 + 5] = (PSOURCE)[i_dx + 4]; \ + } \ +} + +#define JU_INSERTCOPY6(PDEST,PSOURCE,POP1,OFFSET,INDEX) \ +assert((long) (POP1) > 0); \ +assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ +{ \ + Word_t o_ff; \ + \ + for (o_ff = 0; o_ff < (OFFSET); o_ff++) \ + { \ + Word_t i_dx = o_ff * 6; \ + (PDEST)[i_dx + 0] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4] = (PSOURCE)[i_dx + 4]; \ + (PDEST)[i_dx + 5] = (PSOURCE)[i_dx + 5]; \ + } \ + JU_COPY6_LONG_TO_PINDEX(&((PDEST)[(OFFSET) * 6]), INDEX); \ + \ + for (/* null */; o_ff < (POP1); o_ff++) \ + { \ + Word_t i_dx = o_ff * 6; \ + (PDEST)[i_dx + 0 + 6] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1 + 6] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2 + 6] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3 + 6] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4 + 6] = (PSOURCE)[i_dx + 4]; \ + (PDEST)[i_dx + 5 + 6] = (PSOURCE)[i_dx + 5]; \ + } \ +} + +#define JU_INSERTCOPY7(PDEST,PSOURCE,POP1,OFFSET,INDEX) \ +assert((long) (POP1) > 0); \ +assert((Word_t) (OFFSET) <= (Word_t) (POP1)); \ +{ \ + Word_t o_ff; \ + \ + for (o_ff = 0; o_ff < (OFFSET); o_ff++) \ + { \ + Word_t i_dx = o_ff * 7; \ + (PDEST)[i_dx + 0] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4] = (PSOURCE)[i_dx + 4]; \ + (PDEST)[i_dx + 5] = (PSOURCE)[i_dx + 5]; \ + (PDEST)[i_dx + 6] = (PSOURCE)[i_dx + 6]; \ + } \ + JU_COPY7_LONG_TO_PINDEX(&((PDEST)[(OFFSET) * 7]), INDEX); \ + \ + for (/* null */; o_ff < (POP1); o_ff++) \ + { \ + Word_t i_dx = o_ff * 7; \ + (PDEST)[i_dx + 0 + 7] = (PSOURCE)[i_dx + 0]; \ + (PDEST)[i_dx + 1 + 7] = (PSOURCE)[i_dx + 1]; \ + (PDEST)[i_dx + 2 + 7] = (PSOURCE)[i_dx + 2]; \ + (PDEST)[i_dx + 3 + 7] = (PSOURCE)[i_dx + 3]; \ + (PDEST)[i_dx + 4 + 7] = (PSOURCE)[i_dx + 4]; \ + (PDEST)[i_dx + 5 + 7] = (PSOURCE)[i_dx + 5]; \ + (PDEST)[i_dx + 6 + 7] = (PSOURCE)[i_dx + 6]; \ + } \ +} + +#endif // JU_64BIT + +// Counterparts to the above for deleting an Index: + +#define JU_DELETECOPY(PDEST,PSOURCE,POP1,OFFSET,IGNORE) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) < (Word_t) (POP1)); \ + { \ + Word_t i_offset; \ + \ + for (i_offset = 0; i_offset < (OFFSET); ++i_offset) \ + (PDEST)[i_offset] = (PSOURCE)[i_offset]; \ + \ + for (++i_offset; i_offset < (POP1); ++i_offset) \ + (PDEST)[i_offset - 1] = (PSOURCE)[i_offset]; \ + } + +// Variation for odd-byte-sized (non-native) Indexes, where cIS = Index Size; +// copy byte-by-byte: +// +// Note: There are no endian issues here because bytes are just shifted as-is, +// not converted to/from an Index. +// +// Note: If cIS == 1, JU_DELETECOPY_ODD == JU_DELETECOPY, at least in concept. + +#define JU_DELETECOPY_ODD(PDEST,PSOURCE,POP1,OFFSET,cIS) \ + assert((long) (POP1) > 0); \ + assert((Word_t) (OFFSET) < (Word_t) (POP1)); \ + { \ + uint8_t *_Pdest = (uint8_t *) (PDEST); \ + uint8_t *_Psource = (uint8_t *) (PSOURCE); \ + Word_t b_off; \ + \ + for (b_off = 0; b_off < ((OFFSET) * (cIS)); ++b_off) \ + *_Pdest++ = *_Psource++; \ + \ + _Psource += (cIS); \ + \ + for (b_off += (cIS); b_off < ((POP1) * (cIS)); ++b_off) \ + *_Pdest++ = *_Psource++; \ + } + + +// GENERIC RETURN CODE HANDLING FOR JUDY1 (NO VALUE AREAS) AND JUDYL (VALUE +// AREAS): +// +// This common code hides Judy1 versus JudyL details of how to return various +// conditions, including a pointer to a value area for JudyL. +// +// First, define an internal variation of JERR called JERRI (I = int) to make +// lint happy. We accidentally shipped to 11.11 OEUR with all functions that +// return int or Word_t using JERR, which is type Word_t, for errors. Lint +// complains about this for functions that return int. So, internally use +// JERRI for error returns from the int functions. Experiments show that +// callers which compare int Foo() to (Word_t) JERR (~0UL) are OK, since JERRI +// sign-extends to match JERR. + +#define JERRI ((int) ~0) // see above. + +#ifdef JUDY1 + +#define JU_RET_FOUND return(1) +#define JU_RET_NOTFOUND return(0) + +// For Judy1, these all "fall through" to simply JU_RET_FOUND, since there is no +// value area pointer to return: + +#define JU_RET_FOUND_LEAFW(PJLW,POP1,OFFSET) JU_RET_FOUND + +#define JU_RET_FOUND_JPM(Pjpm) JU_RET_FOUND +#define JU_RET_FOUND_PVALUE(Pjv,OFFSET) JU_RET_FOUND +#ifndef JU_64BIT +#define JU_RET_FOUND_LEAF1(Pjll,POP1,OFFSET) JU_RET_FOUND +#endif +#define JU_RET_FOUND_LEAF2(Pjll,POP1,OFFSET) JU_RET_FOUND +#define JU_RET_FOUND_LEAF3(Pjll,POP1,OFFSET) JU_RET_FOUND +#ifdef JU_64BIT +#define JU_RET_FOUND_LEAF4(Pjll,POP1,OFFSET) JU_RET_FOUND +#define JU_RET_FOUND_LEAF5(Pjll,POP1,OFFSET) JU_RET_FOUND +#define JU_RET_FOUND_LEAF6(Pjll,POP1,OFFSET) JU_RET_FOUND +#define JU_RET_FOUND_LEAF7(Pjll,POP1,OFFSET) JU_RET_FOUND +#endif +#define JU_RET_FOUND_IMM_01(Pjp) JU_RET_FOUND +#define JU_RET_FOUND_IMM(Pjp,OFFSET) JU_RET_FOUND + +// Note: No JudyL equivalent: + +#define JU_RET_FOUND_FULLPOPU1 JU_RET_FOUND +#define JU_RET_FOUND_LEAF_B1(PJLB,SUBEXP,OFFSET) JU_RET_FOUND + +#else // JUDYL + +// JU_RET_FOUND // see below; must NOT be defined for JudyL. +#define JU_RET_NOTFOUND return((PPvoid_t) NULL) + +// For JudyL, the location of the value area depends on the JP type and other +// factors: +// +// TBD: The value areas should be accessed via data structures, here and in +// Dougs code, not by hard-coded address calculations. +// +// This is useful in insert/delete code when the value area is returned from +// lower levels in the JPM: + +#define JU_RET_FOUND_JPM(Pjpm) return((PPvoid_t) ((Pjpm)->jpm_PValue)) + +// This is useful in insert/delete code when the value area location is already +// computed: + +#define JU_RET_FOUND_PVALUE(Pjv,OFFSET) return((PPvoid_t) ((Pjv) + OFFSET)) + +#define JU_RET_FOUND_LEAFW(PJLW,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAFWVALUEAREA(PJLW, POP1) + (OFFSET))) + +#define JU_RET_FOUND_LEAF1(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF1VALUEAREA(Pjll, POP1) + (OFFSET))) +#define JU_RET_FOUND_LEAF2(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF2VALUEAREA(Pjll, POP1) + (OFFSET))) +#define JU_RET_FOUND_LEAF3(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF3VALUEAREA(Pjll, POP1) + (OFFSET))) +#ifdef JU_64BIT +#define JU_RET_FOUND_LEAF4(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF4VALUEAREA(Pjll, POP1) + (OFFSET))) +#define JU_RET_FOUND_LEAF5(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF5VALUEAREA(Pjll, POP1) + (OFFSET))) +#define JU_RET_FOUND_LEAF6(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF6VALUEAREA(Pjll, POP1) + (OFFSET))) +#define JU_RET_FOUND_LEAF7(Pjll,POP1,OFFSET) \ + return((PPvoid_t) (JL_LEAF7VALUEAREA(Pjll, POP1) + (OFFSET))) +#endif + +// Note: Here jp_Addr is a value area itself and not an address, so P_JV() is +// not needed: + +#define JU_RET_FOUND_IMM_01(PJP) return((PPvoid_t) (&((PJP)->jp_Addr))) + +// Note: Here jp_Addr is a pointer to a separately-mallocd value area, so +// P_JV() is required; likewise for JL_JLB_PVALUE: + +#define JU_RET_FOUND_IMM(PJP,OFFSET) \ + return((PPvoid_t) (P_JV((PJP)->jp_Addr) + (OFFSET))) + +#define JU_RET_FOUND_LEAF_B1(PJLB,SUBEXP,OFFSET) \ + return((PPvoid_t) (P_JV(JL_JLB_PVALUE(PJLB, SUBEXP)) + (OFFSET))) + +#endif // JUDYL + + +// GENERIC ERROR HANDLING: +// +// This is complicated by variations in the needs of the callers of these +// macros. Only use JU_SET_ERRNO() for PJError, because it can be null; use +// JU_SET_ERRNO_NONNULL() for Pjpm, which is never null, and also in other +// cases where the pointer is known not to be null (to save dead branches). +// +// Note: Most cases of JU_ERRNO_OVERRUN or JU_ERRNO_CORRUPT should result in +// an assertion failure in debug code, so they are more likely to be caught, so +// do that here in each macro. + +#define JU_SET_ERRNO(PJError, JErrno) \ + { \ + assert((JErrno) != JU_ERRNO_OVERRUN); \ + assert((JErrno) != JU_ERRNO_CORRUPT); \ + \ + if (PJError != (PJError_t) NULL) \ + { \ + JU_ERRNO(PJError) = (JErrno); \ + JU_ERRID(PJError) = __LINE__; \ + } \ + } + +// Variation for callers who know already that PJError is non-null; and, it can +// also be Pjpm (both PJError_t and Pjpm_t have je_* fields), so only assert it +// for null, not cast to any specific pointer type: + +#define JU_SET_ERRNO_NONNULL(PJError, JErrno) \ + { \ + assert((JErrno) != JU_ERRNO_OVERRUN); \ + assert((JErrno) != JU_ERRNO_CORRUPT); \ + assert(PJError); \ + \ + JU_ERRNO(PJError) = (JErrno); \ + JU_ERRID(PJError) = __LINE__; \ + } + +// Variation to copy error info from a (required) JPM to an (optional) +// PJError_t: +// +// Note: The assertions above about JU_ERRNO_OVERRUN and JU_ERRNO_CORRUPT +// should have already popped, so they are not needed here. + +#define JU_COPY_ERRNO(PJError, Pjpm) \ + { \ + if (PJError) \ + { \ + JU_ERRNO(PJError) = (uint8_t)JU_ERRNO(Pjpm); \ + JU_ERRID(PJError) = JU_ERRID(Pjpm); \ + } \ + } + +// For JErrno parameter to previous macros upon return from Judy*Alloc*(): +// +// The memory allocator returns an address of 0 for out of memory, +// 1..sizeof(Word_t)-1 for corruption (an invalid pointer), otherwise a valid +// pointer. + +#define JU_ALLOC_ERRNO(ADDR) \ + (((void *) (ADDR) != (void *) NULL) ? JU_ERRNO_OVERRUN : JU_ERRNO_NOMEM) + +#define JU_CHECKALLOC(Type,Ptr,Retval) \ + if ((Ptr) < (Type) sizeof(Word_t)) \ + { \ + JU_SET_ERRNO(PJError, JU_ALLOC_ERRNO(Ptr)); \ + return(Retval); \ + } + +// Leaf search routines + +#ifdef JU_NOINLINE + +int j__udySearchLeaf1(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); +int j__udySearchLeaf2(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); +int j__udySearchLeaf3(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); + +#ifdef JU_64BIT + +int j__udySearchLeaf4(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); +int j__udySearchLeaf5(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); +int j__udySearchLeaf6(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); +int j__udySearchLeaf7(Pjll_t Pjll, Word_t LeafPop1, Word_t Index); + +#endif // JU_64BIT + +int j__udySearchLeafW(Pjlw_t Pjlw, Word_t LeafPop1, Word_t Index); + +#else // complier support for inline + +#ifdef JU_WIN +static __inline int j__udySearchLeaf1(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf1(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNATIVE(uint8_t, Pjll, LeafPop1, Index); } + +#ifdef JU_WIN +static __inline int j__udySearchLeaf2(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf2(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNATIVE(uint16_t, Pjll, LeafPop1, Index); } + +#ifdef JU_WIN +static __inline int j__udySearchLeaf3(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf3(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNONNAT(Pjll, LeafPop1, Index, 3, JU_COPY3_PINDEX_TO_LONG); } + +#ifdef JU_64BIT + +#ifdef JU_WIN +static __inline int j__udySearchLeaf4(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf4(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNATIVE(uint32_t, Pjll, LeafPop1, Index); } + +#ifdef JU_WIN +static __inline int j__udySearchLeaf5(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf5(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNONNAT(Pjll, LeafPop1, Index, 5, JU_COPY5_PINDEX_TO_LONG); } + +#ifdef JU_WIN +static __inline int j__udySearchLeaf6(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf6(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNONNAT(Pjll, LeafPop1, Index, 6, JU_COPY6_PINDEX_TO_LONG); } + +#ifdef JU_WIN +static __inline int j__udySearchLeaf7(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeaf7(Pjll_t Pjll, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNONNAT(Pjll, LeafPop1, Index, 7, JU_COPY7_PINDEX_TO_LONG); } + +#endif // JU_64BIT + +#ifdef JU_WIN +static __inline int j__udySearchLeafW(Pjlw_t Pjlw, Word_t LeafPop1, Word_t Index) +#else +static inline int j__udySearchLeafW(Pjlw_t Pjlw, Word_t LeafPop1, Word_t Index) +#endif +{ SEARCHLEAFNATIVE(Word_t, Pjlw, LeafPop1, Index); } + +#endif // compiler support for inline + +#endif // ! _JUDYPRIVATE_INCLUDED diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h b/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h new file mode 100644 index 000000000..5b4704899 --- /dev/null +++ b/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h @@ -0,0 +1,485 @@ +#ifndef _JUDYPRIVATE1L_INCLUDED +#define _JUDYPRIVATE1L_INCLUDED +// _________________ +// +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.31 $ $Source: /judy/src/JudyCommon/JudyPrivate1L.h $ + +// **************************************************************************** +// Declare common cJU_* names for JP Types that occur in both Judy1 and JudyL, +// for use by code that ifdefs JUDY1 and JUDYL. Only JP Types common to both +// Judy1 and JudyL are #defined here with equivalent cJU_* names. JP Types +// unique to only Judy1 or JudyL are listed in comments, so the type lists +// match the Judy1.h and JudyL.h files. +// +// This file also defines cJU_* for other JP-related constants and functions +// that some shared JUDY1/JUDYL code finds handy. +// +// At least in principle this file should be included AFTER Judy1.h or JudyL.h. +// +// WARNING: This file must be kept consistent with the enums in Judy1.h and +// JudyL.h. +// +// TBD: You might think, why not define common cJU_* enums in, say, +// JudyPrivate.h, and then inherit them into superset enums in Judy1.h and +// JudyL.h? The problem is that the enum lists for each class (cJ1_* and +// cJL_*) must be numerically "packed" into the correct order, for two reasons: +// (1) allow the compiler to generate "tight" switch statements with no wasted +// slots (although this is not very big), and (2) allow calculations using the +// enum values, although this is also not an issue if the calculations are only +// within each cJ*_JPIMMED_*_* class and the members are packed within the +// class. + +#ifdef JUDY1 + +#define cJU_JRPNULL cJ1_JRPNULL +#define cJU_JPNULL1 cJ1_JPNULL1 +#define cJU_JPNULL2 cJ1_JPNULL2 +#define cJU_JPNULL3 cJ1_JPNULL3 +#ifdef JU_64BIT +#define cJU_JPNULL4 cJ1_JPNULL4 +#define cJU_JPNULL5 cJ1_JPNULL5 +#define cJU_JPNULL6 cJ1_JPNULL6 +#define cJU_JPNULL7 cJ1_JPNULL7 +#endif +#define cJU_JPNULLMAX cJ1_JPNULLMAX +#define cJU_JPBRANCH_L2 cJ1_JPBRANCH_L2 +#define cJU_JPBRANCH_L3 cJ1_JPBRANCH_L3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_L4 cJ1_JPBRANCH_L4 +#define cJU_JPBRANCH_L5 cJ1_JPBRANCH_L5 +#define cJU_JPBRANCH_L6 cJ1_JPBRANCH_L6 +#define cJU_JPBRANCH_L7 cJ1_JPBRANCH_L7 +#endif +#define cJU_JPBRANCH_L cJ1_JPBRANCH_L +#define j__U_BranchBJPPopToWords j__1_BranchBJPPopToWords +#define cJU_JPBRANCH_B2 cJ1_JPBRANCH_B2 +#define cJU_JPBRANCH_B3 cJ1_JPBRANCH_B3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_B4 cJ1_JPBRANCH_B4 +#define cJU_JPBRANCH_B5 cJ1_JPBRANCH_B5 +#define cJU_JPBRANCH_B6 cJ1_JPBRANCH_B6 +#define cJU_JPBRANCH_B7 cJ1_JPBRANCH_B7 +#endif +#define cJU_JPBRANCH_B cJ1_JPBRANCH_B +#define cJU_JPBRANCH_U2 cJ1_JPBRANCH_U2 +#define cJU_JPBRANCH_U3 cJ1_JPBRANCH_U3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_U4 cJ1_JPBRANCH_U4 +#define cJU_JPBRANCH_U5 cJ1_JPBRANCH_U5 +#define cJU_JPBRANCH_U6 cJ1_JPBRANCH_U6 +#define cJU_JPBRANCH_U7 cJ1_JPBRANCH_U7 +#endif +#define cJU_JPBRANCH_U cJ1_JPBRANCH_U +#ifndef JU_64BIT +#define cJU_JPLEAF1 cJ1_JPLEAF1 +#endif +#define cJU_JPLEAF2 cJ1_JPLEAF2 +#define cJU_JPLEAF3 cJ1_JPLEAF3 +#ifdef JU_64BIT +#define cJU_JPLEAF4 cJ1_JPLEAF4 +#define cJU_JPLEAF5 cJ1_JPLEAF5 +#define cJU_JPLEAF6 cJ1_JPLEAF6 +#define cJU_JPLEAF7 cJ1_JPLEAF7 +#endif +#define cJU_JPLEAF_B1 cJ1_JPLEAF_B1 +// cJ1_JPFULLPOPU1 +#define cJU_JPIMMED_1_01 cJ1_JPIMMED_1_01 +#define cJU_JPIMMED_2_01 cJ1_JPIMMED_2_01 +#define cJU_JPIMMED_3_01 cJ1_JPIMMED_3_01 +#ifdef JU_64BIT +#define cJU_JPIMMED_4_01 cJ1_JPIMMED_4_01 +#define cJU_JPIMMED_5_01 cJ1_JPIMMED_5_01 +#define cJU_JPIMMED_6_01 cJ1_JPIMMED_6_01 +#define cJU_JPIMMED_7_01 cJ1_JPIMMED_7_01 +#endif +#define cJU_JPIMMED_1_02 cJ1_JPIMMED_1_02 +#define cJU_JPIMMED_1_03 cJ1_JPIMMED_1_03 +#define cJU_JPIMMED_1_04 cJ1_JPIMMED_1_04 +#define cJU_JPIMMED_1_05 cJ1_JPIMMED_1_05 +#define cJU_JPIMMED_1_06 cJ1_JPIMMED_1_06 +#define cJU_JPIMMED_1_07 cJ1_JPIMMED_1_07 +#ifdef JU_64BIT +// cJ1_JPIMMED_1_08 +// cJ1_JPIMMED_1_09 +// cJ1_JPIMMED_1_10 +// cJ1_JPIMMED_1_11 +// cJ1_JPIMMED_1_12 +// cJ1_JPIMMED_1_13 +// cJ1_JPIMMED_1_14 +// cJ1_JPIMMED_1_15 +#endif +#define cJU_JPIMMED_2_02 cJ1_JPIMMED_2_02 +#define cJU_JPIMMED_2_03 cJ1_JPIMMED_2_03 +#ifdef JU_64BIT +// cJ1_JPIMMED_2_04 +// cJ1_JPIMMED_2_05 +// cJ1_JPIMMED_2_06 +// cJ1_JPIMMED_2_07 +#endif +#define cJU_JPIMMED_3_02 cJ1_JPIMMED_3_02 +#ifdef JU_64BIT +// cJ1_JPIMMED_3_03 +// cJ1_JPIMMED_3_04 +// cJ1_JPIMMED_3_05 +// cJ1_JPIMMED_4_02 +// cJ1_JPIMMED_4_03 +// cJ1_JPIMMED_5_02 +// cJ1_JPIMMED_5_03 +// cJ1_JPIMMED_6_02 +// cJ1_JPIMMED_7_02 +#endif +#define cJU_JPIMMED_CAP cJ1_JPIMMED_CAP + +#else // JUDYL **************************************************************** + +#define cJU_JRPNULL cJL_JRPNULL +#define cJU_JPNULL1 cJL_JPNULL1 +#define cJU_JPNULL2 cJL_JPNULL2 +#define cJU_JPNULL3 cJL_JPNULL3 +#ifdef JU_64BIT +#define cJU_JPNULL4 cJL_JPNULL4 +#define cJU_JPNULL5 cJL_JPNULL5 +#define cJU_JPNULL6 cJL_JPNULL6 +#define cJU_JPNULL7 cJL_JPNULL7 +#endif +#define cJU_JPNULLMAX cJL_JPNULLMAX +#define cJU_JPBRANCH_L2 cJL_JPBRANCH_L2 +#define cJU_JPBRANCH_L3 cJL_JPBRANCH_L3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_L4 cJL_JPBRANCH_L4 +#define cJU_JPBRANCH_L5 cJL_JPBRANCH_L5 +#define cJU_JPBRANCH_L6 cJL_JPBRANCH_L6 +#define cJU_JPBRANCH_L7 cJL_JPBRANCH_L7 +#endif +#define cJU_JPBRANCH_L cJL_JPBRANCH_L +#define j__U_BranchBJPPopToWords j__L_BranchBJPPopToWords +#define cJU_JPBRANCH_B2 cJL_JPBRANCH_B2 +#define cJU_JPBRANCH_B3 cJL_JPBRANCH_B3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_B4 cJL_JPBRANCH_B4 +#define cJU_JPBRANCH_B5 cJL_JPBRANCH_B5 +#define cJU_JPBRANCH_B6 cJL_JPBRANCH_B6 +#define cJU_JPBRANCH_B7 cJL_JPBRANCH_B7 +#endif +#define cJU_JPBRANCH_B cJL_JPBRANCH_B +#define cJU_JPBRANCH_U2 cJL_JPBRANCH_U2 +#define cJU_JPBRANCH_U3 cJL_JPBRANCH_U3 +#ifdef JU_64BIT +#define cJU_JPBRANCH_U4 cJL_JPBRANCH_U4 +#define cJU_JPBRANCH_U5 cJL_JPBRANCH_U5 +#define cJU_JPBRANCH_U6 cJL_JPBRANCH_U6 +#define cJU_JPBRANCH_U7 cJL_JPBRANCH_U7 +#endif +#define cJU_JPBRANCH_U cJL_JPBRANCH_U +#define cJU_JPLEAF1 cJL_JPLEAF1 +#define cJU_JPLEAF2 cJL_JPLEAF2 +#define cJU_JPLEAF3 cJL_JPLEAF3 +#ifdef JU_64BIT +#define cJU_JPLEAF4 cJL_JPLEAF4 +#define cJU_JPLEAF5 cJL_JPLEAF5 +#define cJU_JPLEAF6 cJL_JPLEAF6 +#define cJU_JPLEAF7 cJL_JPLEAF7 +#endif +#define cJU_JPLEAF_B1 cJL_JPLEAF_B1 +#define cJU_JPIMMED_1_01 cJL_JPIMMED_1_01 +#define cJU_JPIMMED_2_01 cJL_JPIMMED_2_01 +#define cJU_JPIMMED_3_01 cJL_JPIMMED_3_01 +#ifdef JU_64BIT +#define cJU_JPIMMED_4_01 cJL_JPIMMED_4_01 +#define cJU_JPIMMED_5_01 cJL_JPIMMED_5_01 +#define cJU_JPIMMED_6_01 cJL_JPIMMED_6_01 +#define cJU_JPIMMED_7_01 cJL_JPIMMED_7_01 +#endif +#define cJU_JPIMMED_1_02 cJL_JPIMMED_1_02 +#define cJU_JPIMMED_1_03 cJL_JPIMMED_1_03 +#ifdef JU_64BIT +#define cJU_JPIMMED_1_04 cJL_JPIMMED_1_04 +#define cJU_JPIMMED_1_05 cJL_JPIMMED_1_05 +#define cJU_JPIMMED_1_06 cJL_JPIMMED_1_06 +#define cJU_JPIMMED_1_07 cJL_JPIMMED_1_07 +#define cJU_JPIMMED_2_02 cJL_JPIMMED_2_02 +#define cJU_JPIMMED_2_03 cJL_JPIMMED_2_03 +#define cJU_JPIMMED_3_02 cJL_JPIMMED_3_02 +#endif +#define cJU_JPIMMED_CAP cJL_JPIMMED_CAP + +#endif // JUDYL + + +// **************************************************************************** +// cJU*_ other than JP types: + +#ifdef JUDY1 + +#define cJU_LEAFW_MAXPOP1 cJ1_LEAFW_MAXPOP1 +#ifndef JU_64BIT +#define cJU_LEAF1_MAXPOP1 cJ1_LEAF1_MAXPOP1 +#endif +#define cJU_LEAF2_MAXPOP1 cJ1_LEAF2_MAXPOP1 +#define cJU_LEAF3_MAXPOP1 cJ1_LEAF3_MAXPOP1 +#ifdef JU_64BIT +#define cJU_LEAF4_MAXPOP1 cJ1_LEAF4_MAXPOP1 +#define cJU_LEAF5_MAXPOP1 cJ1_LEAF5_MAXPOP1 +#define cJU_LEAF6_MAXPOP1 cJ1_LEAF6_MAXPOP1 +#define cJU_LEAF7_MAXPOP1 cJ1_LEAF7_MAXPOP1 +#endif +#define cJU_IMMED1_MAXPOP1 cJ1_IMMED1_MAXPOP1 +#define cJU_IMMED2_MAXPOP1 cJ1_IMMED2_MAXPOP1 +#define cJU_IMMED3_MAXPOP1 cJ1_IMMED3_MAXPOP1 +#ifdef JU_64BIT +#define cJU_IMMED4_MAXPOP1 cJ1_IMMED4_MAXPOP1 +#define cJU_IMMED5_MAXPOP1 cJ1_IMMED5_MAXPOP1 +#define cJU_IMMED6_MAXPOP1 cJ1_IMMED6_MAXPOP1 +#define cJU_IMMED7_MAXPOP1 cJ1_IMMED7_MAXPOP1 +#endif + +#define JU_LEAF1POPTOWORDS(Pop1) J1_LEAF1POPTOWORDS(Pop1) +#define JU_LEAF2POPTOWORDS(Pop1) J1_LEAF2POPTOWORDS(Pop1) +#define JU_LEAF3POPTOWORDS(Pop1) J1_LEAF3POPTOWORDS(Pop1) +#ifdef JU_64BIT +#define JU_LEAF4POPTOWORDS(Pop1) J1_LEAF4POPTOWORDS(Pop1) +#define JU_LEAF5POPTOWORDS(Pop1) J1_LEAF5POPTOWORDS(Pop1) +#define JU_LEAF6POPTOWORDS(Pop1) J1_LEAF6POPTOWORDS(Pop1) +#define JU_LEAF7POPTOWORDS(Pop1) J1_LEAF7POPTOWORDS(Pop1) +#endif +#define JU_LEAFWPOPTOWORDS(Pop1) J1_LEAFWPOPTOWORDS(Pop1) + +#ifndef JU_64BIT +#define JU_LEAF1GROWINPLACE(Pop1) J1_LEAF1GROWINPLACE(Pop1) +#endif +#define JU_LEAF2GROWINPLACE(Pop1) J1_LEAF2GROWINPLACE(Pop1) +#define JU_LEAF3GROWINPLACE(Pop1) J1_LEAF3GROWINPLACE(Pop1) +#ifdef JU_64BIT +#define JU_LEAF4GROWINPLACE(Pop1) J1_LEAF4GROWINPLACE(Pop1) +#define JU_LEAF5GROWINPLACE(Pop1) J1_LEAF5GROWINPLACE(Pop1) +#define JU_LEAF6GROWINPLACE(Pop1) J1_LEAF6GROWINPLACE(Pop1) +#define JU_LEAF7GROWINPLACE(Pop1) J1_LEAF7GROWINPLACE(Pop1) +#endif +#define JU_LEAFWGROWINPLACE(Pop1) J1_LEAFWGROWINPLACE(Pop1) + +#define j__udyCreateBranchL j__udy1CreateBranchL +#define j__udyCreateBranchB j__udy1CreateBranchB +#define j__udyCreateBranchU j__udy1CreateBranchU +#define j__udyCascade1 j__udy1Cascade1 +#define j__udyCascade2 j__udy1Cascade2 +#define j__udyCascade3 j__udy1Cascade3 +#ifdef JU_64BIT +#define j__udyCascade4 j__udy1Cascade4 +#define j__udyCascade5 j__udy1Cascade5 +#define j__udyCascade6 j__udy1Cascade6 +#define j__udyCascade7 j__udy1Cascade7 +#endif +#define j__udyCascadeL j__udy1CascadeL +#define j__udyInsertBranch j__udy1InsertBranch + +#define j__udyBranchBToBranchL j__udy1BranchBToBranchL +#ifndef JU_64BIT +#define j__udyLeafB1ToLeaf1 j__udy1LeafB1ToLeaf1 +#endif +#define j__udyLeaf1ToLeaf2 j__udy1Leaf1ToLeaf2 +#define j__udyLeaf2ToLeaf3 j__udy1Leaf2ToLeaf3 +#ifndef JU_64BIT +#define j__udyLeaf3ToLeafW j__udy1Leaf3ToLeafW +#else +#define j__udyLeaf3ToLeaf4 j__udy1Leaf3ToLeaf4 +#define j__udyLeaf4ToLeaf5 j__udy1Leaf4ToLeaf5 +#define j__udyLeaf5ToLeaf6 j__udy1Leaf5ToLeaf6 +#define j__udyLeaf6ToLeaf7 j__udy1Leaf6ToLeaf7 +#define j__udyLeaf7ToLeafW j__udy1Leaf7ToLeafW +#endif + +#define jpm_t j1pm_t +#define Pjpm_t Pj1pm_t + +#define jlb_t j1lb_t +#define Pjlb_t Pj1lb_t + +#define JU_JLB_BITMAP J1_JLB_BITMAP + +#define j__udyAllocJPM j__udy1AllocJ1PM +#define j__udyAllocJBL j__udy1AllocJBL +#define j__udyAllocJBB j__udy1AllocJBB +#define j__udyAllocJBBJP j__udy1AllocJBBJP +#define j__udyAllocJBU j__udy1AllocJBU +#ifndef JU_64BIT +#define j__udyAllocJLL1 j__udy1AllocJLL1 +#endif +#define j__udyAllocJLL2 j__udy1AllocJLL2 +#define j__udyAllocJLL3 j__udy1AllocJLL3 +#ifdef JU_64BIT +#define j__udyAllocJLL4 j__udy1AllocJLL4 +#define j__udyAllocJLL5 j__udy1AllocJLL5 +#define j__udyAllocJLL6 j__udy1AllocJLL6 +#define j__udyAllocJLL7 j__udy1AllocJLL7 +#endif +#define j__udyAllocJLW j__udy1AllocJLW +#define j__udyAllocJLB1 j__udy1AllocJLB1 +#define j__udyFreeJPM j__udy1FreeJ1PM +#define j__udyFreeJBL j__udy1FreeJBL +#define j__udyFreeJBB j__udy1FreeJBB +#define j__udyFreeJBBJP j__udy1FreeJBBJP +#define j__udyFreeJBU j__udy1FreeJBU +#ifndef JU_64BIT +#define j__udyFreeJLL1 j__udy1FreeJLL1 +#endif +#define j__udyFreeJLL2 j__udy1FreeJLL2 +#define j__udyFreeJLL3 j__udy1FreeJLL3 +#ifdef JU_64BIT +#define j__udyFreeJLL4 j__udy1FreeJLL4 +#define j__udyFreeJLL5 j__udy1FreeJLL5 +#define j__udyFreeJLL6 j__udy1FreeJLL6 +#define j__udyFreeJLL7 j__udy1FreeJLL7 +#endif +#define j__udyFreeJLW j__udy1FreeJLW +#define j__udyFreeJLB1 j__udy1FreeJLB1 +#define j__udyFreeSM j__udy1FreeSM + +#define j__uMaxWords j__u1MaxWords + +#ifdef DEBUG +#define JudyCheckPop Judy1CheckPop +#endif + +#else // JUDYL **************************************************************** + +#define cJU_LEAFW_MAXPOP1 cJL_LEAFW_MAXPOP1 +#define cJU_LEAF1_MAXPOP1 cJL_LEAF1_MAXPOP1 +#define cJU_LEAF2_MAXPOP1 cJL_LEAF2_MAXPOP1 +#define cJU_LEAF3_MAXPOP1 cJL_LEAF3_MAXPOP1 +#ifdef JU_64BIT +#define cJU_LEAF4_MAXPOP1 cJL_LEAF4_MAXPOP1 +#define cJU_LEAF5_MAXPOP1 cJL_LEAF5_MAXPOP1 +#define cJU_LEAF6_MAXPOP1 cJL_LEAF6_MAXPOP1 +#define cJU_LEAF7_MAXPOP1 cJL_LEAF7_MAXPOP1 +#endif +#define cJU_IMMED1_MAXPOP1 cJL_IMMED1_MAXPOP1 +#define cJU_IMMED2_MAXPOP1 cJL_IMMED2_MAXPOP1 +#define cJU_IMMED3_MAXPOP1 cJL_IMMED3_MAXPOP1 +#ifdef JU_64BIT +#define cJU_IMMED4_MAXPOP1 cJL_IMMED4_MAXPOP1 +#define cJU_IMMED5_MAXPOP1 cJL_IMMED5_MAXPOP1 +#define cJU_IMMED6_MAXPOP1 cJL_IMMED6_MAXPOP1 +#define cJU_IMMED7_MAXPOP1 cJL_IMMED7_MAXPOP1 +#endif + +#define JU_LEAF1POPTOWORDS(Pop1) JL_LEAF1POPTOWORDS(Pop1) +#define JU_LEAF2POPTOWORDS(Pop1) JL_LEAF2POPTOWORDS(Pop1) +#define JU_LEAF3POPTOWORDS(Pop1) JL_LEAF3POPTOWORDS(Pop1) +#ifdef JU_64BIT +#define JU_LEAF4POPTOWORDS(Pop1) JL_LEAF4POPTOWORDS(Pop1) +#define JU_LEAF5POPTOWORDS(Pop1) JL_LEAF5POPTOWORDS(Pop1) +#define JU_LEAF6POPTOWORDS(Pop1) JL_LEAF6POPTOWORDS(Pop1) +#define JU_LEAF7POPTOWORDS(Pop1) JL_LEAF7POPTOWORDS(Pop1) +#endif +#define JU_LEAFWPOPTOWORDS(Pop1) JL_LEAFWPOPTOWORDS(Pop1) + +#define JU_LEAF1GROWINPLACE(Pop1) JL_LEAF1GROWINPLACE(Pop1) +#define JU_LEAF2GROWINPLACE(Pop1) JL_LEAF2GROWINPLACE(Pop1) +#define JU_LEAF3GROWINPLACE(Pop1) JL_LEAF3GROWINPLACE(Pop1) +#ifdef JU_64BIT +#define JU_LEAF4GROWINPLACE(Pop1) JL_LEAF4GROWINPLACE(Pop1) +#define JU_LEAF5GROWINPLACE(Pop1) JL_LEAF5GROWINPLACE(Pop1) +#define JU_LEAF6GROWINPLACE(Pop1) JL_LEAF6GROWINPLACE(Pop1) +#define JU_LEAF7GROWINPLACE(Pop1) JL_LEAF7GROWINPLACE(Pop1) +#endif +#define JU_LEAFWGROWINPLACE(Pop1) JL_LEAFWGROWINPLACE(Pop1) + +#define j__udyCreateBranchL j__udyLCreateBranchL +#define j__udyCreateBranchB j__udyLCreateBranchB +#define j__udyCreateBranchU j__udyLCreateBranchU +#define j__udyCascade1 j__udyLCascade1 +#define j__udyCascade2 j__udyLCascade2 +#define j__udyCascade3 j__udyLCascade3 +#ifdef JU_64BIT +#define j__udyCascade4 j__udyLCascade4 +#define j__udyCascade5 j__udyLCascade5 +#define j__udyCascade6 j__udyLCascade6 +#define j__udyCascade7 j__udyLCascade7 +#endif +#define j__udyCascadeL j__udyLCascadeL +#define j__udyInsertBranch j__udyLInsertBranch + +#define j__udyBranchBToBranchL j__udyLBranchBToBranchL +#define j__udyLeafB1ToLeaf1 j__udyLLeafB1ToLeaf1 +#define j__udyLeaf1ToLeaf2 j__udyLLeaf1ToLeaf2 +#define j__udyLeaf2ToLeaf3 j__udyLLeaf2ToLeaf3 +#ifndef JU_64BIT +#define j__udyLeaf3ToLeafW j__udyLLeaf3ToLeafW +#else +#define j__udyLeaf3ToLeaf4 j__udyLLeaf3ToLeaf4 +#define j__udyLeaf4ToLeaf5 j__udyLLeaf4ToLeaf5 +#define j__udyLeaf5ToLeaf6 j__udyLLeaf5ToLeaf6 +#define j__udyLeaf6ToLeaf7 j__udyLLeaf6ToLeaf7 +#define j__udyLeaf7ToLeafW j__udyLLeaf7ToLeafW +#endif + +#define jpm_t jLpm_t +#define Pjpm_t PjLpm_t + +#define jlb_t jLlb_t +#define Pjlb_t PjLlb_t + +#define JU_JLB_BITMAP JL_JLB_BITMAP + +#define j__udyAllocJPM j__udyLAllocJLPM +#define j__udyAllocJBL j__udyLAllocJBL +#define j__udyAllocJBB j__udyLAllocJBB +#define j__udyAllocJBBJP j__udyLAllocJBBJP +#define j__udyAllocJBU j__udyLAllocJBU +#define j__udyAllocJLL1 j__udyLAllocJLL1 +#define j__udyAllocJLL2 j__udyLAllocJLL2 +#define j__udyAllocJLL3 j__udyLAllocJLL3 +#ifdef JU_64BIT +#define j__udyAllocJLL4 j__udyLAllocJLL4 +#define j__udyAllocJLL5 j__udyLAllocJLL5 +#define j__udyAllocJLL6 j__udyLAllocJLL6 +#define j__udyAllocJLL7 j__udyLAllocJLL7 +#endif +#define j__udyAllocJLW j__udyLAllocJLW +#define j__udyAllocJLB1 j__udyLAllocJLB1 +// j__udyLAllocJV +#define j__udyFreeJPM j__udyLFreeJLPM +#define j__udyFreeJBL j__udyLFreeJBL +#define j__udyFreeJBB j__udyLFreeJBB +#define j__udyFreeJBBJP j__udyLFreeJBBJP +#define j__udyFreeJBU j__udyLFreeJBU +#define j__udyFreeJLL1 j__udyLFreeJLL1 +#define j__udyFreeJLL2 j__udyLFreeJLL2 +#define j__udyFreeJLL3 j__udyLFreeJLL3 +#ifdef JU_64BIT +#define j__udyFreeJLL4 j__udyLFreeJLL4 +#define j__udyFreeJLL5 j__udyLFreeJLL5 +#define j__udyFreeJLL6 j__udyLFreeJLL6 +#define j__udyFreeJLL7 j__udyLFreeJLL7 +#endif +#define j__udyFreeJLW j__udyLFreeJLW +#define j__udyFreeJLB1 j__udyLFreeJLB1 +#define j__udyFreeSM j__udyLFreeSM +// j__udyLFreeJV + +#define j__uMaxWords j__uLMaxWords + +#ifdef DEBUG +#define JudyCheckPop JudyLCheckPop +#endif + +#endif // JUDYL + +#endif // _JUDYPRIVATE1L_INCLUDED diff --git a/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h b/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h new file mode 100644 index 000000000..10295ba95 --- /dev/null +++ b/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h @@ -0,0 +1,788 @@ +#ifndef _JUDY_PRIVATE_BRANCH_INCLUDED +#define _JUDY_PRIVATE_BRANCH_INCLUDED +// _________________ +// +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 1.2 $ $Source: /home/doug/judy-1.0.5_min/test/../src/JudyCommon/RCS/JudyPrivateBranch.h,v $ +// +// Header file for all Judy sources, for global but private (non-exported) +// declarations specific to branch support. +// +// See also the "Judy Shop Manual" (try judy/doc/int/JudyShopManual.*). + + +// **************************************************************************** +// JUDY POINTER (JP) SUPPORT +// **************************************************************************** +// +// This "rich pointer" object is pivotal to Judy execution. +// +// JP CONTAINING OTHER THAN IMMEDIATE INDEXES: +// +// If the JP points to a linear or bitmap leaf, jp_DcdPopO contains the +// Population-1 in LSbs and Decode (Dcd) bytes in the MSBs. (In practice the +// Decode bits are masked off while accessing the Pop0 bits.) +// +// The Decode Size, the number of Dcd bytes available, is encoded in jpo_Type. +// It can also be thought of as the number of states "skipped" in the SM, where +// each state decodes 8 bits = 1 byte. +// +// TBD: Dont need two structures, except possibly to force jp_Type to highest +// address! +// +// Note: The jpo_u union is not required by HP-UX or Linux but Win32 because +// the cl.exe compiler otherwise refuses to pack a bitfield (DcdPopO) with +// anything else, even with the -Zp option. This is pretty ugly, but +// fortunately portable, and its all hide-able by macros (see below). + +typedef struct J_UDY_POINTER_OTHERS // JPO. + { + Word_t j_po_Addr; // first word: Pjp_t, Word_t, etc. + union { + Word_t j_po_Addr1; + uint8_t j_po_DcdP0[sizeof(Word_t) - 1]; + uint8_t j_po_Bytes[sizeof(Word_t)]; // last byte = jp_Type. + } jpo_u; + } jpo_t; + + +// JP CONTAINING IMMEDIATE INDEXES: +// +// j_pi_1Index[] plus j_pi_LIndex[] together hold as many N-byte (1..3-byte +// [1..7-byte]) Indexes as will fit in sizeof(jpi_t) less 1 byte for j_pi_Type +// (that is, 7..1 [15..1] Indexes). +// +// For Judy1, j_pi_1Index[] is used and j_pi_LIndex[] is not used. +// For JudyL, j_pi_LIndex[] is used and j_pi_1Index[] is not used. +// +// Note: Actually when Pop1 = 1, jpi_t is not used, and the least bytes of the +// single Index are stored in j_po_DcdPopO, for both Judy1 and JudyL, so for +// JudyL the j_po_Addr field can hold the target value. +// +// TBD: Revise this structure to not overload j_po_DcdPopO this way? The +// current arrangement works, its just confusing. + +typedef struct _JUDY_POINTER_IMMEDL + { + Word_t j_pL_Addr; + uint8_t j_pL_LIndex[sizeof(Word_t) - 1]; // see above. + uint8_t j_pL_Type; + } jpL_t; + +typedef struct _JUDY_POINTER_IMMED1 + { + uint8_t j_p1_1Index[(2 * sizeof(Word_t)) - 1]; + uint8_t j_p1_Type; + } jp1_t; + +// UNION OF JP TYPES: +// +// A branch is an array of cJU_BRANCHUNUMJPS (256) of this object, or an +// alternate data type such as: A linear branch which is a list of 2..7 JPs, +// or a bitmap branch which contains 8 lists of 0..32 JPs. JPs reside only in +// branches of a Judy SM. + +typedef union J_UDY_POINTER // JP. + { + jpo_t j_po; // other than immediate indexes. + jpL_t j_pL; // immediate indexes. + jp1_t j_p1; // immediate indexes. + } jp_t, *Pjp_t; + +// For coding convenience: +// +// Note, jp_Type has the same bits in jpo_t jpL_t and jp1_t. + +#define jp_1Index j_p1.j_p1_1Index // for storing Indexes in first word. +#define jp_LIndex j_pL.j_pL_LIndex // for storing Indexes in second word. +#define jp_Addr j_po.j_po_Addr +#define jp_Addr1 j_po.jpo_u.j_po_Addr1 +//#define jp_DcdPop0 j_po.jpo_u.j_po_DcdPop0 +#define jp_Addr1 j_po.jpo_u.j_po_Addr1 +//#define jp_Type j_po.jpo_u.j_po_Bytes[sizeof(Word_t) - 1] +#define jp_Type j_p1.j_p1_Type +#define jp_DcdP0 j_po.jpo_u.j_po_DcdP0 + + +// **************************************************************************** +// JUDY POINTER (JP) -- RELATED MACROS AND CONSTANTS +// **************************************************************************** + +// EXTRACT VALUES FROM JP: +// +// Masks for the bytes in the Dcd and Pop0 parts of jp_DcdPopO: +// +// cJU_DCDMASK() consists of a mask that excludes the (LSb) Pop0 bytes and +// also, just to be safe, the top byte of the word, since jp_DcdPopO is 1 byte +// less than a full word. +// +// Note: These are constant macros (cJU) because cPopBytes should be a +// constant. Also note cPopBytes == state in the SM. + +#define cJU_POP0MASK(cPopBytes) JU_LEASTBYTESMASK(cPopBytes) + +#define cJU_DCDMASK(cPopBytes) \ + ((cJU_ALLONES >> cJU_BITSPERBYTE) & (~cJU_POP0MASK(cPopBytes))) + +// Mask off the high byte from INDEX to it can be compared to DcdPopO: + +#define JU_TRIMTODCDSIZE(INDEX) ((cJU_ALLONES >> cJU_BITSPERBYTE) & (INDEX)) + +// Get from jp_DcdPopO the Pop0 for various branch JP Types: +// +// Note: There are no simple macros for cJU_BRANCH* Types because their +// populations must be added up and dont reside in an already-calculated +// place. + +#define JU_JPBRANCH_POP0(PJP,cPopBytes) \ + (JU_JPDCDPOP0(PJP) & cJU_POP0MASK(cPopBytes)) + +// METHOD FOR DETERMINING IF OBJECTS HAVE ROOM TO GROW: +// +// J__U_GROWCK() is a generic method to determine if an object can grow in +// place, based on whether the next population size (one more) would use the +// same space. + +#define J__U_GROWCK(POP1,MAXPOP1,POPTOWORDS) \ + (((POP1) != (MAXPOP1)) && (POPTOWORDS[POP1] == POPTOWORDS[(POP1) + 1])) + +#define JU_BRANCHBJPGROWINPLACE(NumJPs) \ + J__U_GROWCK(NumJPs, cJU_BITSPERSUBEXPB, j__U_BranchBJPPopToWords) + + +// DETERMINE IF AN INDEX IS (NOT) IN A JPS EXPANSE: + +#define JU_DCDNOTMATCHINDEX(INDEX,PJP,POP0BYTES) \ + (((INDEX) ^ JU_JPDCDPOP0(PJP)) & cJU_DCDMASK(POP0BYTES)) + + +// NUMBER OF JPs IN AN UNCOMPRESSED BRANCH: +// +// An uncompressed branch is simply an array of 256 Judy Pointers (JPs). It is +// a minimum cacheline fill object. Define it here before its first needed. + +#define cJU_BRANCHUNUMJPS cJU_SUBEXPPERSTATE + + +// **************************************************************************** +// JUDY BRANCH LINEAR (JBL) SUPPORT +// **************************************************************************** +// +// A linear branch is a way of compressing empty expanses (null JPs) out of an +// uncompressed 256-way branch, when the number of populated expanses is so +// small that even a bitmap branch is excessive. +// +// The maximum number of JPs in a Judy linear branch: +// +// Note: This number results in a 1-cacheline sized structure. Previous +// versions had a larger struct so a linear branch didnt become a bitmap +// branch until the memory consumed was even, but for speed, its better to +// switch "sooner" and keep a linear branch fast. + +#define cJU_BRANCHLMAXJPS 7 + + +// LINEAR BRANCH STRUCT: +// +// 1-byte count, followed by array of byte-sized expanses, followed by JPs. + +typedef struct J__UDY_BRANCH_LINEAR + { + uint8_t jbl_NumJPs; // num of JPs (Pjp_t), 1..N. + uint8_t jbl_Expanse[cJU_BRANCHLMAXJPS]; // 1..7 MSbs of pop exps. + jp_t jbl_jp [cJU_BRANCHLMAXJPS]; // JPs for populated exps. + } jbl_t, * Pjbl_t; + + +// **************************************************************************** +// JUDY BRANCH BITMAP (JBB) SUPPORT +// **************************************************************************** +// +// A bitmap branch is a way of compressing empty expanses (null JPs) out of +// uncompressed 256-way branch. This costs 1 additional cache line fill, but +// can save a lot of memory when it matters most, near the leaves, and +// typically there will be only one at most in the path to any Index (leaf). +// +// The bitmap indicates which of the cJU_BRANCHUNUMJPS (256) JPs in the branch +// are NOT null, that is, their expanses are populated. The jbb_t also +// contains N pointers to "mini" Judy branches ("subexpanses") of up to M JPs +// each (see BITMAP_BRANCHMxN, for example, BITMAP_BRANCH32x8), where M x N = +// cJU_BRANCHUNUMJPS. These are dynamically allocated and never contain +// cJ*_JPNULL* jp_Types. An empty subexpanse is represented by no bit sets in +// the corresponding subexpanse bitmap, in which case the corresponding +// jbbs_Pjp pointers value is unused. +// +// Note that the number of valid JPs in each 1-of-N subexpanses is determined +// by POPULATION rather than by EXPANSE -- the desired outcome to save memory +// when near the leaves. Note that the memory required for 185 JPs is about as +// much as an uncompressed 256-way branch, therefore 184 is set as the maximum. +// However, it is expected that a conversion to an uncompressed 256-way branch +// will normally take place before this limit is reached for other reasons, +// such as improving performance when the "wasted" memory is well amortized by +// the population under the branch, preserving an acceptable overall +// bytes/Index in the Judy array. +// +// The number of pointers to arrays of JPs in the Judy bitmap branch: +// +// Note: The numbers below are the same in both 32 and 64 bit systems. + +#define cJU_BRANCHBMAXJPS 184 // maximum JPs for bitmap branches. + +// Convenience wrappers for referencing BranchB bitmaps or JP subarray +// pointers: +// +// Note: JU_JBB_PJP produces a "raw" memory address that must pass through +// P_JP before use, except when freeing memory: + +#define JU_JBB_BITMAP(Pjbb, SubExp) ((Pjbb)->jbb_jbbs[SubExp].jbbs_Bitmap) +#define JU_JBB_PJP( Pjbb, SubExp) ((Pjbb)->jbb_jbbs[SubExp].jbbs_Pjp) + +#define JU_SUBEXPB(Digit) (((Digit) / cJU_BITSPERSUBEXPB) & (cJU_NUMSUBEXPB-1)) + +#define JU_BITMAPTESTB(Pjbb, Index) \ + (JU_JBB_BITMAP(Pjbb, JU_SUBEXPB(Index)) & JU_BITPOSMASKB(Index)) + +#define JU_BITMAPSETB(Pjbb, Index) \ + (JU_JBB_BITMAP(Pjbb, JU_SUBEXPB(Index)) |= JU_BITPOSMASKB(Index)) + +// Note: JU_BITMAPCLEARB is not defined because the code does it a faster way. + +typedef struct J__UDY_BRANCH_BITMAP_SUBEXPANSE + { + BITMAPB_t jbbs_Bitmap; + Pjp_t jbbs_Pjp; + + } jbbs_t; + +typedef struct J__UDY_BRANCH_BITMAP + { + jbbs_t jbb_jbbs [cJU_NUMSUBEXPB]; +#ifdef SUBEXPCOUNTS + Word_t jbb_subPop1[cJU_NUMSUBEXPB]; +#endif + } jbb_t, * Pjbb_t; + +#define JU_BRANCHJP_NUMJPSTOWORDS(NumJPs) (j__U_BranchBJPPopToWords[NumJPs]) + +#ifdef SUBEXPCOUNTS +#define cJU_NUMSUBEXPU 16 // number of subexpanse counts. +#endif + + +// **************************************************************************** +// JUDY BRANCH UNCOMPRESSED (JBU) SUPPORT +// **************************************************************************** + +// Convenience wrapper for referencing BranchU JPs: +// +// Note: This produces a non-"raw" address already passed through P_JBU(). + +#define JU_JBU_PJP(Pjp,Index,Level) \ + (&((P_JBU((Pjp)->jp_Addr))->jbu_jp[JU_DIGITATSTATE(Index, Level)])) +#define JU_JBU_PJP0(Pjp) \ + (&((P_JBU((Pjp)->jp_Addr))->jbu_jp[0])) + +typedef struct J__UDY_BRANCH_UNCOMPRESSED + { + jp_t jbu_jp [cJU_BRANCHUNUMJPS]; // JPs for populated exp. +#ifdef SUBEXPCOUNTS + Word_t jbu_subPop1[cJU_NUMSUBEXPU]; +#endif + } jbu_t, * Pjbu_t; + + +// **************************************************************************** +// OTHER SUPPORT FOR JUDY STATE MACHINES (SMs) +// **************************************************************************** + +// OBJECT SIZES IN WORDS: +// +// Word_ts per various JudyL structures that have constant sizes. +// cJU_WORDSPERJP should always be 2; this is fundamental to the Judy +// structures. + +#define cJU_WORDSPERJP (sizeof(jp_t) / cJU_BYTESPERWORD) +#define cJU_WORDSPERCL (cJU_BYTESPERCL / cJU_BYTESPERWORD) + + +// OPPORTUNISTIC UNCOMPRESSION: +// +// Define populations at which a BranchL or BranchB must convert to BranchU. +// Earlier conversion is possible with good memory efficiency -- see below. + +#ifndef NO_BRANCHU + +// Max population below BranchL, then convert to BranchU: + +#define JU_BRANCHL_MAX_POP 1000 + +// Minimum global population increment before next conversion of a BranchB to a +// BranchU: +// +// This is was done to allow malloc() to coalesce memory before the next big +// (~512 words) allocation. + +#define JU_BTOU_POP_INCREMENT 300 + +// Min/max population below BranchB, then convert to BranchU: + +#define JU_BRANCHB_MIN_POP 135 +#define JU_BRANCHB_MAX_POP 750 + +#else // NO_BRANCHU + +// These are set up to have conservative conversion schedules to BranchU: + +#define JU_BRANCHL_MAX_POP (-1UL) +#define JU_BTOU_POP_INCREMENT 300 +#define JU_BRANCHB_MIN_POP 1000 +#define JU_BRANCHB_MAX_POP (-1UL) + +#endif // NO_BRANCHU + + +// MISCELLANEOUS MACROS: + +// Get N most significant bits from the shifted Index word: +// +// As Index words are decoded, they are shifted left so only relevant, +// undecoded Index bits remain. + +#define JU_BITSFROMSFTIDX(SFTIDX, N) ((SFTIDX) >> (cJU_BITSPERWORD - (N))) + +// TBD: I have my doubts about the necessity of these macros (dlb): + +// Produce 1-digit mask at specified state: + +#define cJU_MASKATSTATE(State) (0xffL << (((State) - 1) * cJU_BITSPERBYTE)) + +// Get byte (digit) from Index at the specified state, right justified: +// +// Note: State must be 1..cJU_ROOTSTATE, and Digits must be 1..(cJU_ROOTSTATE +// - 1), but theres no way to assert these within an expression. + +#define JU_DIGITATSTATE(Index,cState) \ + ((uint8_t)((Index) >> (((cState) - 1) * cJU_BITSPERBYTE))) + +// Similarly, place byte (digit) at correct position for the specified state: +// +// Note: Cast digit to a Word_t first so there are no complaints or problems +// about shifting it more than 32 bits on a 64-bit system, say, when it is a +// uint8_t from jbl_Expanse[]. (Believe it or not, the C standard says to +// promote an unsigned char to a signed int; -Ac does not do this, but -Ae +// does.) +// +// Also, to make lint happy, cast the whole result again because apparently +// shifting a Word_t does not result in a Word_t! + +#define JU_DIGITTOSTATE(Digit,cState) \ + ((Word_t) (((Word_t) (Digit)) << (((cState) - 1) * cJU_BITSPERBYTE))) + +#endif // ! _JUDY_PRIVATE_BRANCH_INCLUDED + + +#ifdef TEST_INSDEL + +// **************************************************************************** +// TEST CODE FOR INSERT/DELETE MACROS +// **************************************************************************** +// +// To use this, compile a temporary *.c file containing: +// +// #define DEBUG +// #define JUDY_ASSERT +// #define TEST_INSDEL +// #include "JudyPrivate.h" +// #include "JudyPrivateBranch.h" +// +// Use a command like this: cc -Ae +DD64 -I. -I JudyCommon -o t t.c +// For best results, include +DD64 on a 64-bit system. +// +// This test code exercises some tricky macros, but the output must be studied +// manually to verify it. Assume that for even-index testing, whole words +// (Word_t) suffices. + +#include <stdio.h> + +#define INDEXES 3 // in each array. + + +// **************************************************************************** +// I N I T +// +// Set up variables for next test. See usage. + +FUNCTION void Init ( + int base, + PWord_t PeIndex, + PWord_t PoIndex, + PWord_t Peleaf, // always whole words. +#ifndef JU_64BIT + uint8_t * Poleaf3) +#else + uint8_t * Poleaf3, + uint8_t * Poleaf5, + uint8_t * Poleaf6, + uint8_t * Poleaf7) +#endif +{ + int offset; + + *PeIndex = 99; + + for (offset = 0; offset <= INDEXES; ++offset) + Peleaf[offset] = base + offset; + + for (offset = 0; offset < (INDEXES + 1) * 3; ++offset) + Poleaf3[offset] = base + offset; + +#ifndef JU_64BIT + *PoIndex = (91 << 24) | (92 << 16) | (93 << 8) | 94; +#else + + *PoIndex = (91L << 56) | (92L << 48) | (93L << 40) | (94L << 32) + | (95L << 24) | (96L << 16) | (97L << 8) | 98L; + + for (offset = 0; offset < (INDEXES + 1) * 5; ++offset) + Poleaf5[offset] = base + offset; + + for (offset = 0; offset < (INDEXES + 1) * 6; ++offset) + Poleaf6[offset] = base + offset; + + for (offset = 0; offset < (INDEXES + 1) * 7; ++offset) + Poleaf7[offset] = base + offset; +#endif + +} // Init() + + +// **************************************************************************** +// P R I N T L E A F +// +// Print the byte values in a leaf. + +FUNCTION void PrintLeaf ( + char * Label, // for output. + int IOffset, // insertion offset in array. + int Indsize, // index size in bytes. + uint8_t * PLeaf) // array of Index bytes. +{ + int offset; // in PLeaf. + int byte; // in one word. + + (void) printf("%s %u: ", Label, IOffset); + + for (offset = 0; offset <= INDEXES; ++offset) + { + for (byte = 0; byte < Indsize; ++byte) + (void) printf("%2d", PLeaf[(offset * Indsize) + byte]); + + (void) printf(" "); + } + + (void) printf("\n"); + +} // PrintLeaf() + + +// **************************************************************************** +// M A I N +// +// Test program. + +FUNCTION main() +{ + Word_t eIndex; // even, to insert. + Word_t oIndex; // odd, to insert. + Word_t eleaf [ INDEXES + 1]; // even leaf, index size 4. + uint8_t oleaf3[(INDEXES + 1) * 3]; // odd leaf, index size 3. +#ifdef JU_64BIT + uint8_t oleaf5[(INDEXES + 1) * 5]; // odd leaf, index size 5. + uint8_t oleaf6[(INDEXES + 1) * 6]; // odd leaf, index size 6. + uint8_t oleaf7[(INDEXES + 1) * 7]; // odd leaf, index size 7. +#endif + Word_t eleaf_2 [ INDEXES + 1]; // same, but second arrays: + uint8_t oleaf3_2[(INDEXES + 1) * 3]; +#ifdef JU_64BIT + uint8_t oleaf5_2[(INDEXES + 1) * 5]; + uint8_t oleaf6_2[(INDEXES + 1) * 6]; + uint8_t oleaf7_2[(INDEXES + 1) * 7]; +#endif + int ioffset; // index insertion offset. + +#ifndef JU_64BIT +#define INIT Init( 0, & eIndex, & oIndex, eleaf, oleaf3) +#define INIT2 INIT; Init(50, & eIndex, & oIndex, eleaf_2, oleaf3_2) +#else +#define INIT Init( 0, & eIndex, & oIndex, eleaf, oleaf3, \ + oleaf5, oleaf6, oleaf7) +#define INIT2 INIT; Init(50, & eIndex, & oIndex, eleaf_2, oleaf3_2, \ + oleaf5_2, oleaf6_2, oleaf7_2) +#endif + +#define WSIZE sizeof (Word_t) // shorthand. + +#ifdef PRINTALL // to turn on "noisy" printouts. +#define PRINTLEAF(Label,IOffset,Indsize,PLeaf) \ + PrintLeaf(Label,IOffset,Indsize,PLeaf) +#else +#define PRINTLEAF(Label,IOffset,Indsize,PLeaf) \ + if (ioffset == 0) \ + PrintLeaf(Label,IOffset,Indsize,PLeaf) +#endif + + (void) printf( +"In each case, tests operate on an initial array of %d indexes. Even-index\n" +"tests set index values to 0,1,2...; odd-index tests set byte values to\n" +"0,1,2... Inserted indexes have a value of 99 or else byte values 91,92,...\n", + INDEXES); + + (void) puts("\nJU_INSERTINPLACE():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, WSIZE, (uint8_t *) eleaf); + JU_INSERTINPLACE(eleaf, INDEXES, ioffset, eIndex); + PrintLeaf("After ", ioffset, WSIZE, (uint8_t *) eleaf); + } + + (void) puts("\nJU_INSERTINPLACE3():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 3, oleaf3); + JU_INSERTINPLACE3(oleaf3, INDEXES, ioffset, oIndex); + PrintLeaf("After ", ioffset, 3, oleaf3); + } + +#ifdef JU_64BIT + (void) puts("\nJU_INSERTINPLACE5():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 5, oleaf5); + JU_INSERTINPLACE5(oleaf5, INDEXES, ioffset, oIndex); + PrintLeaf("After ", ioffset, 5, oleaf5); + } + + (void) puts("\nJU_INSERTINPLACE6():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 6, oleaf6); + JU_INSERTINPLACE6(oleaf6, INDEXES, ioffset, oIndex); + PrintLeaf("After ", ioffset, 6, oleaf6); + } + + (void) puts("\nJU_INSERTINPLACE7():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 7, oleaf7); + JU_INSERTINPLACE7(oleaf7, INDEXES, ioffset, oIndex); + PrintLeaf("After ", ioffset, 7, oleaf7); + } +#endif // JU_64BIT + + (void) puts("\nJU_DELETEINPLACE():"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, WSIZE, (uint8_t *) eleaf); + JU_DELETEINPLACE(eleaf, INDEXES, ioffset); + PrintLeaf("After ", ioffset, WSIZE, (uint8_t *) eleaf); + } + + (void) puts("\nJU_DELETEINPLACE_ODD(3):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 3, oleaf3); + JU_DELETEINPLACE_ODD(oleaf3, INDEXES, ioffset, 3); + PrintLeaf("After ", ioffset, 3, oleaf3); + } + +#ifdef JU_64BIT + (void) puts("\nJU_DELETEINPLACE_ODD(5):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 5, oleaf5); + JU_DELETEINPLACE_ODD(oleaf5, INDEXES, ioffset, 5); + PrintLeaf("After ", ioffset, 5, oleaf5); + } + + (void) puts("\nJU_DELETEINPLACE_ODD(6):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 6, oleaf6); + JU_DELETEINPLACE_ODD(oleaf6, INDEXES, ioffset, 6); + PrintLeaf("After ", ioffset, 6, oleaf6); + } + + (void) puts("\nJU_DELETEINPLACE_ODD(7):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT; + PRINTLEAF("Before", ioffset, 7, oleaf7); + JU_DELETEINPLACE_ODD(oleaf7, INDEXES, ioffset, 7); + PrintLeaf("After ", ioffset, 7, oleaf7); + } +#endif // JU_64BIT + + (void) puts("\nJU_INSERTCOPY():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, WSIZE, (uint8_t *) eleaf); + PRINTLEAF("Before, dest", ioffset, WSIZE, (uint8_t *) eleaf_2); + JU_INSERTCOPY(eleaf_2, eleaf, INDEXES, ioffset, eIndex); + PRINTLEAF("After, src ", ioffset, WSIZE, (uint8_t *) eleaf); + PrintLeaf("After, dest", ioffset, WSIZE, (uint8_t *) eleaf_2); + } + + (void) puts("\nJU_INSERTCOPY3():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 3, oleaf3); + PRINTLEAF("Before, dest", ioffset, 3, oleaf3_2); + JU_INSERTCOPY3(oleaf3_2, oleaf3, INDEXES, ioffset, oIndex); + PRINTLEAF("After, src ", ioffset, 3, oleaf3); + PrintLeaf("After, dest", ioffset, 3, oleaf3_2); + } + +#ifdef JU_64BIT + (void) puts("\nJU_INSERTCOPY5():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 5, oleaf5); + PRINTLEAF("Before, dest", ioffset, 5, oleaf5_2); + JU_INSERTCOPY5(oleaf5_2, oleaf5, INDEXES, ioffset, oIndex); + PRINTLEAF("After, src ", ioffset, 5, oleaf5); + PrintLeaf("After, dest", ioffset, 5, oleaf5_2); + } + + (void) puts("\nJU_INSERTCOPY6():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 6, oleaf6); + PRINTLEAF("Before, dest", ioffset, 6, oleaf6_2); + JU_INSERTCOPY6(oleaf6_2, oleaf6, INDEXES, ioffset, oIndex); + PRINTLEAF("After, src ", ioffset, 6, oleaf6); + PrintLeaf("After, dest", ioffset, 6, oleaf6_2); + } + + (void) puts("\nJU_INSERTCOPY7():"); + + for (ioffset = 0; ioffset <= INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 7, oleaf7); + PRINTLEAF("Before, dest", ioffset, 7, oleaf7_2); + JU_INSERTCOPY7(oleaf7_2, oleaf7, INDEXES, ioffset, oIndex); + PRINTLEAF("After, src ", ioffset, 7, oleaf7); + PrintLeaf("After, dest", ioffset, 7, oleaf7_2); + } +#endif // JU_64BIT + + (void) puts("\nJU_DELETECOPY():"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, WSIZE, (uint8_t *) eleaf); + PRINTLEAF("Before, dest", ioffset, WSIZE, (uint8_t *) eleaf_2); + JU_DELETECOPY(eleaf_2, eleaf, INDEXES, ioffset, ignore); + PRINTLEAF("After, src ", ioffset, WSIZE, (uint8_t *) eleaf); + PrintLeaf("After, dest", ioffset, WSIZE, (uint8_t *) eleaf_2); + } + + (void) puts("\nJU_DELETECOPY_ODD(3):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 3, oleaf3); + PRINTLEAF("Before, dest", ioffset, 3, oleaf3_2); + JU_DELETECOPY_ODD(oleaf3_2, oleaf3, INDEXES, ioffset, 3); + PRINTLEAF("After, src ", ioffset, 3, oleaf3); + PrintLeaf("After, dest", ioffset, 3, oleaf3_2); + } + +#ifdef JU_64BIT + (void) puts("\nJU_DELETECOPY_ODD(5):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 5, oleaf5); + PRINTLEAF("Before, dest", ioffset, 5, oleaf5_2); + JU_DELETECOPY_ODD(oleaf5_2, oleaf5, INDEXES, ioffset, 5); + PRINTLEAF("After, src ", ioffset, 5, oleaf5); + PrintLeaf("After, dest", ioffset, 5, oleaf5_2); + } + + (void) puts("\nJU_DELETECOPY_ODD(6):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 6, oleaf6); + PRINTLEAF("Before, dest", ioffset, 6, oleaf6_2); + JU_DELETECOPY_ODD(oleaf6_2, oleaf6, INDEXES, ioffset, 6); + PRINTLEAF("After, src ", ioffset, 6, oleaf6); + PrintLeaf("After, dest", ioffset, 6, oleaf6_2); + } + + (void) puts("\nJU_DELETECOPY_ODD(7):"); + + for (ioffset = 0; ioffset < INDEXES; ++ioffset) + { + INIT2; + PRINTLEAF("Before, src ", ioffset, 7, oleaf7); + PRINTLEAF("Before, dest", ioffset, 7, oleaf7_2); + JU_DELETECOPY_ODD(oleaf7_2, oleaf7, INDEXES, ioffset, 7); + PRINTLEAF("After, src ", ioffset, 7, oleaf7); + PrintLeaf("After, dest", ioffset, 7, oleaf7_2); + } +#endif // JU_64BIT + + return(0); + +} // main() + +#endif // TEST_INSDEL diff --git a/libnetdata/libjudy/src/JudyHS/JudyHS.c b/libnetdata/libjudy/src/JudyHS/JudyHS.c new file mode 100644 index 000000000..21191babc --- /dev/null +++ b/libnetdata/libjudy/src/JudyHS/JudyHS.c @@ -0,0 +1,771 @@ +// @(#) $Revision: 4.1 $ $Source: /judy/src/JudyHS/JudyHS.c +//======================================================================= +// Author Douglas L. Baskins, Dec 2003. +// Permission to use this code is freely granted, provided that this +// statement is retained. +// email - doug@sourcejudy.com -or- dougbaskins@yahoo.com +//======================================================================= + +#include <string.h> // for memcmp(), memcpy() + +#include <Judy.h> // for JudyL* routines/macros + +/* + This routine is a very fast "string" version of an ADT that stores + (JudyHSIns()), retrieves (JudyHSGet()), deletes (JudyHSDel()) and + frees the entire ADT (JudyHSFreeArray()) strings. It uses the "Judy + arrays" JudyL() API as the main workhorse. The length of the string + is included in the calling parameters so that strings with embedded + \0s can be used. The string lengths can be from 0 bytes to whatever + malloc() can handle (~2GB). + + Compile: + + cc -O JudyHS.c -c needs to link with -lJudy (libJudy.a) + + Note: in gcc version 3.3.1, -O2 generates faster code than -O + Note: in gcc version 3.3.2, -O3 generates faster code than -O2 + + NOTES: + +1) There may be some performance issues with 64 bit machines, because I + have not characterized that it yet. + +2) It appears that a modern CPU (>2Ghz) that the instruction times are + much faster that a RAM access, so building up a word from bytes takes + no longer that a whole word access. I am taking advantage of this to + make this code endian neutral. A side effect of this is strings do + not need to be aligned, nor tested to be on to a word boundry. In + older and in slow (RISC) machines, this may be a performance issue. + I have given up trying to optimize for machines that have very slow + mpy, mod, variable shifts and call returns. + +3) JudyHS is very scalable from 1 string to billions (with enough RAM). + The memory usage is also scales with population. I have attempted to + combine the best characteristics of JudyL arrays with Hashing methods + and well designed modern processors (such as the 1.3Ghz Intel + Centrino this is being written on). + + HOW JudyHS WORKS: ( 4[8] means 4 bytes in 32 bit machine and 8 in 64) + + A) A JudyL array is used to separate strings of equal lengths into + their own structures (a different hash table is used for each length + of string). The additional time overhead is very near zero because + of the CPU cache. The space efficiency is improved because the + length need not be stored with the string (ls_t). The "JLHash" ADT + in the test program "StringCompare" is verification of both these + assumptions. + + B) A 32 bit hash value is produced from the string. Many thanks to + the Internet and the author (Bob Jenkins) for coming up with a very + good and fast universal string hash. Next the 32 bit hash number is + used as an Index to another JudyL array. Notice that one (1) JudyL + array is used as a hash table per each string length. If there are + no hash collisions (normally) then the string is copied to a + structure (ls_t) along with room for storing a Value. A flag is + added to the pointer to note it is pointing to a ls_t structure. + Since the lengths of the strings are the same, there is no need to + stored length of string in the ls_t structure. This saves about a + word per string of memory. + + C) When there is a hashing collision (very rare), a JudyL array is + used to decode the next 4[8] bytes of the string. That is, the next + 4[8] bytes of the string are used as the Index. This process is + repeated until the remaining string is unique. The remaining string + (if any) is stored in a (now smaller) ls_t structure. If the + remaining string is less or equal to 4[8] bytes, then the ls_t + structure is not needed and the Value area in the JudyL array is + used. A compile option -DDONOTUSEHASH is available to test this + structure without using hashing (only the JudyL tree is used). This + is equivalent to having all strings hashed to the same bucket. The + speed is still better than all other tree based ADTs I have tested. + An added benefit of this is a very fast "hash collision" resolving. + It could foil hackers that exploit the slow synonym (linked-list) + collision handling property used with most hashing algorithms. If + this is not a necessary property, then a simpler ADT "JLHash" that is + documented the the test program "StringCompare.c" may be used with a + little loss of memory efficiency (because it includes the string + length with the ls_t structure). JudyHS was written to be the + fastest, very scalable, memory efficient, general purpose string ADT + possible. (However, I would like to eat those words someday). (dlb) + +*/ + +#ifdef EXAMPLE_CODE +#include <stdio.h> +#include <unistd.h> +#include <string.h> + +#include <Judy.h> + +//#include "JudyHS.h" // for Judy.h without JudyHS*() + +// By Doug Baskins Apr 2004 - for JudyHS man page + +#define MAXLINE 1000000 /* max length of line */ +char Index[MAXLINE]; // string to check + +int // Usage: CheckDupLines < file +main() +{ + Pvoid_t PJArray = (PWord_t)NULL; // Judy array. + PWord_t PValue; // ^ Judy array element. + Word_t Bytes; // size of JudyHS array. + Word_t LineNumb = 0; // current line number + Word_t Dups = 0; // number of duplicate lines + + while (fgets(Index, MAXLINE, stdin) != (char *)NULL) + { + LineNumb++; // line number + +// store string into array + JHSI(PValue, PJArray, Index, strlen(Index)); + if (*PValue) // check if duplicate + { + Dups++; // count duplicates + printf("Duplicate lines %lu:%lu:%s", *PValue, LineNumb, Index); + } + else + { + *PValue = LineNumb; // store Line number + } + } + printf("%lu Duplicates, free JudyHS array of %lu Lines\n", + Dups, LineNumb - Dups); + JHSFA(Bytes, PJArray); // free array + printf("The JudyHS array allocated %lu bytes of memory\n", Bytes); + return (0); +} +#endif // EXAMPLE_CODE + +// Note: Use JLAP_INVALID, which is non-zero, to mark pointers to a ls_t +// This makes it compatable with previous versions of JudyL() + +#define IS_PLS(PLS) (((Word_t) (PLS)) & JLAP_INVALID) +#define CLEAR_PLS(PLS) (((Word_t) (PLS)) & (~JLAP_INVALID)) +#define SET_PLS(PLS) (((Word_t) (PLS)) | JLAP_INVALID) + +#define WORDSIZE (sizeof(Word_t)) + +// this is the struct used for "leaf" strings. Note that +// the Value is followed by a "variable" length ls_String array. +// +typedef struct L_EAFSTRING +{ + Word_t ls_Value; // Value area (cannot change size) + uint8_t ls_String[WORDSIZE]; // to fill out to a Word_t size +} ls_t , *Pls_t; + +#define LS_STRUCTOVD (sizeof(ls_t) - WORDSIZE) + +// Calculate size of ls_t including the string of length of LEN. +// +#define LS_WORDLEN(LEN) (((LEN) + LS_STRUCTOVD + WORDSIZE - 1) / WORDSIZE) + +// Copy from 0..4[8] bytes from string to a Word_t +// NOTE: the copy in in little-endian order to take advantage of improved +// memory efficiency of JudyLIns() with smaller numbers +// +#define COPYSTRING4toWORD(WORD,STR,LEN) \ +{ \ + WORD = 0; \ + switch(LEN) \ + { \ + default: /* four and greater */ \ + case 4: \ + WORD += (Word_t)(((uint8_t *)(STR))[3] << 24); \ + case 3: \ + WORD += (Word_t)(((uint8_t *)(STR))[2] << 16); \ + case 2: \ + WORD += (Word_t)(((uint8_t *)(STR))[1] << 8); \ + case 1: \ + WORD += (Word_t)(((uint8_t *)(STR))[0]); \ + case 0: break; \ + } \ +} + +#ifdef JU_64BIT + +// copy from 0..8 bytes from string to Word_t +// +#define COPYSTRING8toWORD(WORD,STR,LEN) \ +{ \ + WORD = 0UL; \ + switch(LEN) \ + { \ + default: /* eight and greater */ \ + case 8: \ + WORD += ((Word_t)((uint8_t *)(STR))[7] << 56); \ + case 7: \ + WORD += ((Word_t)((uint8_t *)(STR))[6] << 48); \ + case 6: \ + WORD += ((Word_t)((uint8_t *)(STR))[5] << 40); \ + case 5: \ + WORD += ((Word_t)((uint8_t *)(STR))[4] << 32); \ + case 4: \ + WORD += ((Word_t)((uint8_t *)(STR))[3] << 24); \ + case 3: \ + WORD += ((Word_t)((uint8_t *)(STR))[2] << 16); \ + case 2: \ + WORD += ((Word_t)((uint8_t *)(STR))[1] << 8); \ + case 1: \ + WORD += ((Word_t)((uint8_t *)(STR))[0]); \ + case 0: break; \ + } \ +} + +#define COPYSTRINGtoWORD COPYSTRING8toWORD + +#else // JU_32BIT + +#define COPYSTRINGtoWORD COPYSTRING4toWORD + +#endif // JU_32BIT + +// set JError_t locally + +#define JU_SET_ERRNO(PJERROR, JERRNO) \ +{ \ + if (PJERROR != (PJError_t) NULL) \ + { \ + if (JERRNO) \ + JU_ERRNO(PJError) = (JERRNO); \ + JU_ERRID(PJERROR) = __LINE__; \ + } \ +} + +//======================================================================= +// This routine must hash string to 24..32 bits. The "goodness" of +// the hash is not as important as its speed. +//======================================================================= + +// hash to no more than 32 bits + +// extern Word_t gHmask; for hash bits experiments + +#define JUDYHASHSTR(HVALUE,STRING,LENGTH) \ +{ \ + uint8_t *p_ = (uint8_t *)(STRING); \ + uint8_t *q_ = p_ + (LENGTH); \ + uint32_t c_ = 0; \ + for (; p_ != q_; ++p_) \ + { \ + c_ = (c_ * 31) + *p_; \ + } \ +/* c_ &= gHmask; see above */ \ + (HVALUE) = c_; \ +} + +// Find String of Len in JudyHS structure, return pointer to associated Value + +PPvoid_t +JudyHSGet(Pcvoid_t PArray, // pointer (^) to structure + void * Str, // pointer to string + Word_t Len // length of string + ) +{ + uint8_t *String = (uint8_t *)Str; + PPvoid_t PPValue; // pointer to Value + Word_t Index; // 4[8] bytes of String + + JLG(PPValue, PArray, Len); // find hash table for strings of Len + if (PPValue == (PPvoid_t) NULL) + return ((PPvoid_t) NULL); // no strings of this Len + +// check for caller error (null pointer) +// + if ((String == (void *) NULL) && (Len != 0)) + return ((PPvoid_t) NULL); // avoid null-pointer dereference + +#ifndef DONOTUSEHASH + if (Len > WORDSIZE) // Hash table not necessary with short + { + uint32_t HValue; // hash of input string + JUDYHASHSTR(HValue, String, Len); // hash to no more than 32 bits + JLG(PPValue, *PPValue, (Word_t)HValue); // get ^ to hash bucket + if (PPValue == (PPvoid_t) NULL) + return ((PPvoid_t) NULL); // no entry in Hash table + } +#endif // DONOTUSEHASH + +/* + Each JudyL array decodes 4[8] bytes of the string. Since the hash + collisions occur very infrequently, the performance is not important. + However, even if the Hash code is not used this method still is + significantly faster than common tree methods (AVL, Red-Black, Splay, + b-tree, etc..). You can compare it yourself with #define DONOTUSEHASH + 1 or putting -DDONOTUSEHASH in the cc line. Use the "StringCompare.c" + code to compare (9Dec2003 dlb). +*/ + while (Len > WORDSIZE) // traverse tree of JudyL arrays + { + if (IS_PLS(*PPValue)) // ^ to JudyL array or ls_t struct? + { + Pls_t Pls; // ls_t struct, termination of tree + Pls = (Pls_t) CLEAR_PLS(*PPValue); // remove flag from ^ + +// if remaining string matches, return ^ to Value, else NULL + + if (memcmp(String, Pls->ls_String, Len) == 0) + return ((PPvoid_t) (&(Pls->ls_Value))); + else + return ((PPvoid_t) NULL); // string does not match + } + else + { + COPYSTRINGtoWORD(Index, String, WORDSIZE); + + JLG(PPValue, *PPValue, Index); // decode next 4[8] bytes + if (PPValue == (PPvoid_t) NULL) // if NULL array, bail out + return ((PPvoid_t) NULL); // string does not match + + String += WORDSIZE; // advance + Len -= WORDSIZE; + } + } + +// Get remaining 1..4[8] bytes left in string + + COPYSTRINGtoWORD(Index, String, Len); + JLG(PPValue, *PPValue, Index); // decode last 1-4[8] bytes + return (PPValue); +} + +// Add string to a tree of JudyL arrays (all lengths must be same) + +static PPvoid_t +insStrJudyLTree(uint8_t * String, // string to add to tree of JudyL arrays + Word_t Len, // length of string + PPvoid_t PPValue, // pointer to root pointer + PJError_t PJError // for returning error info + ) +{ + Word_t Index; // next 4[8] bytes of String + + while (Len > WORDSIZE) // add to JudyL tree + { +// CASE 1, pointer is to a NULL, make a new ls_t leaf + + if (*PPValue == (Pvoid_t)NULL) + { + Pls_t Pls; // memory for a ls_t + Pls = (Pls_t) JudyMalloc(LS_WORDLEN(Len)); + if (Pls == NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NOMEM); + return (PPJERR); + } + Pls->ls_Value = 0; // clear Value word + memcpy(Pls->ls_String, String, Len); // copy to new struct + *PPValue = (Pvoid_t)SET_PLS(Pls); // mark pointer + return ((PPvoid_t) (&Pls->ls_Value)); // return ^ to Value + } // no exit here +// CASE 2: is a ls_t, free (and shorten), then decode into JudyL tree + + if (IS_PLS(*PPValue)) // pointer to a ls_t? (leaf) + { + Pls_t Pls; // ^ to ls_t + uint8_t *String0; // ^ to string in ls_t + Word_t Index0; // 4[8] bytes in string + Word_t FreeLen; // length of ls_t + PPvoid_t PPsplit; + + FreeLen = LS_WORDLEN(Len); // length of ls_t + + Pls = (Pls_t) CLEAR_PLS(*PPValue); // demangle ^ to ls_t + String0 = Pls->ls_String; + if (memcmp(String, String0, Len) == 0) // check if match? + { + return ((PPvoid_t) (&Pls->ls_Value)); // yes, duplicate + } + + *PPValue = NULL; // clear ^ to ls_t and make JudyL + +// This do loop is technically not required, saves multiple JudyFree() +// when storing already sorted strings into structure + + do // decode next 4[8] bytes of string + { // with a JudyL array +// Note: string0 is always aligned + + COPYSTRINGtoWORD(Index0, String0, WORDSIZE); + String0 += WORDSIZE; + COPYSTRINGtoWORD(Index, String, WORDSIZE); + String += WORDSIZE; + Len -= WORDSIZE; + PPsplit = PPValue; // save for split below + PPValue = JudyLIns(PPValue, Index0, PJError); + if (PPValue == PPJERR) + { + JU_SET_ERRNO(PJError, 0); + return (PPJERR); + } + + } while ((Index0 == Index) && (Len > WORDSIZE)); + +// finish storing remainder of string that was in the ls_t + + PPValue = insStrJudyLTree(String0, Len, PPValue, PJError); + if (PPValue == PPJERR) + { + return (PPJERR); + } +// copy old Value to Value in new struct + + *(PWord_t)PPValue = Pls->ls_Value; + +// free the string buffer (ls_t) + + JudyFree((Pvoid_t)Pls, FreeLen); + PPValue = JudyLIns(PPsplit, Index, PJError); + if (PPValue == PPJERR) + { + JU_SET_ERRNO(PJError, 0); + return (PPValue); + } + +// finish remainder of newly inserted string + + PPValue = insStrJudyLTree(String, Len, PPValue, PJError); + return (PPValue); + } // no exit here +// CASE 3, more JudyL arrays, decode to next tree + + COPYSTRINGtoWORD(Index, String, WORDSIZE); + Len -= WORDSIZE; + String += WORDSIZE; + + PPValue = JudyLIns(PPValue, Index, PJError); // next 4[8] bytes + if (PPValue == PPJERR) + { + JU_SET_ERRNO(PJError, 0); + return (PPValue); + } + } +// this is done outside of loop so "Len" can be an unsigned number + + COPYSTRINGtoWORD(Index, String, Len); + PPValue = JudyLIns(PPValue, Index, PJError); // remaining 4[8] bytes + + return (PPValue); +} + + +// Insert string to JudyHS structure, return pointer to associated Value + +PPvoid_t +JudyHSIns(PPvoid_t PPArray, // ^ to JudyHashArray name + void * Str, // pointer to string + Word_t Len, // length of string + PJError_t PJError // optional, for returning error info + ) +{ + uint8_t * String = (uint8_t *)Str; + PPvoid_t PPValue; + +// string can only be NULL if Len is 0. + + if ((String == (uint8_t *) NULL) && (Len != 0UL)) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + return (PPJERR); + } + JLG(PPValue, *PPArray, Len); // JudyL hash table for strings of Len + if (PPValue == (PPvoid_t) NULL) // make new if missing, (very rare) + { + PPValue = JudyLIns(PPArray, Len, PJError); + if (PPValue == PPJERR) + { + JU_SET_ERRNO(PJError, 0); + return (PPJERR); + } + } +#ifndef DONOTUSEHASH + if (Len > WORDSIZE) + { + uint32_t HValue; // hash of input string + JUDYHASHSTR(HValue, String, Len); // hash to no more than 32 bits + PPValue = JudyLIns(PPValue, (Word_t)HValue, PJError); + if (PPValue == PPJERR) + { + JU_SET_ERRNO(PJError, 0); + return (PPJERR); + } + } +#endif // DONOTUSEHASH + + PPValue = insStrJudyLTree(String, Len, PPValue, PJError); // add string + return (PPValue); // ^ to Value +} + +// Delete string from tree of JudyL arrays (all Lens must be same) + +static int +delStrJudyLTree(uint8_t * String, // delete from tree of JudyL arrays + Word_t Len, // length of string + PPvoid_t PPValue, // ^ to hash bucket + PJError_t PJError // for returning error info + ) +{ + PPvoid_t PPValueN; // next pointer + Word_t Index; + int Ret; // -1=failed, 1=success, 2=quit del + + if (IS_PLS(*PPValue)) // is pointer to ls_t? + { + Pls_t Pls; + Pls = (Pls_t) CLEAR_PLS(*PPValue); // demangle pointer + JudyFree((Pvoid_t)Pls, LS_WORDLEN(Len)); // free the ls_t + + *PPValue = (Pvoid_t)NULL; // clean pointer + return (1); // successfully deleted + } + + if (Len > WORDSIZE) // delete from JudyL tree, not leaf + { + COPYSTRINGtoWORD(Index, String, WORDSIZE); // get Index + JLG(PPValueN, *PPValue, Index); // get pointer to next JudyL array + + String += WORDSIZE; // advance to next 4[8] bytes + Len -= WORDSIZE; + + Ret = delStrJudyLTree(String, Len, PPValueN, PJError); + if (Ret != 1) return(Ret); + + if (*PPValueN == (PPvoid_t) NULL) + { +// delete JudyL element from tree + + Ret = JudyLDel(PPValue, Index, PJError); + } + } + else + { + COPYSTRINGtoWORD(Index, String, Len); // get leaf element + +// delete last 1-4[8] bytes from leaf element + + Ret = JudyLDel(PPValue, Index, PJError); + } + return (Ret); +} + +// Delete string from JHS structure + +int +JudyHSDel(PPvoid_t PPArray, // ^ to JudyHashArray struct + void * Str, // pointer to string + Word_t Len, // length of string + PJError_t PJError // optional, for returning error info + ) +{ + uint8_t * String = (uint8_t *)Str; + PPvoid_t PPBucket, PPHtble; + int Ret; // return bool from Delete routine +#ifndef DONOTUSEHASH + uint32_t HValue = 0; // hash value of input string +#endif // DONOTUSEHASH + + if (PPArray == NULL) + return (0); // no pointer, return not found + +// This is a little slower than optimum method, but not much in new CPU +// Verify that string is in the structure -- simplifies future assumptions + + if (JudyHSGet(*PPArray, String, Len) == (PPvoid_t) NULL) + return (0); // string not found, return + +// string is in structure, so testing for absence is not necessary + + JLG(PPHtble, *PPArray, Len); // JudyL hash table for strings of Len + +#ifdef DONOTUSEHASH + PPBucket = PPHtble; // simulate below code +#else // USEHASH + if (Len > WORDSIZE) + { + JUDYHASHSTR(HValue, String, Len); // hash to no more than 32 bits + +// get pointer to hash bucket + + JLG(PPBucket, *PPHtble, (Word_t)HValue); + } + else + { + PPBucket = PPHtble; // no bucket to JLGet + } +#endif // USEHASH + +// delete from JudyL tree +// + Ret = delStrJudyLTree(String, Len, PPBucket, PJError); + if (Ret != 1) + { + JU_SET_ERRNO(PJError, 0); + return(-1); + } +// handle case of missing JudyL array from hash table and length table + + if (*PPBucket == (Pvoid_t)NULL) // if JudyL tree gone + { +#ifndef DONOTUSEHASH + if (Len > WORDSIZE) + { +// delete entry in Hash table + + Ret = JudyLDel(PPHtble, (Word_t)HValue, PJError); + if (Ret != 1) + { + JU_SET_ERRNO(PJError, 0); + return(-1); + } + } +#endif // USEHASH + if (*PPHtble == (PPvoid_t) NULL) // if Hash table gone + { +// delete entry from the String length table + + Ret = JudyLDel(PPArray, Len, PJError); + if (Ret != 1) + { + JU_SET_ERRNO(PJError, 0); + return(-1); + } + } + } + return (1); // success +} + +static Word_t +delJudyLTree(PPvoid_t PPValue, // ^ to JudyL root pointer + Word_t Len, // length of string + PJError_t PJError) // for returning error info +{ + Word_t bytes_freed = 0; // bytes freed at point + Word_t bytes_total = 0; // accumulated bytes freed + PPvoid_t PPValueN; + +// Pointer is to another tree of JudyL arrays or ls_t struct + + if (Len > WORDSIZE) // more depth to tree + { + Word_t NEntry; + +// Pointer is to a ls_t struct + + if (IS_PLS(*PPValue)) + { + Pls_t Pls; + Word_t freewords; + + freewords = LS_WORDLEN(Len); // calculate length + Pls = (Pls_t)CLEAR_PLS(*PPValue); // demangle pointer + +// *PPValue = (Pvoid_t)NULL; // clean pointer + JudyFree((Pvoid_t)Pls, freewords); // free the ls_t + + return(freewords * WORDSIZE); + } +// else +// Walk all the entrys in the JudyL array + + NEntry = 0; // start at beginning + for (PPValueN = JudyLFirst(*PPValue, &NEntry, PJError); + (PPValueN != (PPvoid_t) NULL) && (PPValueN != PPJERR); + PPValueN = JudyLNext(*PPValue, &NEntry, PJError)) + { +// recurse to the next level in the tree of arrays + + bytes_freed = delJudyLTree(PPValueN, Len - WORDSIZE, PJError); + if (bytes_freed == JERR) return(JERR); + bytes_total += bytes_freed; + } + if (PPValueN == PPJERR) return(JERR); + +// now free this JudyL array + + bytes_freed = JudyLFreeArray(PPValue, PJError); + if (bytes_freed == JERR) return(JERR); + bytes_total += bytes_freed; + + return(bytes_total); // return amount freed + } +// else + +// Pointer to simple JudyL array + + bytes_freed = JudyLFreeArray(PPValue, PJError); + + return(bytes_freed); +} + + +Word_t // bytes freed +JudyHSFreeArray(PPvoid_t PPArray, // ^ to JudyHashArray struct + PJError_t PJError // optional, for returning error info + ) +{ + Word_t Len; // start at beginning + Word_t bytes_freed; // bytes freed at this level. + Word_t bytes_total; // bytes total at all levels. + PPvoid_t PPHtble; + + if (PPArray == NULL) + return (0); // no pointer, return none + +// Walk the string length table for subsidary hash structs +// NOTE: This is necessary to determine the depth of the tree + + bytes_freed = 0; + bytes_total = 0; + Len = 0; // walk to length table + + for (PPHtble = JudyLFirst(*PPArray, &Len, PJError); + (PPHtble != (PPvoid_t) NULL) && (PPHtble != PPJERR); + PPHtble = JudyLNext(*PPArray, &Len, PJError)) + { + PPvoid_t PPValueH; + +#ifndef DONOTUSEHASH + if (Len > WORDSIZE) + { + Word_t HEntry = 0; // walk the hash tables + + for (PPValueH = JudyLFirst(*PPHtble, &HEntry, PJError); + (PPValueH != (PPvoid_t) NULL) && (PPValueH != PPJERR); + PPValueH = JudyLNext(*PPHtble, &HEntry, PJError)) + { + bytes_freed = delJudyLTree(PPValueH, Len, PJError); + if (bytes_freed == JERR) return(JERR); + bytes_total += bytes_freed; + } + + if (PPValueH == PPJERR) return(JERR); + +// free the Hash table for this length of string + + bytes_freed = JudyLFreeArray(PPHtble, PJError); + if (bytes_freed == JERR) return(JERR); + bytes_total += bytes_freed; + } + else +#endif // DONOTUSEHASH + { + PPValueH = PPHtble; // simulate hash table + + bytes_freed = delJudyLTree(PPValueH, Len, PJError); + if (bytes_freed == JERR) return(JERR); + bytes_total += bytes_freed; + } + } + if (PPHtble == PPJERR) return(JERR); + +// free the length table + + bytes_freed = JudyLFreeArray(PPArray, PJError); + if (bytes_freed == JERR) return(JERR); + + bytes_total += bytes_freed; + + return(bytes_total); // return bytes freed +} diff --git a/libnetdata/libjudy/src/JudyL/JudyL.h b/libnetdata/libjudy/src/JudyL/JudyL.h new file mode 100644 index 000000000..d901969d6 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyL.h @@ -0,0 +1,505 @@ +#ifndef _JUDYL_INCLUDED +#define _JUDYL_INCLUDED +// _________________ +// +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.41 $ $Source: /judy/src/JudyL/JudyL.h $ + +// **************************************************************************** +// JUDYL -- SMALL/LARGE AND/OR CLUSTERED/SPARSE ARRAYS +// +// -by- +// +// Douglas L. Baskins +// doug@sourcejudy.com +// +// Judy arrays are designed to be used instead of arrays. The performance +// suggests the reason why Judy arrays are thought of as arrays, instead of +// trees. They are remarkably memory efficient at all populations. +// Implemented as a hybrid digital tree (but really a state machine, see +// below), Judy arrays feature fast insert/retrievals, fast near neighbor +// searching, and contain a population tree for extremely fast ordinal related +// retrievals. +// +// CONVENTIONS: +// +// - The comments here refer to 32-bit [64-bit] systems. +// +// - BranchL, LeafL refer to linear branches and leaves (small populations), +// except LeafL does not actually appear as such; rather, Leaf1..3 [Leaf1..7] +// is used to represent leaf Index sizes, and LeafW refers to a Leaf with +// full (Long) word Indexes, which is also a type of linear leaf. Note that +// root-level LeafW (Leaf4 [Leaf8]) leaves are called LEAFW. +// +// - BranchB, LeafB1 refer to bitmap branches and leaves (intermediate +// populations). +// +// - BranchU refers to uncompressed branches. An uncompressed branch has 256 +// JPs, some of which could be null. Note: All leaves are compressed (and +// sorted), or else an expanse is full (FullPopu), so there is no LeafU +// equivalent to BranchU. +// +// - "Popu" is short for "Population". +// - "Pop1" refers to actual population (base 1). +// - "Pop0" refers to Pop1 - 1 (base 0), the way populations are stored in data +// structures. +// +// - Branches and Leaves are both named by the number of bytes in their Pop0 +// field. In the case of Leaves, the same number applies to the Index sizes. +// +// - The representation of many numbers as hex is a relatively safe and +// portable way to get desired bitpatterns as unsigned longs. +// +// - Some preprocessors cant handle single apostrophe characters within +// #ifndef code, so here, delete all instead. + + +#include "JudyPrivate.h" // includes Judy.h in turn. +#include "JudyPrivateBranch.h" // support for branches. + + +// **************************************************************************** +// JUDYL ROOT POINTER (JRP) AND JUDYL POINTER (JP) TYPE FIELDS +// **************************************************************************** + +typedef enum // uint8_t -- but C does not support this type of enum. +{ + +// JP NULL TYPES: +// +// There is a series of cJL_JPNULL* Types because each one pre-records a +// different Index Size for when the first Index is inserted in the previously +// null JP. They must start >= 8 (three bits). +// +// Note: These Types must be in sequential order for doing relative +// calculations between them. + + cJL_JPNULL1 = 1, + // Index Size 1[1] byte when 1 Index inserted. + cJL_JPNULL2, // Index Size 2[2] bytes when 1 Index inserted. + cJL_JPNULL3, // Index Size 3[3] bytes when 1 Index inserted. + +#ifndef JU_64BIT +#define cJL_JPNULLMAX cJL_JPNULL3 +#else + cJL_JPNULL4, // Index Size 4[4] bytes when 1 Index inserted. + cJL_JPNULL5, // Index Size 5[5] bytes when 1 Index inserted. + cJL_JPNULL6, // Index Size 6[6] bytes when 1 Index inserted. + cJL_JPNULL7, // Index Size 7[7] bytes when 1 Index inserted. +#define cJL_JPNULLMAX cJL_JPNULL7 +#endif + + +// JP BRANCH TYPES: +// +// Note: There are no state-1 branches; only leaves reside at state 1. + +// Linear branches: +// +// Note: These Types must be in sequential order for doing relative +// calculations between them. + + cJL_JPBRANCH_L2, // 2[2] bytes Pop0, 1[5] bytes Dcd. + cJL_JPBRANCH_L3, // 3[3] bytes Pop0, 0[4] bytes Dcd. + +#ifdef JU_64BIT + cJL_JPBRANCH_L4, // [4] bytes Pop0, [3] bytes Dcd. + cJL_JPBRANCH_L5, // [5] bytes Pop0, [2] bytes Dcd. + cJL_JPBRANCH_L6, // [6] bytes Pop0, [1] byte Dcd. + cJL_JPBRANCH_L7, // [7] bytes Pop0, [0] bytes Dcd. +#endif + + cJL_JPBRANCH_L, // note: DcdPopO field not used. + +// Bitmap branches: +// +// Note: These Types must be in sequential order for doing relative +// calculations between them. + + cJL_JPBRANCH_B2, // 2[2] bytes Pop0, 1[5] bytes Dcd. + cJL_JPBRANCH_B3, // 3[3] bytes Pop0, 0[4] bytes Dcd. + +#ifdef JU_64BIT + cJL_JPBRANCH_B4, // [4] bytes Pop0, [3] bytes Dcd. + cJL_JPBRANCH_B5, // [5] bytes Pop0, [2] bytes Dcd. + cJL_JPBRANCH_B6, // [6] bytes Pop0, [1] byte Dcd. + cJL_JPBRANCH_B7, // [7] bytes Pop0, [0] bytes Dcd. +#endif + + cJL_JPBRANCH_B, // note: DcdPopO field not used. + +// Uncompressed branches: +// +// Note: These Types must be in sequential order for doing relative +// calculations between them. + + cJL_JPBRANCH_U2, // 2[2] bytes Pop0, 1[5] bytes Dcd. + cJL_JPBRANCH_U3, // 3[3] bytes Pop0, 0[4] bytes Dcd. + +#ifdef JU_64BIT + cJL_JPBRANCH_U4, // [4] bytes Pop0, [3] bytes Dcd. + cJL_JPBRANCH_U5, // [5] bytes Pop0, [2] bytes Dcd. + cJL_JPBRANCH_U6, // [6] bytes Pop0, [1] byte Dcd. + cJL_JPBRANCH_U7, // [7] bytes Pop0, [0] bytes Dcd. +#endif + + cJL_JPBRANCH_U, // note: DcdPopO field not used. + + +// JP LEAF TYPES: + +// Linear leaves: +// +// Note: These Types must be in sequential order for doing relative +// calculations between them. +// +// Note: There is no full-word (4-byte [8-byte]) Index leaf under a JP because +// non-root-state leaves only occur under branches that decode at least one +// byte. Full-word, root-state leaves are under a JRP, not a JP. However, in +// the code a "fake" JP can be created temporarily above a root-state leaf. + + cJL_JPLEAF1, // 1[1] byte Pop0, 2 bytes Dcd. + cJL_JPLEAF2, // 2[2] bytes Pop0, 1[5] bytes Dcd. + cJL_JPLEAF3, // 3[3] bytes Pop0, 0[4] bytes Dcd. + +#ifdef JU_64BIT + cJL_JPLEAF4, // [4] bytes Pop0, [3] bytes Dcd. + cJL_JPLEAF5, // [5] bytes Pop0, [2] bytes Dcd. + cJL_JPLEAF6, // [6] bytes Pop0, [1] byte Dcd. + cJL_JPLEAF7, // [7] bytes Pop0, [0] bytes Dcd. +#endif + +// Bitmap leaf; Index Size == 1: +// +// Note: These are currently only supported at state 1. At other states the +// bitmap would grow from 256 to 256^2, 256^3, ... bits, which would not be +// efficient.. + + cJL_JPLEAF_B1, // 1[1] byte Pop0, 2[6] bytes Dcd. + +// Full population; Index Size == 1 virtual leaf: +// +// Note: JudyL has no cJL_JPFULLPOPU1 equivalent to cJ1_JPFULLPOPU1, because +// in the JudyL case this could result in a values-only leaf of up to 256 words +// (value areas) that would be slow to insert/delete. + + +// JP IMMEDIATES; leaves (Indexes) stored inside a JP: +// +// The second numeric suffix is the Pop1 for each type. As the Index Size +// increases, the maximum possible population decreases. +// +// Note: These Types must be in sequential order in each group (Index Size), +// and the groups in correct order too, for doing relative calculations between +// them. For example, since these Types enumerate the Pop1 values (unlike +// other JP Types where there is a Pop0 value in the JP), the maximum Pop1 for +// each Index Size is computable. +// +// All enums equal or above this point are cJL_JPIMMEDs. + + cJL_JPIMMED_1_01, // Index Size = 1, Pop1 = 1. + cJL_JPIMMED_2_01, // Index Size = 2, Pop1 = 1. + cJL_JPIMMED_3_01, // Index Size = 3, Pop1 = 1. + +#ifdef JU_64BIT + cJL_JPIMMED_4_01, // Index Size = 4, Pop1 = 1. + cJL_JPIMMED_5_01, // Index Size = 5, Pop1 = 1. + cJL_JPIMMED_6_01, // Index Size = 6, Pop1 = 1. + cJL_JPIMMED_7_01, // Index Size = 7, Pop1 = 1. +#endif + + cJL_JPIMMED_1_02, // Index Size = 1, Pop1 = 2. + cJL_JPIMMED_1_03, // Index Size = 1, Pop1 = 3. + +#ifdef JU_64BIT + cJL_JPIMMED_1_04, // Index Size = 1, Pop1 = 4. + cJL_JPIMMED_1_05, // Index Size = 1, Pop1 = 5. + cJL_JPIMMED_1_06, // Index Size = 1, Pop1 = 6. + cJL_JPIMMED_1_07, // Index Size = 1, Pop1 = 7. + + cJL_JPIMMED_2_02, // Index Size = 2, Pop1 = 2. + cJL_JPIMMED_2_03, // Index Size = 2, Pop1 = 3. + + cJL_JPIMMED_3_02, // Index Size = 3, Pop1 = 2. +#endif + +// This special Type is merely a sentinel for doing relative calculations. +// This value should not be used in switch statements (to avoid allocating code +// for it), which is also why it appears at the end of the enum list. + + cJL_JPIMMED_CAP + +} jpL_Type_t; + + +// RELATED VALUES: + +// Index Size (state) for leaf JP, and JP type based on Index Size (state): + +#define JL_LEAFINDEXSIZE(jpType) ((jpType) - cJL_JPLEAF1 + 1) +#define JL_LEAFTYPE(IndexSize) ((IndexSize) + cJL_JPLEAF1 - 1) + + +// MAXIMUM POPULATIONS OF LINEAR LEAVES: + +#ifndef JU_64BIT // 32-bit + +#define J_L_MAXB (sizeof(Word_t) * 64) +#define ALLOCSIZES { 3, 5, 7, 11, 15, 23, 32, 47, 64, TERMINATOR } // in words. +#define cJL_LEAF1_MAXWORDS (32) // max Leaf1 size in words. + +// Note: cJL_LEAF1_MAXPOP1 is chosen such that the index portion is less than +// 32 bytes -- the number of bytes the index takes in a bitmap leaf. + +#define cJL_LEAF1_MAXPOP1 \ + ((cJL_LEAF1_MAXWORDS * cJU_BYTESPERWORD)/(1 + cJU_BYTESPERWORD)) +#define cJL_LEAF2_MAXPOP1 (J_L_MAXB / (2 + cJU_BYTESPERWORD)) +#define cJL_LEAF3_MAXPOP1 (J_L_MAXB / (3 + cJU_BYTESPERWORD)) +#define cJL_LEAFW_MAXPOP1 \ + ((J_L_MAXB - cJU_BYTESPERWORD) / (2 * cJU_BYTESPERWORD)) + +#else // 64-bit + +#define J_L_MAXB (sizeof(Word_t) * 64) +#define ALLOCSIZES { 3, 5, 7, 11, 15, 23, 32, 47, 64, TERMINATOR } // in words. +#define cJL_LEAF1_MAXWORDS (15) // max Leaf1 size in words. + +#define cJL_LEAF1_MAXPOP1 \ + ((cJL_LEAF1_MAXWORDS * cJU_BYTESPERWORD)/(1 + cJU_BYTESPERWORD)) +#define cJL_LEAF2_MAXPOP1 (J_L_MAXB / (2 + cJU_BYTESPERWORD)) +#define cJL_LEAF3_MAXPOP1 (J_L_MAXB / (3 + cJU_BYTESPERWORD)) +#define cJL_LEAF4_MAXPOP1 (J_L_MAXB / (4 + cJU_BYTESPERWORD)) +#define cJL_LEAF5_MAXPOP1 (J_L_MAXB / (5 + cJU_BYTESPERWORD)) +#define cJL_LEAF6_MAXPOP1 (J_L_MAXB / (6 + cJU_BYTESPERWORD)) +#define cJL_LEAF7_MAXPOP1 (J_L_MAXB / (7 + cJU_BYTESPERWORD)) +#define cJL_LEAFW_MAXPOP1 \ + ((J_L_MAXB - cJU_BYTESPERWORD) / (2 * cJU_BYTESPERWORD)) + +#endif // 64-bit + + +// MAXIMUM POPULATIONS OF IMMEDIATE JPs: +// +// These specify the maximum Population of immediate JPs with various Index +// Sizes (== sizes of remaining undecoded Index bits). Since the JP Types enum +// already lists all the immediates in order by state and size, calculate these +// values from it to avoid redundancy. + +#define cJL_IMMED1_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 1) // 3 [7]. +#define cJL_IMMED2_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 2) // 1 [3]. +#define cJL_IMMED3_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 3) // 1 [2]. + +#ifdef JU_64BIT +#define cJL_IMMED4_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 4) // [1]. +#define cJL_IMMED5_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 5) // [1]. +#define cJL_IMMED6_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 6) // [1]. +#define cJL_IMMED7_MAXPOP1 ((cJU_BYTESPERWORD - 1) / 7) // [1]. +#endif + + +// **************************************************************************** +// JUDYL LEAF BITMAP (JLLB) SUPPORT +// **************************************************************************** +// +// Assemble bitmap leaves out of smaller units that put bitmap subexpanses +// close to their associated pointers. Why not just use a bitmap followed by a +// series of pointers? (See 4.27.) Turns out this wastes a cache fill on +// systems with smaller cache lines than the assumed value cJU_WORDSPERCL. + +#define JL_JLB_BITMAP(Pjlb, Subexp) ((Pjlb)->jLlb_jLlbs[Subexp].jLlbs_Bitmap) +#define JL_JLB_PVALUE(Pjlb, Subexp) ((Pjlb)->jLlb_jLlbs[Subexp].jLlbs_PValue) + +typedef struct J__UDYL_LEAF_BITMAP_SUBEXPANSE +{ + BITMAPL_t jLlbs_Bitmap; + Pjv_t jLlbs_PValue; + +} jLlbs_t; + +typedef struct J__UDYL_LEAF_BITMAP +{ + jLlbs_t jLlb_jLlbs[cJU_NUMSUBEXPL]; + +} jLlb_t, * PjLlb_t; + +// Words per bitmap leaf: + +#define cJL_WORDSPERLEAFB1 (sizeof(jLlb_t) / cJU_BYTESPERWORD) + + +// **************************************************************************** +// MEMORY ALLOCATION SUPPORT +// **************************************************************************** + +// ARRAY-GLOBAL INFORMATION: +// +// At the cost of an occasional additional cache fill, this object, which is +// pointed at by a JRP and in turn points to a JP_BRANCH*, carries array-global +// information about a JudyL array that has sufficient population to amortize +// the cost. The jpm_Pop0 field prevents having to add up the total population +// for the array in insert, delete, and count code. The jpm_JP field prevents +// having to build a fake JP for entry to a state machine; however, the +// jp_DcdPopO field in jpm_JP, being one byte too small, is not used. +// +// Note: Struct fields are ordered to keep "hot" data in the first 8 words +// (see left-margin comments) for machines with 8-word cache lines, and to keep +// sub-word fields together for efficient packing. + +typedef struct J_UDYL_POPULATION_AND_MEMORY +{ +/* 1 */ Word_t jpm_Pop0; // total population-1 in array. +/* 2 */ jp_t jpm_JP; // JP to first branch; see above. +/* 4 */ Word_t jpm_LastUPop0; // last jpm_Pop0 when convert to BranchU +/* 7 */ Pjv_t jpm_PValue; // pointer to value to return. +// Note: Field names match PJError_t for convenience in macros: +/* 8 */ char je_Errno; // one of the enums in Judy.h. +/* 8/9 */ int je_ErrID; // often an internal source line number. +/* 9/10 */ Word_t jpm_TotalMemWords; // words allocated in array. +} jLpm_t, *PjLpm_t; + + +// TABLES FOR DETERMINING IF LEAVES HAVE ROOM TO GROW: +// +// These tables indicate if a given memory chunk can support growth of a given +// object into wasted (rounded-up) memory in the chunk. Note: This violates +// the hiddenness of the JudyMalloc code. + +extern const uint8_t j__L_Leaf1PopToWords[cJL_LEAF1_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf2PopToWords[cJL_LEAF2_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf3PopToWords[cJL_LEAF3_MAXPOP1 + 1]; +#ifdef JU_64BIT +extern const uint8_t j__L_Leaf4PopToWords[cJL_LEAF4_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf5PopToWords[cJL_LEAF5_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf6PopToWords[cJL_LEAF6_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf7PopToWords[cJL_LEAF7_MAXPOP1 + 1]; +#endif +extern const uint8_t j__L_LeafWPopToWords[cJL_LEAFW_MAXPOP1 + 1]; +extern const uint8_t j__L_LeafVPopToWords[]; + +// These tables indicate where value areas start: + +extern const uint8_t j__L_Leaf1Offset [cJL_LEAF1_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf2Offset [cJL_LEAF2_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf3Offset [cJL_LEAF3_MAXPOP1 + 1]; +#ifdef JU_64BIT +extern const uint8_t j__L_Leaf4Offset [cJL_LEAF4_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf5Offset [cJL_LEAF5_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf6Offset [cJL_LEAF6_MAXPOP1 + 1]; +extern const uint8_t j__L_Leaf7Offset [cJL_LEAF7_MAXPOP1 + 1]; +#endif +extern const uint8_t j__L_LeafWOffset [cJL_LEAFW_MAXPOP1 + 1]; + +// Also define macros to hide the details in the code using these tables. + +#define JL_LEAF1GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF1_MAXPOP1, j__L_Leaf1PopToWords) +#define JL_LEAF2GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF2_MAXPOP1, j__L_Leaf2PopToWords) +#define JL_LEAF3GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF3_MAXPOP1, j__L_Leaf3PopToWords) +#ifdef JU_64BIT +#define JL_LEAF4GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF4_MAXPOP1, j__L_Leaf4PopToWords) +#define JL_LEAF5GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF5_MAXPOP1, j__L_Leaf5PopToWords) +#define JL_LEAF6GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF6_MAXPOP1, j__L_Leaf6PopToWords) +#define JL_LEAF7GROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAF7_MAXPOP1, j__L_Leaf7PopToWords) +#endif +#define JL_LEAFWGROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJL_LEAFW_MAXPOP1, j__L_LeafWPopToWords) +#define JL_LEAFVGROWINPLACE(Pop1) \ + J__U_GROWCK(Pop1, cJU_BITSPERSUBEXPL, j__L_LeafVPopToWords) + +#define JL_LEAF1VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf1Offset[Pop1]) +#define JL_LEAF2VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf2Offset[Pop1]) +#define JL_LEAF3VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf3Offset[Pop1]) +#ifdef JU_64BIT +#define JL_LEAF4VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf4Offset[Pop1]) +#define JL_LEAF5VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf5Offset[Pop1]) +#define JL_LEAF6VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf6Offset[Pop1]) +#define JL_LEAF7VALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_Leaf7Offset[Pop1]) +#endif +#define JL_LEAFWVALUEAREA(Pjv,Pop1) (((PWord_t)(Pjv)) + j__L_LeafWOffset[Pop1]) + +#define JL_LEAF1POPTOWORDS(Pop1) (j__L_Leaf1PopToWords[Pop1]) +#define JL_LEAF2POPTOWORDS(Pop1) (j__L_Leaf2PopToWords[Pop1]) +#define JL_LEAF3POPTOWORDS(Pop1) (j__L_Leaf3PopToWords[Pop1]) +#ifdef JU_64BIT +#define JL_LEAF4POPTOWORDS(Pop1) (j__L_Leaf4PopToWords[Pop1]) +#define JL_LEAF5POPTOWORDS(Pop1) (j__L_Leaf5PopToWords[Pop1]) +#define JL_LEAF6POPTOWORDS(Pop1) (j__L_Leaf6PopToWords[Pop1]) +#define JL_LEAF7POPTOWORDS(Pop1) (j__L_Leaf7PopToWords[Pop1]) +#endif +#define JL_LEAFWPOPTOWORDS(Pop1) (j__L_LeafWPopToWords[Pop1]) +#define JL_LEAFVPOPTOWORDS(Pop1) (j__L_LeafVPopToWords[Pop1]) + + +// FUNCTIONS TO ALLOCATE OBJECTS: + +PjLpm_t j__udyLAllocJLPM(void); // constant size. + +Pjbl_t j__udyLAllocJBL( PjLpm_t); // constant size. +Pjbb_t j__udyLAllocJBB( PjLpm_t); // constant size. +Pjp_t j__udyLAllocJBBJP(Word_t, PjLpm_t); +Pjbu_t j__udyLAllocJBU( PjLpm_t); // constant size. + +Pjll_t j__udyLAllocJLL1( Word_t, PjLpm_t); +Pjll_t j__udyLAllocJLL2( Word_t, PjLpm_t); +Pjll_t j__udyLAllocJLL3( Word_t, PjLpm_t); + +#ifdef JU_64BIT +Pjll_t j__udyLAllocJLL4( Word_t, PjLpm_t); +Pjll_t j__udyLAllocJLL5( Word_t, PjLpm_t); +Pjll_t j__udyLAllocJLL6( Word_t, PjLpm_t); +Pjll_t j__udyLAllocJLL7( Word_t, PjLpm_t); +#endif + +Pjlw_t j__udyLAllocJLW( Word_t ); // no PjLpm_t needed. +PjLlb_t j__udyLAllocJLB1( PjLpm_t); // constant size. +Pjv_t j__udyLAllocJV( Word_t, PjLpm_t); + + +// FUNCTIONS TO FREE OBJECTS: + +void j__udyLFreeJLPM( PjLpm_t, PjLpm_t); // constant size. + +void j__udyLFreeJBL( Pjbl_t, PjLpm_t); // constant size. +void j__udyLFreeJBB( Pjbb_t, PjLpm_t); // constant size. +void j__udyLFreeJBBJP(Pjp_t, Word_t, PjLpm_t); +void j__udyLFreeJBU( Pjbu_t, PjLpm_t); // constant size. + +void j__udyLFreeJLL1( Pjll_t, Word_t, PjLpm_t); +void j__udyLFreeJLL2( Pjll_t, Word_t, PjLpm_t); +void j__udyLFreeJLL3( Pjll_t, Word_t, PjLpm_t); + +#ifdef JU_64BIT +void j__udyLFreeJLL4( Pjll_t, Word_t, PjLpm_t); +void j__udyLFreeJLL5( Pjll_t, Word_t, PjLpm_t); +void j__udyLFreeJLL6( Pjll_t, Word_t, PjLpm_t); +void j__udyLFreeJLL7( Pjll_t, Word_t, PjLpm_t); +#endif + +void j__udyLFreeJLW( Pjlw_t, Word_t, PjLpm_t); +void j__udyLFreeJLB1( PjLlb_t, PjLpm_t); // constant size. +void j__udyLFreeJV( Pjv_t, Word_t, PjLpm_t); +void j__udyLFreeSM( Pjp_t, PjLpm_t); // everything below Pjp. + +#endif // ! _JUDYL_INCLUDED diff --git a/libnetdata/libjudy/src/JudyL/JudyLByCount.c b/libnetdata/libjudy/src/JudyL/JudyLByCount.c new file mode 100644 index 000000000..c5a004796 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLByCount.c @@ -0,0 +1,954 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.28 $ $Source: /judy/src/JudyCommon/JudyByCount.c $ +// +// Judy*ByCount() function for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DNOSMARTJBB, -DNOSMARTJBU, and/or -DNOSMARTJLB to build a +// version with cache line optimizations deleted, for testing. +// +// Judy*ByCount() is a conceptual although not literal inverse of Judy*Count(). +// Judy*Count() takes a pair of Indexes, and allows finding the ordinal of a +// given Index (that is, its position in the list of valid indexes from the +// beginning) as a degenerate case, because in general the count between two +// Indexes, inclusive, is not always just the difference in their ordinals. +// However, it suffices for Judy*ByCount() to simply be an ordinal-to-Index +// mapper. +// +// Note: Like Judy*Count(), this code must "count sideways" in branches, which +// can result in a lot of cache line fills. However, unlike Judy*Count(), this +// code does not receive a specific Index, hence digit, where to start in each +// branch, so it cant accurately calculate cache line fills required in each +// direction. The best it can do is an approximation based on the total +// population of the expanse (pop1 from Pjp) and the ordinal of the target +// Index (see SETOFFSET()) within the expanse. +// +// Compile with -DSMARTMETRICS to obtain global variables containing smart +// cache line metrics. Note: Dont turn this on simultaneously for this file +// and JudyCount.c because they export the same globals. +// **************************************************************************** + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +// These are imported from JudyCount.c: +// +// TBD: Should this be in common code? Exported from a header file? + +#ifdef JUDY1 +extern Word_t j__udy1JPPop1(const Pjp_t Pjp); +#define j__udyJPPop1 j__udy1JPPop1 +#else +extern Word_t j__udyLJPPop1(const Pjp_t Pjp); +#define j__udyJPPop1 j__udyLJPPop1 +#endif + +// Avoid duplicate symbols since this file is multi-compiled: + +#ifdef SMARTMETRICS +#ifdef JUDY1 +Word_t jbb_upward = 0; // counts of directions taken: +Word_t jbb_downward = 0; +Word_t jbu_upward = 0; +Word_t jbu_downward = 0; +Word_t jlb_upward = 0; +Word_t jlb_downward = 0; +#else +extern Word_t jbb_upward; +extern Word_t jbb_downward; +extern Word_t jbu_upward; +extern Word_t jbu_downward; +extern Word_t jlb_upward; +extern Word_t jlb_downward; +#endif +#endif + + +// **************************************************************************** +// J U D Y 1 B Y C O U N T +// J U D Y L B Y C O U N T +// +// See the manual entry. + +#ifdef JUDY1 +FUNCTION int Judy1ByCount +#else +FUNCTION PPvoid_t JudyLByCount +#endif + ( + Pcvoid_t PArray, // root pointer to first branch/leaf in SM. + Word_t Count, // ordinal of Index to find, 1..MAX. + Word_t * PIndex, // to return found Index. + PJError_t PJError // optional, for returning error info. + ) +{ + Word_t Count0; // Count, base-0, to match pop0. + Word_t state; // current state in SM. + Word_t pop1; // of current branch or leaf, or of expanse. + Word_t pop1lower; // pop1 of expanses (JPs) below that for Count. + Word_t digit; // current word in branch. + Word_t jpcount; // JPs in a BranchB subexpanse. + long jpnum; // JP number in a branch (base 0). + long subexp; // for stepping through layer 1 (subexpanses). + int offset; // index ordinal within a leaf, base 0. + + Pjp_t Pjp; // current JP in branch. + Pjll_t Pjll; // current Judy linear leaf. + + +// CHECK FOR EMPTY ARRAY OR NULL PINDEX: + + if (PArray == (Pvoid_t) NULL) JU_RET_NOTFOUND; + + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +// Convert Count to Count0; assume special case of Count = 0 maps to ~0, as +// desired, to represent the last index in a full array: +// +// Note: Think of Count0 as a reliable "number of Indexes below the target." + + Count0 = Count - 1; + assert((Count || Count0 == ~0)); // ensure CPU is sane about 0 - 1. + pop1lower = 0; + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + + if (Count0 > Pjlw[0]) JU_RET_NOTFOUND; // too high. + + *PIndex = Pjlw[Count]; // Index, base 1. + + JU_RET_FOUND_LEAFW(Pjlw, Pjlw[0] + 1, Count0); + } + else + { + Pjpm_t Pjpm = P_JPM(PArray); + + if (Count0 > (Pjpm->jpm_Pop0)) JU_RET_NOTFOUND; // too high. + + Pjp = &(Pjpm->jpm_JP); + pop1 = (Pjpm->jpm_Pop0) + 1; + +// goto SMByCount; + } + +// COMMON CODE: +// +// Prepare to handle a root-level or lower-level branch: Save the current +// state, obtain the total population for the branch in a state-dependent way, +// and then branch to common code for multiple cases. +// +// For root-level branches, the state is always cJU_ROOTSTATE, and the array +// population must already be set in pop1; it is not available in jp_DcdPopO. +// +// Note: The total population is only needed in cases where the common code +// "counts down" instead of up to minimize cache line fills. However, its +// available cheaply, and its better to do it with a constant shift (constant +// state value) instead of a variable shift later "when needed". + +#define PREPB_ROOT(Next) \ + state = cJU_ROOTSTATE; \ + goto Next + +// Use PREPB_DCD() to first copy the Dcd bytes to *PIndex if there are any +// (only if state < cJU_ROOTSTATE - 1): + +#define PREPB_DCD(Pjp,cState,Next) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + PREPB((Pjp), cState, Next) + +#define PREPB(Pjp,cState,Next) \ + state = (cState); \ + pop1 = JU_JPBRANCH_POP0(Pjp, (cState)) + 1; \ + goto Next + +// Calculate whether the ordinal of an Index within a given expanse falls in +// the lower or upper half of the expanses population, taking care with +// unsigned math and boundary conditions: +// +// Note: Assume the ordinal falls within the expanses population, that is, +// 0 < (Count - Pop1lower) <= Pop1exp (assuming infinite math). +// +// Note: If the ordinal is the middle element, it doesnt matter whether +// LOWERHALF() is TRUE or FALSE. + +#define LOWERHALF(Count0,Pop1lower,Pop1exp) \ + (((Count0) - (Pop1lower)) < ((Pop1exp) / 2)) + +// Calculate the (signed) offset within a leaf to the desired ordinal (Count - +// Pop1lower; offset is one less), and optionally ensure its in range: + +#define SETOFFSET(Offset,Count0,Pop1lower,Pjp) \ + (Offset) = (Count0) - (Pop1lower); \ + assert((Offset) >= 0); \ + assert((Offset) <= JU_JPLEAF_POP0(Pjp)) + +// Variations for immediate indexes, with and without pop1-specific assertions: + +#define SETOFFSET_IMM_CK(Offset,Count0,Pop1lower,cPop1) \ + (Offset) = (Count0) - (Pop1lower); \ + assert((Offset) >= 0); \ + assert((Offset) < (cPop1)) + +#define SETOFFSET_IMM(Offset,Count0,Pop1lower) \ + (Offset) = (Count0) - (Pop1lower) + + +// STATE MACHINE -- TRAVERSE TREE: +// +// In branches, look for the expanse (digit), if any, where the total pop1 +// below or at that expanse would meet or exceed Count, meaning the Index must +// be in this expanse. + +SMByCount: // return here for next branch/leaf. + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH; count populations in JPs in the JBL upwards until finding the +// expanse (digit) containing Count, and "recurse". +// +// Note: There are no null JPs in a JBL; watch out for pop1 == 0. +// +// Note: A JBL should always fit in one cache line => no need to count up +// versus down to save cache line fills. +// +// TBD: The previous is no longer true. Consider enhancing this code to count +// up/down, but it can wait for a later tuning phase. In the meantime, PREPB() +// sets pop1 for the whole array, but that value is not used here. 001215: +// Maybe its true again? + + case cJU_JPBRANCH_L2: PREPB_DCD(Pjp, 2, BranchL); +#ifndef JU_64BIT + case cJU_JPBRANCH_L3: PREPB( Pjp, 3, BranchL); +#else + case cJU_JPBRANCH_L3: PREPB_DCD(Pjp, 3, BranchL); + case cJU_JPBRANCH_L4: PREPB_DCD(Pjp, 4, BranchL); + case cJU_JPBRANCH_L5: PREPB_DCD(Pjp, 5, BranchL); + case cJU_JPBRANCH_L6: PREPB_DCD(Pjp, 6, BranchL); + case cJU_JPBRANCH_L7: PREPB( Pjp, 7, BranchL); +#endif + case cJU_JPBRANCH_L: PREPB_ROOT( BranchL); + { + Pjbl_t Pjbl; + +// Common code (state-independent) for all cases of linear branches: + +BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + + for (jpnum = 0; jpnum < (Pjbl->jbl_NumJPs); ++jpnum) + { + if ((pop1 = j__udyJPPop1((Pjbl->jbl_jp) + jpnum)) + == cJU_ALLONES) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + assert(pop1 != 0); + +// Warning: pop1lower and pop1 are unsigned, so do not subtract 1 and compare +// >=, but instead use the following expression: + + if (pop1lower + pop1 > Count0) // Index is in this expanse. + { + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[jpnum], state); + Pjp = (Pjbl->jbl_jp) + jpnum; + goto SMByCount; // look under this expanse. + } + + pop1lower += pop1; // add this JPs pop1. + } + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // should never get here. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // case cJU_JPBRANCH_L + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH; count populations in JPs in the JBB upwards or downwards +// until finding the expanse (digit) containing Count, and "recurse". +// +// Note: There are no null JPs in a JBB; watch out for pop1 == 0. + + case cJU_JPBRANCH_B2: PREPB_DCD(Pjp, 2, BranchB); +#ifndef JU_64BIT + case cJU_JPBRANCH_B3: PREPB( Pjp, 3, BranchB); +#else + case cJU_JPBRANCH_B3: PREPB_DCD(Pjp, 3, BranchB); + case cJU_JPBRANCH_B4: PREPB_DCD(Pjp, 4, BranchB); + case cJU_JPBRANCH_B5: PREPB_DCD(Pjp, 5, BranchB); + case cJU_JPBRANCH_B6: PREPB_DCD(Pjp, 6, BranchB); + case cJU_JPBRANCH_B7: PREPB( Pjp, 7, BranchB); +#endif + case cJU_JPBRANCH_B: PREPB_ROOT( BranchB); + { + Pjbb_t Pjbb; + +// Common code (state-independent) for all cases of bitmap branches: + +BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + +// Shorthand for one subexpanse in a bitmap and for one JP in a bitmap branch: +// +// Note: BMPJP0 exists separately to support assertions. + +#define BMPJP0(Subexp) (P_JP(JU_JBB_PJP(Pjbb, Subexp))) +#define BMPJP(Subexp,JPnum) (BMPJP0(Subexp) + (JPnum)) + + +// Common code for descending through a JP: +// +// Determine the digit for the expanse and save it in *PIndex; then "recurse". + +#define JBB_FOUNDEXPANSE \ + { \ + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb,subexp), jpnum); \ + JU_SETDIGIT(*PIndex, digit, state); \ + Pjp = BMPJP(subexp, jpnum); \ + goto SMByCount; \ + } + + +#ifndef NOSMARTJBB // enable to turn off smart code for comparison purposes. + +// FIGURE OUT WHICH DIRECTION CAUSES FEWER CACHE LINE FILLS; adding the pop1s +// in JPs upwards, or subtracting the pop1s in JPs downwards: +// +// See header comments about limitations of this for Judy*ByCount(). + +#endif + +// COUNT UPWARD, adding each "below" JPs pop1: + +#ifndef NOSMARTJBB // enable to turn off smart code for comparison purposes. + + if (LOWERHALF(Count0, pop1lower, pop1)) + { +#endif +#ifdef SMARTMETRICS + ++jbb_upward; +#endif + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + if ((jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb,subexp))) + && (BMPJP0(subexp) == (Pjp_t) NULL)) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // null ptr. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +// Note: An empty subexpanse (jpcount == 0) is handled "for free": + + for (jpnum = 0; jpnum < jpcount; ++jpnum) + { + if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum))) + == cJU_ALLONES) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + assert(pop1 != 0); + +// Warning: pop1lower and pop1 are unsigned, see earlier comment: + + if (pop1lower + pop1 > Count0) + JBB_FOUNDEXPANSE; // Index is in this expanse. + + pop1lower += pop1; // add this JPs pop1. + } + } +#ifndef NOSMARTJBB // enable to turn off smart code for comparison purposes. + } + + +// COUNT DOWNWARD, subtracting each "above" JPs pop1 from the whole expanses +// pop1: + + else + { +#ifdef SMARTMETRICS + ++jbb_downward; +#endif + pop1lower += pop1; // add whole branch to start. + + for (subexp = cJU_NUMSUBEXPB - 1; subexp >= 0; --subexp) + { + if ((jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp))) + && (BMPJP0(subexp) == (Pjp_t) NULL)) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // null ptr. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +// Note: An empty subexpanse (jpcount == 0) is handled "for free": + + for (jpnum = jpcount - 1; jpnum >= 0; --jpnum) + { + if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum))) + == cJU_ALLONES) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + assert(pop1 != 0); + +// Warning: pop1lower and pop1 are unsigned, see earlier comment: + + pop1lower -= pop1; + +// Beware unsigned math problems: + + if ((pop1lower == 0) || (pop1lower - 1 < Count0)) + JBB_FOUNDEXPANSE; // Index is in this expanse. + } + } + } +#endif // NOSMARTJBB + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // should never get here. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // case cJU_JPBRANCH_B + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH; count populations in JPs in the JBU upwards or +// downwards until finding the expanse (digit) containing Count, and "recurse". + + case cJU_JPBRANCH_U2: PREPB_DCD(Pjp, 2, BranchU); +#ifndef JU_64BIT + case cJU_JPBRANCH_U3: PREPB( Pjp, 3, BranchU); +#else + case cJU_JPBRANCH_U3: PREPB_DCD(Pjp, 3, BranchU); + case cJU_JPBRANCH_U4: PREPB_DCD(Pjp, 4, BranchU); + case cJU_JPBRANCH_U5: PREPB_DCD(Pjp, 5, BranchU); + case cJU_JPBRANCH_U6: PREPB_DCD(Pjp, 6, BranchU); + case cJU_JPBRANCH_U7: PREPB( Pjp, 7, BranchU); +#endif + case cJU_JPBRANCH_U: PREPB_ROOT( BranchU); + { + Pjbu_t Pjbu; + +// Common code (state-independent) for all cases of uncompressed branches: + +BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + +// Common code for descending through a JP: +// +// Save the digit for the expanse in *PIndex, then "recurse". + +#define JBU_FOUNDEXPANSE \ + { \ + JU_SETDIGIT(*PIndex, jpnum, state); \ + Pjp = (Pjbu->jbu_jp) + jpnum; \ + goto SMByCount; \ + } + + +#ifndef NOSMARTJBU // enable to turn off smart code for comparison purposes. + +// FIGURE OUT WHICH DIRECTION CAUSES FEWER CACHE LINE FILLS; adding the pop1s +// in JPs upwards, or subtracting the pop1s in JPs downwards: +// +// See header comments about limitations of this for Judy*ByCount(). + +#endif + +// COUNT UPWARD, simply adding the pop1 of each JP: + +#ifndef NOSMARTJBU // enable to turn off smart code for comparison purposes. + + if (LOWERHALF(Count0, pop1lower, pop1)) + { +#endif +#ifdef SMARTMETRICS + ++jbu_upward; +#endif + + for (jpnum = 0; jpnum < cJU_BRANCHUNUMJPS; ++jpnum) + { + // shortcut, save a function call: + + if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX) + continue; + + if ((pop1 = j__udyJPPop1((Pjbu->jbu_jp) + jpnum)) + == cJU_ALLONES) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + assert(pop1 != 0); + +// Warning: pop1lower and pop1 are unsigned, see earlier comment: + + if (pop1lower + pop1 > Count0) + JBU_FOUNDEXPANSE; // Index is in this expanse. + + pop1lower += pop1; // add this JPs pop1. + } +#ifndef NOSMARTJBU // enable to turn off smart code for comparison purposes. + } + + +// COUNT DOWNWARD, subtracting the pop1 of each JP above from the whole +// expanses pop1: + + else + { +#ifdef SMARTMETRICS + ++jbu_downward; +#endif + pop1lower += pop1; // add whole branch to start. + + for (jpnum = cJU_BRANCHUNUMJPS - 1; jpnum >= 0; --jpnum) + { + // shortcut, save a function call: + + if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX) + continue; + + if ((pop1 = j__udyJPPop1(Pjbu->jbu_jp + jpnum)) + == cJU_ALLONES) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + assert(pop1 != 0); + +// Warning: pop1lower and pop1 are unsigned, see earlier comment: + + pop1lower -= pop1; + +// Beware unsigned math problems: + + if ((pop1lower == 0) || (pop1lower - 1 < Count0)) + JBU_FOUNDEXPANSE; // Index is in this expanse. + } + } +#endif // NOSMARTJBU + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // should never get here. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // case cJU_JPBRANCH_U + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Return the Index at the proper ordinal (see SETOFFSET()) in the leaf. First +// copy Dcd bytes, if there are any (only if state < cJU_ROOTSTATE - 1), to +// *PIndex. +// +// Note: The preceding branch traversal code MIGHT set pop1 for this expanse +// (linear leaf) as a side-effect, but dont depend on that (for JUDYL, which +// is the only cases that need it anyway). + +#define PREPL_DCD(cState) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + PREPL + +#ifdef JUDY1 +#define PREPL_SETPOP1 // not needed in any cases. +#else +#define PREPL_SETPOP1 pop1 = JU_JPLEAF_POP0(Pjp) + 1 +#endif + +#define PREPL \ + Pjll = P_JLL(Pjp->jp_Addr); \ + PREPL_SETPOP1; \ + SETOFFSET(offset, Count0, pop1lower, Pjp) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + PREPL_DCD(1); + JU_SETDIGIT1(*PIndex, ((uint8_t *) Pjll)[offset]); + JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + + case cJU_JPLEAF2: + + PREPL_DCD(2); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + +#ifndef JU_64BIT + case cJU_JPLEAF3: + { + Word_t lsb; + PREPL; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + +#else + case cJU_JPLEAF3: + { + Word_t lsb; + PREPL_DCD(3); + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + + case cJU_JPLEAF4: + + PREPL_DCD(4); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + + case cJU_JPLEAF5: + { + Word_t lsb; + PREPL_DCD(5); + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + } + + case cJU_JPLEAF6: + { + Word_t lsb; + PREPL_DCD(6); + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + } + + case cJU_JPLEAF7: + { + Word_t lsb; + PREPL; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_LEAF7(Pjll, pop1, offset); + } +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Return the Index at the proper ordinal (see SETOFFSET()) in the leaf by +// counting bits. First copy Dcd bytes (always present since state 1 < +// cJU_ROOTSTATE) to *PIndex. +// +// Note: The preceding branch traversal code MIGHT set pop1 for this expanse +// (bitmap leaf) as a side-effect, but dont depend on that. + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; + + JU_SETDCD(*PIndex, Pjp, 1); + Pjlb = P_JLB(Pjp->jp_Addr); + pop1 = JU_JPLEAF_POP0(Pjp) + 1; + +// COUNT UPWARD, adding the pop1 of each subexpanse: +// +// The entire bitmap should fit in one cache line, but still try to save some +// CPU time by counting the fewest possible number of subexpanses from the +// bitmap. +// +// See header comments about limitations of this for Judy*ByCount(). + +#ifndef NOSMARTJLB // enable to turn off smart code for comparison purposes. + + if (LOWERHALF(Count0, pop1lower, pop1)) + { +#endif +#ifdef SMARTMETRICS + ++jlb_upward; +#endif + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + { + pop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + +// Warning: pop1lower and pop1 are unsigned, see earlier comment: + + if (pop1lower + pop1 > Count0) + goto LeafB1; // Index is in this subexpanse. + + pop1lower += pop1; // add this subexpanses pop1. + } +#ifndef NOSMARTJLB // enable to turn off smart code for comparison purposes. + } + + +// COUNT DOWNWARD, subtracting each "above" subexpanses pop1 from the whole +// expanses pop1: + + else + { +#ifdef SMARTMETRICS + ++jlb_downward; +#endif + pop1lower += pop1; // add whole leaf to start. + + for (subexp = cJU_NUMSUBEXPL - 1; subexp >= 0; --subexp) + { + pop1lower -= j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + +// Beware unsigned math problems: + + if ((pop1lower == 0) || (pop1lower - 1 < Count0)) + goto LeafB1; // Index is in this subexpanse. + } + } +#endif // NOSMARTJLB + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // should never get here. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + +// RETURN INDEX FOUND: +// +// Come here with subexp set to the correct subexpanse, and pop1lower set to +// the sum for all lower expanses and subexpanses in the Judy tree. Calculate +// and save in *PIndex the digit corresponding to the ordinal in this +// subexpanse. + +LeafB1: + SETOFFSET(offset, Count0, pop1lower, Pjp); + JU_BITMAPDIGITL(digit, subexp, JU_JLB_BITMAP(Pjlb, subexp), offset); + JU_SETDIGIT1(*PIndex, digit); + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + offset)) + + } // case cJU_JPLEAF_B1 + + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// Copy Dcd bytes (always present since state 1 < cJU_ROOTSTATE) to *PIndex, +// then set the appropriate digit for the ordinal (see SETOFFSET()) in the leaf +// as the LSB in *PIndex. + + case cJ1_JPFULLPOPU1: + + JU_SETDCD(*PIndex, Pjp, 1); + SETOFFSET(offset, Count0, pop1lower, Pjp); + assert(offset >= 0); + assert(offset <= cJU_JPFULLPOPU1_POP0); + JU_SETDIGIT1(*PIndex, offset); + JU_RET_FOUND_FULLPOPU1; +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: +// +// Locate the Index with the proper ordinal (see SETOFFSET()) in the Immediate, +// depending on leaf Index Size and pop1. Note: There are no Dcd bytes in an +// Immediate JP, but in a cJU_JPIMMED_*_01 JP, the field holds the least bytes +// of the immediate Index. + +#define SET_01(cState) JU_SETDIGITS(*PIndex, JU_JPDCDPOP0(Pjp), cState) + + case cJU_JPIMMED_1_01: SET_01(1); goto Imm_01; + case cJU_JPIMMED_2_01: SET_01(2); goto Imm_01; + case cJU_JPIMMED_3_01: SET_01(3); goto Imm_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: SET_01(4); goto Imm_01; + case cJU_JPIMMED_5_01: SET_01(5); goto Imm_01; + case cJU_JPIMMED_6_01: SET_01(6); goto Imm_01; + case cJU_JPIMMED_7_01: SET_01(7); goto Imm_01; +#endif + +Imm_01: + + DBGCODE(SETOFFSET_IMM_CK(offset, Count0, pop1lower, 1);) + JU_RET_FOUND_IMM_01(Pjp); + +// Shorthand for where to find start of Index bytes array: + +#ifdef JUDY1 +#define PJI (Pjp->jp_1Index) +#else +#define PJI (Pjp->jp_LIndex) +#endif + +// Optional code to check the remaining ordinal (see SETOFFSET_IMM()) against +// the Index Size of the Immediate: + +#ifndef DEBUG // simple placeholder: +#define IMM(cPop1,Next) \ + goto Next +#else // extra pop1-specific checking: +#define IMM(cPop1,Next) \ + SETOFFSET_IMM_CK(offset, Count0, pop1lower, cPop1); \ + goto Next +#endif + + case cJU_JPIMMED_1_02: IMM( 2, Imm1); + case cJU_JPIMMED_1_03: IMM( 3, Imm1); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: IMM( 4, Imm1); + case cJU_JPIMMED_1_05: IMM( 5, Imm1); + case cJU_JPIMMED_1_06: IMM( 6, Imm1); + case cJU_JPIMMED_1_07: IMM( 7, Imm1); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: IMM( 8, Imm1); + case cJ1_JPIMMED_1_09: IMM( 9, Imm1); + case cJ1_JPIMMED_1_10: IMM(10, Imm1); + case cJ1_JPIMMED_1_11: IMM(11, Imm1); + case cJ1_JPIMMED_1_12: IMM(12, Imm1); + case cJ1_JPIMMED_1_13: IMM(13, Imm1); + case cJ1_JPIMMED_1_14: IMM(14, Imm1); + case cJ1_JPIMMED_1_15: IMM(15, Imm1); +#endif + +Imm1: SETOFFSET_IMM(offset, Count0, pop1lower); + JU_SETDIGIT1(*PIndex, ((uint8_t *) PJI)[offset]); + JU_RET_FOUND_IMM(Pjp, offset); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: IMM(2, Imm2); + case cJU_JPIMMED_2_03: IMM(3, Imm2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: IMM(4, Imm2); + case cJ1_JPIMMED_2_05: IMM(5, Imm2); + case cJ1_JPIMMED_2_06: IMM(6, Imm2); + case cJ1_JPIMMED_2_07: IMM(7, Imm2); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +Imm2: SETOFFSET_IMM(offset, Count0, pop1lower); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: IMM(2, Imm3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: IMM(3, Imm3); + case cJ1_JPIMMED_3_04: IMM(4, Imm3); + case cJ1_JPIMMED_3_05: IMM(5, Imm3); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +Imm3: + { + Word_t lsb; + SETOFFSET_IMM(offset, Count0, pop1lower); + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: IMM(2, Imm4); + case cJ1_JPIMMED_4_03: IMM(3, Imm4); + +Imm4: SETOFFSET_IMM(offset, Count0, pop1lower); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); + + case cJ1_JPIMMED_5_02: IMM(2, Imm5); + case cJ1_JPIMMED_5_03: IMM(3, Imm5); + +Imm5: + { + Word_t lsb; + SETOFFSET_IMM(offset, Count0, pop1lower); + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_6_02: IMM(2, Imm6); + +Imm6: + { + Word_t lsb; + SETOFFSET_IMM(offset, Count0, pop1lower); + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_7_02: IMM(2, Imm7); + +Imm7: + { + Word_t lsb; + SETOFFSET_IMM(offset, Count0, pop1lower); + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif // (JUDY1 && JU_64BIT) + + +// ---------------------------------------------------------------------------- +// UNEXPECTED JP TYPES: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SMByCount switch. + + /*NOTREACHED*/ + +} // Judy1ByCount() / JudyLByCount() diff --git a/libnetdata/libjudy/src/JudyL/JudyLCascade.c b/libnetdata/libjudy/src/JudyL/JudyLCascade.c new file mode 100644 index 000000000..6b52ddf5f --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLCascade.c @@ -0,0 +1,1942 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.38 $ $Source: /judy/src/JudyCommon/JudyCascade.c $ + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +extern int j__udyCreateBranchL(Pjp_t, Pjp_t, uint8_t *, Word_t, Pvoid_t); +extern int j__udyCreateBranchB(Pjp_t, Pjp_t, uint8_t *, Word_t, Pvoid_t); + +DBGCODE(extern void JudyCheckSorted(Pjll_t Pjll, Word_t Pop1, long IndexSize);) + +static const jbb_t StageJBBZero; // zeroed versions of namesake struct. + +// TBD: There are multiple copies of (some of) these CopyWto3, Copy3toW, +// CopyWto7 and Copy7toW functions in Judy1Cascade.c, JudyLCascade.c, and +// JudyDecascade.c. These static functions should probably be moved to a +// common place, made macros, or something to avoid having four copies. + + +// **************************************************************************** +// __ J U D Y C O P Y X T O W + + +FUNCTION static void j__udyCopy3toW( + PWord_t PDest, + uint8_t * PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY3_PINDEX_TO_LONG(*PDest, PSrc); + PSrc += 3; + PDest += 1; + + } while(--LeafIndexes); + +} //j__udyCopy3toW() + + +#ifdef JU_64BIT + +FUNCTION static void j__udyCopy4toW( + PWord_t PDest, + uint32_t * PSrc, + Word_t LeafIndexes) +{ + do { *PDest++ = *PSrc++; + } while(--LeafIndexes); + +} // j__udyCopy4toW() + + +FUNCTION static void j__udyCopy5toW( + PWord_t PDest, + uint8_t * PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY5_PINDEX_TO_LONG(*PDest, PSrc); + PSrc += 5; + PDest += 1; + + } while(--LeafIndexes); + +} // j__udyCopy5toW() + + +FUNCTION static void j__udyCopy6toW( + PWord_t PDest, + uint8_t * PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY6_PINDEX_TO_LONG(*PDest, PSrc); + PSrc += 6; + PDest += 1; + + } while(--LeafIndexes); + +} // j__udyCopy6toW() + + +FUNCTION static void j__udyCopy7toW( + PWord_t PDest, + uint8_t * PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY7_PINDEX_TO_LONG(*PDest, PSrc); + PSrc += 7; + PDest += 1; + + } while(--LeafIndexes); + +} // j__udyCopy7toW() + +#endif // JU_64BIT + + +// **************************************************************************** +// __ J U D Y C O P Y W T O X + + +FUNCTION static void j__udyCopyWto3( + uint8_t * PDest, + PWord_t PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY3_LONG_TO_PINDEX(PDest, *PSrc); + PSrc += 1; + PDest += 3; + + } while(--LeafIndexes); + +} // j__udyCopyWto3() + + +#ifdef JU_64BIT + +FUNCTION static void j__udyCopyWto4( + uint8_t * PDest, + PWord_t PSrc, + Word_t LeafIndexes) +{ + uint32_t *PDest32 = (uint32_t *)PDest; + + do + { + *PDest32 = *PSrc; + PSrc += 1; + PDest32 += 1; + } while(--LeafIndexes); + +} // j__udyCopyWto4() + + +FUNCTION static void j__udyCopyWto5( + uint8_t * PDest, + PWord_t PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY5_LONG_TO_PINDEX(PDest, *PSrc); + PSrc += 1; + PDest += 5; + + } while(--LeafIndexes); + +} // j__udyCopyWto5() + + +FUNCTION static void j__udyCopyWto6( + uint8_t * PDest, + PWord_t PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY6_LONG_TO_PINDEX(PDest, *PSrc); + PSrc += 1; + PDest += 6; + + } while(--LeafIndexes); + +} // j__udyCopyWto6() + + +FUNCTION static void j__udyCopyWto7( + uint8_t * PDest, + PWord_t PSrc, + Word_t LeafIndexes) +{ + do + { + JU_COPY7_LONG_TO_PINDEX(PDest, *PSrc); + PSrc += 1; + PDest += 7; + + } while(--LeafIndexes); + +} // j__udyCopyWto7() + +#endif // JU_64BIT + + +// **************************************************************************** +// COMMON CODE (MACROS): +// +// Free objects in an array of valid JPs, StageJP[ExpCnt] == last one may +// include Immeds, which are ignored. + +#define FREEALLEXIT(ExpCnt,StageJP,Pjpm) \ + { \ + Word_t _expct = (ExpCnt); \ + while (_expct--) j__udyFreeSM(&((StageJP)[_expct]), Pjpm); \ + return(-1); \ + } + +// Clear the array that keeps track of the number of JPs in a subexpanse: + +#define ZEROJP(SubJPCount) \ + { \ + int ii; \ + for (ii = 0; ii < cJU_NUMSUBEXPB; ii++) (SubJPCount[ii]) = 0; \ + } + +// **************************************************************************** +// __ J U D Y S T A G E J B B T O J B B +// +// Create a mallocd BranchB (jbb_t) from a staged BranchB while "splaying" a +// single old leaf. Return -1 if out of memory, otherwise 1. + +static int j__udyStageJBBtoJBB( + Pjp_t PjpLeaf, // JP of leaf being splayed. + Pjbb_t PStageJBB, // temp jbb_t on stack. + Pjp_t PjpArray, // array of JPs to splayed new leaves. + uint8_t * PSubCount, // count of JPs for each subexpanse. + Pjpm_t Pjpm) // the jpm_t for JudyAlloc*(). +{ + Pjbb_t PjbbRaw; // pointer to new bitmap branch. + Pjbb_t Pjbb; + Word_t subexp; + +// Get memory for new BranchB: + + if ((PjbbRaw = j__udyAllocJBB(Pjpm)) == (Pjbb_t) NULL) return(-1); + Pjbb = P_JBB(PjbbRaw); + +// Copy staged BranchB into just-allocated BranchB: + + *Pjbb = *PStageJBB; + +// Allocate the JP subarrays (BJP) for the new BranchB: + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; subexp++) + { + Pjp_t PjpRaw; + Pjp_t Pjp; + Word_t NumJP; // number of JPs in each subexpanse. + + if ((NumJP = PSubCount[subexp]) == 0) continue; // empty. + +// Out of memory, back out previous allocations: + + if ((PjpRaw = j__udyAllocJBBJP(NumJP, Pjpm)) == (Pjp_t) NULL) + { + while(subexp--) + { + if ((NumJP = PSubCount[subexp]) == 0) continue; + + PjpRaw = JU_JBB_PJP(Pjbb, subexp); + j__udyFreeJBBJP(PjpRaw, NumJP, Pjpm); + } + j__udyFreeJBB(PjbbRaw, Pjpm); + return(-1); // out of memory. + } + Pjp = P_JP(PjpRaw); + +// Place the JP subarray pointer in the new BranchB, copy subarray JPs, and +// advance to the next subexpanse: + + JU_JBB_PJP(Pjbb, subexp) = PjpRaw; + JU_COPYMEM(Pjp, PjpArray, NumJP); + PjpArray += NumJP; + + } // for each subexpanse. + +// Change the PjpLeaf from Leaf to BranchB: + + PjpLeaf->jp_Addr = (Word_t) PjbbRaw; + PjpLeaf->jp_Type += cJU_JPBRANCH_B2 - cJU_JPLEAF2; // Leaf to BranchB. + + return(1); + +} // j__udyStageJBBtoJBB() + + +// **************************************************************************** +// __ J U D Y J L L 2 T O J L B 1 +// +// Create a LeafB1 (jlb_t = JLB1) from a Leaf2 (2-byte Indexes and for JudyL, +// Word_t Values). Return NULL if out of memory, else a pointer to the new +// LeafB1. +// +// NOTE: Caller must release the Leaf2 that was passed in. + +FUNCTION static Pjlb_t j__udyJLL2toJLB1( + uint16_t * Pjll, // array of 16-bit indexes. +#ifdef JUDYL + Pjv_t Pjv, // array of associated values. +#endif + Word_t LeafPop1, // number of indexes/values. + Pvoid_t Pjpm) // jpm_t for JudyAlloc*()/JudyFree*(). +{ + Pjlb_t PjlbRaw; + Pjlb_t Pjlb; + int offset; +JUDYLCODE(int subexp;) + +// Allocate the LeafB1: + + if ((PjlbRaw = j__udyAllocJLB1(Pjpm)) == (Pjlb_t) NULL) + return((Pjlb_t) NULL); + Pjlb = P_JLB(PjlbRaw); + +// Copy Leaf2 indexes to LeafB1: + + for (offset = 0; offset < LeafPop1; ++offset) + JU_BITMAPSETL(Pjlb, Pjll[offset]); + +#ifdef JUDYL + +// Build LeafVs from bitmap: + + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + { + struct _POINTER_VALUES + { + Word_t pv_Pop1; // size of value area. + Pjv_t pv_Pjv; // raw pointer to value area. + } pv[cJU_NUMSUBEXPL]; + +// Get the population of the subexpanse, and if any, allocate a LeafV: + + pv[subexp].pv_Pop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + + if (pv[subexp].pv_Pop1) + { + Pjv_t Pjvnew; + +// TBD: There is an opportunity to put pop == 1 value in pointer: + + pv[subexp].pv_Pjv = j__udyLAllocJV(pv[subexp].pv_Pop1, Pjpm); + +// Upon out of memory, free all previously allocated: + + if (pv[subexp].pv_Pjv == (Pjv_t) NULL) + { + while(subexp--) + { + if (pv[subexp].pv_Pop1) + { + j__udyLFreeJV(pv[subexp].pv_Pjv, pv[subexp].pv_Pop1, + Pjpm); + } + } + j__udyFreeJLB1(PjlbRaw, Pjpm); + return((Pjlb_t) NULL); + } + + Pjvnew = P_JV(pv[subexp].pv_Pjv); + JU_COPYMEM(Pjvnew, Pjv, pv[subexp].pv_Pop1); + Pjv += pv[subexp].pv_Pop1; // advance value pointer. + +// Place raw pointer to value array in bitmap subexpanse: + + JL_JLB_PVALUE(Pjlb, subexp) = pv[subexp].pv_Pjv; + + } // populated subexpanse. + } // each subexpanse. + +#endif // JUDYL + + return(PjlbRaw); // pointer to LeafB1. + +} // j__udyJLL2toJLB1() + + +// **************************************************************************** +// __ J U D Y C A S C A D E 1 +// +// Create bitmap leaf from 1-byte Indexes and Word_t Values. +// +// TBD: There must be a better way. +// +// Only for JudyL 32 bit: (note, unifdef disallows comment on next line) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + +FUNCTION int j__udyCascade1( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + Word_t DcdP0; + uint8_t * PLeaf; + Pjlb_t PjlbRaw; + Pjlb_t Pjlb; + Word_t Pop1; + Word_t ii; // temp for loop counter +JUDYLCODE(Pjv_t Pjv;) + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF1); + assert((JU_JPDCDPOP0(Pjp) & 0xFF) == (cJU_LEAF1_MAXPOP1-1)); + + PjlbRaw = j__udyAllocJLB1(Pjpm); + if (PjlbRaw == (Pjlb_t) NULL) return(-1); + + Pjlb = P_JLB(PjlbRaw); + PLeaf = (uint8_t *) P_JLL(Pjp->jp_Addr); + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + + JUDYLCODE(Pjv = JL_LEAF1VALUEAREA(PLeaf, Pop1);) + +// Copy 1 byte index Leaf to bitmap Leaf + for (ii = 0; ii < Pop1; ii++) JU_BITMAPSETL(Pjlb, PLeaf[ii]); + +#ifdef JUDYL +// Build 8 subexpanse Value leaves from bitmap + for (ii = 0; ii < cJU_NUMSUBEXPL; ii++) + { +// Get number of Indexes in subexpanse + if ((Pop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, ii)))) + { + Pjv_t PjvnewRaw; // value area of new leaf. + Pjv_t Pjvnew; + + PjvnewRaw = j__udyLAllocJV(Pop1, Pjpm); + if (PjvnewRaw == (Pjv_t) NULL) // out of memory. + { +// Free prevously allocated LeafVs: + while(ii--) + { + if ((Pop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, ii)))) + { + PjvnewRaw = JL_JLB_PVALUE(Pjlb, ii); + j__udyLFreeJV(PjvnewRaw, Pop1, Pjpm); + } + } +// Free the bitmap leaf + j__udyLFreeJLB1(PjlbRaw,Pjpm); + return(-1); + } + Pjvnew = P_JV(PjvnewRaw); + JU_COPYMEM(Pjvnew, Pjv, Pop1); + + Pjv += Pop1; + JL_JLB_PVALUE(Pjlb, ii) = PjvnewRaw; + } + } +#endif // JUDYL + + DcdP0 = JU_JPDCDPOP0(Pjp) | (PLeaf[0] & cJU_DCDMASK(1)); + JU_JPSETADT(Pjp, (Word_t)PjlbRaw, DcdP0, cJU_JPLEAF_B1); + + return(1); // return success + +} // j__udyCascade1() + +#endif // (!(JUDY1 && JU_64BIT)) + + +// **************************************************************************** +// __ J U D Y C A S C A D E 2 +// +// Entry PLeaf of size LeafPop1 is either compressed or splayed with pointer +// returned in Pjp. Entry Levels sizeof(Word_t) down to level 2. +// +// Splay or compress the 2-byte Index Leaf that Pjp point to. Return *Pjp as a +// (compressed) cJU_LEAFB1 or a cJU_BRANCH_*2 + +FUNCTION int j__udyCascade2( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint16_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF2_MAXPOP1]; // JPs of new leaves + uint8_t StageExp [cJU_LEAF2_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF2); + assert((JU_JPDCDPOP0(Pjp) & 0xFFFF) == (cJU_LEAF2_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint16_t *) P_JLL(Pjp->jp_Addr); + +// And its Value area + JUDYLCODE(Pjv = JL_LEAF2VALUEAREA(PLeaf, cJU_LEAF2_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it to a Bitmap Leaf + + CIndex = PLeaf[0]; + if (!JU_DIGITATSTATE(CIndex ^ PLeaf[cJU_LEAF2_MAXPOP1-1], 2)) + { +// cJU_JPLEAF_B1 + Word_t DcdP0; + Pjlb_t PjlbRaw; + PjlbRaw = j__udyJLL2toJLB1(PLeaf, +#ifdef JUDYL + Pjv, +#endif + cJU_LEAF2_MAXPOP1, Pjpm); + if (PjlbRaw == (Pjlb_t)NULL) return(-1); // out of memory + +// Merge in another Dcd byte because compressing + DcdP0 = (CIndex & cJU_DCDMASK(1)) | JU_JPDCDPOP0(Pjp); + JU_JPSETADT(Pjp, (Word_t)PjlbRaw, DcdP0, cJU_JPLEAF_B1); + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 2 byte index Leaf to 1 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF2_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ PLeaf[End], 2)) + ) + { +// Build a leaf below the previous expanse +// + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 2); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 2); + + if (Pop1 == 1) // cJU_JPIMMED_1_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(1)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_1_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_1_01); +#endif // JUDYL + } + else if (Pop1 <= cJU_IMMED1_MAXPOP1) // bigger + { +// cJL_JPIMMED_1_02..3: JudyL 32 +// cJ1_JPIMMED_1_02..7: Judy1 32 +// cJL_JPIMMED_1_02..7: JudyL 64 +// cJ1_JPIMMED_1_02..15: Judy1 64 +#ifdef JUDYL + Pjv_t PjvnewRaw; // value area of leaf. + Pjv_t Pjvnew; + +// Allocate Value area for Immediate Leaf + PjvnewRaw = j__udyLAllocJV(Pop1, Pjpm); + if (PjvnewRaw == (Pjv_t) NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjvnew = P_JV(PjvnewRaw); + +// Copy to Values to Value Leaf + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); + PjpJP->jp_Addr = (Word_t) PjvnewRaw; + +// Copy to JP as an immediate Leaf + JU_COPYMEM(PjpJP->jp_LIndex, PLeaf + Start, + Pop1); +#else + JU_COPYMEM(PjpJP->jp_1Index, PLeaf + Start, + Pop1); +#endif +// Set Type, Population and Index size + PjpJP->jp_Type = cJU_JPIMMED_1_02 + Pop1 - 2; + } + +// 64Bit Judy1 does not have Leaf1: (note, unifdef disallows comment on next +// line) + +#if (! (defined(JUDY1) && defined(JU_64BIT))) + else if (Pop1 <= cJU_LEAF1_MAXPOP1) // still bigger + { +// cJU_JPLEAF1 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Get a new Leaf + PjllRaw = j__udyAllocJLL1(Pop1, Pjpm); + if (PjllRaw == (Pjll_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjll = P_JLL(PjllRaw); +#ifdef JUDYL +// Copy to Values to new Leaf + Pjvnew = JL_LEAF1VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif +// Copy Indexes to new Leaf + JU_COPYMEM((uint8_t *)Pjll, PLeaf+Start, Pop1); + + DBGCODE(JudyCheckSorted(Pjll, Pop1, 1);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(2)) + | + (CIndex & cJU_DCDMASK(2-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF1); + } +#endif // (!(JUDY1 && JU_64BIT)) // Not 64Bit Judy1 + + else // biggest + { +// cJU_JPLEAF_B1 + Word_t DcdP0; + Pjlb_t PjlbRaw; + PjlbRaw = j__udyJLL2toJLB1( + PLeaf + Start, +#ifdef JUDYL + Pjv + Start, +#endif + Pop1, Pjpm); + if (PjlbRaw == (Pjlb_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(2)) + | + (CIndex & cJU_DCDMASK(2-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjlbRaw, DcdP0, + cJU_JPLEAF_B1); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF2_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = PLeaf[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L2; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade2() + + +// **************************************************************************** +// __ J U D Y C A S C A D E 3 +// +// Return *Pjp as a (compressed) cJU_LEAF2, cJU_BRANCH_L3, cJU_BRANCH_B3. + +FUNCTION int j__udyCascade3( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint8_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF3_MAXPOP1]; // JPs of new leaves + Word_t StageA [cJU_LEAF3_MAXPOP1]; + uint8_t StageExp [cJU_LEAF3_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF3); + assert((JU_JPDCDPOP0(Pjp) & 0xFFFFFF) == (cJU_LEAF3_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint8_t *) P_JLL(Pjp->jp_Addr); + +// Extract leaf to Word_t and insert-sort Index into it + j__udyCopy3toW(StageA, PLeaf, cJU_LEAF3_MAXPOP1); + +// Get the address of the Leaf and Value area + JUDYLCODE(Pjv = JL_LEAF3VALUEAREA(PLeaf, cJU_LEAF3_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it (compare 1st, last & Index) + + CIndex = StageA[0]; + if (!JU_DIGITATSTATE(CIndex ^ StageA[cJU_LEAF3_MAXPOP1-1], 3)) + { + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Alloc a 2 byte Index Leaf + PjllRaw = j__udyAllocJLL2(cJU_LEAF3_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy just 2 bytes Indexes to new Leaf +// j__udyCopyWto2((uint16_t *) Pjll, StageA, cJU_LEAF3_MAXPOP1); + JU_COPYMEM ((uint16_t *) Pjll, StageA, cJU_LEAF3_MAXPOP1); +#ifdef JUDYL +// Copy Value area into new Leaf + Pjvnew = JL_LEAF2VALUEAREA(Pjll, cJU_LEAF3_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAF3_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAF3_MAXPOP1, 2);) + +// Form new JP, Pop0 field is unchanged +// Add in another Dcd byte because compressing + DcdP0 = (CIndex & cJU_DCDMASK(2)) | JU_JPDCDPOP0(Pjp); + + JU_JPSETADT(Pjp, (Word_t) PjllRaw, DcdP0, cJU_JPLEAF2); + + return(1); // Success + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 3 byte index Leaf to 2 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF3_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ StageA[End], 3)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 3); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 3); + + if (Pop1 == 1) // cJU_JPIMMED_2_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(2)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_2_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_2_01); +#endif // JUDYL + } +#if (defined(JUDY1) || defined(JU_64BIT)) + else if (Pop1 <= cJU_IMMED2_MAXPOP1) + { +// cJ1_JPIMMED_2_02..3: Judy1 32 +// cJL_JPIMMED_2_02..3: JudyL 64 +// cJ1_JPIMMED_2_02..7: Judy1 64 +#ifdef JUDYL +// Alloc is 1st in case of malloc fail + Pjv_t PjvnewRaw; // value area of new leaf. + Pjv_t Pjvnew; + +// Allocate Value area for Immediate Leaf + PjvnewRaw = j__udyLAllocJV(Pop1, Pjpm); + if (PjvnewRaw == (Pjv_t) NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjvnew = P_JV(PjvnewRaw); + +// Copy to Values to Value Leaf + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); + + PjpJP->jp_Addr = (Word_t) PjvnewRaw; + +// Copy to Index to JP as an immediate Leaf + JU_COPYMEM((uint16_t *) (PjpJP->jp_LIndex), + StageA + Start, Pop1); +#else // JUDY1 + JU_COPYMEM((uint16_t *) (PjpJP->jp_1Index), + StageA + Start, Pop1); +#endif // JUDY1 +// Set Type, Population and Index size + PjpJP->jp_Type = cJU_JPIMMED_2_02 + Pop1 - 2; + } +#endif // (JUDY1 || JU_64BIT) + + else // Make a linear leaf2 + { +// cJU_JPLEAF2 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + + PjllRaw = j__udyAllocJLL2(Pop1, Pjpm); + if (PjllRaw == (Pjll_t) NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjll = P_JLL(PjllRaw); +#ifdef JUDYL +// Copy to Values to new Leaf + Pjvnew = JL_LEAF2VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif +// Copy least 2 bytes per Index of Leaf to new Leaf + JU_COPYMEM((uint16_t *) Pjll, StageA+Start, + Pop1); + + DBGCODE(JudyCheckSorted(Pjll, Pop1, 2);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(3)) + | + (CIndex & cJU_DCDMASK(3-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF2); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF3_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = StageA[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L3; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade3() + + +#ifdef JU_64BIT // JudyCascade[4567] + +// **************************************************************************** +// __ J U D Y C A S C A D E 4 +// +// Cascade from a cJU_JPLEAF4 to one of the following: +// 1. if leaf is in 1 expanse: +// compress it into a JPLEAF3 +// 2. if leaf contains multiple expanses: +// create linear or bitmap branch containing +// each new expanse is either a: +// JPIMMED_3_01 branch +// JPIMMED_3_02 branch +// JPLEAF3 + +FUNCTION int j__udyCascade4( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint32_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF4_MAXPOP1]; // JPs of new leaves + Word_t StageA [cJU_LEAF4_MAXPOP1]; + uint8_t StageExp [cJU_LEAF4_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF4); + assert((JU_JPDCDPOP0(Pjp) & 0xFFFFFFFF) == (cJU_LEAF4_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint32_t *) P_JLL(Pjp->jp_Addr); + +// Extract 4 byte index Leaf to Word_t + j__udyCopy4toW(StageA, PLeaf, cJU_LEAF4_MAXPOP1); + +// Get the address of the Leaf and Value area + JUDYLCODE(Pjv = JL_LEAF4VALUEAREA(PLeaf, cJU_LEAF4_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it (compare 1st, last & Index) + + CIndex = StageA[0]; + if (!JU_DIGITATSTATE(CIndex ^ StageA[cJU_LEAF4_MAXPOP1-1], 4)) + { + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new Leaf. + +// Alloc a 3 byte Index Leaf + PjllRaw = j__udyAllocJLL3(cJU_LEAF4_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy Index area into new Leaf + j__udyCopyWto3((uint8_t *) Pjll, StageA, cJU_LEAF4_MAXPOP1); +#ifdef JUDYL +// Copy Value area into new Leaf + Pjvnew = JL_LEAF3VALUEAREA(Pjll, cJU_LEAF4_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAF4_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAF4_MAXPOP1, 3);) + + DcdP0 = JU_JPDCDPOP0(Pjp) | (CIndex & cJU_DCDMASK(3)); + JU_JPSETADT(Pjp, (Word_t)PjllRaw, DcdP0, cJU_JPLEAF3); + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 4 byte index Leaf to 3 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF4_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ StageA[End], 4)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 4); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 4); + + if (Pop1 == 1) // cJU_JPIMMED_3_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(3)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_3_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_3_01); +#endif // JUDYL + } + else if (Pop1 <= cJU_IMMED3_MAXPOP1) + { +// cJ1_JPIMMED_3_02 : Judy1 32 +// cJL_JPIMMED_3_02 : JudyL 64 +// cJ1_JPIMMED_3_02..5: Judy1 64 + +#ifdef JUDYL +// Alloc is 1st in case of malloc fail + Pjv_t PjvnewRaw; // value area of new leaf. + Pjv_t Pjvnew; + +// Allocate Value area for Immediate Leaf + PjvnewRaw = j__udyLAllocJV(Pop1, Pjpm); + if (PjvnewRaw == (Pjv_t) NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjvnew = P_JV(PjvnewRaw); + +// Copy to Values to Value Leaf + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); + PjpJP->jp_Addr = (Word_t) PjvnewRaw; + +// Copy to Index to JP as an immediate Leaf + j__udyCopyWto3(PjpJP->jp_LIndex, + StageA + Start, Pop1); +#else + j__udyCopyWto3(PjpJP->jp_1Index, + StageA + Start, Pop1); +#endif +// Set type, population and Index size + PjpJP->jp_Type = cJU_JPIMMED_3_02 + Pop1 - 2; + } + else + { +// cJU_JPLEAF3 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + + PjllRaw = j__udyAllocJLL3(Pop1, Pjpm); + if (PjllRaw == (Pjll_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjll = P_JLL(PjllRaw); + +// Copy Indexes to new Leaf + j__udyCopyWto3((uint8_t *) Pjll, StageA + Start, + Pop1); +#ifdef JUDYL +// Copy to Values to new Leaf + Pjvnew = JL_LEAF3VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif + DBGCODE(JudyCheckSorted(Pjll, Pop1, 3);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(4)) + | + (CIndex & cJU_DCDMASK(4-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF3); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF4_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = StageA[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L4; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade4() + + +// **************************************************************************** +// __ J U D Y C A S C A D E 5 +// +// Cascade from a cJU_JPLEAF5 to one of the following: +// 1. if leaf is in 1 expanse: +// compress it into a JPLEAF4 +// 2. if leaf contains multiple expanses: +// create linear or bitmap branch containing +// each new expanse is either a: +// JPIMMED_4_01 branch +// JPLEAF4 + +FUNCTION int j__udyCascade5( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint8_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF5_MAXPOP1]; // JPs of new leaves + Word_t StageA [cJU_LEAF5_MAXPOP1]; + uint8_t StageExp [cJU_LEAF5_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF5); + assert((JU_JPDCDPOP0(Pjp) & 0xFFFFFFFFFF) == (cJU_LEAF5_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint8_t *) P_JLL(Pjp->jp_Addr); + +// Extract 5 byte index Leaf to Word_t + j__udyCopy5toW(StageA, PLeaf, cJU_LEAF5_MAXPOP1); + +// Get the address of the Leaf and Value area + JUDYLCODE(Pjv = JL_LEAF5VALUEAREA(PLeaf, cJU_LEAF5_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it (compare 1st, last & Index) + + CIndex = StageA[0]; + if (!JU_DIGITATSTATE(CIndex ^ StageA[cJU_LEAF5_MAXPOP1-1], 5)) + { + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Alloc a 4 byte Index Leaf + PjllRaw = j__udyAllocJLL4(cJU_LEAF5_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy Index area into new Leaf + j__udyCopyWto4((uint8_t *) Pjll, StageA, cJU_LEAF5_MAXPOP1); +#ifdef JUDYL +// Copy Value area into new Leaf + Pjvnew = JL_LEAF4VALUEAREA(Pjll, cJU_LEAF5_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAF5_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAF5_MAXPOP1, 4);) + + DcdP0 = JU_JPDCDPOP0(Pjp) | (CIndex & cJU_DCDMASK(4)); + JU_JPSETADT(Pjp, (Word_t)PjllRaw, DcdP0, cJU_JPLEAF4); + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 5 byte index Leaf to 4 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF5_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ StageA[End], 5)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 5); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 5); + + if (Pop1 == 1) // cJU_JPIMMED_4_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(4)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_4_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_4_01); +#endif // JUDYL + } +#ifdef JUDY1 + else if (Pop1 <= cJ1_IMMED4_MAXPOP1) + { +// cJ1_JPIMMED_4_02..3: Judy1 64 + +// Copy to Index to JP as an immediate Leaf + j__udyCopyWto4(PjpJP->jp_1Index, + StageA + Start, Pop1); + +// Set pointer, type, population and Index size + PjpJP->jp_Type = cJ1_JPIMMED_4_02 + Pop1 - 2; + } +#endif + else + { +// cJU_JPLEAF4 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Get a new Leaf + PjllRaw = j__udyAllocJLL4(Pop1, Pjpm); + if (PjllRaw == (Pjll_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjll = P_JLL(PjllRaw); + +// Copy Indexes to new Leaf + j__udyCopyWto4((uint8_t *) Pjll, StageA + Start, + Pop1); +#ifdef JUDYL +// Copy to Values to new Leaf + Pjvnew = JL_LEAF4VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif + DBGCODE(JudyCheckSorted(Pjll, Pop1, 4);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(5)) + | + (CIndex & cJU_DCDMASK(5-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF4); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF5_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = StageA[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L5; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade5() + + +// **************************************************************************** +// __ J U D Y C A S C A D E 6 +// +// Cascade from a cJU_JPLEAF6 to one of the following: +// 1. if leaf is in 1 expanse: +// compress it into a JPLEAF5 +// 2. if leaf contains multiple expanses: +// create linear or bitmap branch containing +// each new expanse is either a: +// JPIMMED_5_01 ... JPIMMED_5_03 branch +// JPIMMED_5_01 branch +// JPLEAF5 + +FUNCTION int j__udyCascade6( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint8_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF6_MAXPOP1]; // JPs of new leaves + Word_t StageA [cJU_LEAF6_MAXPOP1]; + uint8_t StageExp [cJU_LEAF6_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF6); + assert((JU_JPDCDPOP0(Pjp) & 0xFFFFFFFFFFFF) == (cJU_LEAF6_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint8_t *) P_JLL(Pjp->jp_Addr); + +// Extract 6 byte index Leaf to Word_t + j__udyCopy6toW(StageA, PLeaf, cJU_LEAF6_MAXPOP1); + +// Get the address of the Leaf and Value area + JUDYLCODE(Pjv = JL_LEAF6VALUEAREA(PLeaf, cJU_LEAF6_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it (compare 1st, last & Index) + + CIndex = StageA[0]; + if (!JU_DIGITATSTATE(CIndex ^ StageA[cJU_LEAF6_MAXPOP1-1], 6)) + { + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Alloc a 5 byte Index Leaf + PjllRaw = j__udyAllocJLL5(cJU_LEAF6_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy Index area into new Leaf + j__udyCopyWto5((uint8_t *) Pjll, StageA, cJU_LEAF6_MAXPOP1); +#ifdef JUDYL +// Copy Value area into new Leaf + Pjvnew = JL_LEAF5VALUEAREA(Pjll, cJU_LEAF6_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAF6_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAF6_MAXPOP1, 5);) + + DcdP0 = JU_JPDCDPOP0(Pjp) | (CIndex & cJU_DCDMASK(5)); + JU_JPSETADT(Pjp, (Word_t)PjllRaw, DcdP0, cJU_JPLEAF5); + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 6 byte index Leaf to 5 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF6_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ StageA[End], 6)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 6); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 6); + + if (Pop1 == 1) // cJU_JPIMMED_5_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(5)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_5_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_5_01); +#endif // JUDYL + } +#ifdef JUDY1 + else if (Pop1 <= cJ1_IMMED5_MAXPOP1) + { +// cJ1_JPIMMED_5_02..3: Judy1 64 + +// Copy to Index to JP as an immediate Leaf + j__udyCopyWto5(PjpJP->jp_1Index, + StageA + Start, Pop1); + +// Set pointer, type, population and Index size + PjpJP->jp_Type = cJ1_JPIMMED_5_02 + Pop1 - 2; + } +#endif + else + { +// cJU_JPLEAF5 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Get a new Leaf + PjllRaw = j__udyAllocJLL5(Pop1, Pjpm); + if (PjllRaw == (Pjll_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjll = P_JLL(PjllRaw); + +// Copy Indexes to new Leaf + j__udyCopyWto5((uint8_t *) Pjll, StageA + Start, + Pop1); + +// Copy to Values to new Leaf +#ifdef JUDYL + Pjvnew = JL_LEAF5VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif + DBGCODE(JudyCheckSorted(Pjll, Pop1, 5);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(6)) + | + (CIndex & cJU_DCDMASK(6-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF5); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF6_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = StageA[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L6; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade6() + + +// **************************************************************************** +// __ J U D Y C A S C A D E 7 +// +// Cascade from a cJU_JPLEAF7 to one of the following: +// 1. if leaf is in 1 expanse: +// compress it into a JPLEAF6 +// 2. if leaf contains multiple expanses: +// create linear or bitmap branch containing +// each new expanse is either a: +// JPIMMED_6_01 ... JPIMMED_6_02 branch +// JPIMMED_6_01 branch +// JPLEAF6 + +FUNCTION int j__udyCascade7( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + uint8_t * PLeaf; // pointer to leaf, explicit type. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAF7_MAXPOP1]; // JPs of new leaves + Word_t StageA [cJU_LEAF7_MAXPOP1]; + uint8_t StageExp [cJU_LEAF7_MAXPOP1]; // Expanses of new leaves + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF7); + assert(JU_JPDCDPOP0(Pjp) == (cJU_LEAF7_MAXPOP1-1)); + +// Get the address of the Leaf + PLeaf = (uint8_t *) P_JLL(Pjp->jp_Addr); + +// Extract 7 byte index Leaf to Word_t + j__udyCopy7toW(StageA, PLeaf, cJU_LEAF7_MAXPOP1); + +// Get the address of the Leaf and Value area + JUDYLCODE(Pjv = JL_LEAF7VALUEAREA(PLeaf, cJU_LEAF7_MAXPOP1);) + +// If Leaf is in 1 expanse -- just compress it (compare 1st, last & Index) + + CIndex = StageA[0]; + if (!JU_DIGITATSTATE(CIndex ^ StageA[cJU_LEAF7_MAXPOP1-1], 7)) + { + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Alloc a 6 byte Index Leaf + PjllRaw = j__udyAllocJLL6(cJU_LEAF7_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy Index area into new Leaf + j__udyCopyWto6((uint8_t *) Pjll, StageA, cJU_LEAF7_MAXPOP1); +#ifdef JUDYL +// Copy Value area into new Leaf + Pjvnew = JL_LEAF6VALUEAREA(Pjll, cJU_LEAF7_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAF7_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAF7_MAXPOP1, 6);) + + DcdP0 = JU_JPDCDPOP0(Pjp) | (CIndex & cJU_DCDMASK(6)); + JU_JPSETADT(Pjp, (Word_t)PjllRaw, DcdP0, cJU_JPLEAF6); + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 7 byte index Leaf to 6 byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAF7_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ StageA[End], 7)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, 7); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, 7); + + if (Pop1 == 1) // cJU_JPIMMED_6_01 + { + Word_t DcdP0; + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(6)) | + CIndex; +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, DcdP0, cJ1_JPIMMED_6_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], DcdP0, + cJL_JPIMMED_6_01); +#endif // JUDYL + } +#ifdef JUDY1 + else if (Pop1 == cJ1_IMMED6_MAXPOP1) + { +// cJ1_JPIMMED_6_02: Judy1 64 + +// Copy to Index to JP as an immediate Leaf + j__udyCopyWto6(PjpJP->jp_1Index, + StageA + Start, 2); + +// Set pointer, type, population and Index size + PjpJP->jp_Type = cJ1_JPIMMED_6_02; + } +#endif + else + { +// cJU_JPLEAF6 + Word_t DcdP0; + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Get a new Leaf + PjllRaw = j__udyAllocJLL6(Pop1, Pjpm); + if (PjllRaw == (Pjll_t)NULL) + FREEALLEXIT(ExpCnt, StageJP, Pjpm); + Pjll = P_JLL(PjllRaw); + +// Copy Indexes to new Leaf + j__udyCopyWto6((uint8_t *) Pjll, StageA + Start, + Pop1); +#ifdef JUDYL +// Copy to Values to new Leaf + Pjvnew = JL_LEAF6VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif + DBGCODE(JudyCheckSorted(Pjll, Pop1, 6);) + + DcdP0 = (JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(7)) + | + (CIndex & cJU_DCDMASK(7-1)) + | + (Pop1 - 1); + + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, DcdP0, + cJU_JPLEAF6); + } + ExpCnt++; +// Done? + if (End == cJU_LEAF7_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = StageA[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L7; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + } + return(1); + +} // j__udyCascade7() + +#endif // JU_64BIT + + +// **************************************************************************** +// __ J U D Y C A S C A D E L +// +// (Compressed) cJU_LEAF3[7], cJ1_JPBRANCH_L. +// +// Cascade from a LEAFW (under Pjp) to one of the following: +// 1. if LEAFW is in 1 expanse: +// create linear branch with a JPLEAF3[7] under it +// 2. LEAFW contains multiple expanses: +// create linear or bitmap branch containing new expanses +// each new expanse is either a: 32 64 +// JPIMMED_3_01 branch Y N +// JPIMMED_7_01 branch N Y +// JPLEAF3 Y N +// JPLEAF7 N Y + +FUNCTION int j__udyCascadeL( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + Pjlw_t Pjlw; // leaf to work on. + Word_t End, Start; // temporaries. + Word_t ExpCnt; // count of expanses of splay. + Word_t CIndex; // current Index word. +JUDYLCODE(Pjv_t Pjv;) // value area of leaf. + +// Temp staging for parts(Leaves) of newly splayed leaf + jp_t StageJP [cJU_LEAFW_MAXPOP1]; + uint8_t StageExp[cJU_LEAFW_MAXPOP1]; + uint8_t SubJPCount[cJU_NUMSUBEXPB]; // JPs in each subexpanse + jbb_t StageJBB; // staged bitmap branch + +// Get the address of the Leaf + Pjlw = P_JLW(Pjp->jp_Addr); + + assert(Pjlw[0] == (cJU_LEAFW_MAXPOP1 - 1)); + +// Get pointer to Value area of old Leaf + JUDYLCODE(Pjv = JL_LEAFWVALUEAREA(Pjlw, cJU_LEAFW_MAXPOP1);) + + Pjlw++; // Now point to Index area + +// If Leaf is in 1 expanse -- first compress it (compare 1st, last & Index): + + CIndex = Pjlw[0]; // also used far below + if (!JU_DIGITATSTATE(CIndex ^ Pjlw[cJU_LEAFW_MAXPOP1 - 1], + cJU_ROOTSTATE)) + { + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. + +// Get the common expanse to all elements in Leaf + StageExp[0] = JU_DIGITATSTATE(CIndex, cJU_ROOTSTATE); + +// Alloc a 3[7] byte Index Leaf +#ifdef JU_64BIT + PjllRaw = j__udyAllocJLL7(cJU_LEAFW_MAXPOP1, Pjpm); + if (PjllRaw == (Pjlb_t)NULL) return(-1); // out of memory + + Pjll = P_JLL(PjllRaw); + +// Copy LEAFW to a cJU_JPLEAF7 + j__udyCopyWto7((uint8_t *) Pjll, Pjlw, cJU_LEAFW_MAXPOP1); +#ifdef JUDYL +// Get the Value area of new Leaf + Pjvnew = JL_LEAF7VALUEAREA(Pjll, cJU_LEAFW_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAFW_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAFW_MAXPOP1, 7);) +#else // 32 Bit + PjllRaw = j__udyAllocJLL3(cJU_LEAFW_MAXPOP1, Pjpm); + if (PjllRaw == (Pjll_t) NULL) return(-1); + + Pjll = P_JLL(PjllRaw); + +// Copy LEAFW to a cJU_JPLEAF3 + j__udyCopyWto3((uint8_t *) Pjll, Pjlw, cJU_LEAFW_MAXPOP1); +#ifdef JUDYL +// Get the Value area of new Leaf + Pjvnew = JL_LEAF3VALUEAREA(Pjll, cJU_LEAFW_MAXPOP1); + JU_COPYMEM(Pjvnew, Pjv, cJU_LEAFW_MAXPOP1); +#endif + DBGCODE(JudyCheckSorted(Pjll, cJU_LEAFW_MAXPOP1, 3);) +#endif // 32 Bit + +// Following not needed because cJU_DCDMASK(3[7]) is == 0 +////// StageJP[0].jp_DcdPopO |= (CIndex & cJU_DCDMASK(3[7])); +#ifdef JU_64BIT + JU_JPSETADT(&(StageJP[0]), (Word_t)PjllRaw, cJU_LEAFW_MAXPOP1-1, + cJU_JPLEAF7); +#else // 32BIT + JU_JPSETADT(&(StageJP[0]), (Word_t)PjllRaw, cJU_LEAFW_MAXPOP1-1, + cJU_JPLEAF3); +#endif // 32BIT +// Create a 1 element Linear branch + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, 1, Pjpm) == -1) + return(-1); + +// Change the type of callers JP + Pjp->jp_Type = cJU_JPBRANCH_L; + + return(1); + } + +// Else in 2+ expanses, splay Leaf into smaller leaves at higher compression + + StageJBB = StageJBBZero; // zero staged bitmap branch + ZEROJP(SubJPCount); + +// Splay the 4[8] byte Index Leaf to 3[7] byte Index Leaves + for (ExpCnt = Start = 0, End = 1; ; End++) + { +// Check if new expanse or last one + if ( (End == cJU_LEAFW_MAXPOP1) + || + (JU_DIGITATSTATE(CIndex ^ Pjlw[End], cJU_ROOTSTATE)) + ) + { +// Build a leaf below the previous expanse + + Pjp_t PjpJP = StageJP + ExpCnt; + Word_t Pop1 = End - Start; + Word_t expanse = JU_DIGITATSTATE(CIndex, cJU_ROOTSTATE); + Word_t subexp = expanse / cJU_BITSPERSUBEXPB; +// +// set the bit that is the current expanse + JU_JBB_BITMAP(&StageJBB, subexp) |= JU_BITPOSMASKB(expanse); +#ifdef SUBEXPCOUNTS + StageJBB.jbb_subPop1[subexp] += Pop1; // pop of subexpanse +#endif +// count number of expanses in each subexpanse + SubJPCount[subexp]++; + +// Save byte expanse of leaf + StageExp[ExpCnt] = JU_DIGITATSTATE(CIndex, + cJU_ROOTSTATE); + + if (Pop1 == 1) // cJU_JPIMMED_3[7]_01 + { +#ifdef JU_64BIT +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, CIndex, cJ1_JPIMMED_7_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], CIndex, + cJL_JPIMMED_7_01); +#endif // JUDYL + +#else // JU_32BIT +#ifdef JUDY1 + JU_JPSETADT(PjpJP, 0, CIndex, cJ1_JPIMMED_3_01); +#else // JUDYL + JU_JPSETADT(PjpJP, Pjv[Start], CIndex, + cJL_JPIMMED_3_01); +#endif // JUDYL +#endif // JU_32BIT + } +#ifdef JUDY1 +#ifdef JU_64BIT + else if (Pop1 <= cJ1_IMMED7_MAXPOP1) +#else + else if (Pop1 <= cJ1_IMMED3_MAXPOP1) +#endif + { +// cJ1_JPIMMED_3_02 : Judy1 32 +// cJ1_JPIMMED_7_02 : Judy1 64 +// Copy to JP as an immediate Leaf +#ifdef JU_64BIT + j__udyCopyWto7(PjpJP->jp_1Index, Pjlw+Start, 2); + PjpJP->jp_Type = cJ1_JPIMMED_7_02; +#else + j__udyCopyWto3(PjpJP->jp_1Index, Pjlw+Start, 2); + PjpJP->jp_Type = cJ1_JPIMMED_3_02; +#endif // 32 Bit + } +#endif // JUDY1 + else // Linear Leaf JPLEAF3[7] + { +// cJU_JPLEAF3[7] + Pjll_t PjllRaw; // pointer to new leaf. + Pjll_t Pjll; + JUDYLCODE(Pjv_t Pjvnew;) // value area of new leaf. +#ifdef JU_64BIT + PjllRaw = j__udyAllocJLL7(Pop1, Pjpm); + if (PjllRaw == (Pjll_t) NULL) return(-1); + Pjll = P_JLL(PjllRaw); + + j__udyCopyWto7((uint8_t *) Pjll, Pjlw + Start, + Pop1); +#ifdef JUDYL + Pjvnew = JL_LEAF7VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif // JUDYL + DBGCODE(JudyCheckSorted(Pjll, Pop1, 7);) +#else // JU_64BIT - 32 Bit + PjllRaw = j__udyAllocJLL3(Pop1, Pjpm); + if (PjllRaw == (Pjll_t) NULL) return(-1); + Pjll = P_JLL(PjllRaw); + + j__udyCopyWto3((uint8_t *) Pjll, Pjlw + Start, + Pop1); +#ifdef JUDYL + Pjvnew = JL_LEAF3VALUEAREA(Pjll, Pop1); + JU_COPYMEM(Pjvnew, Pjv + Start, Pop1); +#endif // JUDYL + DBGCODE(JudyCheckSorted(Pjll, Pop1, 3);) +#endif // 32 Bit + +#ifdef JU_64BIT + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, Pop1 - 1, + cJU_JPLEAF7); +#else // JU_64BIT - 32 Bit + JU_JPSETADT(PjpJP, (Word_t)PjllRaw, Pop1 - 1, + cJU_JPLEAF3); +#endif // 32 Bit + } + ExpCnt++; +// Done? + if (End == cJU_LEAFW_MAXPOP1) break; + +// New Expanse, Start and Count + CIndex = Pjlw[End]; + Start = End; + } + } + +// Now put all the Leaves below a BranchL or BranchB: + if (ExpCnt <= cJU_BRANCHLMAXJPS) // put the Leaves below a BranchL + { + if (j__udyCreateBranchL(Pjp, StageJP, StageExp, ExpCnt, + Pjpm) == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_L; + } + else + { + if (j__udyStageJBBtoJBB(Pjp, &StageJBB, StageJP, SubJPCount, Pjpm) + == -1) FREEALLEXIT(ExpCnt, StageJP, Pjpm); + + Pjp->jp_Type = cJU_JPBRANCH_B; // cJU_LEAFW is out of sequence + } + return(1); + +} // j__udyCascadeL() diff --git a/libnetdata/libjudy/src/JudyL/JudyLCount.c b/libnetdata/libjudy/src/JudyL/JudyLCount.c new file mode 100644 index 000000000..179757f0a --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLCount.c @@ -0,0 +1,1195 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.78 $ $Source: /judy/src/JudyCommon/JudyCount.c $ +// +// Judy*Count() function for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DNOSMARTJBB, -DNOSMARTJBU, and/or -DNOSMARTJLB to build a +// version with cache line optimizations deleted, for testing. +// +// Compile with -DSMARTMETRICS to obtain global variables containing smart +// cache line metrics. Note: Dont turn this on simultaneously for this file +// and JudyByCount.c because they export the same globals. +// +// Judy*Count() returns the "count of Indexes" (inclusive) between the two +// specified limits (Indexes). This code is remarkably fast. It traverses the +// "Judy array" data structure. +// +// This count code is the GENERIC untuned version (minimum code size). It +// might be possible to tuned to a specific architecture to be faster. +// However, in real applications, with a modern machine, it is expected that +// the instruction times will be swamped by cache line fills. +// **************************************************************************** + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + + +// define a phoney that is for sure + +#define cJU_LEAFW cJU_JPIMMED_CAP + +// Avoid duplicate symbols since this file is multi-compiled: + +#ifdef SMARTMETRICS +#ifdef JUDY1 +Word_t jbb_upward = 0; // counts of directions taken: +Word_t jbb_downward = 0; +Word_t jbu_upward = 0; +Word_t jbu_downward = 0; +Word_t jlb_upward = 0; +Word_t jlb_downward = 0; +#else +extern Word_t jbb_upward; +extern Word_t jbb_downward; +extern Word_t jbu_upward; +extern Word_t jbu_downward; +extern Word_t jlb_upward; +extern Word_t jlb_downward; +#endif +#endif + + +// FORWARD DECLARATIONS (prototypes): + +static Word_t j__udy1LCountSM(const Pjp_t Pjp, const Word_t Index, + const Pjpm_t Pjpm); + +// Each of Judy1 and JudyL get their own private (static) version of this +// function: + +static int j__udyCountLeafB1(const Pjll_t Pjll, const Word_t Pop1, + const Word_t Index); + +// These functions are not static because they are exported to Judy*ByCount(): +// +// TBD: Should be made static for performance reasons? And thus duplicated? +// +// Note: There really are two different functions, but for convenience they +// are referred to here with a generic name. + +#ifdef JUDY1 +#define j__udyJPPop1 j__udy1JPPop1 +#else +#define j__udyJPPop1 j__udyLJPPop1 +#endif + +Word_t j__udyJPPop1(const Pjp_t Pjp); + + +// LOCAL ERROR HANDLING: +// +// The Judy*Count() functions are unusual because they return 0 instead of JERR +// for an error. In this source file, define C_JERR for clarity. + +#define C_JERR 0 + + +// **************************************************************************** +// J U D Y 1 C O U N T +// J U D Y L C O U N T +// +// See the manual entry for details. +// +// This code is written recursively, at least at first, because thats much +// simpler; hope its fast enough. + +#ifdef JUDY1 +FUNCTION Word_t Judy1Count +#else +FUNCTION Word_t JudyLCount +#endif + ( + Pcvoid_t PArray, // JRP to first branch/leaf in SM. + Word_t Index1, // starting Index. + Word_t Index2, // ending Index. + PJError_t PJError // optional, for returning error info. + ) +{ + jpm_t fakejpm; // local temporary for small arrays. + Pjpm_t Pjpm; // top JPM or local temporary for error info. + jp_t fakejp; // constructed for calling j__udy1LCountSM(). + Pjp_t Pjp; // JP to pass to j__udy1LCountSM(). + Word_t pop1; // total for the array. + Word_t pop1above1; // indexes at or above Index1, inclusive. + Word_t pop1above2; // indexes at or above Index2, exclusive. + int retcode; // from Judy*First() calls. +JUDYLCODE(PPvoid_t PPvalue); // from JudyLFirst() calls. + + +// CHECK FOR SHORTCUTS: +// +// As documented, return C_JERR if the Judy array is empty or Index1 > Index2. + + if ((PArray == (Pvoid_t) NULL) || (Index1 > Index2)) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NONE); + return(C_JERR); + } + +// If Index1 == Index2, simply check if the specified Index is set; pass +// through the return value from Judy1Test() or JudyLGet() with appropriate +// translations. + + if (Index1 == Index2) + { +#ifdef JUDY1 + retcode = Judy1Test(PArray, Index1, PJError); + + if (retcode == JERRI) return(C_JERR); // pass through error. + + if (retcode == 0) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NONE); + return(C_JERR); + } +#else + PPvalue = JudyLGet(PArray, Index1, PJError); + + if (PPvalue == PPJERR) return(C_JERR); // pass through error. + + if (PPvalue == (PPvoid_t) NULL) // Index is not set. + { + JU_SET_ERRNO(PJError, JU_ERRNO_NONE); + return(C_JERR); + } +#endif + return(1); // single index is set. + } + + +// CHECK JRP TYPE: +// +// Use an if/then for speed rather than a switch, and put the most common cases +// first. +// +// Note: Since even cJU_LEAFW types require counting between two Indexes, +// prepare them here for common code below that calls j__udy1LCountSM(), rather +// than handling them even more specially here. + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + Pjpm = & fakejpm; + Pjp = & fakejp; + Pjp->jp_Addr = (Word_t) Pjlw; + Pjp->jp_Type = cJU_LEAFW; + Pjpm->jpm_Pop0 = Pjlw[0]; // from first word of leaf. + pop1 = Pjpm->jpm_Pop0 + 1; + } + else + { + Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); + pop1 = (Pjpm->jpm_Pop0) + 1; // note: can roll over to 0. + +#if (defined(JUDY1) && (! defined(JU_64BIT))) + if (pop1 == 0) // rare special case of full array: + { + Word_t count = Index2 - Index1 + 1; // can roll over again. + + if (count == 0) + { + JU_SET_ERRNO(PJError, JU_ERRNO_FULL); + return(C_JERR); + } + return(count); + } +#else + assert(pop1); // JudyL or 64-bit cannot create a full array! +#endif + } + + +// COUNT POP1 ABOVE INDEX1, INCLUSIVE: + + assert(pop1); // just to be safe. + + if (Index1 == 0) // shortcut, pop1above1 is entire population: + { + pop1above1 = pop1; + } + else // find first valid Index above Index1, if any: + { +#ifdef JUDY1 + if ((retcode = Judy1First(PArray, & Index1, PJError)) == JERRI) + return(C_JERR); // pass through error. +#else + if ((PPvalue = JudyLFirst(PArray, & Index1, PJError)) == PPJERR) + return(C_JERR); // pass through error. + + retcode = (PPvalue != (PPvoid_t) NULL); // found a next Index. +#endif + +// If theres no Index at or above Index1, just return C_JERR (early exit): + + if (retcode == 0) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NONE); + return(C_JERR); + } + +// If a first/next Index was found, call the counting motor starting with that +// known valid Index, meaning the return should be positive, not C_JERR except +// in case of a real error: + + if ((pop1above1 = j__udy1LCountSM(Pjp, Index1, Pjpm)) == C_JERR) + { + JU_COPY_ERRNO(PJError, Pjpm); // pass through error. + return(C_JERR); + } + } + + +// COUNT POP1 ABOVE INDEX2, EXCLUSIVE, AND RETURN THE DIFFERENCE: +// +// In principle, calculate the ordinal of each Index and take the difference, +// with caution about off-by-one errors due to the specified Indexes being set +// or unset. In practice: +// +// - The ordinals computed here are inverse ordinals, that is, the populations +// ABOVE the specified Indexes (Index1 inclusive, Index2 exclusive), so +// subtract pop1above2 from pop1above1, rather than vice-versa. +// +// - Index1s result already includes a count for Index1 and/or Index2 if +// either is set, so calculate pop1above2 exclusive of Index2. +// +// TBD: If Index1 and Index2 fall in the same expanse in the top-state +// branch(es), would it be faster to walk the SM only once, to their divergence +// point, before calling j__udy1LCountSM() or equivalent? Possibly a non-issue +// if a top-state pop1 becomes stored with each Judy1 array. Also, consider +// whether the first call of j__udy1LCountSM() fills the cache, for common tree +// branches, for the second call. +// +// As for pop1above1, look for shortcuts for special cases when pop1above2 is +// zero. Otherwise call the counting "motor". + + assert(pop1above1); // just to be safe. + + if (Index2++ == cJU_ALLONES) return(pop1above1); // Index2 at limit. + +#ifdef JUDY1 + if ((retcode = Judy1First(PArray, & Index2, PJError)) == JERRI) + return(C_JERR); +#else + if ((PPvalue = JudyLFirst(PArray, & Index2, PJError)) == PPJERR) + return(C_JERR); + + retcode = (PPvalue != (PPvoid_t) NULL); // found a next Index. +#endif + if (retcode == 0) return(pop1above1); // no Index above Index2. + +// Just as for Index1, j__udy1LCountSM() cannot return 0 (locally == C_JERR) +// except in case of a real error: + + if ((pop1above2 = j__udy1LCountSM(Pjp, Index2, Pjpm)) == C_JERR) + { + JU_COPY_ERRNO(PJError, Pjpm); // pass through error. + return(C_JERR); + } + + if (pop1above1 == pop1above2) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NONE); + return(C_JERR); + } + + return(pop1above1 - pop1above2); + +} // Judy1Count() / JudyLCount() + + +// **************************************************************************** +// __ J U D Y 1 L C O U N T S M +// +// Given a pointer to a JP (with invalid jp_DcdPopO at cJU_ROOTSTATE), a known +// valid Index, and a Pjpm for returning error info, recursively visit a Judy +// array state machine (SM) and return the count of Indexes, including Index, +// through the end of the Judy array at this state or below. In case of error +// or a count of 0 (should never happen), return C_JERR with appropriate +// JU_ERRNO in the Pjpm. +// +// Note: This function is not told the current state because its encoded in +// the JP Type. +// +// Method: To minimize cache line fills, while studying each branch, if Index +// resides above the midpoint of the branch (which often consists of multiple +// cache lines), ADD the populations at or above Index; otherwise, SUBTRACT +// from the population of the WHOLE branch (available from the JP) the +// populations at or above Index. This is especially tricky for bitmap +// branches. +// +// Note: Unlike, say, the Ins and Del walk routines, this function returns the +// same type of returns as Judy*Count(), so it can use *_SET_ERRNO*() macros +// the same way. + +FUNCTION static Word_t j__udy1LCountSM( +const Pjp_t Pjp, // top of Judy (sub)SM. +const Word_t Index, // count at or above this Index. +const Pjpm_t Pjpm) // for returning error info. +{ + Pjbl_t Pjbl; // Pjp->jp_Addr masked and cast to types: + Pjbb_t Pjbb; + Pjbu_t Pjbu; + Pjll_t Pjll; // a Judy lower-level linear leaf. + + Word_t digit; // next digit to decode from Index. + long jpnum; // JP number in a branch (base 0). + int offset; // index ordinal within a leaf, base 0. + Word_t pop1; // total population of an expanse. + Word_t pop1above; // to return. + +// Common code to check Decode bits in a JP against the equivalent portion of +// Index; XOR together, then mask bits of interest; must be all 0: +// +// Note: Why does this code only assert() compliance rather than actively +// checking for outliers? Its because Index is supposed to be valid, hence +// always match any Dcd bits traversed. +// +// Note: This assertion turns out to be always true for cState = 3 on 32-bit +// and 7 on 64-bit, but its harmless, probably removed by the compiler. + +#define CHECKDCD(Pjp,cState) \ + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cState)) + +// Common code to prepare to handle a root-level or lower-level branch: +// Extract a state-dependent digit from Index in a "constant" way, obtain the +// total population for the branch in a state-dependent way, and then branch to +// common code for multiple cases: +// +// For root-level branches, the state is always cJU_ROOTSTATE, and the +// population is received in Pjpm->jpm_Pop0. +// +// Note: The total population is only needed in cases where the common code +// "counts up" instead of down to minimize cache line fills. However, its +// available cheaply, and its better to do it with a constant shift (constant +// state value) instead of a variable shift later "when needed". + +#define PREPB_ROOT(Pjp,Next) \ + digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); \ + pop1 = (Pjpm->jpm_Pop0) + 1; \ + goto Next + +#define PREPB(Pjp,cState,Next) \ + digit = JU_DIGITATSTATE(Index, cState); \ + pop1 = JU_JPBRANCH_POP0(Pjp, (cState)) + 1; \ + goto Next + + +// SWITCH ON JP TYPE: +// +// WARNING: For run-time efficiency the following cases replicate code with +// varying constants, rather than using common code with variable values! + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// ROOT-STATE LEAF that starts with a Pop0 word; just count within the leaf: + + case cJU_LEAFW: + { + Pjlw_t Pjlw = P_JLW(Pjp->jp_Addr); // first word of leaf. + + assert((Pjpm->jpm_Pop0) + 1 == Pjlw[0] + 1); // sent correctly. + offset = j__udySearchLeafW(Pjlw + 1, Pjpm->jpm_Pop0 + 1, Index); + assert(offset >= 0); // Index must exist. + assert(offset < (Pjpm->jpm_Pop0) + 1); // Index be in range. + return((Pjpm->jpm_Pop0) + 1 - offset); // INCLUSIVE of Index. + } + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH; count populations in JPs in the JBL ABOVE the next digit in +// Index, and recurse for the next digit in Index: +// +// Note: There are no null JPs in a JBL; watch out for pop1 == 0. +// +// Note: A JBL should always fit in one cache line => no need to count up +// versus down to save cache line fills. (PREPB() sets pop1 for no reason.) + + case cJU_JPBRANCH_L2: CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchL); + case cJU_JPBRANCH_L3: CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchL); + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchL); + case cJU_JPBRANCH_L5: CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchL); + case cJU_JPBRANCH_L6: CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchL); + case cJU_JPBRANCH_L7: CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchL); +#endif + case cJU_JPBRANCH_L: PREPB_ROOT(Pjp, BranchL); + +// Common code (state-independent) for all cases of linear branches: + +BranchL: + + Pjbl = P_JBL(Pjp->jp_Addr); + jpnum = Pjbl->jbl_NumJPs; // above last JP. + pop1above = 0; + + while (digit < (Pjbl->jbl_Expanse[--jpnum])) // still ABOVE digit. + { + if ((pop1 = j__udyJPPop1((Pjbl->jbl_jp) + jpnum)) == cJU_ALLONES) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(C_JERR); + } + + pop1above += pop1; + assert(jpnum > 0); // should find digit. + } + + assert(digit == (Pjbl->jbl_Expanse[jpnum])); // should find digit. + + pop1 = j__udy1LCountSM((Pjbl->jbl_jp) + jpnum, Index, Pjpm); + if (pop1 == C_JERR) return(C_JERR); // pass error up. + + assert(pop1above + pop1); + return(pop1above + pop1); + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH; count populations in JPs in the JBB ABOVE the next digit in +// Index, and recurse for the next digit in Index: +// +// Note: There are no null JPs in a JBB; watch out for pop1 == 0. + + case cJU_JPBRANCH_B2: CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchB); + case cJU_JPBRANCH_B3: CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchB); + case cJU_JPBRANCH_B5: CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchB); + case cJU_JPBRANCH_B6: CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchB); + case cJU_JPBRANCH_B7: CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchB); +#endif + case cJU_JPBRANCH_B: PREPB_ROOT(Pjp, BranchB); + +// Common code (state-independent) for all cases of bitmap branches: + +BranchB: + { + long subexp; // for stepping through layer 1 (subexpanses). + long findsub; // subexpanse containing Index (digit). + Word_t findbit; // bit representing Index (digit). + Word_t lowermask; // bits for indexes at or below Index. + Word_t jpcount; // JPs in a subexpanse. + Word_t clbelow; // cache lines below digits cache line. + Word_t clabove; // cache lines above digits cache line. + + Pjbb = P_JBB(Pjp->jp_Addr); + findsub = digit / cJU_BITSPERSUBEXPB; + findbit = digit % cJU_BITSPERSUBEXPB; + lowermask = JU_MASKLOWERINC(JU_BITPOSMASKB(findbit)); + clbelow = clabove = 0; // initial/default => always downward. + + assert(JU_BITMAPTESTB(Pjbb, digit)); // digit must have a JP. + assert(findsub < cJU_NUMSUBEXPB); // falls in expected range. + +// Shorthand for one subexpanse in a bitmap and for one JP in a bitmap branch: +// +// Note: BMPJP0 exists separately to support assertions. + +#define BMPJP0(Subexp) (P_JP(JU_JBB_PJP(Pjbb, Subexp))) +#define BMPJP(Subexp,JPnum) (BMPJP0(Subexp) + (JPnum)) + +#ifndef NOSMARTJBB // enable to turn off smart code for comparison purposes. + +// FIGURE OUT WHICH DIRECTION CAUSES FEWER CACHE LINE FILLS; adding the pop1s +// in JPs above Indexs JP, or subtracting the pop1s in JPs below Indexs JP. +// +// This is tricky because, while each set bit in the bitmap represents a JP, +// the JPs are scattered over cJU_NUMSUBEXPB subexpanses, each of which can +// contain JPs packed into multiple cache lines, and this code must visit every +// JP either BELOW or ABOVE the JP for Index. +// +// Number of cache lines required to hold a linear list of the given number of +// JPs, assuming the first JP is at the start of a cache line or the JPs in +// jpcount fit wholly within a single cache line, which is ensured by +// JudyMalloc(): + +#define CLPERJPS(jpcount) \ + ((((jpcount) * cJU_WORDSPERJP) + cJU_WORDSPERCL - 1) / cJU_WORDSPERCL) + +// Count cache lines below/above for each subexpanse: + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + +// When at the subexpanse containing Index (digit), add cache lines +// below/above appropriately, excluding the cache line containing the JP for +// Index itself: + + if (subexp < findsub) clbelow += CLPERJPS(jpcount); + else if (subexp > findsub) clabove += CLPERJPS(jpcount); + else // (subexp == findsub) + { + Word_t clfind; // cache line containing Index (digit). + + clfind = CLPERJPS(j__udyCountBitsB( + JU_JBB_BITMAP(Pjbb, subexp) & lowermask)); + + assert(clfind > 0); // digit itself should have 1 CL. + clbelow += clfind - 1; + clabove += CLPERJPS(jpcount) - clfind; + } + } +#endif // ! NOSMARTJBB + +// Note: Its impossible to get through the following "if" without setting +// jpnum -- see some of the assertions below -- but gcc -Wall doesnt know +// this, so preset jpnum to make it happy: + + jpnum = 0; + + +// COUNT POPULATION FOR A BITMAP BRANCH, in whichever direction should result +// in fewer cache line fills: +// +// Note: If the remainder of Index is zero, pop1above is the pop1 of the +// entire expanse and theres no point in recursing to lower levels; but this +// should be so rare that its not worth checking for; +// Judy1Count()/JudyLCount() never even calls the motor for Index == 0 (all +// bytes). + + +// COUNT UPWARD, subtracting each "below or at" JPs pop1 from the whole +// expanses pop1: +// +// Note: If this causes clbelow + 1 cache line fills including JPs cache +// line, thats OK; at worst this is the same as clabove. + + if (clbelow < clabove) + { +#ifdef SMARTMETRICS + ++jbb_upward; +#endif + pop1above = pop1; // subtract JPs at/below Index. + +// Count JPs for which to accrue pop1s in this subexpanse: +// +// TBD: If JU_JBB_BITMAP is cJU_FULLBITMAPB, dont bother counting. + + for (subexp = 0; subexp <= findsub; ++subexp) + { + jpcount = j__udyCountBitsB((subexp < findsub) ? + JU_JBB_BITMAP(Pjbb, subexp) : + JU_JBB_BITMAP(Pjbb, subexp) & lowermask); + + // should always find findbit: + assert((subexp < findsub) || jpcount); + +// Subtract pop1s from JPs BELOW OR AT Index (digit): +// +// Note: The pop1 for Indexs JP itself is partially added back later at a +// lower state. +// +// Note: An empty subexpanse (jpcount == 0) is handled "for free". +// +// Note: Must be null JP subexp pointer in empty subexpanse and non-empty in +// non-empty subexpanse: + + assert( jpcount || (BMPJP0(subexp) == (Pjp_t) NULL)); + assert((! jpcount) || (BMPJP0(subexp) != (Pjp_t) NULL)); + + for (jpnum = 0; jpnum < jpcount; ++jpnum) + { + if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum))) + == cJU_ALLONES) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(C_JERR); + } + + pop1above -= pop1; + } + + jpnum = jpcount - 1; // make correct for digit. + } + } + +// COUNT DOWNWARD, adding each "above" JPs pop1: + + else + { + long jpcountbf; // below findbit, inclusive. +#ifdef SMARTMETRICS + ++jbb_downward; +#endif + pop1above = 0; // add JPs above Index. + jpcountbf = 0; // until subexp == findsub. + +// Count JPs for which to accrue pop1s in this subexpanse: +// +// This is more complicated than counting upward because the scan of digits +// subexpanse must count ALL JPs, to know where to START counting down, and +// ALSO note the offset of digits JP to know where to STOP counting down. + + for (subexp = cJU_NUMSUBEXPB - 1; subexp >= findsub; --subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + + // should always find findbit: + assert((subexp > findsub) || jpcount); + + if (! jpcount) continue; // empty subexpanse, save time. + +// Count JPs below digit, inclusive: + + if (subexp == findsub) + { + jpcountbf = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp) + & lowermask); + } + + // should always find findbit: + assert((subexp > findsub) || jpcountbf); + assert(jpcount >= jpcountbf); // proper relationship. + +// Add pop1s from JPs ABOVE Index (digit): + + // no null JP subexp pointers: + assert(BMPJP0(subexp) != (Pjp_t) NULL); + + for (jpnum = jpcount - 1; jpnum >= jpcountbf; --jpnum) + { + if ((pop1 = j__udyJPPop1(BMPJP(subexp, jpnum))) + == cJU_ALLONES) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(C_JERR); + } + + pop1above += pop1; + } + // jpnum is now correct for digit. + } + } // else. + +// Return the net population ABOVE the digits JP at this state (in this JBB) +// plus the population AT OR ABOVE Index in the SM under the digits JP: + + pop1 = j__udy1LCountSM(BMPJP(findsub, jpnum), Index, Pjpm); + if (pop1 == C_JERR) return(C_JERR); // pass error up. + + assert(pop1above + pop1); + return(pop1above + pop1); + + } // case. + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH; count populations in JPs in the JBU ABOVE the next +// digit in Index, and recurse for the next digit in Index: +// +// Note: If the remainder of Index is zero, pop1above is the pop1 of the +// entire expanse and theres no point in recursing to lower levels; but this +// should be so rare that its not worth checking for; +// Judy1Count()/JudyLCount() never even calls the motor for Index == 0 (all +// bytes). + + case cJU_JPBRANCH_U2: CHECKDCD(Pjp, 2); PREPB(Pjp, 2, BranchU); + case cJU_JPBRANCH_U3: CHECKDCD(Pjp, 3); PREPB(Pjp, 3, BranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: CHECKDCD(Pjp, 4); PREPB(Pjp, 4, BranchU); + case cJU_JPBRANCH_U5: CHECKDCD(Pjp, 5); PREPB(Pjp, 5, BranchU); + case cJU_JPBRANCH_U6: CHECKDCD(Pjp, 6); PREPB(Pjp, 6, BranchU); + case cJU_JPBRANCH_U7: CHECKDCD(Pjp, 7); PREPB(Pjp, 7, BranchU); +#endif + case cJU_JPBRANCH_U: PREPB_ROOT(Pjp, BranchU); + +// Common code (state-independent) for all cases of uncompressed branches: + +BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + +#ifndef NOSMARTJBU // enable to turn off smart code for comparison purposes. + +// FIGURE OUT WHICH WAY CAUSES FEWER CACHE LINE FILLS; adding the JPs above +// Indexs JP, or subtracting the JPs below Indexs JP. +// +// COUNT UPWARD, subtracting the pop1 of each JP BELOW OR AT Index, from the +// whole expanses pop1: + + if (digit < (cJU_BRANCHUNUMJPS / 2)) + { + pop1above = pop1; // subtract JPs below Index. +#ifdef SMARTMETRICS + ++jbu_upward; +#endif + for (jpnum = 0; jpnum <= digit; ++jpnum) + { + if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX) + continue; // shortcut, save a function call. + + if ((pop1 = j__udyJPPop1(Pjbu->jbu_jp + jpnum)) + == cJU_ALLONES) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(C_JERR); + } + + pop1above -= pop1; + } + } + +// COUNT DOWNWARD, simply adding the pop1 of each JP ABOVE Index: + + else +#endif // NOSMARTJBU + { + assert(digit < cJU_BRANCHUNUMJPS); +#ifdef SMARTMETRICS + ++jbu_downward; +#endif + pop1above = 0; // add JPs above Index. + + for (jpnum = cJU_BRANCHUNUMJPS - 1; jpnum > digit; --jpnum) + { + if ((Pjbu->jbu_jp[jpnum].jp_Type) <= cJU_JPNULLMAX) + continue; // shortcut, save a function call. + + if ((pop1 = j__udyJPPop1(Pjbu->jbu_jp + jpnum)) + == cJU_ALLONES) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(C_JERR); + } + + pop1above += pop1; + } + } + + if ((pop1 = j__udy1LCountSM(Pjbu->jbu_jp + digit, Index, Pjpm)) + == C_JERR) return(C_JERR); // pass error up. + + assert(pop1above + pop1); + return(pop1above + pop1); + + +// ---------------------------------------------------------------------------- +// LEAF COUNT MACROS: +// +// LEAF*ABOVE() are common code for different JP types (linear leaves, bitmap +// leaves, and immediates) and different leaf Index Sizes, which result in +// calling different leaf search functions. Linear leaves get the leaf address +// from jp_Addr and the Population from jp_DcdPopO, while immediates use Pjp +// itself as the leaf address and get Population from jp_Type. + +#define LEAFLABOVE(Func) \ + Pjll = P_JLL(Pjp->jp_Addr); \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + LEAFABOVE(Func, Pjll, pop1) + +#define LEAFB1ABOVE(Func) LEAFLABOVE(Func) // different Func, otherwise same. + +#ifdef JUDY1 +#define IMMABOVE(Func,Pop1) \ + Pjll = (Pjll_t) Pjp; \ + LEAFABOVE(Func, Pjll, Pop1) +#else +// Note: For JudyL immediates with >= 2 Indexes, the index bytes are in a +// different place than for Judy1: + +#define IMMABOVE(Func,Pop1) \ + LEAFABOVE(Func, (Pjll_t) (Pjp->jp_LIndex), Pop1) +#endif + +// For all leaf types, the population AT OR ABOVE is the total pop1 less the +// offset of Index; and Index should always be found: + +#define LEAFABOVE(Func,Pjll,Pop1) \ + offset = Func(Pjll, Pop1, Index); \ + assert(offset >= 0); \ + assert(offset < (Pop1)); \ + return((Pop1) - offset) + +// IMMABOVE_01 handles the special case of an immediate JP with 1 index, which +// the search functions arent used for anyway: +// +// The target Index should be the one in this Immediate, in which case the +// count above (inclusive) is always 1. + +#define IMMABOVE_01 \ + assert((JU_JPDCDPOP0(Pjp)) == JU_TRIMTODCDSIZE(Index)); \ + return(1) + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF; search the leaf for Index; size is computed from jp_Type: + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: LEAFLABOVE(j__udySearchLeaf1); +#endif + case cJU_JPLEAF2: LEAFLABOVE(j__udySearchLeaf2); + case cJU_JPLEAF3: LEAFLABOVE(j__udySearchLeaf3); + +#ifdef JU_64BIT + case cJU_JPLEAF4: LEAFLABOVE(j__udySearchLeaf4); + case cJU_JPLEAF5: LEAFLABOVE(j__udySearchLeaf5); + case cJU_JPLEAF6: LEAFLABOVE(j__udySearchLeaf6); + case cJU_JPLEAF7: LEAFLABOVE(j__udySearchLeaf7); +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF; search the leaf for Index: +// +// Since the bitmap describes Indexes digitally rather than linearly, this is +// not really a search, but just a count. + + case cJU_JPLEAF_B1: LEAFB1ABOVE(j__udyCountLeafB1); + + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// Return the count of Indexes AT OR ABOVE Index, which is the total population +// of the expanse (a constant) less the value of the undecoded digit remaining +// in Index (its base-0 offset in the expanse), which yields an inclusive count +// above. +// +// TBD: This only supports a 1-byte full expanse. Should this extract a +// stored value for pop0 and possibly more LSBs of Index, to handle larger full +// expanses? + + case cJ1_JPFULLPOPU1: + return(cJU_JPFULLPOPU1_POP0 + 1 - JU_DIGITATSTATE(Index, 1)); +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: + + case cJU_JPIMMED_1_01: IMMABOVE_01; + case cJU_JPIMMED_2_01: IMMABOVE_01; + case cJU_JPIMMED_3_01: IMMABOVE_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: IMMABOVE_01; + case cJU_JPIMMED_5_01: IMMABOVE_01; + case cJU_JPIMMED_6_01: IMMABOVE_01; + case cJU_JPIMMED_7_01: IMMABOVE_01; +#endif + + case cJU_JPIMMED_1_02: IMMABOVE(j__udySearchLeaf1, 2); + case cJU_JPIMMED_1_03: IMMABOVE(j__udySearchLeaf1, 3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: IMMABOVE(j__udySearchLeaf1, 4); + case cJU_JPIMMED_1_05: IMMABOVE(j__udySearchLeaf1, 5); + case cJU_JPIMMED_1_06: IMMABOVE(j__udySearchLeaf1, 6); + case cJU_JPIMMED_1_07: IMMABOVE(j__udySearchLeaf1, 7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: IMMABOVE(j__udySearchLeaf1, 8); + case cJ1_JPIMMED_1_09: IMMABOVE(j__udySearchLeaf1, 9); + case cJ1_JPIMMED_1_10: IMMABOVE(j__udySearchLeaf1, 10); + case cJ1_JPIMMED_1_11: IMMABOVE(j__udySearchLeaf1, 11); + case cJ1_JPIMMED_1_12: IMMABOVE(j__udySearchLeaf1, 12); + case cJ1_JPIMMED_1_13: IMMABOVE(j__udySearchLeaf1, 13); + case cJ1_JPIMMED_1_14: IMMABOVE(j__udySearchLeaf1, 14); + case cJ1_JPIMMED_1_15: IMMABOVE(j__udySearchLeaf1, 15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: IMMABOVE(j__udySearchLeaf2, 2); + case cJU_JPIMMED_2_03: IMMABOVE(j__udySearchLeaf2, 3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: IMMABOVE(j__udySearchLeaf2, 4); + case cJ1_JPIMMED_2_05: IMMABOVE(j__udySearchLeaf2, 5); + case cJ1_JPIMMED_2_06: IMMABOVE(j__udySearchLeaf2, 6); + case cJ1_JPIMMED_2_07: IMMABOVE(j__udySearchLeaf2, 7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: IMMABOVE(j__udySearchLeaf3, 2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: IMMABOVE(j__udySearchLeaf3, 3); + case cJ1_JPIMMED_3_04: IMMABOVE(j__udySearchLeaf3, 4); + case cJ1_JPIMMED_3_05: IMMABOVE(j__udySearchLeaf3, 5); + + case cJ1_JPIMMED_4_02: IMMABOVE(j__udySearchLeaf4, 2); + case cJ1_JPIMMED_4_03: IMMABOVE(j__udySearchLeaf4, 3); + + case cJ1_JPIMMED_5_02: IMMABOVE(j__udySearchLeaf5, 2); + case cJ1_JPIMMED_5_03: IMMABOVE(j__udySearchLeaf5, 3); + + case cJ1_JPIMMED_6_02: IMMABOVE(j__udySearchLeaf6, 2); + + case cJ1_JPIMMED_7_02: IMMABOVE(j__udySearchLeaf7, 2); +#endif + + +// ---------------------------------------------------------------------------- +// OTHER CASES: + + default: JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); return(C_JERR); + + } // switch on JP type + + /*NOTREACHED*/ + +} // j__udy1LCountSM() + + +// **************************************************************************** +// J U D Y C O U N T L E A F B 1 +// +// This is a private analog of the j__udySearchLeaf*() functions for counting +// in bitmap 1-byte leaves. Since a bitmap leaf describes Indexes digitally +// rather than linearly, this is not really a search, but just a count of the +// valid Indexes == set bits below or including Index, which should be valid. +// Return the "offset" (really the ordinal), 0 .. Pop1 - 1, of Index in Pjll; +// if Indexs bit is not set (which should never happen, so this is DEBUG-mode +// only), return the 1s-complement equivalent (== negative offset minus 1). +// +// Note: The source code for this function looks identical for both Judy1 and +// JudyL, but the JU_JLB_BITMAP macro varies. +// +// Note: For simpler calling, the first arg is of type Pjll_t but then cast to +// Pjlb_t. + +FUNCTION static int j__udyCountLeafB1( +const Pjll_t Pjll, // bitmap leaf, as Pjll_t for consistency. +const Word_t Pop1, // Population of whole leaf. +const Word_t Index) // to which to count. +{ + Pjlb_t Pjlb = (Pjlb_t) Pjll; // to proper type. + Word_t digit = Index & cJU_MASKATSTATE(1); + Word_t findsub = digit / cJU_BITSPERSUBEXPL; + Word_t findbit = digit % cJU_BITSPERSUBEXPL; + int count; // in leaf through Index. + long subexp; // for stepping through subexpanses. + + +// COUNT UPWARD: +// +// The entire bitmap should fit in one cache line, but still try to save some +// CPU time by counting the fewest possible number of subexpanses from the +// bitmap. + +#ifndef NOSMARTJLB // enable to turn off smart code for comparison purposes. + + if (findsub < (cJU_NUMSUBEXPL / 2)) + { +#ifdef SMARTMETRICS + ++jlb_upward; +#endif + count = 0; + + for (subexp = 0; subexp < findsub; ++subexp) + { + count += ((JU_JLB_BITMAP(Pjlb, subexp) == cJU_FULLBITMAPL) ? + cJU_BITSPERSUBEXPL : + j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp))); + } + +// This count includes findbit, which should be set, resulting in a base-1 +// offset: + + count += j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, findsub) + & JU_MASKLOWERINC(JU_BITPOSMASKL(findbit))); + + DBGCODE(if (! JU_BITMAPTESTL(Pjlb, digit)) return(~count);) + assert(count >= 1); + return(count - 1); // convert to base-0 offset. + } +#endif // NOSMARTJLB + + +// COUNT DOWNWARD: +// +// Count the valid Indexes above or at Index, and subtract from Pop1. + +#ifdef SMARTMETRICS + ++jlb_downward; +#endif + count = Pop1; // base-1 for now. + + for (subexp = cJU_NUMSUBEXPL - 1; subexp > findsub; --subexp) + { + count -= ((JU_JLB_BITMAP(Pjlb, subexp) == cJU_FULLBITMAPL) ? + cJU_BITSPERSUBEXPL : + j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp))); + } + +// This count includes findbit, which should be set, resulting in a base-0 +// offset: + + count -= j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, findsub) + & JU_MASKHIGHERINC(JU_BITPOSMASKL(findbit))); + + DBGCODE(if (! JU_BITMAPTESTL(Pjlb, digit)) return(~count);) + assert(count >= 0); // should find Index itself. + return(count); // is already a base-0 offset. + +} // j__udyCountLeafB1() + + +// **************************************************************************** +// J U D Y J P P O P 1 +// +// This function takes any type of JP other than a root-level JP (cJU_LEAFW* or +// cJU_JPBRANCH* with no number suffix) and extracts the Pop1 from it. In some +// sense this is a wrapper around the JU_JP*_POP0 macros. Why write it as a +// function instead of a complex macro containing a trinary? (See version +// Judy1.h version 4.17.) We think its cheaper to call a function containing +// a switch statement with "constant" cases than to do the variable +// calculations in a trinary. +// +// For invalid JP Types return cJU_ALLONES. Note that this is an impossibly +// high Pop1 for any JP below a top level branch. + +FUNCTION Word_t j__udyJPPop1( +const Pjp_t Pjp) // JP to count. +{ + switch (JU_JPTYPE(Pjp)) + { +#ifdef notdef // caller should shortcut and not even call with these: + + case cJU_JPNULL1: + case cJU_JPNULL2: + case cJU_JPNULL3: return(0); +#ifdef JU_64BIT + case cJU_JPNULL4: + case cJU_JPNULL5: + case cJU_JPNULL6: + case cJU_JPNULL7: return(0); +#endif +#endif // notdef + + case cJU_JPBRANCH_L2: + case cJU_JPBRANCH_B2: + case cJU_JPBRANCH_U2: return(JU_JPBRANCH_POP0(Pjp,2) + 1); + + case cJU_JPBRANCH_L3: + case cJU_JPBRANCH_B3: + case cJU_JPBRANCH_U3: return(JU_JPBRANCH_POP0(Pjp,3) + 1); + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + case cJU_JPBRANCH_B4: + case cJU_JPBRANCH_U4: return(JU_JPBRANCH_POP0(Pjp,4) + 1); + + case cJU_JPBRANCH_L5: + case cJU_JPBRANCH_B5: + case cJU_JPBRANCH_U5: return(JU_JPBRANCH_POP0(Pjp,5) + 1); + + case cJU_JPBRANCH_L6: + case cJU_JPBRANCH_B6: + case cJU_JPBRANCH_U6: return(JU_JPBRANCH_POP0(Pjp,6) + 1); + + case cJU_JPBRANCH_L7: + case cJU_JPBRANCH_B7: + case cJU_JPBRANCH_U7: return(JU_JPBRANCH_POP0(Pjp,7) + 1); +#endif + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: +#endif + case cJU_JPLEAF2: + case cJU_JPLEAF3: +#ifdef JU_64BIT + case cJU_JPLEAF4: + case cJU_JPLEAF5: + case cJU_JPLEAF6: + case cJU_JPLEAF7: +#endif + case cJU_JPLEAF_B1: return(JU_JPLEAF_POP0(Pjp) + 1); + +#ifdef JUDY1 + case cJ1_JPFULLPOPU1: return(cJU_JPFULLPOPU1_POP0 + 1); +#endif + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: return(1); +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: return(1); +#endif + + case cJU_JPIMMED_1_02: return(2); + case cJU_JPIMMED_1_03: return(3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: return(4); + case cJU_JPIMMED_1_05: return(5); + case cJU_JPIMMED_1_06: return(6); + case cJU_JPIMMED_1_07: return(7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: return(8); + case cJ1_JPIMMED_1_09: return(9); + case cJ1_JPIMMED_1_10: return(10); + case cJ1_JPIMMED_1_11: return(11); + case cJ1_JPIMMED_1_12: return(12); + case cJ1_JPIMMED_1_13: return(13); + case cJ1_JPIMMED_1_14: return(14); + case cJ1_JPIMMED_1_15: return(15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: return(2); + case cJU_JPIMMED_2_03: return(3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: return(4); + case cJ1_JPIMMED_2_05: return(5); + case cJ1_JPIMMED_2_06: return(6); + case cJ1_JPIMMED_2_07: return(7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: return(2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: return(3); + case cJ1_JPIMMED_3_04: return(4); + case cJ1_JPIMMED_3_05: return(5); + + case cJ1_JPIMMED_4_02: return(2); + case cJ1_JPIMMED_4_03: return(3); + + case cJ1_JPIMMED_5_02: return(2); + case cJ1_JPIMMED_5_03: return(3); + + case cJ1_JPIMMED_6_02: return(2); + + case cJ1_JPIMMED_7_02: return(2); +#endif + + default: return(cJU_ALLONES); + } + + /*NOTREACHED*/ + +} // j__udyJPPop1() diff --git a/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c b/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c new file mode 100644 index 000000000..ffe6b3bde --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c @@ -0,0 +1,314 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.26 $ $Source: /judy/src/JudyCommon/JudyCreateBranch.c $ + +// Branch creation functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + + +// **************************************************************************** +// J U D Y C R E A T E B R A N C H L +// +// Build a BranchL from an array of JPs and associated 1 byte digits +// (expanses). Return with Pjp pointing to the BranchL. Caller must +// deallocate passed arrays, if necessary. +// +// We have no idea what kind of BranchL it is, so caller must set the jp_Type. +// +// Return -1 if error (details in Pjpm), otherwise return 1. + +FUNCTION int j__udyCreateBranchL( + Pjp_t Pjp, // Build JPs from this place + Pjp_t PJPs, // Array of JPs to put into Bitmap branch + uint8_t Exp[], // Array of expanses to put into bitmap + Word_t ExpCnt, // Number of above JPs and Expanses + Pvoid_t Pjpm) +{ + Pjbl_t PjblRaw; // pointer to linear branch. + Pjbl_t Pjbl; + + assert(ExpCnt <= cJU_BRANCHLMAXJPS); + + PjblRaw = j__udyAllocJBL(Pjpm); + if (PjblRaw == (Pjbl_t) NULL) return(-1); + Pjbl = P_JBL(PjblRaw); + +// Build a Linear Branch + Pjbl->jbl_NumJPs = ExpCnt; + +// Copy from the Linear branch from splayed leaves + JU_COPYMEM(Pjbl->jbl_Expanse, Exp, ExpCnt); + JU_COPYMEM(Pjbl->jbl_jp, PJPs, ExpCnt); + +// Pass back new pointer to the Linear branch in JP + Pjp->jp_Addr = (Word_t) PjblRaw; + + return(1); + +} // j__udyCreateBranchL() + + +// **************************************************************************** +// J U D Y C R E A T E B R A N C H B +// +// Build a BranchB from an array of JPs and associated 1 byte digits +// (expanses). Return with Pjp pointing to the BranchB. Caller must +// deallocate passed arrays, if necessary. +// +// We have no idea what kind of BranchB it is, so caller must set the jp_Type. +// +// Return -1 if error (details in Pjpm), otherwise return 1. + +FUNCTION int j__udyCreateBranchB( + Pjp_t Pjp, // Build JPs from this place + Pjp_t PJPs, // Array of JPs to put into Bitmap branch + uint8_t Exp[], // Array of expanses to put into bitmap + Word_t ExpCnt, // Number of above JPs and Expanses + Pvoid_t Pjpm) +{ + Pjbb_t PjbbRaw; // pointer to bitmap branch. + Pjbb_t Pjbb; + Word_t ii, jj; // Temps + uint8_t CurrSubExp; // Current sub expanse for BM + +// This assertion says the number of populated subexpanses is not too large. +// This function is only called when a BranchL overflows to a BranchB or when a +// cascade occurs, meaning a leaf overflows. Either way ExpCnt cant be very +// large, in fact a lot smaller than cJU_BRANCHBMAXJPS. (Otherwise a BranchU +// would be used.) Popping this assertion means something (unspecified) has +// gone very wrong, or else Judys design criteria have changed, although in +// fact there should be no HARM in creating a BranchB with higher actual +// fanout. + + assert(ExpCnt <= cJU_BRANCHBMAXJPS); + +// Get memory for a Bitmap branch + PjbbRaw = j__udyAllocJBB(Pjpm); + if (PjbbRaw == (Pjbb_t) NULL) return(-1); + Pjbb = P_JBB(PjbbRaw); + +// Get 1st "sub" expanse (0..7) of bitmap branch + CurrSubExp = Exp[0] / cJU_BITSPERSUBEXPB; + +// Index thru all 1 byte sized expanses: + + for (jj = ii = 0; ii <= ExpCnt; ii++) + { + Word_t SubExp; // Cannot be a uint8_t + +// Make sure we cover the last one + if (ii == ExpCnt) + { + SubExp = cJU_ALLONES; // Force last one + } + else + { +// Calculate the "sub" expanse of the byte expanse + SubExp = Exp[ii] / cJU_BITSPERSUBEXPB; // Bits 5..7. + +// Set the bit that represents the expanse in Exp[] + JU_JBB_BITMAP(Pjbb, SubExp) |= JU_BITPOSMASKB(Exp[ii]); + } +// Check if a new "sub" expanse range needed + if (SubExp != CurrSubExp) + { +// Get number of JPs in this sub expanse + Word_t NumJP = ii - jj; + Pjp_t PjpRaw; + Pjp_t Pjp; + + PjpRaw = j__udyAllocJBBJP(NumJP, Pjpm); + Pjp = P_JP(PjpRaw); + + if (PjpRaw == (Pjp_t) NULL) // out of memory. + { + +// Free any previous allocations: + + while(CurrSubExp--) + { + NumJP = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, + CurrSubExp)); + if (NumJP) + { + j__udyFreeJBBJP(JU_JBB_PJP(Pjbb, + CurrSubExp), NumJP, Pjpm); + } + } + j__udyFreeJBB(PjbbRaw, Pjpm); + return(-1); + } + +// Place the array of JPs in bitmap branch: + + JU_JBB_PJP(Pjbb, CurrSubExp) = PjpRaw; + +// Copy the JPs to new leaf: + + JU_COPYMEM(Pjp, PJPs + jj, NumJP); + +// On to the next bitmap branch "sub" expanse: + + jj = ii; + CurrSubExp = SubExp; + } + } // for each 1-byte expanse + +// Pass back some of the JP to the new Bitmap branch: + + Pjp->jp_Addr = (Word_t) PjbbRaw; + + return(1); + +} // j__udyCreateBranchB() + + +// **************************************************************************** +// J U D Y C R E A T E B R A N C H U +// +// Build a BranchU from a BranchB. Return with Pjp pointing to the BranchU. +// Free the BranchB and its JP subarrays. +// +// Return -1 if error (details in Pjpm), otherwise return 1. + +FUNCTION int j__udyCreateBranchU( + Pjp_t Pjp, + Pvoid_t Pjpm) +{ + jp_t JPNull; + Pjbu_t PjbuRaw; + Pjbu_t Pjbu; + Pjbb_t PjbbRaw; + Pjbb_t Pjbb; + Word_t ii, jj; + BITMAPB_t BitMap; + Pjp_t PDstJP; +#ifdef JU_STAGED_EXP + jbu_t BranchU; // Staged uncompressed branch +#else + +// Allocate memory for a BranchU: + + PjbuRaw = j__udyAllocJBU(Pjpm); + if (PjbuRaw == (Pjbu_t) NULL) return(-1); + Pjbu = P_JBU(PjbuRaw); +#endif + JU_JPSETADT(&JPNull, 0, 0, JU_JPTYPE(Pjp) - cJU_JPBRANCH_B2 + cJU_JPNULL1); + +// Get the pointer to the BranchB: + + PjbbRaw = (Pjbb_t) (Pjp->jp_Addr); + Pjbb = P_JBB(PjbbRaw); + +// Set the pointer to the Uncompressed branch +#ifdef JU_STAGED_EXP + PDstJP = BranchU.jbu_jp; +#else + PDstJP = Pjbu->jbu_jp; +#endif + for (ii = 0; ii < cJU_NUMSUBEXPB; ii++) + { + Pjp_t PjpA; + Pjp_t PjpB; + + PjpB = PjpA = P_JP(JU_JBB_PJP(Pjbb, ii)); + +// Get the bitmap for this subexpanse + BitMap = JU_JBB_BITMAP(Pjbb, ii); + +// NULL empty subexpanses + if (BitMap == 0) + { +// But, fill with NULLs + for (jj = 0; jj < cJU_BITSPERSUBEXPB; jj++) + { + PDstJP[jj] = JPNull; + } + PDstJP += cJU_BITSPERSUBEXPB; + continue; + } +// Check if Uncompressed subexpanse + if (BitMap == cJU_FULLBITMAPB) + { +// Copy subexpanse to the Uncompressed branch intact + JU_COPYMEM(PDstJP, PjpA, cJU_BITSPERSUBEXPB); + +// Bump to next subexpanse + PDstJP += cJU_BITSPERSUBEXPB; + +// Set length of subexpanse + jj = cJU_BITSPERSUBEXPB; + } + else + { + for (jj = 0; jj < cJU_BITSPERSUBEXPB; jj++) + { +// Copy JP or NULLJP depending on bit + if (BitMap & 1) { *PDstJP = *PjpA++; } + else { *PDstJP = JPNull; } + + PDstJP++; // advance to next JP + BitMap >>= 1; + } + jj = PjpA - PjpB; + } + +// Free the subexpanse: + + j__udyFreeJBBJP(JU_JBB_PJP(Pjbb, ii), jj, Pjpm); + + } // for each JP in BranchU + +#ifdef JU_STAGED_EXP + +// Allocate memory for a BranchU: + + PjbuRaw = j__udyAllocJBU(Pjpm); + if (PjbuRaw == (Pjbu_t) NULL) return(-1); + Pjbu = P_JBU(PjbuRaw); + +// Copy staged branch to newly allocated branch: +// +// TBD: I think this code is broken. + + *Pjbu = BranchU; + +#endif // JU_STAGED_EXP + +// Finally free the BranchB and put the BranchU in its place: + + j__udyFreeJBB(PjbbRaw, Pjpm); + + Pjp->jp_Addr = (Word_t) PjbuRaw; + Pjp->jp_Type += cJU_JPBRANCH_U - cJU_JPBRANCH_B; + + return(1); + +} // j__udyCreateBranchU() diff --git a/libnetdata/libjudy/src/JudyL/JudyLDecascade.c b/libnetdata/libjudy/src/JudyL/JudyLDecascade.c new file mode 100644 index 000000000..39a89eff1 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLDecascade.c @@ -0,0 +1,1206 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.25 $ $Source: /judy/src/JudyCommon/JudyDecascade.c $ +// +// "Decascade" support functions for JudyDel.c: These functions convert +// smaller-index-size leaves to larger-index-size leaves, and also, bitmap +// leaves (LeafB1s) to Leaf1s, and some types of branches to smaller branches +// at the same index size. Some "decascading" occurs explicitly in JudyDel.c, +// but rare or large subroutines appear as functions here, and the overhead to +// call them is negligible. +// +// Compile with one of -DJUDY1 or -DJUDYL. Note: Function names are converted +// to Judy1 or JudyL specific values by external #defines. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#endif +#ifdef JUDYL +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +DBGCODE(extern void JudyCheckSorted(Pjll_t Pjll, Word_t Pop1, long IndexSize);) + + +// **************************************************************************** +// __ J U D Y C O P Y 2 T O 3 +// +// Copy one or more 2-byte Indexes to a series of 3-byte Indexes. + +FUNCTION static void j__udyCopy2to3( + uint8_t * PDest, // to where to copy 3-byte Indexes. + uint16_t * PSrc, // from where to copy 2-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + Word_t Temp; // for building 3-byte Index. + + assert(Pop1); + + do { + Temp = MSByte | *PSrc++; + JU_COPY3_LONG_TO_PINDEX(PDest, Temp); + PDest += 3; + } while (--Pop1); + +} // j__udyCopy2to3() + + +#ifdef JU_64BIT + +// **************************************************************************** +// __ J U D Y C O P Y 3 T O 4 +// +// Copy one or more 3-byte Indexes to a series of 4-byte Indexes. + +FUNCTION static void j__udyCopy3to4( + uint32_t * PDest, // to where to copy 4-byte Indexes. + uint8_t * PSrc, // from where to copy 3-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + Word_t Temp; // for building 4-byte Index. + + assert(Pop1); + + do { + JU_COPY3_PINDEX_TO_LONG(Temp, PSrc); + Temp |= MSByte; + PSrc += 3; + *PDest++ = Temp; // truncates to uint32_t. + } while (--Pop1); + +} // j__udyCopy3to4() + + +// **************************************************************************** +// __ J U D Y C O P Y 4 T O 5 +// +// Copy one or more 4-byte Indexes to a series of 5-byte Indexes. + +FUNCTION static void j__udyCopy4to5( + uint8_t * PDest, // to where to copy 4-byte Indexes. + uint32_t * PSrc, // from where to copy 4-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + Word_t Temp; // for building 5-byte Index. + + assert(Pop1); + + do { + Temp = MSByte | *PSrc++; + JU_COPY5_LONG_TO_PINDEX(PDest, Temp); + PDest += 5; + } while (--Pop1); + +} // j__udyCopy4to5() + + +// **************************************************************************** +// __ J U D Y C O P Y 5 T O 6 +// +// Copy one or more 5-byte Indexes to a series of 6-byte Indexes. + +FUNCTION static void j__udyCopy5to6( + uint8_t * PDest, // to where to copy 6-byte Indexes. + uint8_t * PSrc, // from where to copy 5-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + Word_t Temp; // for building 6-byte Index. + + assert(Pop1); + + do { + JU_COPY5_PINDEX_TO_LONG(Temp, PSrc); + Temp |= MSByte; + JU_COPY6_LONG_TO_PINDEX(PDest, Temp); + PSrc += 5; + PDest += 6; + } while (--Pop1); + +} // j__udyCopy5to6() + + +// **************************************************************************** +// __ J U D Y C O P Y 6 T O 7 +// +// Copy one or more 6-byte Indexes to a series of 7-byte Indexes. + +FUNCTION static void j__udyCopy6to7( + uint8_t * PDest, // to where to copy 6-byte Indexes. + uint8_t * PSrc, // from where to copy 5-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + Word_t Temp; // for building 6-byte Index. + + assert(Pop1); + + do { + JU_COPY6_PINDEX_TO_LONG(Temp, PSrc); + Temp |= MSByte; + JU_COPY7_LONG_TO_PINDEX(PDest, Temp); + PSrc += 6; + PDest += 7; + } while (--Pop1); + +} // j__udyCopy6to7() + +#endif // JU_64BIT + + +#ifndef JU_64BIT // 32-bit + +// **************************************************************************** +// __ J U D Y C O P Y 3 T O W +// +// Copy one or more 3-byte Indexes to a series of longs (words, always 4-byte). + +FUNCTION static void j__udyCopy3toW( + PWord_t PDest, // to where to copy full-word Indexes. + uint8_t * PSrc, // from where to copy 3-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + assert(Pop1); + + do { + JU_COPY3_PINDEX_TO_LONG(*PDest, PSrc); + *PDest++ |= MSByte; + PSrc += 3; + } while (--Pop1); + +} // j__udyCopy3toW() + + +#else // JU_64BIT + +// **************************************************************************** +// __ J U D Y C O P Y 7 T O W +// +// Copy one or more 7-byte Indexes to a series of longs (words, always 8-byte). + +FUNCTION static void j__udyCopy7toW( + PWord_t PDest, // to where to copy full-word Indexes. + uint8_t * PSrc, // from where to copy 7-byte indexes. + Word_t Pop1, // number of Indexes to copy. + Word_t MSByte) // most-significant byte, prefix to each Index. +{ + assert(Pop1); + + do { + JU_COPY7_PINDEX_TO_LONG(*PDest, PSrc); + *PDest++ |= MSByte; + PSrc += 7; + } while (--Pop1); + +} // j__udyCopy7toW() + +#endif // JU_64BIT + + +// **************************************************************************** +// __ J U D Y B R A N C H B T O B R A N C H L +// +// When a BranchB shrinks to have few enough JPs, call this function to convert +// it to a BranchL. Return 1 for success, or -1 for failure (with details in +// Pjpm). + +FUNCTION int j__udyBranchBToBranchL( + Pjp_t Pjp, // points to BranchB to shrink. + Pvoid_t Pjpm) // for global accounting. +{ + Pjbb_t PjbbRaw; // old BranchB to shrink. + Pjbb_t Pjbb; + Pjbl_t PjblRaw; // new BranchL to create. + Pjbl_t Pjbl; + Word_t Digit; // in BranchB. + Word_t NumJPs; // non-null JPs in BranchB. + uint8_t Expanse[cJU_BRANCHLMAXJPS]; // for building jbl_Expanse[]. + Pjp_t Pjpjbl; // current JP in BranchL. + Word_t SubExp; // in BranchB. + + assert(JU_JPTYPE(Pjp) >= cJU_JPBRANCH_B2); + assert(JU_JPTYPE(Pjp) <= cJU_JPBRANCH_B); + + PjbbRaw = (Pjbb_t) (Pjp->jp_Addr); + Pjbb = P_JBB(PjbbRaw); + +// Copy 1-byte subexpanse digits from BranchB to temporary buffer for BranchL, +// for each bit set in the BranchB: +// +// TBD: The following supports variable-sized linear branches, but they are no +// longer variable; this could be simplified to save the copying. +// +// TBD: Since cJU_BRANCHLMAXJP == 7 now, and cJU_BRANCHUNUMJPS == 256, the +// following might be inefficient; is there a faster way to do it? At least +// skip wholly empty subexpanses? + + for (NumJPs = Digit = 0; Digit < cJU_BRANCHUNUMJPS; ++Digit) + { + if (JU_BITMAPTESTB(Pjbb, Digit)) + { + Expanse[NumJPs++] = Digit; + assert(NumJPs <= cJU_BRANCHLMAXJPS); // required of caller. + } + } + +// Allocate and populate the BranchL: + + if ((PjblRaw = j__udyAllocJBL(Pjpm)) == (Pjbl_t) NULL) return(-1); + Pjbl = P_JBL(PjblRaw); + + JU_COPYMEM(Pjbl->jbl_Expanse, Expanse, NumJPs); + + Pjbl->jbl_NumJPs = NumJPs; + DBGCODE(JudyCheckSorted((Pjll_t) (Pjbl->jbl_Expanse), NumJPs, 1);) + +// Copy JPs from each BranchB subexpanse subarray: + + Pjpjbl = P_JP(Pjbl->jbl_jp); // start at first JP in array. + + for (SubExp = 0; SubExp < cJU_NUMSUBEXPB; ++SubExp) + { + Pjp_t PjpRaw = JU_JBB_PJP(Pjbb, SubExp); // current Pjp. + Pjp_t Pjp; + + if (PjpRaw == (Pjp_t) NULL) continue; // skip empty subexpanse. + Pjp = P_JP(PjpRaw); + + NumJPs = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, SubExp)); + assert(NumJPs); + JU_COPYMEM(Pjpjbl, Pjp, NumJPs); // one subarray at a time. + + Pjpjbl += NumJPs; + j__udyFreeJBBJP(PjpRaw, NumJPs, Pjpm); // subarray. + } + j__udyFreeJBB(PjbbRaw, Pjpm); // BranchB itself. + +// Finish up: Calculate new JP type (same index size = level in new class), +// and tie new BranchB into parent JP: + + Pjp->jp_Type += cJU_JPBRANCH_L - cJU_JPBRANCH_B; + Pjp->jp_Addr = (Word_t) PjblRaw; + + return(1); + +} // j__udyBranchBToBranchL() + + +#ifdef notdef + +// **************************************************************************** +// __ J U D Y B R A N C H U T O B R A N C H B +// +// When a BranchU shrinks to need little enough memory, call this function to +// convert it to a BranchB to save memory (at the cost of some speed). Return +// 1 for success, or -1 for failure (with details in Pjpm). +// +// TBD: Fill out if/when needed. Not currently used in JudyDel.c for reasons +// explained there. + +FUNCTION int j__udyBranchUToBranchB( + Pjp_t Pjp, // points to BranchU to shrink. + Pvoid_t Pjpm) // for global accounting. +{ + assert(FALSE); + return(1); +} +#endif // notdef + + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + +// **************************************************************************** +// __ J U D Y L E A F B 1 T O L E A F 1 +// +// Shrink a bitmap leaf (cJU_LEAFB1) to linear leaf (cJU_JPLEAF1). +// Return 1 for success, or -1 for failure (with details in Pjpm). +// +// Note: This function is different than the other JudyLeaf*ToLeaf*() +// functions because it receives a Pjp, not just a leaf, and handles its own +// allocation and free, in order to allow the caller to continue with a LeafB1 +// if allocation fails. + +FUNCTION int j__udyLeafB1ToLeaf1( + Pjp_t Pjp, // points to LeafB1 to shrink. + Pvoid_t Pjpm) // for global accounting. +{ + Pjlb_t PjlbRaw; // bitmap in old leaf. + Pjlb_t Pjlb; + Pjll_t PjllRaw; // new Leaf1. + uint8_t * Pleaf1; // Leaf1 pointer type. + Word_t Digit; // in LeafB1 bitmap. +#ifdef JUDYL + Pjv_t PjvNew; // value area in new Leaf1. + Word_t Pop1; + Word_t SubExp; +#endif + + assert(JU_JPTYPE(Pjp) == cJU_JPLEAF_B1); + assert(((JU_JPDCDPOP0(Pjp) & 0xFF) + 1) == cJU_LEAF1_MAXPOP1); + +// Allocate JPLEAF1 and prepare pointers: + + if ((PjllRaw = j__udyAllocJLL1(cJU_LEAF1_MAXPOP1, Pjpm)) == 0) + return(-1); + + Pleaf1 = (uint8_t *) P_JLL(PjllRaw); + PjlbRaw = (Pjlb_t) (Pjp->jp_Addr); + Pjlb = P_JLB(PjlbRaw); + JUDYLCODE(PjvNew = JL_LEAF1VALUEAREA(Pleaf1, cJL_LEAF1_MAXPOP1);) + +// Copy 1-byte indexes from old LeafB1 to new Leaf1: + + for (Digit = 0; Digit < cJU_BRANCHUNUMJPS; ++Digit) + if (JU_BITMAPTESTL(Pjlb, Digit)) + *Pleaf1++ = Digit; + +#ifdef JUDYL + +// Copy all old-LeafB1 value areas from value subarrays to new Leaf1: + + for (SubExp = 0; SubExp < cJU_NUMSUBEXPL; ++SubExp) + { + Pjv_t PjvRaw = JL_JLB_PVALUE(Pjlb, SubExp); + Pjv_t Pjv = P_JV(PjvRaw); + + if (Pjv == (Pjv_t) NULL) continue; // skip empty subarray. + + Pop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, SubExp)); // subarray. + assert(Pop1); + + JU_COPYMEM(PjvNew, Pjv, Pop1); // copy value areas. + j__udyLFreeJV(PjvRaw, Pop1, Pjpm); + PjvNew += Pop1; // advance through new. + } + + assert((((Word_t) Pleaf1) - (Word_t) P_JLL(PjllRaw)) + == (PjvNew - JL_LEAF1VALUEAREA(P_JLL(PjllRaw), cJL_LEAF1_MAXPOP1))); +#endif // JUDYL + + DBGCODE(JudyCheckSorted((Pjll_t) P_JLL(PjllRaw), + (((Word_t) Pleaf1) - (Word_t) P_JLL(PjllRaw)), 1);) + +// Finish up: Free the old LeafB1 and plug the new Leaf1 into the JP: +// +// Note: jp_DcdPopO does not change here. + + j__udyFreeJLB1(PjlbRaw, Pjpm); + + Pjp->jp_Addr = (Word_t) PjllRaw; + Pjp->jp_Type = cJU_JPLEAF1; + + return(1); + +} // j__udyLeafB1ToLeaf1() + +#endif // (JUDYL || (! JU_64BIT)) + + +// **************************************************************************** +// __ J U D Y L E A F 1 T O L E A F 2 +// +// Copy 1-byte Indexes from a LeafB1 or Leaf1 to 2-byte Indexes in a Leaf2. +// Pjp MUST be one of: cJU_JPLEAF_B1, cJU_JPLEAF1, or cJU_JPIMMED_1_*. +// Return number of Indexes copied. +// +// TBD: In this and all following functions, the caller should already be able +// to compute the Pop1 return value, so why return it? + +FUNCTION Word_t j__udyLeaf1ToLeaf2( + uint16_t * PLeaf2, // destination uint16_t * Index portion of leaf. +#ifdef JUDYL + Pjv_t Pjv2, // destination value part of leaf. +#endif + Pjp_t Pjp, // 1-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. + Word_t Offset; // in linear leaf list. +JUDYLCODE(Pjv_t Pjv1Raw;) // source object value area. +JUDYLCODE(Pjv_t Pjv1;) + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF_B1: + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb = P_JLB(Pjp->jp_Addr); + Word_t Digit; // in LeafB1 bitmap. + JUDYLCODE(Word_t SubExp;) // in LeafB1. + + Pop1 = JU_JPBRANCH_POP0(Pjp, 1) + 1; assert(Pop1); + +// Copy 1-byte indexes from old LeafB1 to new Leaf2, including splicing in +// the missing MSByte needed in the Leaf2: + + for (Digit = 0; Digit < cJU_BRANCHUNUMJPS; ++Digit) + if (JU_BITMAPTESTL(Pjlb, Digit)) + *PLeaf2++ = MSByte | Digit; + +#ifdef JUDYL + +// Copy all old-LeafB1 value areas from value subarrays to new Leaf2: + + for (SubExp = 0; SubExp < cJU_NUMSUBEXPL; ++SubExp) + { + Word_t SubExpPop1; + + Pjv1Raw = JL_JLB_PVALUE(Pjlb, SubExp); + if (Pjv1Raw == (Pjv_t) NULL) continue; // skip empty. + Pjv1 = P_JV(Pjv1Raw); + + SubExpPop1 = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, SubExp)); + assert(SubExpPop1); + + JU_COPYMEM(Pjv2, Pjv1, SubExpPop1); // copy value areas. + j__udyLFreeJV(Pjv1Raw, SubExpPop1, Pjpm); + Pjv2 += SubExpPop1; // advance through new. + } +#endif // JUDYL + + j__udyFreeJLB1((Pjlb_t) (Pjp->jp_Addr), Pjpm); // LeafB1 itself. + return(Pop1); + + } // case cJU_JPLEAF_B1 + + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + +// JPLEAF1: + + case cJU_JPLEAF1: + { + uint8_t * PLeaf1 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPBRANCH_POP0(Pjp, 1) + 1; assert(Pop1); + JUDYLCODE(Pjv1 = JL_LEAF1VALUEAREA(PLeaf1, Pop1);) + +// Copy all Index bytes including splicing in missing MSByte needed in Leaf2 +// (plus, for JudyL, value areas): + + for (Offset = 0; Offset < Pop1; ++Offset) + { + PLeaf2[Offset] = MSByte | PLeaf1[Offset]; + JUDYLCODE(Pjv2[Offset] = Pjv1[Offset];) + } + j__udyFreeJLL1((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } +#endif // (JUDYL || (! JU_64BIT)) + + +// JPIMMED_1_01: +// +// Note: jp_DcdPopO has 3 [7] bytes of Index (all but most significant byte), +// so the assignment to PLeaf2[] truncates and MSByte is not needed. + + case cJU_JPIMMED_1_01: + { + PLeaf2[0] = JU_JPDCDPOP0(Pjp); // see above. + JUDYLCODE(Pjv2[0] = Pjp->jp_Addr;) + return(1); + } + + +// JPIMMED_1_0[2+]: + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + { + Pop1 = JU_JPTYPE(Pjp) - cJU_JPIMMED_1_02 + 2; assert(Pop1); + JUDYLCODE(Pjv1Raw = (Pjv_t) (Pjp->jp_Addr);) + JUDYLCODE(Pjv1 = P_JV(Pjv1Raw);) + + for (Offset = 0; Offset < Pop1; ++Offset) + { +#ifdef JUDY1 + PLeaf2[Offset] = MSByte | Pjp->jp_1Index[Offset]; +#else + PLeaf2[Offset] = MSByte | Pjp->jp_LIndex[Offset]; + Pjv2 [Offset] = Pjv1[Offset]; +#endif + } + JUDYLCODE(j__udyLFreeJV(Pjv1Raw, Pop1, Pjpm);) + return(Pop1); + } + + +// UNEXPECTED CASES, including JPNULL1, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf1ToLeaf2() + + +// ***************************************************************************** +// __ J U D Y L E A F 2 T O L E A F 3 +// +// Copy 2-byte Indexes from a Leaf2 to 3-byte Indexes in a Leaf3. +// Pjp MUST be one of: cJU_JPLEAF2 or cJU_JPIMMED_2_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-3 branch to a +// Leaf3, the branch has no narrow pointers under it, meaning only level-2 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf2ToLeaf3( + uint8_t * PLeaf3, // destination "uint24_t *" Index part of leaf. +#ifdef JUDYL + Pjv_t Pjv3, // destination value part of leaf. +#endif + Pjp_t Pjp, // 2-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +#if (defined(JUDYL) && defined(JU_64BIT)) + Pjv_t Pjv2Raw; // source object value area. +#endif +JUDYLCODE(Pjv_t Pjv2;) + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF2: + + case cJU_JPLEAF2: + { + uint16_t * PLeaf2 = (uint16_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; assert(Pop1); + j__udyCopy2to3(PLeaf3, PLeaf2, Pop1, MSByte); +#ifdef JUDYL + Pjv2 = JL_LEAF2VALUEAREA(PLeaf2, Pop1); + JU_COPYMEM(Pjv3, Pjv2, Pop1); +#endif + j__udyFreeJLL2((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_2_01: +// +// Note: jp_DcdPopO has 3 [7] bytes of Index (all but most significant byte), +// so the "assignment" to PLeaf3[] is exact [truncates] and MSByte is not +// needed. + + case cJU_JPIMMED_2_01: + { + JU_COPY3_LONG_TO_PINDEX(PLeaf3, JU_JPDCDPOP0(Pjp)); // see above. + JUDYLCODE(Pjv3[0] = Pjp->jp_Addr;) + return(1); + } + + +// JPIMMED_2_0[2+]: + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + { + JUDY1CODE(uint16_t * PLeaf2 = (uint16_t *) (Pjp->jp_1Index);) + JUDYLCODE(uint16_t * PLeaf2 = (uint16_t *) (Pjp->jp_LIndex);) + + Pop1 = JU_JPTYPE(Pjp) - cJU_JPIMMED_2_02 + 2; assert(Pop1); + j__udyCopy2to3(PLeaf3, PLeaf2, Pop1, MSByte); +#ifdef JUDYL + Pjv2Raw = (Pjv_t) (Pjp->jp_Addr); + Pjv2 = P_JV(Pjv2Raw); + JU_COPYMEM(Pjv3, Pjv2, Pop1); + j__udyLFreeJV(Pjv2Raw, Pop1, Pjpm); +#endif + return(Pop1); + } +#endif // (JUDY1 || JU_64BIT) + + +// UNEXPECTED CASES, including JPNULL2, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf2ToLeaf3() + + +#ifdef JU_64BIT + +// **************************************************************************** +// __ J U D Y L E A F 3 T O L E A F 4 +// +// Copy 3-byte Indexes from a Leaf3 to 4-byte Indexes in a Leaf4. +// Pjp MUST be one of: cJU_JPLEAF3 or cJU_JPIMMED_3_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-4 branch to a +// Leaf4, the branch has no narrow pointers under it, meaning only level-3 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf3ToLeaf4( + uint32_t * PLeaf4, // destination uint32_t * Index part of leaf. +#ifdef JUDYL + Pjv_t Pjv4, // destination value part of leaf. +#endif + Pjp_t Pjp, // 3-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv3Raw;) // source object value area. +JUDYLCODE(Pjv_t Pjv3;) + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF3: + + case cJU_JPLEAF3: + { + uint8_t * PLeaf3 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; assert(Pop1); + j__udyCopy3to4(PLeaf4, (uint8_t *) PLeaf3, Pop1, MSByte); +#ifdef JUDYL + Pjv3 = JL_LEAF3VALUEAREA(PLeaf3, Pop1); + JU_COPYMEM(Pjv4, Pjv3, Pop1); +#endif + j__udyFreeJLL3((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_3_01: +// +// Note: jp_DcdPopO has 7 bytes of Index (all but most significant byte), so +// the assignment to PLeaf4[] truncates and MSByte is not needed. + + case cJU_JPIMMED_3_01: + { + PLeaf4[0] = JU_JPDCDPOP0(Pjp); // see above. + JUDYLCODE(Pjv4[0] = Pjp->jp_Addr;) + return(1); + } + + +// JPIMMED_3_0[2+]: + + case cJU_JPIMMED_3_02: +#ifdef JUDY1 + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: +#endif + { + JUDY1CODE(uint8_t * PLeaf3 = (uint8_t *) (Pjp->jp_1Index);) + JUDYLCODE(uint8_t * PLeaf3 = (uint8_t *) (Pjp->jp_LIndex);) + + JUDY1CODE(Pop1 = JU_JPTYPE(Pjp) - cJU_JPIMMED_3_02 + 2;) + JUDYLCODE(Pop1 = 2;) + + j__udyCopy3to4(PLeaf4, PLeaf3, Pop1, MSByte); +#ifdef JUDYL + Pjv3Raw = (Pjv_t) (Pjp->jp_Addr); + Pjv3 = P_JV(Pjv3Raw); + JU_COPYMEM(Pjv4, Pjv3, Pop1); + j__udyLFreeJV(Pjv3Raw, Pop1, Pjpm); +#endif + return(Pop1); + } + + +// UNEXPECTED CASES, including JPNULL3, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf3ToLeaf4() + + +// Note: In all following j__udyLeaf*ToLeaf*() functions, JPIMMED_*_0[2+] +// cases exist for Judy1 (&& 64-bit) only. JudyL has no equivalent Immeds. + + +// ***************************************************************************** +// __ J U D Y L E A F 4 T O L E A F 5 +// +// Copy 4-byte Indexes from a Leaf4 to 5-byte Indexes in a Leaf5. +// Pjp MUST be one of: cJU_JPLEAF4 or cJU_JPIMMED_4_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-5 branch to a +// Leaf5, the branch has no narrow pointers under it, meaning only level-4 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf4ToLeaf5( + uint8_t * PLeaf5, // destination "uint40_t *" Index part of leaf. +#ifdef JUDYL + Pjv_t Pjv5, // destination value part of leaf. +#endif + Pjp_t Pjp, // 4-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv4;) // source object value area. + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF4: + + case cJU_JPLEAF4: + { + uint32_t * PLeaf4 = (uint32_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; assert(Pop1); + j__udyCopy4to5(PLeaf5, PLeaf4, Pop1, MSByte); +#ifdef JUDYL + Pjv4 = JL_LEAF4VALUEAREA(PLeaf4, Pop1); + JU_COPYMEM(Pjv5, Pjv4, Pop1); +#endif + j__udyFreeJLL4((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_4_01: +// +// Note: jp_DcdPopO has 7 bytes of Index (all but most significant byte), so +// the assignment to PLeaf5[] truncates and MSByte is not needed. + + case cJU_JPIMMED_4_01: + { + JU_COPY5_LONG_TO_PINDEX(PLeaf5, JU_JPDCDPOP0(Pjp)); // see above. + JUDYLCODE(Pjv5[0] = Pjp->jp_Addr;) + return(1); + } + + +#ifdef JUDY1 + +// JPIMMED_4_0[4+]: + + case cJ1_JPIMMED_4_02: + case cJ1_JPIMMED_4_03: + { + uint32_t * PLeaf4 = (uint32_t *) (Pjp->jp_1Index); + + Pop1 = JU_JPTYPE(Pjp) - cJ1_JPIMMED_4_02 + 2; + j__udyCopy4to5(PLeaf5, PLeaf4, Pop1, MSByte); + return(Pop1); + } +#endif // JUDY1 + + +// UNEXPECTED CASES, including JPNULL4, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf4ToLeaf5() + + +// **************************************************************************** +// __ J U D Y L E A F 5 T O L E A F 6 +// +// Copy 5-byte Indexes from a Leaf5 to 6-byte Indexes in a Leaf6. +// Pjp MUST be one of: cJU_JPLEAF5 or cJU_JPIMMED_5_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-6 branch to a +// Leaf6, the branch has no narrow pointers under it, meaning only level-5 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf5ToLeaf6( + uint8_t * PLeaf6, // destination uint8_t * Index part of leaf. +#ifdef JUDYL + Pjv_t Pjv6, // destination value part of leaf. +#endif + Pjp_t Pjp, // 5-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv5;) // source object value area. + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF5: + + case cJU_JPLEAF5: + { + uint8_t * PLeaf5 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; assert(Pop1); + j__udyCopy5to6(PLeaf6, PLeaf5, Pop1, MSByte); +#ifdef JUDYL + Pjv5 = JL_LEAF5VALUEAREA(PLeaf5, Pop1); + JU_COPYMEM(Pjv6, Pjv5, Pop1); +#endif + j__udyFreeJLL5((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_5_01: +// +// Note: jp_DcdPopO has 7 bytes of Index (all but most significant byte), so +// the assignment to PLeaf6[] truncates and MSByte is not needed. + + case cJU_JPIMMED_5_01: + { + JU_COPY6_LONG_TO_PINDEX(PLeaf6, JU_JPDCDPOP0(Pjp)); // see above. + JUDYLCODE(Pjv6[0] = Pjp->jp_Addr;) + return(1); + } + + +#ifdef JUDY1 + +// JPIMMED_5_0[2+]: + + case cJ1_JPIMMED_5_02: + case cJ1_JPIMMED_5_03: + { + uint8_t * PLeaf5 = (uint8_t *) (Pjp->jp_1Index); + + Pop1 = JU_JPTYPE(Pjp) - cJ1_JPIMMED_5_02 + 2; + j__udyCopy5to6(PLeaf6, PLeaf5, Pop1, MSByte); + return(Pop1); + } +#endif // JUDY1 + + +// UNEXPECTED CASES, including JPNULL5, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf5ToLeaf6() + + +// ***************************************************************************** +// __ J U D Y L E A F 6 T O L E A F 7 +// +// Copy 6-byte Indexes from a Leaf2 to 7-byte Indexes in a Leaf7. +// Pjp MUST be one of: cJU_JPLEAF6 or cJU_JPIMMED_6_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-7 branch to a +// Leaf7, the branch has no narrow pointers under it, meaning only level-6 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf6ToLeaf7( + uint8_t * PLeaf7, // destination "uint24_t *" Index part of leaf. +#ifdef JUDYL + Pjv_t Pjv7, // destination value part of leaf. +#endif + Pjp_t Pjp, // 6-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv6;) // source object value area. + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF6: + + case cJU_JPLEAF6: + { + uint8_t * PLeaf6 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyCopy6to7(PLeaf7, PLeaf6, Pop1, MSByte); +#ifdef JUDYL + Pjv6 = JL_LEAF6VALUEAREA(PLeaf6, Pop1); + JU_COPYMEM(Pjv7, Pjv6, Pop1); +#endif + j__udyFreeJLL6((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_6_01: +// +// Note: jp_DcdPopO has 7 bytes of Index (all but most significant byte), so +// the "assignment" to PLeaf7[] is exact and MSByte is not needed. + + case cJU_JPIMMED_6_01: + { + JU_COPY7_LONG_TO_PINDEX(PLeaf7, JU_JPDCDPOP0(Pjp)); // see above. + JUDYLCODE(Pjv7[0] = Pjp->jp_Addr;) + return(1); + } + + +#ifdef JUDY1 + +// JPIMMED_6_02: + + case cJ1_JPIMMED_6_02: + { + uint8_t * PLeaf6 = (uint8_t *) (Pjp->jp_1Index); + + j__udyCopy6to7(PLeaf7, PLeaf6, /* Pop1 = */ 2, MSByte); + return(2); + } +#endif // JUDY1 + + +// UNEXPECTED CASES, including JPNULL6, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf6ToLeaf7() + +#endif // JU_64BIT + + +#ifndef JU_64BIT // 32-bit version first + +// **************************************************************************** +// __ J U D Y L E A F 3 T O L E A F W +// +// Copy 3-byte Indexes from a Leaf3 to 4-byte Indexes in a LeafW. Pjp MUST be +// one of: cJU_JPLEAF3 or cJU_JPIMMED_3_*. Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-L branch to a +// LeafW, the branch has no narrow pointers under it, meaning only level-3 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf3ToLeafW( + Pjlw_t Pjlw, // destination Index part of leaf. +#ifdef JUDYL + Pjv_t PjvW, // destination value part of leaf. +#endif + Pjp_t Pjp, // 3-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv3;) // source object value area. + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF3: + + case cJU_JPLEAF3: + { + uint8_t * PLeaf3 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyCopy3toW((PWord_t) Pjlw, PLeaf3, Pop1, MSByte); +#ifdef JUDYL + Pjv3 = JL_LEAF3VALUEAREA(PLeaf3, Pop1); + JU_COPYMEM(PjvW, Pjv3, Pop1); +#endif + j__udyFreeJLL3((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_3_01: +// +// Note: jp_DcdPopO has 3 bytes of Index (all but most significant byte), and +// MSByte must be ord in. + + case cJU_JPIMMED_3_01: + { + Pjlw[0] = MSByte | JU_JPDCDPOP0(Pjp); // see above. + JUDYLCODE(PjvW[0] = Pjp->jp_Addr;) + return(1); + } + + +#ifdef JUDY1 + +// JPIMMED_3_02: + + case cJU_JPIMMED_3_02: + { + uint8_t * PLeaf3 = (uint8_t *) (Pjp->jp_1Index); + + j__udyCopy3toW((PWord_t) Pjlw, PLeaf3, /* Pop1 = */ 2, MSByte); + return(2); + } +#endif // JUDY1 + + +// UNEXPECTED CASES, including JPNULL3, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf3ToLeafW() + + +#else // JU_64BIT + + +// **************************************************************************** +// __ J U D Y L E A F 7 T O L E A F W +// +// Copy 7-byte Indexes from a Leaf7 to 8-byte Indexes in a LeafW. +// Pjp MUST be one of: cJU_JPLEAF7 or cJU_JPIMMED_7_*. +// Return number of Indexes copied. +// +// Note: By the time this function is called to compress a level-L branch to a +// LeafW, the branch has no narrow pointers under it, meaning only level-7 +// objects are below it and must be handled here. + +FUNCTION Word_t j__udyLeaf7ToLeafW( + Pjlw_t Pjlw, // destination Index part of leaf. +#ifdef JUDYL + Pjv_t PjvW, // destination value part of leaf. +#endif + Pjp_t Pjp, // 7-byte-index object from which to copy. + Word_t MSByte, // most-significant byte, prefix to each Index. + Pvoid_t Pjpm) // for global accounting. +{ + Word_t Pop1; // Indexes in leaf. +JUDYLCODE(Pjv_t Pjv7;) // source object value area. + + switch (JU_JPTYPE(Pjp)) + { + + +// JPLEAF7: + + case cJU_JPLEAF7: + { + uint8_t * PLeaf7 = (uint8_t *) P_JLL(Pjp->jp_Addr); + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyCopy7toW((PWord_t) Pjlw, PLeaf7, Pop1, MSByte); +#ifdef JUDYL + Pjv7 = JL_LEAF7VALUEAREA(PLeaf7, Pop1); + JU_COPYMEM(PjvW, Pjv7, Pop1); +#endif + j__udyFreeJLL7((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + return(Pop1); + } + + +// JPIMMED_7_01: +// +// Note: jp_DcdPopO has 7 bytes of Index (all but most significant byte), and +// MSByte must be ord in. + + case cJU_JPIMMED_7_01: + { + Pjlw[0] = MSByte | JU_JPDCDPOP0(Pjp); // see above. + JUDYLCODE(PjvW[0] = Pjp->jp_Addr;) + return(1); + } + + +#ifdef JUDY1 + +// JPIMMED_7_02: + + case cJ1_JPIMMED_7_02: + { + uint8_t * PLeaf7 = (uint8_t *) (Pjp->jp_1Index); + + j__udyCopy7toW((PWord_t) Pjlw, PLeaf7, /* Pop1 = */ 2, MSByte); + return(2); + } +#endif + + +// UNEXPECTED CASES, including JPNULL7, should be handled by caller: + + default: assert(FALSE); break; + + } // switch + + return(0); + +} // j__udyLeaf7ToLeafW() + +#endif // JU_64BIT diff --git a/libnetdata/libjudy/src/JudyL/JudyLDel.c b/libnetdata/libjudy/src/JudyL/JudyLDel.c new file mode 100644 index 000000000..ced4b5fb3 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLDel.c @@ -0,0 +1,2146 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.68 $ $Source: /judy/src/JudyCommon/JudyDel.c $ +// +// Judy1Unset() and JudyLDel() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// About HYSTERESIS: In the Judy code, hysteresis means leaving around a +// nominally suboptimal (not maximally compressed) data structure after a +// deletion. As a result, the shape of the tree for two identical index sets +// can differ depending on the insert/delete path taken to arrive at the index +// sets. The purpose is to minimize worst-case behavior (thrashing) that could +// result from a series of intermixed insertions and deletions. It also makes +// for MUCH simpler code, because instead of performing, "delete and then +// compress," it can say, "compress and then delete," where due to hysteresis, +// compression is not even attempted until the object IS compressible. +// +// In some cases the code has no choice and it must "ungrow" a data structure +// across a "phase transition" boundary without hysteresis. In other cases the +// amount (such as "hysteresis = 1") is indicated by the number of JP deletions +// (in branches) or index deletions (in leaves) that can occur in succession +// before compressing the data structure. (It appears that hysteresis <= 1 in +// all cases.) +// +// In general no hysteresis occurs when the data structure type remains the +// same but the allocated memory chunk for the node must shrink, because the +// relationship is hardwired and theres no way to know how much memory is +// allocated to a given data structure. Hysteresis = 0 in all these cases. +// +// TBD: Could this code be faster if memory chunk hysteresis were supported +// somehow along with data structure type hysteresis? +// +// TBD: Should some of the assertions here be converted to product code that +// returns JU_ERRNO_CORRUPT? +// +// TBD: Dougs code had an odd mix of function-wide and limited-scope +// variables. Should some of the function-wide variables appear only in +// limited scopes, or more likely, vice-versa? + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +DBGCODE(extern void JudyCheckPop(Pvoid_t PArray);) +DBGCODE(extern void JudyCheckSorted(Pjll_t Pjll, Word_t Pop1, long IndexSize);) + +#ifdef TRACEJP +#include "JudyPrintJP.c" +#endif + +// These are defined to generic values in JudyCommon/JudyPrivateTypes.h: +// +// TBD: These should be exported from a header file, but perhaps not, as they +// are only used here, and exported from JudyDecascade.c, which is a separate +// file for profiling reasons (to prevent inlining), but which potentially +// could be merged with this file, either in SoftCM or at compile-time: + +#ifdef JUDY1 + +extern int j__udy1BranchBToBranchL(Pjp_t Pjp, Pvoid_t Pjpm); +#ifndef JU_64BIT +extern int j__udy1LeafB1ToLeaf1(Pjp_t, Pvoid_t); +#endif +extern Word_t j__udy1Leaf1ToLeaf2(uint16_t *, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udy1Leaf2ToLeaf3(uint8_t *, Pjp_t, Word_t, Pvoid_t); +#ifndef JU_64BIT +extern Word_t j__udy1Leaf3ToLeafW(Pjlw_t, Pjp_t, Word_t, Pvoid_t); +#else +extern Word_t j__udy1Leaf3ToLeaf4(uint32_t *, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udy1Leaf4ToLeaf5(uint8_t *, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udy1Leaf5ToLeaf6(uint8_t *, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udy1Leaf6ToLeaf7(uint8_t *, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udy1Leaf7ToLeafW(Pjlw_t, Pjp_t, Word_t, Pvoid_t); +#endif + +#else // JUDYL + +extern int j__udyLBranchBToBranchL(Pjp_t Pjp, Pvoid_t Pjpm); +extern int j__udyLLeafB1ToLeaf1(Pjp_t, Pvoid_t); +extern Word_t j__udyLLeaf1ToLeaf2(uint16_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udyLLeaf2ToLeaf3(uint8_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +#ifndef JU_64BIT +extern Word_t j__udyLLeaf3ToLeafW(Pjlw_t, Pjv_t, Pjp_t, Word_t, Pvoid_t); +#else +extern Word_t j__udyLLeaf3ToLeaf4(uint32_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udyLLeaf4ToLeaf5(uint8_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udyLLeaf5ToLeaf6(uint8_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udyLLeaf6ToLeaf7(uint8_t *, Pjv_t, Pjp_t, Word_t, Pvoid_t); +extern Word_t j__udyLLeaf7ToLeafW(Pjlw_t, Pjv_t, Pjp_t, Word_t, Pvoid_t); +#endif + +#endif // JUDYL + +// For convenience in the calling code; "M1" means "minus one": + +#ifndef JU_64BIT +#define j__udyLeafM1ToLeafW j__udyLeaf3ToLeafW +#else +#define j__udyLeafM1ToLeafW j__udyLeaf7ToLeafW +#endif + + +// **************************************************************************** +// __ J U D Y D E L W A L K +// +// Given a pointer to a JP, an Index known to be valid, the number of bytes +// left to decode (== level in the tree), and a pointer to a global JPM, walk a +// Judy (sub)tree to do an unset/delete of that index, and possibly modify the +// JPM. This function is only called internally, and recursively. Unlike +// Judy1Test() and JudyLGet(), the extra time required for recursion should be +// negligible compared with the total. +// +// Return values: +// +// -1 error; details in JPM +// +// 0 Index already deleted (should never happen, Index is known to be valid) +// +// 1 previously valid Index deleted +// +// 2 same as 1, but in addition the JP now points to a BranchL containing a +// single JP, which should be compressed into the parent branch (if there +// is one, which is not the case for a top-level branch under a JPM) + +DBGCODE(uint8_t parentJPtype;) // parent branch JP type. + +FUNCTION static int j__udyDelWalk( + Pjp_t Pjp, // current JP under which to delete. + Word_t Index, // to delete. + Word_t ParentLevel, // of parent branch. + Pjpm_t Pjpm) // for returning info to top level. +{ + Word_t pop1; // of a leaf. + Word_t level; // of a leaf. + uint8_t digit; // from Index, in current branch. + Pjll_t PjllnewRaw; // address of newly allocated leaf. + Pjll_t Pjllnew; + int offset; // within a branch. + int retcode; // return code: -1, 0, 1, 2. +JUDYLCODE(Pjv_t PjvRaw;) // value area. +JUDYLCODE(Pjv_t Pjv;) + + DBGCODE(level = 0;) + +ContinueDelWalk: // for modifying state without recursing. + +#ifdef TRACEJP + JudyPrintJP(Pjp, "d", __LINE__); +#endif + + switch (JU_JPTYPE(Pjp)) // entry: Pjp, Index. + { + + +// **************************************************************************** +// LINEAR BRANCH: +// +// MACROS FOR COMMON CODE: +// +// Check for population too high to compress a branch to a leaf, meaning just +// descend through the branch, with a purposeful off-by-one error that +// constitutes hysteresis = 1. In other words, do not compress until the +// branchs CURRENT population fits in the leaf, even BEFORE deleting one +// index. +// +// Next is a label for branch-type-specific common code. Variables pop1, +// level, digit, and Index are in the context. + +#define JU_BRANCH_KEEP(cLevel,MaxPop1,Next) \ + if (pop1 > (MaxPop1)) /* hysteresis = 1 */ \ + { \ + assert((cLevel) >= 2); \ + level = (cLevel); \ + digit = JU_DIGITATSTATE(Index, cLevel); \ + goto Next; \ + } + +// Support for generic calling of JudyLeaf*ToLeaf*() functions: +// +// Note: Cannot use JUDYLCODE() because this contains a comma. + +#ifdef JUDY1 +#define JU_PVALUEPASS // null. +#else +#define JU_PVALUEPASS Pjv, +#endif + +// During compression to a leaf, check if a JP contains nothing but a +// cJU_JPIMMED_*_01, in which case shortcut calling j__udyLeaf*ToLeaf*(): +// +// Copy the index bytes from the jp_DcdPopO field (with possible truncation), +// and continue the branch-JP-walk loop. Variables Pjp and Pleaf are in the +// context. + +#define JU_BRANCH_COPY_IMMED_EVEN(cLevel,Pjp,ignore) \ + if (JU_JPTYPE(Pjp) == cJU_JPIMMED_1_01 + (cLevel) - 2) \ + { \ + *Pleaf++ = JU_JPDCDPOP0(Pjp); \ + JUDYLCODE(*Pjv++ = (Pjp)->jp_Addr;) \ + continue; /* for-loop */ \ + } + +#define JU_BRANCH_COPY_IMMED_ODD(cLevel,Pjp,CopyIndex) \ + if (JU_JPTYPE(Pjp) == cJU_JPIMMED_1_01 + (cLevel) - 2) \ + { \ + CopyIndex(Pleaf, (Word_t) (JU_JPDCDPOP0(Pjp))); \ + Pleaf += (cLevel); /* index size = level */ \ + JUDYLCODE(*Pjv++ = (Pjp)->jp_Addr;) \ + continue; /* for-loop */ \ + } + +// Compress a BranchL into a leaf one index size larger: +// +// Allocate a new leaf, walk the JPs in the old BranchL and pack their contents +// into the new leaf (of type NewJPType), free the old BranchL, and finally +// restart the switch to delete Index from the new leaf. (Note that all +// BranchLs are the same size.) Variables Pjp, Pjpm, Pleaf, digit, and pop1 +// are in the context. + +#define JU_BRANCHL_COMPRESS(cLevel,LeafType,MaxPop1,NewJPType, \ + LeafToLeaf,Alloc,ValueArea, \ + CopyImmed,CopyIndex) \ + { \ + LeafType Pleaf; \ + Pjbl_t PjblRaw; \ + Pjbl_t Pjbl; \ + Word_t numJPs; \ + \ + if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ + Pleaf = (LeafType) Pjllnew; \ + JUDYLCODE(Pjv = ValueArea(Pleaf, MaxPop1);) \ + \ + PjblRaw = (Pjbl_t) (Pjp->jp_Addr); \ + Pjbl = P_JBL(PjblRaw); \ + numJPs = Pjbl->jbl_NumJPs; \ + \ + for (offset = 0; offset < numJPs; ++offset) \ + { \ + CopyImmed(cLevel, (Pjbl->jbl_jp) + offset, CopyIndex); \ + \ + pop1 = LeafToLeaf(Pleaf, JU_PVALUEPASS \ + (Pjbl->jbl_jp) + offset, \ + JU_DIGITTOSTATE(Pjbl->jbl_Expanse[offset], \ + cLevel), (Pvoid_t) Pjpm); \ + Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1)); \ + JUDYLCODE(Pjv += pop1;) \ + } \ + assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \ + JUDYLCODE(assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));) \ + DBGCODE(JudyCheckSorted(Pjllnew, MaxPop1, cLevel);) \ + \ + j__udyFreeJBL(PjblRaw, Pjpm); \ + \ + Pjp->jp_Type = (NewJPType); \ + Pjp->jp_Addr = (Word_t) PjllnewRaw; \ + goto ContinueDelWalk; /* delete from new leaf */ \ + } + +// Overall common code for initial BranchL deletion handling: +// +// Assert that Index is in the branch, then see if the BranchL should be kept +// or else compressed to a leaf. Variables Index, Pjp, and pop1 are in the +// context. + +#define JU_BRANCHL(cLevel,MaxPop1,LeafType,NewJPType, \ + LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex) \ + \ + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cLevel)); \ + assert(ParentLevel > (cLevel)); \ + \ + pop1 = JU_JPBRANCH_POP0(Pjp, cLevel) + 1; \ + JU_BRANCH_KEEP(cLevel, MaxPop1, BranchLKeep); \ + assert(pop1 == (MaxPop1)); \ + \ + JU_BRANCHL_COMPRESS(cLevel, LeafType, MaxPop1, NewJPType, \ + LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex) + + +// END OF MACROS, START OF CASES: + + case cJU_JPBRANCH_L2: + + JU_BRANCHL(2, cJU_LEAF2_MAXPOP1, uint16_t *, cJU_JPLEAF2, + j__udyLeaf1ToLeaf2, j__udyAllocJLL2, JL_LEAF2VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_L3: + + JU_BRANCHL(3, cJU_LEAF3_MAXPOP1, uint8_t *, cJU_JPLEAF3, + j__udyLeaf2ToLeaf3, j__udyAllocJLL3, JL_LEAF3VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY3_LONG_TO_PINDEX); + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + + JU_BRANCHL(4, cJU_LEAF4_MAXPOP1, uint32_t *, cJU_JPLEAF4, + j__udyLeaf3ToLeaf4, j__udyAllocJLL4, JL_LEAF4VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_L5: + + JU_BRANCHL(5, cJU_LEAF5_MAXPOP1, uint8_t *, cJU_JPLEAF5, + j__udyLeaf4ToLeaf5, j__udyAllocJLL5, JL_LEAF5VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY5_LONG_TO_PINDEX); + + case cJU_JPBRANCH_L6: + + JU_BRANCHL(6, cJU_LEAF6_MAXPOP1, uint8_t *, cJU_JPLEAF6, + j__udyLeaf5ToLeaf6, j__udyAllocJLL6, JL_LEAF6VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY6_LONG_TO_PINDEX); + + case cJU_JPBRANCH_L7: + + JU_BRANCHL(7, cJU_LEAF7_MAXPOP1, uint8_t *, cJU_JPLEAF7, + j__udyLeaf6ToLeaf7, j__udyAllocJLL7, JL_LEAF7VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY7_LONG_TO_PINDEX); +#endif // JU_64BIT + +// A top-level BranchL is different and cannot use JU_BRANCHL(): Dont try to +// compress to a (LEAFW) leaf yet, but leave this for a later deletion +// (hysteresis > 0); and the next JP type depends on the system word size; so +// dont use JU_BRANCH_KEEP(): + + case cJU_JPBRANCH_L: + { + Pjbl_t Pjbl; + Word_t numJPs; + + level = cJU_ROOTSTATE; + digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + + // fall through: + + +// COMMON CODE FOR KEEPING AND DESCENDING THROUGH A BRANCHL: +// +// Come here with level and digit set. + +BranchLKeep: + Pjbl = P_JBL(Pjp->jp_Addr); + numJPs = Pjbl->jbl_NumJPs; + assert(numJPs > 0); + DBGCODE(parentJPtype = JU_JPTYPE(Pjp);) + +// Search for a match to the digit (valid Index => must find digit): + + for (offset = 0; (Pjbl->jbl_Expanse[offset]) != digit; ++offset) + assert(offset < numJPs - 1); + + Pjp = (Pjbl->jbl_jp) + offset; + +// If not at a (deletable) JPIMMED_*_01, continue the walk (to descend through +// the BranchL): + + assert(level >= 2); + if ((JU_JPTYPE(Pjp)) != cJU_JPIMMED_1_01 + level - 2) break; + +// At JPIMMED_*_01: Ensure the index is in the right expanse, then delete the +// Immed from the BranchL: +// +// Note: A BranchL has a fixed size and format regardless of numJPs. + + assert(JU_JPDCDPOP0(Pjp) == JU_TRIMTODCDSIZE(Index)); + + JU_DELETEINPLACE(Pjbl->jbl_Expanse, numJPs, offset, ignore); + JU_DELETEINPLACE(Pjbl->jbl_jp, numJPs, offset, ignore); + + DBGCODE(JudyCheckSorted((Pjll_t) (Pjbl->jbl_Expanse), + numJPs - 1, 1);) + +// If only one index left in the BranchL, indicate this to the caller: + + return ((--(Pjbl->jbl_NumJPs) <= 1) ? 2 : 1); + + } // case cJU_JPBRANCH_L. + + +// **************************************************************************** +// BITMAP BRANCH: +// +// MACROS FOR COMMON CODE: +// +// Note the reuse of common macros here, defined earlier: JU_BRANCH_KEEP(), +// JU_PVALUE*. +// +// Compress a BranchB into a leaf one index size larger: +// +// Allocate a new leaf, walk the JPs in the old BranchB (one bitmap subexpanse +// at a time) and pack their contents into the new leaf (of type NewJPType), +// free the old BranchB, and finally restart the switch to delete Index from +// the new leaf. Variables Pjp, Pjpm, Pleaf, digit, and pop1 are in the +// context. +// +// Note: Its no accident that the interface to JU_BRANCHB_COMPRESS() is +// identical to JU_BRANCHL_COMPRESS(). Only the details differ in how to +// traverse the branchs JPs. + +#define JU_BRANCHB_COMPRESS(cLevel,LeafType,MaxPop1,NewJPType, \ + LeafToLeaf,Alloc,ValueArea, \ + CopyImmed,CopyIndex) \ + { \ + LeafType Pleaf; \ + Pjbb_t PjbbRaw; /* BranchB to compress */ \ + Pjbb_t Pjbb; \ + Word_t subexp; /* current subexpanse number */ \ + BITMAPB_t bitmap; /* portion for this subexpanse */ \ + Pjp_t Pjp2Raw; /* one subexpanses subarray */ \ + Pjp_t Pjp2; \ + \ + if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ + Pleaf = (LeafType) Pjllnew; \ + JUDYLCODE(Pjv = ValueArea(Pleaf, MaxPop1);) \ + \ + PjbbRaw = (Pjbb_t) (Pjp->jp_Addr); \ + Pjbb = P_JBB(PjbbRaw); \ + \ + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) \ + { \ + if ((bitmap = JU_JBB_BITMAP(Pjbb, subexp)) == 0) \ + continue; /* empty subexpanse */ \ + \ + digit = subexp * cJU_BITSPERSUBEXPB; \ + Pjp2Raw = JU_JBB_PJP(Pjbb, subexp); \ + Pjp2 = P_JP(Pjp2Raw); \ + assert(Pjp2 != (Pjp_t) NULL); \ + \ + for (offset = 0; bitmap != 0; bitmap >>= 1, ++digit) \ + { \ + if (! (bitmap & 1)) \ + continue; /* empty sub-subexpanse */ \ + \ + ++offset; /* before any continue */ \ + \ + CopyImmed(cLevel, Pjp2 + offset - 1, CopyIndex); \ + \ + pop1 = LeafToLeaf(Pleaf, JU_PVALUEPASS \ + Pjp2 + offset - 1, \ + JU_DIGITTOSTATE(digit, cLevel), \ + (Pvoid_t) Pjpm); \ + Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1)); \ + JUDYLCODE(Pjv += pop1;) \ + } \ + j__udyFreeJBBJP(Pjp2Raw, /* pop1 = */ offset, Pjpm); \ + } \ + assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \ + JUDYLCODE(assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));) \ + DBGCODE(JudyCheckSorted(Pjllnew, MaxPop1, cLevel);) \ + \ + j__udyFreeJBB(PjbbRaw, Pjpm); \ + \ + Pjp->jp_Type = (NewJPType); \ + Pjp->jp_Addr = (Word_t) PjllnewRaw; \ + goto ContinueDelWalk; /* delete from new leaf */ \ + } + +// Overall common code for initial BranchB deletion handling: +// +// Assert that Index is in the branch, then see if the BranchB should be kept +// or else compressed to a leaf. Variables Index, Pjp, and pop1 are in the +// context. + +#define JU_BRANCHB(cLevel,MaxPop1,LeafType,NewJPType, \ + LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex) \ + \ + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cLevel)); \ + assert(ParentLevel > (cLevel)); \ + \ + pop1 = JU_JPBRANCH_POP0(Pjp, cLevel) + 1; \ + JU_BRANCH_KEEP(cLevel, MaxPop1, BranchBKeep); \ + assert(pop1 == (MaxPop1)); \ + \ + JU_BRANCHB_COMPRESS(cLevel, LeafType, MaxPop1, NewJPType, \ + LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex) + + +// END OF MACROS, START OF CASES: +// +// Note: Its no accident that the macro calls for these cases is nearly +// identical to the code for BranchLs. + + case cJU_JPBRANCH_B2: + + JU_BRANCHB(2, cJU_LEAF2_MAXPOP1, uint16_t *, cJU_JPLEAF2, + j__udyLeaf1ToLeaf2, j__udyAllocJLL2, JL_LEAF2VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_B3: + + JU_BRANCHB(3, cJU_LEAF3_MAXPOP1, uint8_t *, cJU_JPLEAF3, + j__udyLeaf2ToLeaf3, j__udyAllocJLL3, JL_LEAF3VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY3_LONG_TO_PINDEX); + +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + + JU_BRANCHB(4, cJU_LEAF4_MAXPOP1, uint32_t *, cJU_JPLEAF4, + j__udyLeaf3ToLeaf4, j__udyAllocJLL4, JL_LEAF4VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_B5: + + JU_BRANCHB(5, cJU_LEAF5_MAXPOP1, uint8_t *, cJU_JPLEAF5, + j__udyLeaf4ToLeaf5, j__udyAllocJLL5, JL_LEAF5VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY5_LONG_TO_PINDEX); + + case cJU_JPBRANCH_B6: + + JU_BRANCHB(6, cJU_LEAF6_MAXPOP1, uint8_t *, cJU_JPLEAF6, + j__udyLeaf5ToLeaf6, j__udyAllocJLL6, JL_LEAF6VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY6_LONG_TO_PINDEX); + + case cJU_JPBRANCH_B7: + + JU_BRANCHB(7, cJU_LEAF7_MAXPOP1, uint8_t *, cJU_JPLEAF7, + j__udyLeaf6ToLeaf7, j__udyAllocJLL7, JL_LEAF7VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY7_LONG_TO_PINDEX); +#endif // JU_64BIT + +// A top-level BranchB is different and cannot use JU_BRANCHB(): Dont try to +// compress to a (LEAFW) leaf yet, but leave this for a later deletion +// (hysteresis > 0); and the next JP type depends on the system word size; so +// dont use JU_BRANCH_KEEP(): + + case cJU_JPBRANCH_B: + { + Pjbb_t Pjbb; // BranchB to modify. + Word_t subexp; // current subexpanse number. + Word_t subexp2; // in second-level loop. + BITMAPB_t bitmap; // portion for this subexpanse. + BITMAPB_t bitmask; // with digits bit set. + Pjp_t Pjp2Raw; // one subexpanses subarray. + Pjp_t Pjp2; + Word_t numJPs; // in one subexpanse. + + level = cJU_ROOTSTATE; + digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + + // fall through: + + +// COMMON CODE FOR KEEPING AND DESCENDING THROUGH A BRANCHB: +// +// Come here with level and digit set. + +BranchBKeep: + Pjbb = P_JBB(Pjp->jp_Addr); + subexp = digit / cJU_BITSPERSUBEXPB; + bitmap = JU_JBB_BITMAP(Pjbb, subexp); + bitmask = JU_BITPOSMASKB(digit); + assert(bitmap & bitmask); // Index valid => digits bit is set. + DBGCODE(parentJPtype = JU_JPTYPE(Pjp);) + +// Compute digits offset into the bitmap, with a fast method if all bits are +// set: + + offset = ((bitmap == (cJU_FULLBITMAPB)) ? + digit % cJU_BITSPERSUBEXPB : + j__udyCountBitsB(bitmap & JU_MASKLOWEREXC(bitmask))); + + Pjp2Raw = JU_JBB_PJP(Pjbb, subexp); + Pjp2 = P_JP(Pjp2Raw); + assert(Pjp2 != (Pjp_t) NULL); // valid subexpanse pointer. + +// If not at a (deletable) JPIMMED_*_01, continue the walk (to descend through +// the BranchB): + + if (JU_JPTYPE(Pjp2 + offset) != cJU_JPIMMED_1_01 + level - 2) + { + Pjp = Pjp2 + offset; + break; + } + +// At JPIMMED_*_01: Ensure the index is in the right expanse, then delete the +// Immed from the BranchB: + + assert(JU_JPDCDPOP0(Pjp2 + offset) + == JU_TRIMTODCDSIZE(Index)); + +// If only one index is left in the subexpanse, free the JP array: + + if ((numJPs = j__udyCountBitsB(bitmap)) == 1) + { + j__udyFreeJBBJP(Pjp2Raw, /* pop1 = */ 1, Pjpm); + JU_JBB_PJP(Pjbb, subexp) = (Pjp_t) NULL; + } + +// Shrink JP array in-place: + + else if (JU_BRANCHBJPGROWINPLACE(numJPs - 1)) + { + assert(numJPs > 0); + JU_DELETEINPLACE(Pjp2, numJPs, offset, ignore); + } + +// JP array would end up too large; compress it to a smaller one: + + else + { + Pjp_t PjpnewRaw; + Pjp_t Pjpnew; + + if ((PjpnewRaw = j__udyAllocJBBJP(numJPs - 1, Pjpm)) + == (Pjp_t) NULL) return(-1); + Pjpnew = P_JP(PjpnewRaw); + + JU_DELETECOPY(Pjpnew, Pjp2, numJPs, offset, ignore); + j__udyFreeJBBJP(Pjp2Raw, numJPs, Pjpm); // old. + + JU_JBB_PJP(Pjbb, subexp) = PjpnewRaw; + } + +// Clear digits bit in the bitmap: + + JU_JBB_BITMAP(Pjbb, subexp) ^= bitmask; + +// If the current subexpanse alone is still too large for a BranchL (with +// hysteresis = 1), the delete is all done: + + if (numJPs > cJU_BRANCHLMAXJPS) return(1); + +// Consider shrinking the current BranchB to a BranchL: +// +// Check the numbers of JPs in other subexpanses in the BranchL. Upon reaching +// the critical number of numJPs (which could be right at the start; again, +// with hysteresis = 1), its faster to just watch for any non-empty subexpanse +// than to count bits in each subexpanse. Upon finding too many JPs, give up +// on shrinking the BranchB. + + for (subexp2 = 0; subexp2 < cJU_NUMSUBEXPB; ++subexp2) + { + if (subexp2 == subexp) continue; // skip current subexpanse. + + if ((numJPs == cJU_BRANCHLMAXJPS) ? + JU_JBB_BITMAP(Pjbb, subexp2) : + ((numJPs += j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp2))) + > cJU_BRANCHLMAXJPS)) + { + return(1); // too many JPs, cannot shrink. + } + } + +// Shrink current BranchB to a BranchL: +// +// Note: In this rare case, ignore the return value, do not pass it to the +// caller, because the deletion is already successfully completed and the +// caller(s) must decrement population counts. The only errors expected from +// this call are JU_ERRNO_NOMEM and JU_ERRNO_OVERRUN, neither of which is worth +// forwarding from this point. See also 4.1, 4.8, and 4.15 of this file. + + (void) j__udyBranchBToBranchL(Pjp, Pjpm); + return(1); + + } // case. + + +// **************************************************************************** +// UNCOMPRESSED BRANCH: +// +// MACROS FOR COMMON CODE: +// +// Note the reuse of common macros here, defined earlier: JU_PVALUE*. +// +// Compress a BranchU into a leaf one index size larger: +// +// Allocate a new leaf, walk the JPs in the old BranchU and pack their contents +// into the new leaf (of type NewJPType), free the old BranchU, and finally +// restart the switch to delete Index from the new leaf. Variables Pjp, Pjpm, +// digit, and pop1 are in the context. +// +// Note: Its no accident that the interface to JU_BRANCHU_COMPRESS() is +// nearly identical to JU_BRANCHL_COMPRESS(); just NullJPType is added. The +// details differ in how to traverse the branchs JPs -- +// +// -- and also, what to do upon encountering a cJU_JPIMMED_*_01 JP. In +// BranchLs and BranchBs the JP must be deleted, but in a BranchU its merely +// converted to a null JP, and this is done by other switch cases, so the "keep +// branch" situation is simpler here and JU_BRANCH_KEEP() is not used. Also, +// theres no code to convert a BranchU to a BranchB since counting the JPs in +// a BranchU is (at least presently) expensive, and besides, keeping around a +// BranchU is form of hysteresis. + +#define JU_BRANCHU_COMPRESS(cLevel,LeafType,MaxPop1,NullJPType,NewJPType, \ + LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex) \ + { \ + LeafType Pleaf; \ + Pjbu_t PjbuRaw = (Pjbu_t) (Pjp->jp_Addr); \ + Pjp_t Pjp2 = JU_JBU_PJP0(Pjp); \ + Word_t ldigit; /* larger than uint8_t */ \ + \ + if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ + Pleaf = (LeafType) Pjllnew; \ + JUDYLCODE(Pjv = ValueArea(Pleaf, MaxPop1);) \ + \ + for (ldigit = 0; ldigit < cJU_BRANCHUNUMJPS; ++ldigit, ++Pjp2) \ + { \ + /* fast-process common types: */ \ + if (JU_JPTYPE(Pjp2) == (NullJPType)) continue; \ + CopyImmed(cLevel, Pjp2, CopyIndex); \ + \ + pop1 = LeafToLeaf(Pleaf, JU_PVALUEPASS Pjp2, \ + JU_DIGITTOSTATE(ldigit, cLevel), \ + (Pvoid_t) Pjpm); \ + Pleaf = (LeafType) (((Word_t) Pleaf) + ((cLevel) * pop1)); \ + JUDYLCODE(Pjv += pop1;) \ + } \ + assert(((((Word_t) Pleaf) - ((Word_t) Pjllnew)) / (cLevel)) == (MaxPop1)); \ + JUDYLCODE(assert((Pjv - ValueArea(Pjllnew, MaxPop1)) == (MaxPop1));) \ + DBGCODE(JudyCheckSorted(Pjllnew, MaxPop1, cLevel);) \ + \ + j__udyFreeJBU(PjbuRaw, Pjpm); \ + \ + Pjp->jp_Type = (NewJPType); \ + Pjp->jp_Addr = (Word_t) PjllnewRaw; \ + goto ContinueDelWalk; /* delete from new leaf */ \ + } + +// Overall common code for initial BranchU deletion handling: +// +// Assert that Index is in the branch, then see if a BranchU should be kept or +// else compressed to a leaf. Variables level, Index, Pjp, and pop1 are in the +// context. +// +// Note: BranchU handling differs from BranchL and BranchB as described above. + +#define JU_BRANCHU(cLevel,MaxPop1,LeafType,NullJPType,NewJPType, \ + LeafToLeaf,Alloc,ValueArea,CopyImmed,CopyIndex) \ + \ + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cLevel)); \ + assert(ParentLevel > (cLevel)); \ + DBGCODE(parentJPtype = JU_JPTYPE(Pjp);) \ + \ + pop1 = JU_JPBRANCH_POP0(Pjp, cLevel) + 1; \ + \ + if (pop1 > (MaxPop1)) /* hysteresis = 1 */ \ + { \ + level = (cLevel); \ + Pjp = P_JP(Pjp->jp_Addr) + JU_DIGITATSTATE(Index, cLevel);\ + break; /* descend to next level */ \ + } \ + assert(pop1 == (MaxPop1)); \ + \ + JU_BRANCHU_COMPRESS(cLevel, LeafType, MaxPop1, NullJPType, NewJPType, \ + LeafToLeaf, Alloc, ValueArea, CopyImmed, CopyIndex) + + +// END OF MACROS, START OF CASES: +// +// Note: Its no accident that the macro calls for these cases is nearly +// identical to the code for BranchLs, with the addition of cJU_JPNULL* +// parameters only needed for BranchUs. + + case cJU_JPBRANCH_U2: + + JU_BRANCHU(2, cJU_LEAF2_MAXPOP1, uint16_t *, + cJU_JPNULL1, cJU_JPLEAF2, + j__udyLeaf1ToLeaf2, j__udyAllocJLL2, JL_LEAF2VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_U3: + + JU_BRANCHU(3, cJU_LEAF3_MAXPOP1, uint8_t *, + cJU_JPNULL2, cJU_JPLEAF3, + j__udyLeaf2ToLeaf3, j__udyAllocJLL3, JL_LEAF3VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY3_LONG_TO_PINDEX); + +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: + + JU_BRANCHU(4, cJU_LEAF4_MAXPOP1, uint32_t *, + cJU_JPNULL3, cJU_JPLEAF4, + j__udyLeaf3ToLeaf4, j__udyAllocJLL4, JL_LEAF4VALUEAREA, + JU_BRANCH_COPY_IMMED_EVEN, ignore); + + case cJU_JPBRANCH_U5: + + JU_BRANCHU(5, cJU_LEAF5_MAXPOP1, uint8_t *, + cJU_JPNULL4, cJU_JPLEAF5, + j__udyLeaf4ToLeaf5, j__udyAllocJLL5, JL_LEAF5VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY5_LONG_TO_PINDEX); + + case cJU_JPBRANCH_U6: + + JU_BRANCHU(6, cJU_LEAF6_MAXPOP1, uint8_t *, + cJU_JPNULL5, cJU_JPLEAF6, + j__udyLeaf5ToLeaf6, j__udyAllocJLL6, JL_LEAF6VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY6_LONG_TO_PINDEX); + + case cJU_JPBRANCH_U7: + + JU_BRANCHU(7, cJU_LEAF7_MAXPOP1, uint8_t *, + cJU_JPNULL6, cJU_JPLEAF7, + j__udyLeaf6ToLeaf7, j__udyAllocJLL7, JL_LEAF7VALUEAREA, + JU_BRANCH_COPY_IMMED_ODD, JU_COPY7_LONG_TO_PINDEX); +#endif // JU_64BIT + +// A top-level BranchU is different and cannot use JU_BRANCHU(): Dont try to +// compress to a (LEAFW) leaf yet, but leave this for a later deletion +// (hysteresis > 0); just descend through the BranchU: + + case cJU_JPBRANCH_U: + + DBGCODE(parentJPtype = JU_JPTYPE(Pjp);) + + level = cJU_ROOTSTATE; + Pjp = P_JP(Pjp->jp_Addr) + JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + break; + + +// **************************************************************************** +// LINEAR LEAF: +// +// State transitions while deleting an Index, the inverse of the similar table +// that appears in JudyIns.c: +// +// Note: In JudyIns.c this table is not needed and does not appear until the +// Immed handling code; because once a Leaf is reached upon growing the tree, +// the situation remains simpler, but for deleting indexes, the complexity +// arises when leaves must compress to Immeds. +// +// Note: There are other transitions possible too, not shown here, such as to +// a leaf one level higher. +// +// (Yes, this is very terse... Study it and it will make sense.) +// (Note, parts of this diagram are repeated below for quick reference.) +// +// reformat JP here for Judy1 only, from word-1 to word-2 +// | +// JUDY1 && JU_64BIT JUDY1 || JU_64BIT | +// V +// (*) Leaf1 [[ => 1_15..08 ] => 1_07 => ... => 1_04 ] => 1_03 => 1_02 => 1_01 +// Leaf2 [[ => 2_07..04 ] => 2_03 => 2_02 ] => 2_01 +// Leaf3 [[ => 3_05..03 ] => 3_02 ] => 3_01 +// JU_64BIT only: +// Leaf4 [[ => 4_03..02 ]] => 4_01 +// Leaf5 [[ => 5_03..02 ]] => 5_01 +// Leaf6 [[ => 6_02 ]] => 6_01 +// Leaf7 [[ => 7_02 ]] => 7_01 +// +// (*) For Judy1 & 64-bit, go directly from a LeafB1 to cJU_JPIMMED_1_15; skip +// Leaf1, as described in Judy1.h regarding cJ1_JPLEAF1. +// +// MACROS FOR COMMON CODE: +// +// (De)compress a LeafX into a LeafY one index size (cIS) larger (X+1 = Y): +// +// This is only possible when the current leaf is under a narrow pointer +// ((ParentLevel - 1) > cIS) and its population fits in a higher-level leaf. +// Variables ParentLevel, pop1, PjllnewRaw, Pjllnew, Pjpm, and Index are in the +// context. +// +// Note: Doing an "uplevel" doesnt occur until the old leaf can be compressed +// up one level BEFORE deleting an index; that is, hysteresis = 1. +// +// Note: LeafType, MaxPop1, NewJPType, and Alloc refer to the up-level leaf, +// not the current leaf. +// +// Note: 010327: Fixed bug where the jp_DcdPopO next-uplevel digit (byte) +// above the current Pop0 value was not being cleared. When upleveling, one +// digit in jp_DcdPopO "moves" from being part of the Dcd subfield to the Pop0 +// subfield, but since a leaf maxpop1 is known to be <= 1 byte in size, the new +// Pop0 byte should always be zero. This is easy to overlook because +// JU_JPLEAF_POP0() "knows" to only use the LSB of Pop0 (for efficiency) and +// ignore the other bytes... Until someone uses cJU_POP0MASK() instead of +// JU_JPLEAF_POP0(), such as in JudyInsertBranch.c. +// +// TBD: Should JudyInsertBranch.c use JU_JPLEAF_POP0() rather than +// cJU_POP0MASK(), for efficiency? Does it know for sure its a narrow pointer +// under the leaf? Not necessarily. + +#define JU_LEAF_UPLEVEL(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf, \ + Alloc,ValueArea) \ + \ + assert(((ParentLevel - 1) == (cIS)) || (pop1 >= (MaxPop1))); \ + \ + if (((ParentLevel - 1) > (cIS)) /* under narrow pointer */ \ + && (pop1 == (MaxPop1))) /* hysteresis = 1 */ \ + { \ + Word_t D_cdP0; \ + if ((PjllnewRaw = Alloc(MaxPop1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ + JUDYLCODE(Pjv = ValueArea((LeafType) Pjllnew, MaxPop1);) \ + \ + (void) LeafToLeaf((LeafType) Pjllnew, JU_PVALUEPASS Pjp, \ + Index & cJU_DCDMASK(cIS), /* TBD, Doug says */ \ + (Pvoid_t) Pjpm); \ + DBGCODE(JudyCheckSorted(Pjllnew, MaxPop1, cIS + 1);) \ + \ + D_cdP0 = (~cJU_MASKATSTATE((cIS) + 1)) & JU_JPDCDPOP0(Pjp); \ + JU_JPSETADT(Pjp, (Word_t)PjllnewRaw, D_cdP0, NewJPType); \ + goto ContinueDelWalk; /* delete from new leaf */ \ + } + + +// For Leaf3, only support JU_LEAF_UPLEVEL on a 64-bit system, and for Leaf7, +// there is no JU_LEAF_UPLEVEL: +// +// Note: Theres no way here to go from Leaf3 [Leaf7] to LEAFW on a 32-bit +// [64-bit] system. Thats handled in the main code, because its different in +// that a JPM is involved. + +#ifndef JU_64BIT // 32-bit. +#define JU_LEAF_UPLEVEL64(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf, \ + Alloc,ValueArea) // null. +#else +#define JU_LEAF_UPLEVEL64(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf, \ + Alloc,ValueArea) \ + JU_LEAF_UPLEVEL (cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf, \ + Alloc,ValueArea) +#define JU_LEAF_UPLEVEL_NONE(cIS,LeafType,MaxPop1,NewJPType,LeafToLeaf, \ + Alloc,ValueArea) // null. +#endif + +// Compress a Leaf* with pop1 = 2, or a JPIMMED_*_02, into a JPIMMED_*_01: +// +// Copy whichever Index is NOT being deleted (and assert that the other one is +// found; Index must be valid). This requires special handling of the Index +// bytes (and value area). Variables Pjp, Index, offset, and Pleaf are in the +// context, offset is modified to the undeleted Index, and Pjp is modified +// including jp_Addr. + + +#define JU_TOIMMED_01_EVEN(cIS,ignore1,ignore2) \ +{ \ + Word_t D_cdP0; \ + Word_t A_ddr = 0; \ + uint8_t T_ype = JU_JPTYPE(Pjp); \ + offset = (Pleaf[0] == JU_LEASTBYTES(Index, cIS)); /* undeleted Ind */ \ + assert(Pleaf[offset ? 0 : 1] == JU_LEASTBYTES(Index, cIS)); \ + D_cdP0 = (Index & cJU_DCDMASK(cIS)) | Pleaf[offset]; \ +JUDYLCODE(A_ddr = Pjv[offset];) \ + JU_JPSETADT(Pjp, A_ddr, D_cdP0, T_ype); \ +} + +#define JU_TOIMMED_01_ODD(cIS,SearchLeaf,CopyPIndex) \ + { \ + Word_t D_cdP0; \ + Word_t A_ddr = 0; \ + uint8_t T_ype = JU_JPTYPE(Pjp); \ + \ + offset = SearchLeaf(Pleaf, 2, Index); \ + assert(offset >= 0); /* Index must be valid */ \ + CopyPIndex(D_cdP0, & (Pleaf[offset ? 0 : cIS])); \ + D_cdP0 |= Index & cJU_DCDMASK(cIS); \ + JUDYLCODE(A_ddr = Pjv[offset ? 0 : 1];) \ + JU_JPSETADT(Pjp, A_ddr, D_cdP0, T_ype); \ + } + + +// Compress a Leaf* into a JPIMMED_*_0[2+]: +// +// This occurs as soon as its possible, with hysteresis = 0. Variables pop1, +// Pleaf, offset, and Pjpm are in the context. +// +// TBD: Explain why hysteresis = 0 here, rather than > 0. Probably because +// the insert code assumes if the population is small enough, an Immed is used, +// not a leaf. +// +// The differences between Judy1 and JudyL with respect to value area handling +// are just too large for completely common code between them... Oh well, some +// big ifdefs follow. + +#ifdef JUDY1 + +#define JU_LEAF_TOIMMED(cIS,LeafType,MaxPop1,BaseJPType,ignore1,\ + ignore2,ignore3,ignore4, \ + DeleteCopy,FreeLeaf) \ + \ + assert(pop1 > (MaxPop1)); \ + \ + if ((pop1 - 1) == (MaxPop1)) /* hysteresis = 0 */ \ + { \ + Pjll_t PjllRaw = (Pjll_t) (Pjp->jp_Addr); \ + DeleteCopy((LeafType) (Pjp->jp_1Index), Pleaf, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted((Pjll_t) (Pjp->jp_1Index), pop1-1, cIS);) \ + Pjp->jp_Type = (BaseJPType) - 1 + (MaxPop1) - 1; \ + FreeLeaf(PjllRaw, pop1, Pjpm); \ + return(1); \ + } + +#else // JUDYL + +// Pjv is also in the context. + +#define JU_LEAF_TOIMMED(cIS,LeafType,MaxPop1,BaseJPType,ignore1,\ + ignore2,ignore3,ignore4, \ + DeleteCopy,FreeLeaf) \ + \ + assert(pop1 > (MaxPop1)); \ + \ + if ((pop1 - 1) == (MaxPop1)) /* hysteresis = 0 */ \ + { \ + Pjll_t PjllRaw = (Pjll_t) (Pjp->jp_Addr); \ + Pjv_t PjvnewRaw; \ + Pjv_t Pjvnew; \ + \ + if ((PjvnewRaw = j__udyLAllocJV(pop1 - 1, Pjpm)) \ + == (Pjv_t) NULL) return(-1); \ + JUDYLCODE(Pjvnew = P_JV(PjvnewRaw);) \ + \ + DeleteCopy((LeafType) (Pjp->jp_LIndex), Pleaf, pop1, offset, cIS); \ + JU_DELETECOPY(Pjvnew, Pjv, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted((Pjll_t) (Pjp->jp_LIndex), pop1-1, cIS);) \ + FreeLeaf(PjllRaw, pop1, Pjpm); \ + Pjp->jp_Addr = (Word_t) PjvnewRaw; \ + Pjp->jp_Type = (BaseJPType) - 2 + (MaxPop1); \ + return(1); \ + } + +// A complicating factor for JudyL & 32-bit is that Leaf2..3, and for JudyL & +// 64-bit Leaf 4..7, go directly to an Immed*_01, where the value is stored in +// jp_Addr and not in a separate LeafV. For efficiency, use the following +// macro in cases where it can apply; it is rigged to do the right thing. +// Unfortunately, this requires the calling code to "know" the transition table +// and call the right macro. +// +// This variant compresses a Leaf* with pop1 = 2 into a JPIMMED_*_01: + +#define JU_LEAF_TOIMMED_01(cIS,LeafType,MaxPop1,ignore,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) \ + \ + assert(pop1 > (MaxPop1)); \ + \ + if ((pop1 - 1) == (MaxPop1)) /* hysteresis = 0 */ \ + { \ + Pjll_t PjllRaw = (Pjll_t) (Pjp->jp_Addr); \ + ToImmed(cIS, SearchLeaf, CopyPIndex); \ + FreeLeaf(PjllRaw, pop1, Pjpm); \ + Pjp->jp_Type = (Immed01JPType); \ + return(1); \ + } +#endif // JUDYL + +// See comments above about these: +// +// Note: Here "23" means index size 2 or 3, and "47" means 4..7. + +#if (defined(JUDY1) || defined(JU_64BIT)) +#define JU_LEAF_TOIMMED_23(cIS,LeafType,MaxPop1,BaseJPType,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) \ + JU_LEAF_TOIMMED( cIS,LeafType,MaxPop1,BaseJPType,ignore1, \ + ignore2,ignore3,ignore4, \ + DeleteCopy,FreeLeaf) +#else // JUDYL && 32-bit +#define JU_LEAF_TOIMMED_23(cIS,LeafType,MaxPop1,BaseJPType,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) \ + JU_LEAF_TOIMMED_01(cIS,LeafType,MaxPop1,ignore,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) +#endif + +#ifdef JU_64BIT +#ifdef JUDY1 +#define JU_LEAF_TOIMMED_47(cIS,LeafType,MaxPop1,BaseJPType,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) \ + JU_LEAF_TOIMMED( cIS,LeafType,MaxPop1,BaseJPType,ignore1, \ + ignore2,ignore3,ignore4, \ + DeleteCopy,FreeLeaf) +#else // JUDYL && 64-bit +#define JU_LEAF_TOIMMED_47(cIS,LeafType,MaxPop1,BaseJPType,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) \ + JU_LEAF_TOIMMED_01(cIS,LeafType,MaxPop1,ignore,Immed01JPType, \ + ToImmed,SearchLeaf,CopyPIndex, \ + DeleteCopy,FreeLeaf) +#endif // JUDYL +#endif // JU_64BIT + +// Compress a Leaf* in place: +// +// Here hysteresis = 0 (no memory is wasted). Variables pop1, Pleaf, and +// offset, and for JudyL, Pjv, are in the context. + +#ifdef JUDY1 +#define JU_LEAF_INPLACE(cIS,GrowInPlace,DeleteInPlace) \ + if (GrowInPlace(pop1 - 1)) /* hysteresis = 0 */ \ + { \ + DeleteInPlace(Pleaf, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted(Pleaf, pop1 - 1, cIS);) \ + return(1); \ + } +#else +#define JU_LEAF_INPLACE(cIS,GrowInPlace,DeleteInPlace) \ + if (GrowInPlace(pop1 - 1)) /* hysteresis = 0 */ \ + { \ + DeleteInPlace(Pleaf, pop1, offset, cIS); \ +/**/ JU_DELETEINPLACE(Pjv, pop1, offset, ignore); \ + DBGCODE(JudyCheckSorted(Pleaf, pop1 - 1, cIS);) \ + return(1); \ + } +#endif + +// Compress a Leaf* into a smaller memory object of the same JP type: +// +// Variables PjllnewRaw, Pjllnew, Pleafpop1, Pjpm, PleafRaw, Pleaf, and offset +// are in the context. + +#ifdef JUDY1 + +#define JU_LEAF_SHRINK(cIS,LeafType,DeleteCopy,Alloc,FreeLeaf,ValueArea) \ + if ((PjllnewRaw = Alloc(pop1 - 1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ + DeleteCopy((LeafType) Pjllnew, Pleaf, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted(Pjllnew, pop1 - 1, cIS);) \ + FreeLeaf(PleafRaw, pop1, Pjpm); \ + Pjp->jp_Addr = (Word_t) PjllnewRaw; \ + return(1) + +#else // JUDYL + +#define JU_LEAF_SHRINK(cIS,LeafType,DeleteCopy,Alloc,FreeLeaf,ValueArea) \ + { \ +/**/ Pjv_t Pjvnew; \ + \ + if ((PjllnewRaw = Alloc(pop1 - 1, Pjpm)) == 0) return(-1); \ + Pjllnew = P_JLL(PjllnewRaw); \ +/**/ Pjvnew = ValueArea(Pjllnew, pop1 - 1); \ + DeleteCopy((LeafType) Pjllnew, Pleaf, pop1, offset, cIS); \ +/**/ JU_DELETECOPY(Pjvnew, Pjv, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted(Pjllnew, pop1 - 1, cIS);) \ + FreeLeaf(PleafRaw, pop1, Pjpm); \ + Pjp->jp_Addr = (Word_t) PjllnewRaw; \ + return(1); \ + } +#endif // JUDYL + +// Overall common code for Leaf* deletion handling: +// +// See if the leaf can be: +// - (de)compressed to one a level higher (JU_LEAF_UPLEVEL()), or if not, +// - compressed to an Immediate JP (JU_LEAF_TOIMMED()), or if not, +// - shrunk in place (JU_LEAF_INPLACE()), or if none of those, then +// - shrink the leaf to a smaller chunk of memory (JU_LEAF_SHRINK()). +// +// Variables Pjp, pop1, Index, and offset are in the context. +// The *Up parameters refer to a leaf one level up, if there is any. + +#define JU_LEAF(cIS, \ + UpLevel, \ + LeafTypeUp,MaxPop1Up,LeafJPTypeUp,LeafToLeaf, \ + AllocUp,ValueAreaUp, \ + LeafToImmed,ToImmed,CopyPIndex, \ + LeafType,ImmedMaxPop1,ImmedBaseJPType,Immed01JPType, \ + SearchLeaf,GrowInPlace,DeleteInPlace,DeleteCopy, \ + Alloc,FreeLeaf,ValueArea) \ + { \ + Pjll_t PleafRaw; \ + LeafType Pleaf; \ + \ + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, cIS)); \ + assert(ParentLevel > (cIS)); \ + \ + PleafRaw = (Pjll_t) (Pjp->jp_Addr); \ + Pleaf = (LeafType) P_JLL(PleafRaw); \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + \ + UpLevel(cIS, LeafTypeUp, MaxPop1Up, LeafJPTypeUp, \ + LeafToLeaf, AllocUp, ValueAreaUp); \ + \ + offset = SearchLeaf(Pleaf, pop1, Index); \ + assert(offset >= 0); /* Index must be valid */ \ + JUDYLCODE(Pjv = ValueArea(Pleaf, pop1);) \ + \ + LeafToImmed(cIS, LeafType, ImmedMaxPop1, \ + ImmedBaseJPType, Immed01JPType, \ + ToImmed, SearchLeaf, CopyPIndex, \ + DeleteCopy, FreeLeaf); \ + \ + JU_LEAF_INPLACE(cIS, GrowInPlace, DeleteInPlace); \ + \ + JU_LEAF_SHRINK(cIS, LeafType, DeleteCopy, Alloc, FreeLeaf, \ + ValueArea); \ + } + +// END OF MACROS, START OF CASES: +// +// (*) Leaf1 [[ => 1_15..08 ] => 1_07 => ... => 1_04 ] => 1_03 => 1_02 => 1_01 + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + JU_LEAF(1, + JU_LEAF_UPLEVEL, uint16_t *, cJU_LEAF2_MAXPOP1, cJU_JPLEAF2, + j__udyLeaf1ToLeaf2, j__udyAllocJLL2, JL_LEAF2VALUEAREA, + JU_LEAF_TOIMMED, ignore, ignore, + uint8_t *, cJU_IMMED1_MAXPOP1, + cJU_JPIMMED_1_02, cJU_JPIMMED_1_01, j__udySearchLeaf1, + JU_LEAF1GROWINPLACE, JU_DELETEINPLACE, JU_DELETECOPY, + j__udyAllocJLL1, j__udyFreeJLL1, JL_LEAF1VALUEAREA); +#endif + +// A complicating factor is that for JudyL & 32-bit, a Leaf2 must go directly +// to an Immed 2_01 and a Leaf3 must go directly to an Immed 3_01: +// +// Leaf2 [[ => 2_07..04 ] => 2_03 => 2_02 ] => 2_01 +// Leaf3 [[ => 3_05..03 ] => 3_02 ] => 3_01 +// +// Hence use JU_LEAF_TOIMMED_23 instead of JU_LEAF_TOIMMED in the cases below, +// and also the parameters ToImmed and, for odd index sizes, CopyPIndex, are +// required. + + case cJU_JPLEAF2: + + JU_LEAF(2, + JU_LEAF_UPLEVEL, uint8_t *, cJU_LEAF3_MAXPOP1, cJU_JPLEAF3, + j__udyLeaf2ToLeaf3, j__udyAllocJLL3, JL_LEAF3VALUEAREA, + JU_LEAF_TOIMMED_23, JU_TOIMMED_01_EVEN, ignore, + uint16_t *, cJU_IMMED2_MAXPOP1, + cJU_JPIMMED_2_02, cJU_JPIMMED_2_01, j__udySearchLeaf2, + JU_LEAF2GROWINPLACE, JU_DELETEINPLACE, JU_DELETECOPY, + j__udyAllocJLL2, j__udyFreeJLL2, JL_LEAF2VALUEAREA); + +// On 32-bit there is no transition to "uplevel" for a Leaf3, so use +// JU_LEAF_UPLEVEL64 instead of JU_LEAF_UPLEVEL: + + case cJU_JPLEAF3: + + JU_LEAF(3, + JU_LEAF_UPLEVEL64, uint32_t *, cJU_LEAF4_MAXPOP1, + cJU_JPLEAF4, + j__udyLeaf3ToLeaf4, j__udyAllocJLL4, JL_LEAF4VALUEAREA, + JU_LEAF_TOIMMED_23, + JU_TOIMMED_01_ODD, JU_COPY3_PINDEX_TO_LONG, + uint8_t *, cJU_IMMED3_MAXPOP1, + cJU_JPIMMED_3_02, cJU_JPIMMED_3_01, j__udySearchLeaf3, + JU_LEAF3GROWINPLACE, JU_DELETEINPLACE_ODD, + JU_DELETECOPY_ODD, + j__udyAllocJLL3, j__udyFreeJLL3, JL_LEAF3VALUEAREA); + +#ifdef JU_64BIT + +// A complicating factor is that for JudyL & 64-bit, a Leaf[4-7] must go +// directly to an Immed [4-7]_01: +// +// Leaf4 [[ => 4_03..02 ]] => 4_01 +// Leaf5 [[ => 5_03..02 ]] => 5_01 +// Leaf6 [[ => 6_02 ]] => 6_01 +// Leaf7 [[ => 7_02 ]] => 7_01 +// +// Hence use JU_LEAF_TOIMMED_47 instead of JU_LEAF_TOIMMED in the cases below. + + case cJU_JPLEAF4: + + JU_LEAF(4, + JU_LEAF_UPLEVEL, uint8_t *, cJU_LEAF5_MAXPOP1, cJU_JPLEAF5, + j__udyLeaf4ToLeaf5, j__udyAllocJLL5, JL_LEAF5VALUEAREA, + JU_LEAF_TOIMMED_47, JU_TOIMMED_01_EVEN, ignore, + uint32_t *, cJU_IMMED4_MAXPOP1, + cJ1_JPIMMED_4_02, cJU_JPIMMED_4_01, j__udySearchLeaf4, + JU_LEAF4GROWINPLACE, JU_DELETEINPLACE, JU_DELETECOPY, + j__udyAllocJLL4, j__udyFreeJLL4, JL_LEAF4VALUEAREA); + + case cJU_JPLEAF5: + + JU_LEAF(5, + JU_LEAF_UPLEVEL, uint8_t *, cJU_LEAF6_MAXPOP1, cJU_JPLEAF6, + j__udyLeaf5ToLeaf6, j__udyAllocJLL6, JL_LEAF6VALUEAREA, + JU_LEAF_TOIMMED_47, + JU_TOIMMED_01_ODD, JU_COPY5_PINDEX_TO_LONG, + uint8_t *, cJU_IMMED5_MAXPOP1, + cJ1_JPIMMED_5_02, cJU_JPIMMED_5_01, j__udySearchLeaf5, + JU_LEAF5GROWINPLACE, JU_DELETEINPLACE_ODD, + JU_DELETECOPY_ODD, + j__udyAllocJLL5, j__udyFreeJLL5, JL_LEAF5VALUEAREA); + + case cJU_JPLEAF6: + + JU_LEAF(6, + JU_LEAF_UPLEVEL, uint8_t *, cJU_LEAF7_MAXPOP1, cJU_JPLEAF7, + j__udyLeaf6ToLeaf7, j__udyAllocJLL7, JL_LEAF7VALUEAREA, + JU_LEAF_TOIMMED_47, + JU_TOIMMED_01_ODD, JU_COPY6_PINDEX_TO_LONG, + uint8_t *, cJU_IMMED6_MAXPOP1, + cJ1_JPIMMED_6_02, cJU_JPIMMED_6_01, j__udySearchLeaf6, + JU_LEAF6GROWINPLACE, JU_DELETEINPLACE_ODD, + JU_DELETECOPY_ODD, + j__udyAllocJLL6, j__udyFreeJLL6, JL_LEAF6VALUEAREA); + +// There is no transition to "uplevel" for a Leaf7, so use JU_LEAF_UPLEVEL_NONE +// instead of JU_LEAF_UPLEVEL, and ignore all of the parameters to that macro: + + case cJU_JPLEAF7: + + JU_LEAF(7, + JU_LEAF_UPLEVEL_NONE, ignore1, ignore2, ignore3, ignore4, + ignore5, ignore6, + JU_LEAF_TOIMMED_47, + JU_TOIMMED_01_ODD, JU_COPY7_PINDEX_TO_LONG, + uint8_t *, cJU_IMMED7_MAXPOP1, + cJ1_JPIMMED_7_02, cJU_JPIMMED_7_01, j__udySearchLeaf7, + JU_LEAF7GROWINPLACE, JU_DELETEINPLACE_ODD, + JU_DELETECOPY_ODD, + j__udyAllocJLL7, j__udyFreeJLL7, JL_LEAF7VALUEAREA); +#endif // JU_64BIT + + +// **************************************************************************** +// BITMAP LEAF: + + case cJU_JPLEAF_B1: + { +#ifdef JUDYL + Pjv_t PjvnewRaw; // new value area. + Pjv_t Pjvnew; + Word_t subexp; // 1 of 8 subexpanses in bitmap. + Pjlb_t Pjlb; // pointer to bitmap part of the leaf. + BITMAPL_t bitmap; // for one subexpanse. + BITMAPL_t bitmask; // bit set for Indexs digit. +#endif + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, 1)); + assert(ParentLevel > 1); + // valid Index: + assert(JU_BITMAPTESTL(P_JLB(Pjp->jp_Addr), Index)); + + pop1 = JU_JPLEAF_POP0(Pjp) + 1; + +// Like a Leaf1, see if its under a narrow pointer and can become a Leaf2 +// (hysteresis = 1): + + JU_LEAF_UPLEVEL(1, uint16_t *, cJU_LEAF2_MAXPOP1, cJU_JPLEAF2, + j__udyLeaf1ToLeaf2, j__udyAllocJLL2, + JL_LEAF2VALUEAREA); + +#if (defined(JUDY1) && defined(JU_64BIT)) + +// Handle the unusual special case, on Judy1 64-bit only, where a LeafB1 goes +// directly to a JPIMMED_1_15; as described in comments in Judy1.h and +// JudyIns.c. Copy 1-byte indexes from old LeafB1 to the Immed: + + if ((pop1 - 1) == cJU_IMMED1_MAXPOP1) // hysteresis = 0. + { + Pjlb_t PjlbRaw; // bitmap in old leaf. + Pjlb_t Pjlb; + uint8_t * Pleafnew; // JPIMMED as a pointer. + Word_t ldigit; // larger than uint8_t. + + PjlbRaw = (Pjlb_t) (Pjp->jp_Addr); + Pjlb = P_JLB(PjlbRaw); + Pleafnew = Pjp->jp_1Index; + + JU_BITMAPCLEARL(Pjlb, Index); // unset Indexs bit. + +// TBD: This is very slow, there must be a better way: + + for (ldigit = 0; ldigit < cJU_BRANCHUNUMJPS; ++ldigit) + { + if (JU_BITMAPTESTL(Pjlb, ldigit)) + { + *Pleafnew++ = ldigit; + assert(Pleafnew - (Pjp->jp_1Index) + <= cJU_IMMED1_MAXPOP1); + } + } + + DBGCODE(JudyCheckSorted((Pjll_t) (Pjp->jp_1Index), + cJU_IMMED1_MAXPOP1, 1);) + j__udyFreeJLB1(PjlbRaw, Pjpm); + + Pjp->jp_Type = cJ1_JPIMMED_1_15; + return(1); + } + +#else // (JUDYL || (! JU_64BIT)) + +// Compress LeafB1 to a Leaf1: +// +// Note: 4.37 of this file contained alternate code for Judy1 only that simply +// cleared the bit and allowed the LeafB1 to go below cJU_LEAF1_MAXPOP1. This +// was the ONLY case where a malloc failure was not fatal; however, it violated +// the critical assumption that the tree is always kept in least-compressed +// form. + + if (pop1 == cJU_LEAF1_MAXPOP1) // hysteresis = 1. + { + if (j__udyLeafB1ToLeaf1(Pjp, Pjpm) == -1) return(-1); + goto ContinueDelWalk; // delete Index in new Leaf1. + } +#endif // (JUDYL || (! JU_64BIT)) + +#ifdef JUDY1 + // unset Indexs bit: + + JU_BITMAPCLEARL(P_JLB(Pjp->jp_Addr), Index); +#else // JUDYL + +// This is very different from Judy1 because of the need to manage the value +// area: +// +// Get last byte to decode from Index, and pointer to bitmap leaf: + + digit = JU_DIGITATSTATE(Index, 1); + Pjlb = P_JLB(Pjp->jp_Addr); + +// Prepare additional values: + + subexp = digit / cJU_BITSPERSUBEXPL; // which subexpanse. + bitmap = JU_JLB_BITMAP(Pjlb, subexp); // subexps 32-bit map. + PjvRaw = JL_JLB_PVALUE(Pjlb, subexp); // corresponding values. + Pjv = P_JV(PjvRaw); + bitmask = JU_BITPOSMASKL(digit); // mask for Index. + + assert(bitmap & bitmask); // Index must be valid. + + if (bitmap == cJU_FULLBITMAPL) // full bitmap, take shortcut: + { + pop1 = cJU_BITSPERSUBEXPL; + offset = digit % cJU_BITSPERSUBEXPL; + } + else // compute subexpanse pop1 and value area offset: + { + pop1 = j__udyCountBitsL(bitmap); + offset = j__udyCountBitsL(bitmap & (bitmask - 1)); + } + +// Handle solitary Index remaining in subexpanse: + + if (pop1 == 1) + { + j__udyLFreeJV(PjvRaw, 1, Pjpm); + + JL_JLB_PVALUE(Pjlb, subexp) = (Pjv_t) NULL; + JU_JLB_BITMAP(Pjlb, subexp) = 0; + + return(1); + } + +// Shrink value area in place or move to a smaller value area: + + if (JL_LEAFVGROWINPLACE(pop1 - 1)) // hysteresis = 0. + { + JU_DELETEINPLACE(Pjv, pop1, offset, ignore); + } + else + { + if ((PjvnewRaw = j__udyLAllocJV(pop1 - 1, Pjpm)) + == (Pjv_t) NULL) return(-1); + Pjvnew = P_JV(PjvnewRaw); + + JU_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore); + j__udyLFreeJV(PjvRaw, pop1, Pjpm); + JL_JLB_PVALUE(Pjlb, subexp) = (Pjv_t) PjvnewRaw; + } + + JU_JLB_BITMAP(Pjlb, subexp) ^= bitmask; // clear Indexs bit. + +#endif // JUDYL + + return(1); + + } // case. + + +#ifdef JUDY1 + +// **************************************************************************** +// FULL POPULATION LEAF: +// +// Convert to a LeafB1 and delete the index. Hysteresis = 0; none is possible. +// +// Note: Earlier the second assertion below said, "== 2", but in fact the +// parent could be at a higher level if a fullpop is under a narrow pointer. + + case cJ1_JPFULLPOPU1: + { + Pjlb_t PjlbRaw; + Pjlb_t Pjlb; + Word_t subexp; + + assert(! JU_DCDNOTMATCHINDEX(Index, Pjp, 2)); + assert(ParentLevel > 1); // see above. + + if ((PjlbRaw = j__udyAllocJLB1(Pjpm)) == (Pjlb_t) NULL) + return(-1); + Pjlb = P_JLB(PjlbRaw); + +// Fully populate the leaf, then unset Indexs bit: + + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + JU_JLB_BITMAP(Pjlb, subexp) = cJU_FULLBITMAPL; + + JU_BITMAPCLEARL(Pjlb, Index); + + Pjp->jp_Addr = (Word_t) PjlbRaw; + Pjp->jp_Type = cJU_JPLEAF_B1; + + return(1); + } +#endif // JUDY1 + + +// **************************************************************************** +// IMMEDIATE JP: +// +// If theres just the one Index in the Immed, convert the JP to a JPNULL* +// (should only happen in a BranchU); otherwise delete the Index from the +// Immed. See the state transitions table elsewhere in this file for a summary +// of which Immed types must be handled. Hysteresis = 0; none is possible with +// Immeds. +// +// MACROS FOR COMMON CODE: +// +// Single Index remains in cJU_JPIMMED_*_01; convert JP to null: +// +// Variables Pjp and parentJPtype are in the context. +// +// Note: cJU_JPIMMED_*_01 should only be encountered in BranchUs, not in +// BranchLs or BranchBs (where its improper to merely modify the JP to be a +// null JP); that is, BranchL and BranchB code should have already handled +// any cJU_JPIMMED_*_01 by different means. + +#define JU_IMMED_01(NewJPType,ParentJPType) \ + \ + assert(parentJPtype == (ParentJPType)); \ + assert(JU_JPDCDPOP0(Pjp) == JU_TRIMTODCDSIZE(Index)); \ + JU_JPSETADT(Pjp, 0, 0, NewJPType); \ + return(1) + +// Convert cJ*_JPIMMED_*_02 to cJU_JPIMMED_*_01: +// +// Move the undeleted Index, whichever does not match the least bytes of Index, +// from undecoded-bytes-only (in jp_1Index or jp_LIndex as appropriate) to +// jp_DcdPopO (full-field). Pjp, Index, and offset are in the context. + +#define JU_IMMED_02(cIS,LeafType,NewJPType) \ + { \ + LeafType Pleaf; \ + \ + assert((ParentLevel - 1) == (cIS)); \ + JUDY1CODE(Pleaf = (LeafType) (Pjp->jp_1Index);) \ + JUDYLCODE(Pleaf = (LeafType) (Pjp->jp_LIndex);) \ + JUDYLCODE(PjvRaw = (Pjv_t) (Pjp->jp_Addr);) \ + JUDYLCODE(Pjv = P_JV(PjvRaw);) \ + JU_TOIMMED_01_EVEN(cIS, ignore, ignore); \ + JUDYLCODE(j__udyLFreeJV(PjvRaw, 2, Pjpm);) \ + Pjp->jp_Type = (NewJPType); \ + return(1); \ + } + +#if (defined(JUDY1) || defined(JU_64BIT)) + +// Variation for "odd" cJ*_JPIMMED_*_02 JP types, which are very different from +// "even" types because they use leaf search code and odd-copy macros: +// +// Note: JudyL 32-bit has no "odd" JPIMMED_*_02 types. + +#define JU_IMMED_02_ODD(cIS,NewJPType,SearchLeaf,CopyPIndex) \ + { \ + uint8_t * Pleaf; \ + \ + assert((ParentLevel - 1) == (cIS)); \ + JUDY1CODE(Pleaf = (uint8_t *) (Pjp->jp_1Index);) \ + JUDYLCODE(Pleaf = (uint8_t *) (Pjp->jp_LIndex);) \ + JUDYLCODE(PjvRaw = (Pjv_t) (Pjp->jp_Addr);) \ + JUDYLCODE(Pjv = P_JV(PjvRaw);) \ + JU_TOIMMED_01_ODD(cIS, SearchLeaf, CopyPIndex); \ + JUDYLCODE(j__udyLFreeJV(PjvRaw, 2, Pjpm);) \ + Pjp->jp_Type = (NewJPType); \ + return(1); \ + } +#endif // (JUDY1 || JU_64BIT) + +// Core code for deleting one Index (and for JudyL, its value area) from a +// larger Immed: +// +// Variables Pleaf, pop1, and offset are in the context. + +#ifdef JUDY1 +#define JU_IMMED_DEL(cIS,DeleteInPlace) \ + DeleteInPlace(Pleaf, pop1, offset, cIS); \ + DBGCODE(JudyCheckSorted(Pleaf, pop1 - 1, cIS);) + +#else // JUDYL + +// For JudyL the value area might need to be shrunk: + +#define JU_IMMED_DEL(cIS,DeleteInPlace) \ + \ + if (JL_LEAFVGROWINPLACE(pop1 - 1)) /* hysteresis = 0 */ \ + { \ + DeleteInPlace( Pleaf, pop1, offset, cIS); \ + JU_DELETEINPLACE(Pjv, pop1, offset, ignore); \ + DBGCODE(JudyCheckSorted(Pleaf, pop1 - 1, cIS);) \ + } \ + else \ + { \ + Pjv_t PjvnewRaw; \ + Pjv_t Pjvnew; \ + \ + if ((PjvnewRaw = j__udyLAllocJV(pop1 - 1, Pjpm)) \ + == (Pjv_t) NULL) return(-1); \ + Pjvnew = P_JV(PjvnewRaw); \ + \ + DeleteInPlace(Pleaf, pop1, offset, cIS); \ + JU_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore); \ + DBGCODE(JudyCheckSorted(Pleaf, pop1 - 1, cIS);) \ + j__udyLFreeJV(PjvRaw, pop1, Pjpm); \ + \ + (Pjp->jp_Addr) = (Word_t) PjvnewRaw; \ + } +#endif // JUDYL + +// Delete one Index from a larger Immed where no restructuring is required: +// +// Variables pop1, Pjp, offset, and Index are in the context. + +#define JU_IMMED(cIS,LeafType,BaseJPType,SearchLeaf,DeleteInPlace) \ + { \ + LeafType Pleaf; \ + \ + assert((ParentLevel - 1) == (cIS)); \ + JUDY1CODE(Pleaf = (LeafType) (Pjp->jp_1Index);) \ + JUDYLCODE(Pleaf = (LeafType) (Pjp->jp_LIndex);) \ + JUDYLCODE(PjvRaw = (Pjv_t) (Pjp->jp_Addr);) \ + JUDYLCODE(Pjv = P_JV(PjvRaw);) \ + pop1 = (JU_JPTYPE(Pjp)) - (BaseJPType) + 2; \ + offset = SearchLeaf(Pleaf, pop1, Index); \ + assert(offset >= 0); /* Index must be valid */ \ + \ + JU_IMMED_DEL(cIS, DeleteInPlace); \ + --(Pjp->jp_Type); \ + return(1); \ + } + + +// END OF MACROS, START OF CASES: + +// Single Index remains in Immed; convert JP to null: + + case cJU_JPIMMED_1_01: JU_IMMED_01(cJU_JPNULL1, cJU_JPBRANCH_U2); + case cJU_JPIMMED_2_01: JU_IMMED_01(cJU_JPNULL2, cJU_JPBRANCH_U3); +#ifndef JU_64BIT + case cJU_JPIMMED_3_01: JU_IMMED_01(cJU_JPNULL3, cJU_JPBRANCH_U); +#else + case cJU_JPIMMED_3_01: JU_IMMED_01(cJU_JPNULL3, cJU_JPBRANCH_U4); + case cJU_JPIMMED_4_01: JU_IMMED_01(cJU_JPNULL4, cJU_JPBRANCH_U5); + case cJU_JPIMMED_5_01: JU_IMMED_01(cJU_JPNULL5, cJU_JPBRANCH_U6); + case cJU_JPIMMED_6_01: JU_IMMED_01(cJU_JPNULL6, cJU_JPBRANCH_U7); + case cJU_JPIMMED_7_01: JU_IMMED_01(cJU_JPNULL7, cJU_JPBRANCH_U); +#endif + +// Multiple Indexes remain in the Immed JP; delete the specified Index: + + case cJU_JPIMMED_1_02: + + JU_IMMED_02(1, uint8_t *, cJU_JPIMMED_1_01); + + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + JU_IMMED(1, uint8_t *, cJU_JPIMMED_1_02, + j__udySearchLeaf1, JU_DELETEINPLACE); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + + JU_IMMED_02(2, uint16_t *, cJU_JPIMMED_2_01); + + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + JU_IMMED(2, uint16_t *, cJU_JPIMMED_2_02, + j__udySearchLeaf2, JU_DELETEINPLACE); + + case cJU_JPIMMED_3_02: + + JU_IMMED_02_ODD(3, cJU_JPIMMED_3_01, + j__udySearchLeaf3, JU_COPY3_PINDEX_TO_LONG); + +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: + + JU_IMMED(3, uint8_t *, cJU_JPIMMED_3_02, + j__udySearchLeaf3, JU_DELETEINPLACE_ODD); + + case cJ1_JPIMMED_4_02: + + JU_IMMED_02(4, uint32_t *, cJU_JPIMMED_4_01); + + case cJ1_JPIMMED_4_03: + + JU_IMMED(4, uint32_t *, cJ1_JPIMMED_4_02, + j__udySearchLeaf4, JU_DELETEINPLACE); + + case cJ1_JPIMMED_5_02: + + JU_IMMED_02_ODD(5, cJU_JPIMMED_5_01, + j__udySearchLeaf5, JU_COPY5_PINDEX_TO_LONG); + + case cJ1_JPIMMED_5_03: + + JU_IMMED(5, uint8_t *, cJ1_JPIMMED_5_02, + j__udySearchLeaf5, JU_DELETEINPLACE_ODD); + + case cJ1_JPIMMED_6_02: + + JU_IMMED_02_ODD(6, cJU_JPIMMED_6_01, + j__udySearchLeaf6, JU_COPY6_PINDEX_TO_LONG); + + case cJ1_JPIMMED_7_02: + + JU_IMMED_02_ODD(7, cJU_JPIMMED_7_01, + j__udySearchLeaf7, JU_COPY7_PINDEX_TO_LONG); + +#endif // (JUDY1 && JU_64BIT) + + +// **************************************************************************** +// INVALID JP TYPE: + + default: JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); return(-1); + + } // switch + + +// PROCESS JP -- RECURSIVELY: +// +// For non-Immed JP types, if successful, post-decrement the population count +// at this level, or collapse a BranchL if necessary by copying the remaining +// JP in the BranchL to the parent (hysteresis = 0), which implicitly creates a +// narrow pointer if there was not already one in the hierarchy. + + assert(level); + retcode = j__udyDelWalk(Pjp, Index, level, Pjpm); + assert(retcode != 0); // should never happen. + + if ((JU_JPTYPE(Pjp)) < cJU_JPIMMED_1_01) // not an Immed. + { + switch (retcode) + { + case 1: + { + jp_t JP = *Pjp; + Word_t DcdP0; + + DcdP0 = JU_JPDCDPOP0(Pjp) - 1; // decrement count. + JU_JPSETADT(Pjp, JP.jp_Addr, DcdP0, JU_JPTYPE(&JP)); + break; + } + case 2: // collapse BranchL to single JP; see above: + { + Pjbl_t PjblRaw = (Pjbl_t) (Pjp->jp_Addr); + Pjbl_t Pjbl = P_JBL(PjblRaw); + + *Pjp = Pjbl->jbl_jp[0]; + j__udyFreeJBL(PjblRaw, Pjpm); + retcode = 1; + } + } + } + + return(retcode); + +} // j__udyDelWalk() + + +// **************************************************************************** +// J U D Y 1 U N S E T +// J U D Y L D E L +// +// Main entry point. See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1Unset +#else +FUNCTION int JudyLDel +#endif + ( + PPvoid_t PPArray, // in which to delete. + Word_t Index, // to delete. + PJError_t PJError // optional, for returning error info. + ) +{ + Word_t pop1; // population of leaf. + int offset; // at which to delete Index. + JUDY1CODE(int retcode;) // return code from Judy1Test(). +JUDYLCODE(PPvoid_t PPvalue;) // pointer from JudyLGet(). + + +// CHECK FOR NULL ARRAY POINTER (error by caller): + + if (PPArray == (PPvoid_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPPARRAY); + return(JERRI); + } + + +// CHECK IF INDEX IS INVALID: +// +// If so, theres nothing to do. This saves a lot of time. Pass through +// PJError, if any, from the "get" function. + +#ifdef JUDY1 + if ((retcode = Judy1Test(*PPArray, Index, PJError)) == JERRI) + return (JERRI); + + if (retcode == 0) return(0); +#else + if ((PPvalue = JudyLGet(*PPArray, Index, PJError)) == PPJERR) + return (JERRI); + + if (PPvalue == (PPvoid_t) NULL) return(0); +#endif + + +// **************************************************************************** +// PROCESS TOP LEVEL (LEAFW) BRANCHES AND LEAVES: + +// **************************************************************************** +// LEAFW LEAF, OTHER SIZE: +// +// Shrink or convert the leaf as necessary. Hysteresis = 0; none is possible. + + if (JU_LEAFW_POP0(*PPArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + JUDYLCODE(Pjv_t Pjv;) // current value area. + JUDYLCODE(Pjv_t Pjvnew;) // value area in new leaf. + Pjlw_t Pjlw = P_JLW(*PPArray); // first word of leaf. + Pjlw_t Pjlwnew; // replacement leaf. + pop1 = Pjlw[0] + 1; // first word of leaf is pop0. + +// Delete single (last) Index from array: + + if (pop1 == 1) + { + j__udyFreeJLW(Pjlw, /* pop1 = */ 1, (Pjpm_t) NULL); + *PPArray = (Pvoid_t) NULL; + return(1); + } + +// Locate Index in compressible leaf: + + offset = j__udySearchLeafW(Pjlw + 1, pop1, Index); + assert(offset >= 0); // Index must be valid. + + JUDYLCODE(Pjv = JL_LEAFWVALUEAREA(Pjlw, pop1);) + +// Delete Index in-place: +// +// Note: "Grow in place from pop1 - 1" is the logical inverse of, "shrink in +// place from pop1." Also, Pjlw points to the count word, so skip that for +// doing the deletion. + + if (JU_LEAFWGROWINPLACE(pop1 - 1)) + { + JU_DELETEINPLACE(Pjlw + 1, pop1, offset, ignore); +#ifdef JUDYL // also delete from value area: + JU_DELETEINPLACE(Pjv, pop1, offset, ignore); +#endif + DBGCODE(JudyCheckSorted((Pjll_t) (Pjlw + 1), pop1 - 1, + cJU_ROOTSTATE);) + --(Pjlw[0]); // decrement population. + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + } + +// Allocate new leaf for use in either case below: + + Pjlwnew = j__udyAllocJLW(pop1 - 1); + JU_CHECKALLOC(Pjlw_t, Pjlwnew, JERRI); + +// Shrink to smaller LEAFW: +// +// Note: Skip the first word = pop0 in each leaf. + + Pjlwnew[0] = (pop1 - 1) - 1; + JU_DELETECOPY(Pjlwnew + 1, Pjlw + 1, pop1, offset, ignore); + +#ifdef JUDYL // also delete from value area: + Pjvnew = JL_LEAFWVALUEAREA(Pjlwnew, pop1 - 1); + JU_DELETECOPY(Pjvnew, Pjv, pop1, offset, ignore); +#endif + DBGCODE(JudyCheckSorted(Pjlwnew + 1, pop1 - 1, cJU_ROOTSTATE);) + + j__udyFreeJLW(Pjlw, pop1, (Pjpm_t) NULL); + +//// *PPArray = (Pvoid_t) Pjlwnew | cJU_LEAFW); + *PPArray = (Pvoid_t) Pjlwnew; + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + + } + else + + +// **************************************************************************** +// JRP BRANCH: +// +// Traverse through the JPM to do the deletion unless the population is small +// enough to convert immediately to a LEAFW. + + { + Pjpm_t Pjpm; + Pjp_t Pjp; // top-level JP to process. + Word_t digit; // in a branch. + JUDYLCODE(Pjv_t Pjv;) // to value area. + Pjlw_t Pjlwnew; // replacement leaf. + DBGCODE(Pjlw_t Pjlwnew_orig;) + + Pjpm = P_JPM(*PPArray); // top object in array (tree). + Pjp = &(Pjpm->jpm_JP); // next object (first branch or leaf). + + assert(((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_L) + || ((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_B) + || ((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_U)); + +// WALK THE TREE +// +// Note: Recursive code in j__udyDelWalk() knows how to collapse a lower-level +// BranchL containing a single JP into the parent JP as a narrow pointer, but +// the code here cant do that for a top-level BranchL. The result can be +// PArray -> JPM -> BranchL containing a single JP. This situation is +// unavoidable because a JPM cannot contain a narrow pointer; the BranchL is +// required in order to hold the top digit decoded, and it does not collapse to +// a LEAFW until the population is low enough. +// +// TBD: Should we add a topdigit field to JPMs so they can hold narrow +// pointers? + + if (j__udyDelWalk(Pjp, Index, cJU_ROOTSTATE, Pjpm) == -1) + { + JU_COPY_ERRNO(PJError, Pjpm); + return(JERRI); + } + + --(Pjpm->jpm_Pop0); // success; decrement total population. + + if ((Pjpm->jpm_Pop0 + 1) != cJU_LEAFW_MAXPOP1) + { + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + } + +// COMPRESS A BRANCH[LBU] TO A LEAFW: +// + Pjlwnew = j__udyAllocJLW(cJU_LEAFW_MAXPOP1); + JU_CHECKALLOC(Pjlw_t, Pjlwnew, JERRI); + +// Plug leaf into root pointer and set population count: + +//// *PPArray = (Pvoid_t) ((Word_t) Pjlwnew | cJU_LEAFW); + *PPArray = (Pvoid_t) Pjlwnew; +#ifdef JUDYL // prepare value area: + Pjv = JL_LEAFWVALUEAREA(Pjlwnew, cJU_LEAFW_MAXPOP1); +#endif + *Pjlwnew++ = cJU_LEAFW_MAXPOP1 - 1; // set pop0. + DBGCODE(Pjlwnew_orig = Pjlwnew;) + + switch (JU_JPTYPE(Pjp)) + { + +// JPBRANCH_L: Copy each JPs indexes to the new LEAFW and free the old +// branch: + + case cJU_JPBRANCH_L: + { + Pjbl_t PjblRaw = (Pjbl_t) (Pjp->jp_Addr); + Pjbl_t Pjbl = P_JBL(PjblRaw); + + for (offset = 0; offset < Pjbl->jbl_NumJPs; ++offset) + { + pop1 = j__udyLeafM1ToLeafW(Pjlwnew, JU_PVALUEPASS + (Pjbl->jbl_jp) + offset, + JU_DIGITTOSTATE(Pjbl->jbl_Expanse[offset], + cJU_BYTESPERWORD), + (Pvoid_t) Pjpm); + Pjlwnew += pop1; // advance through indexes. + JUDYLCODE(Pjv += pop1;) // advance through values. + } + j__udyFreeJBL(PjblRaw, Pjpm); + + assert(Pjlwnew == Pjlwnew_orig + cJU_LEAFW_MAXPOP1); + break; // delete Index from new LEAFW. + } + +// JPBRANCH_B: Copy each JPs indexes to the new LEAFW and free the old +// branch, including each JP subarray: + + case cJU_JPBRANCH_B: + { + Pjbb_t PjbbRaw = (Pjbb_t) (Pjp->jp_Addr); + Pjbb_t Pjbb = P_JBB(PjbbRaw); + Word_t subexp; // current subexpanse number. + BITMAPB_t bitmap; // portion for this subexpanse. + Pjp_t Pjp2Raw; // one subexpanses subarray. + Pjp_t Pjp2; + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + if ((bitmap = JU_JBB_BITMAP(Pjbb, subexp)) == 0) + continue; // skip empty subexpanse. + + digit = subexp * cJU_BITSPERSUBEXPB; + Pjp2Raw = JU_JBB_PJP(Pjbb, subexp); + Pjp2 = P_JP(Pjp2Raw); + assert(Pjp2 != (Pjp_t) NULL); + +// Walk through bits for all possible sub-subexpanses (digits); increment +// offset for each populated subexpanse; until no more set bits: + + for (offset = 0; bitmap != 0; bitmap >>= 1, ++digit) + { + if (! (bitmap & 1)) // skip empty sub-subexpanse. + continue; + + pop1 = j__udyLeafM1ToLeafW(Pjlwnew, JU_PVALUEPASS + Pjp2 + offset, + JU_DIGITTOSTATE(digit, cJU_BYTESPERWORD), + (Pvoid_t) Pjpm); + Pjlwnew += pop1; // advance through indexes. + JUDYLCODE(Pjv += pop1;) // advance through values. + ++offset; + } + j__udyFreeJBBJP(Pjp2Raw, /* pop1 = */ offset, Pjpm); + } + j__udyFreeJBB(PjbbRaw, Pjpm); + + assert(Pjlwnew == Pjlwnew_orig + cJU_LEAFW_MAXPOP1); + break; // delete Index from new LEAFW. + + } // case cJU_JPBRANCH_B. + + +// JPBRANCH_U: Copy each JPs indexes to the new LEAFW and free the old +// branch: + + case cJU_JPBRANCH_U: + { + Pjbu_t PjbuRaw = (Pjbu_t) (Pjp->jp_Addr); + Pjbu_t Pjbu = P_JBU(PjbuRaw); + Word_t ldigit; // larger than uint8_t. + + for (Pjp = Pjbu->jbu_jp, ldigit = 0; + ldigit < cJU_BRANCHUNUMJPS; + ++Pjp, ++ldigit) + { + +// Shortcuts, to save a little time for possibly big branches: + + if ((JU_JPTYPE(Pjp)) == cJU_JPNULLMAX) // skip null JP. + continue; + +// TBD: Should the following shortcut also be used in BranchL and BranchB +// code? + +#ifndef JU_64BIT + if ((JU_JPTYPE(Pjp)) == cJU_JPIMMED_3_01) +#else + if ((JU_JPTYPE(Pjp)) == cJU_JPIMMED_7_01) +#endif + { // single Immed: + *Pjlwnew++ = JU_DIGITTOSTATE(ldigit, cJU_BYTESPERWORD) + | JU_JPDCDPOP0(Pjp); // rebuild Index. +#ifdef JUDYL + *Pjv++ = Pjp->jp_Addr; // copy value area. +#endif + continue; + } + + pop1 = j__udyLeafM1ToLeafW(Pjlwnew, JU_PVALUEPASS + Pjp, JU_DIGITTOSTATE(ldigit, cJU_BYTESPERWORD), + (Pvoid_t) Pjpm); + Pjlwnew += pop1; // advance through indexes. + JUDYLCODE(Pjv += pop1;) // advance through values. + } + j__udyFreeJBU(PjbuRaw, Pjpm); + + assert(Pjlwnew == Pjlwnew_orig + cJU_LEAFW_MAXPOP1); + break; // delete Index from new LEAFW. + + } // case cJU_JPBRANCH_U. + + +// INVALID JP TYPE in jpm_t struct + + default: JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(JERRI); + + } // end switch on sub-JP type. + + DBGCODE(JudyCheckSorted((Pjll_t) Pjlwnew_orig, cJU_LEAFW_MAXPOP1, + cJU_ROOTSTATE);) + +// FREE JPM (no longer needed): + + j__udyFreeJPM(Pjpm, (Pjpm_t) NULL); + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + + } + /*NOTREACHED*/ + +} // Judy1Unset() / JudyLDel() diff --git a/libnetdata/libjudy/src/JudyL/JudyLFirst.c b/libnetdata/libjudy/src/JudyL/JudyLFirst.c new file mode 100644 index 000000000..aaf6639cf --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLFirst.c @@ -0,0 +1,213 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.12 $ $Source: /judy/src/JudyCommon/JudyFirst.c $ +// +// Judy*First[Empty]() and Judy*Last[Empty]() routines for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// These are inclusive versions of Judy*Next[Empty]() and Judy*Prev[Empty](). + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + + +// **************************************************************************** +// J U D Y 1 F I R S T +// J U D Y L F I R S T +// +// See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1First +#else +FUNCTION PPvoid_t JudyLFirst +#endif + ( + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError // optional, for returning error info. + ) +{ + if (PIndex == (PWord_t) NULL) // caller error: + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +#ifdef JUDY1 + switch (Judy1Test(PArray, *PIndex, PJError)) + { + case 1: return(1); // found *PIndex itself. + case 0: return(Judy1Next(PArray, PIndex, PJError)); + default: return(JERRI); + } +#else + { + PPvoid_t PValue; + + if ((PValue = JudyLGet(PArray, *PIndex, PJError)) == PPJERR) + return(PPJERR); + + if (PValue != (PPvoid_t) NULL) return(PValue); // found *PIndex. + + return(JudyLNext(PArray, PIndex, PJError)); + } +#endif + +} // Judy1First() / JudyLFirst() + + +// **************************************************************************** +// J U D Y 1 L A S T +// J U D Y L L A S T +// +// See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1Last( +#else +FUNCTION PPvoid_t JudyLLast( +#endif + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError) // optional, for returning error info. +{ + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); // caller error. + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +#ifdef JUDY1 + switch (Judy1Test(PArray, *PIndex, PJError)) + { + case 1: return(1); // found *PIndex itself. + case 0: return(Judy1Prev(PArray, PIndex, PJError)); + default: return(JERRI); + } +#else + { + PPvoid_t PValue; + + if ((PValue = JudyLGet(PArray, *PIndex, PJError)) == PPJERR) + return(PPJERR); + + if (PValue != (PPvoid_t) NULL) return(PValue); // found *PIndex. + + return(JudyLPrev(PArray, PIndex, PJError)); + } +#endif + +} // Judy1Last() / JudyLLast() + + +// **************************************************************************** +// J U D Y 1 F I R S T E M P T Y +// J U D Y L F I R S T E M P T Y +// +// See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1FirstEmpty( +#else +FUNCTION int JudyLFirstEmpty( +#endif + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError) // optional, for returning error info. +{ + if (PIndex == (PWord_t) NULL) // caller error: + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + return(JERRI); + } + +#ifdef JUDY1 + switch (Judy1Test(PArray, *PIndex, PJError)) + { + case 0: return(1); // found *PIndex itself. + case 1: return(Judy1NextEmpty(PArray, PIndex, PJError)); + default: return(JERRI); + } +#else + { + PPvoid_t PValue; + + if ((PValue = JudyLGet(PArray, *PIndex, PJError)) == PPJERR) + return(JERRI); + + if (PValue == (PPvoid_t) NULL) return(1); // found *PIndex. + + return(JudyLNextEmpty(PArray, PIndex, PJError)); + } +#endif + +} // Judy1FirstEmpty() / JudyLFirstEmpty() + + +// **************************************************************************** +// J U D Y 1 L A S T E M P T Y +// J U D Y L L A S T E M P T Y +// +// See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1LastEmpty( +#else +FUNCTION int JudyLLastEmpty( +#endif + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError) // optional, for returning error info. +{ + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); // caller error. + return(JERRI); + } + +#ifdef JUDY1 + switch (Judy1Test(PArray, *PIndex, PJError)) + { + case 0: return(1); // found *PIndex itself. + case 1: return(Judy1PrevEmpty(PArray, PIndex, PJError)); + default: return(JERRI); + } +#else + { + PPvoid_t PValue; + + if ((PValue = JudyLGet(PArray, *PIndex, PJError)) == PPJERR) + return(JERRI); + + if (PValue == (PPvoid_t) NULL) return(1); // found *PIndex. + + return(JudyLPrevEmpty(PArray, PIndex, PJError)); + } +#endif + +} // Judy1LastEmpty() / JudyLLastEmpty() diff --git a/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c b/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c new file mode 100644 index 000000000..34fac509e --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c @@ -0,0 +1,363 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.51 $ $Source: /judy/src/JudyCommon/JudyFreeArray.c $ +// +// Judy1FreeArray() and JudyLFreeArray() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// Return the number of bytes freed from the array. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +DBGCODE(extern void JudyCheckPop(Pvoid_t PArray);) + + +// **************************************************************************** +// J U D Y 1 F R E E A R R A Y +// J U D Y L F R E E A R R A Y +// +// See the Judy*(3C) manual entry for details. +// +// This code is written recursively, at least at first, because thats much +// simpler. Hope its fast enough. + +#ifdef JUDY1 +FUNCTION Word_t Judy1FreeArray +#else +FUNCTION Word_t JudyLFreeArray +#endif + ( + PPvoid_t PPArray, // array to free. + PJError_t PJError // optional, for returning error info. + ) +{ + jpm_t jpm; // local to accumulate free statistics. + +// CHECK FOR NULL POINTER (error by caller): + + if (PPArray == (PPvoid_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPPARRAY); + return(JERR); + } + + DBGCODE(JudyCheckPop(*PPArray);) + +// Zero jpm.jpm_Pop0 (meaning the array will be empty in a moment) for accurate +// logging in TRACEMI2. + + jpm.jpm_Pop0 = 0; // see above. + jpm.jpm_TotalMemWords = 0; // initialize memory freed. + +// Empty array: + + if (P_JLW(*PPArray) == (Pjlw_t) NULL) return(0); + +// PROCESS TOP LEVEL "JRP" BRANCHES AND LEAF: + + if (JU_LEAFW_POP0(*PPArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(*PPArray); // first word of leaf. + + j__udyFreeJLW(Pjlw, Pjlw[0] + 1, &jpm); + *PPArray = (Pvoid_t) NULL; // make an empty array. + return (-(jpm.jpm_TotalMemWords * cJU_BYTESPERWORD)); // see above. + } + else + +// Rootstate leaves: just free the leaf: + +// Common code for returning the amount of memory freed. +// +// Note: In a an ordinary LEAFW, pop0 = *PPArray[0]. +// +// Accumulate (negative) words freed, while freeing objects. +// Return the positive bytes freed. + + { + Pjpm_t Pjpm = P_JPM(*PPArray); + Word_t TotalMem = Pjpm->jpm_TotalMemWords; + + j__udyFreeSM(&(Pjpm->jpm_JP), &jpm); // recurse through tree. + j__udyFreeJPM(Pjpm, &jpm); + +// Verify the array was not corrupt. This means that amount of memory freed +// (which is negative) is equal to the initial amount: + + if (TotalMem + jpm.jpm_TotalMemWords) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + return(JERR); + } + + *PPArray = (Pvoid_t) NULL; // make an empty array. + return (TotalMem * cJU_BYTESPERWORD); + } + +} // Judy1FreeArray() / JudyLFreeArray() + + +// **************************************************************************** +// __ J U D Y F R E E S M +// +// Given a pointer to a JP, recursively visit and free (depth first) all nodes +// in a Judy array BELOW the JP, but not the JP itself. Accumulate in *Pjpm +// the total words freed (as a negative value). "SM" = State Machine. +// +// Note: Corruption is not detected at this level because during a FreeArray, +// if the code hasnt already core dumped, its better to remain silent, even +// if some memory has not been freed, than to bother the caller about the +// corruption. TBD: Is this true? If not, must list all legitimate JPNULL +// and JPIMMED above first, and revert to returning bool_t (see 4.34). + +FUNCTION void j__udyFreeSM( + Pjp_t Pjp, // top of Judy (top-state). + Pjpm_t Pjpm) // to return words freed. +{ + Word_t Pop1; + + switch (JU_JPTYPE(Pjp)) + { + +#ifdef JUDY1 + +// FULL EXPANSE -- nothing to free for this jp_Type. + + case cJ1_JPFULLPOPU1: + break; +#endif + +// JUDY BRANCH -- free the sub-tree depth first: + +// LINEAR BRANCH -- visit each JP in the JBLs list, then free the JBL: +// +// Note: There are no null JPs in a JBL. + + case cJU_JPBRANCH_L: + case cJU_JPBRANCH_L2: + case cJU_JPBRANCH_L3: +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + case cJU_JPBRANCH_L5: + case cJU_JPBRANCH_L6: + case cJU_JPBRANCH_L7: +#endif // JU_64BIT + { + Pjbl_t Pjbl = P_JBL(Pjp->jp_Addr); + Word_t offset; + + for (offset = 0; offset < Pjbl->jbl_NumJPs; ++offset) + j__udyFreeSM((Pjbl->jbl_jp) + offset, Pjpm); + + j__udyFreeJBL((Pjbl_t) (Pjp->jp_Addr), Pjpm); + break; + } + + +// BITMAP BRANCH -- visit each JP in the JBBs list based on the bitmap, also +// +// Note: There are no null JPs in a JBB. + + case cJU_JPBRANCH_B: + case cJU_JPBRANCH_B2: + case cJU_JPBRANCH_B3: +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + case cJU_JPBRANCH_B5: + case cJU_JPBRANCH_B6: + case cJU_JPBRANCH_B7: +#endif // JU_64BIT + { + Word_t subexp; + Word_t offset; + Word_t jpcount; + + Pjbb_t Pjbb = P_JBB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + + if (jpcount) + { + for (offset = 0; offset < jpcount; ++offset) + { + j__udyFreeSM(P_JP(JU_JBB_PJP(Pjbb, subexp)) + offset, + Pjpm); + } + j__udyFreeJBBJP(JU_JBB_PJP(Pjbb, subexp), jpcount, Pjpm); + } + } + j__udyFreeJBB((Pjbb_t) (Pjp->jp_Addr), Pjpm); + + break; + } + + +// UNCOMPRESSED BRANCH -- visit each JP in the JBU array, then free the JBU +// itself: +// +// Note: Null JPs are handled during recursion at a lower state. + + case cJU_JPBRANCH_U: + case cJU_JPBRANCH_U2: + case cJU_JPBRANCH_U3: +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: + case cJU_JPBRANCH_U5: + case cJU_JPBRANCH_U6: + case cJU_JPBRANCH_U7: +#endif // JU_64BIT + { + Word_t offset; + Pjbu_t Pjbu = P_JBU(Pjp->jp_Addr); + + for (offset = 0; offset < cJU_BRANCHUNUMJPS; ++offset) + j__udyFreeSM((Pjbu->jbu_jp) + offset, Pjpm); + + j__udyFreeJBU((Pjbu_t) (Pjp->jp_Addr), Pjpm); + break; + } + + +// -- Cases below here terminate and do not recurse. -- + + +// LINEAR LEAF -- just free the leaf; size is computed from jp_Type: +// +// Note: cJU_JPLEAF1 is a special case, see discussion in ../Judy1/Judy1.h + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL1((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; +#endif + + case cJU_JPLEAF2: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL2((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + + case cJU_JPLEAF3: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL3((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + +#ifdef JU_64BIT + case cJU_JPLEAF4: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL4((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + + case cJU_JPLEAF5: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL5((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + + case cJU_JPLEAF6: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL6((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + + case cJU_JPLEAF7: + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + j__udyFreeJLL7((Pjll_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; +#endif // JU_64BIT + + +// BITMAP LEAF -- free sub-expanse arrays of JPs, then free the JBB. + + case cJU_JPLEAF_B1: + { +#ifdef JUDYL + Word_t subexp; + Word_t jpcount; + Pjlb_t Pjlb = P_JLB(Pjp->jp_Addr); + +// Free the value areas in the bitmap leaf: + + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + { + jpcount = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + + if (jpcount) + j__udyLFreeJV(JL_JLB_PVALUE(Pjlb, subexp), jpcount, Pjpm); + } +#endif // JUDYL + + j__udyFreeJLB1((Pjlb_t) (Pjp->jp_Addr), Pjpm); + break; + + } // case cJU_JPLEAF_B1 + +#ifdef JUDYL + + +// IMMED*: +// +// For JUDYL, all non JPIMMED_*_01s have a LeafV which must be freed: + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#ifdef JU_64BIT + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif + Pop1 = JU_JPTYPE(Pjp) - cJU_JPIMMED_1_02 + 2; + j__udyLFreeJV((Pjv_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + +#ifdef JU_64BIT + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: + + Pop1 = JU_JPTYPE(Pjp) - cJU_JPIMMED_2_02 + 2; + j__udyLFreeJV((Pjv_t) (Pjp->jp_Addr), Pop1, Pjpm); + break; + + case cJU_JPIMMED_3_02: + j__udyLFreeJV((Pjv_t) (Pjp->jp_Addr), 2, Pjpm); + break; + +#endif // JU_64BIT +#endif // JUDYL + + +// OTHER JPNULL, JPIMMED, OR UNEXPECTED TYPE -- nothing to free for this type: +// +// Note: Lump together no-op and invalid JP types; see function header +// comments. + + default: break; + + } // switch (JU_JPTYPE(Pjp)) + +} // j__udyFreeSM() diff --git a/libnetdata/libjudy/src/JudyL/JudyLGet.c b/libnetdata/libjudy/src/JudyL/JudyLGet.c new file mode 100644 index 000000000..0bb9971cc --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLGet.c @@ -0,0 +1,1094 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.43 $ $Source: /judy/src/JudyCommon/JudyGet.c $ +// +// Judy1Test() and JudyLGet() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +#ifdef TRACEJPR // different macro name, for "retrieval" only. +#include "JudyPrintJP.c" +#endif + + +// **************************************************************************** +// J U D Y 1 T E S T +// J U D Y L G E T +// +// See the manual entry for details. Note support for "shortcut" entries to +// trees known to start with a JPM. + +#ifdef JUDY1 + +#ifdef JUDYGETINLINE +FUNCTION int j__udy1Test +#else +FUNCTION int Judy1Test +#endif + +#else // JUDYL + +#ifdef JUDYGETINLINE +FUNCTION PPvoid_t j__udyLGet +#else +FUNCTION PPvoid_t JudyLGet +#endif + +#endif // JUDYL + ( +#ifdef JUDYGETINLINE + Pvoid_t PArray, // from which to retrieve. + Word_t Index // to retrieve. +#else + Pcvoid_t PArray, // from which to retrieve. + Word_t Index, // to retrieve. + PJError_t PJError // optional, for returning error info. +#endif + ) +{ + Pjp_t Pjp; // current JP while walking the tree. + Pjpm_t Pjpm; // for global accounting. + uint8_t Digit; // byte just decoded from Index. + Word_t Pop1; // leaf population (number of indexes). + Pjll_t Pjll; // pointer to LeafL. + DBGCODE(uint8_t ParentJPType;) + +#ifndef JUDYGETINLINE + + if (PArray == (Pcvoid_t) NULL) // empty array. + { + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + } + +// **************************************************************************** +// PROCESS TOP LEVEL BRANCHES AND LEAF: + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + int posidx; // signed offset in leaf. + + Pop1 = Pjlw[0] + 1; + posidx = j__udySearchLeafW(Pjlw + 1, Pop1, Index); + + if (posidx >= 0) + { + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAFWVALUEAREA(Pjlw, Pop1) + posidx));) + } + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + } + +#endif // ! JUDYGETINLINE + + Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); // top branch is below JPM. + +// **************************************************************************** +// WALK THE JUDY TREE USING A STATE MACHINE: + +ContinueWalk: // for going down one level; come here with Pjp set. + +#ifdef TRACEJPR + JudyPrintJP(Pjp, "g", __LINE__); +#endif + switch (JU_JPTYPE(Pjp)) + { + +// Ensure the switch table starts at 0 for speed; otherwise more code is +// executed: + + case 0: goto ReturnCorrupt; // save a little code. + + +// **************************************************************************** +// JPNULL*: +// +// Note: These are legitimate in a BranchU (only) and do not constitute a +// fault. + + case cJU_JPNULL1: + case cJU_JPNULL2: + case cJU_JPNULL3: +#ifdef JU_64BIT + case cJU_JPNULL4: + case cJU_JPNULL5: + case cJU_JPNULL6: + case cJU_JPNULL7: +#endif + assert(ParentJPType >= cJU_JPBRANCH_U2); + assert(ParentJPType <= cJU_JPBRANCH_U); + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + + +// **************************************************************************** +// JPBRANCH_L*: +// +// Note: The use of JU_DCDNOTMATCHINDEX() in branches is not strictly +// required,since this can be done at leaf level, but it costs nothing to do it +// sooner, and it aborts an unnecessary traversal sooner. + + case cJU_JPBRANCH_L2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + Digit = JU_DIGITATSTATE(Index, 2); + goto JudyBranchL; + + case cJU_JPBRANCH_L3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + Digit = JU_DIGITATSTATE(Index, 3); + goto JudyBranchL; + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + Digit = JU_DIGITATSTATE(Index, 4); + goto JudyBranchL; + + case cJU_JPBRANCH_L5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + Digit = JU_DIGITATSTATE(Index, 5); + goto JudyBranchL; + + case cJU_JPBRANCH_L6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + Digit = JU_DIGITATSTATE(Index, 6); + goto JudyBranchL; + + case cJU_JPBRANCH_L7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Digit = JU_DIGITATSTATE(Index, 7); + goto JudyBranchL; + +#endif // JU_64BIT + + case cJU_JPBRANCH_L: + { + Pjbl_t Pjbl; + int posidx; + + Digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + +// Common code for all BranchLs; come here with Digit set: + +JudyBranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + + posidx = 0; + + do { + if (Pjbl->jbl_Expanse[posidx] == Digit) + { // found Digit; continue traversal: + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = Pjbl->jbl_jp + posidx; + goto ContinueWalk; + } + } while (++posidx != Pjbl->jbl_NumJPs); + + break; + } + + +// **************************************************************************** +// JPBRANCH_B*: + + case cJU_JPBRANCH_B2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + Digit = JU_DIGITATSTATE(Index, 2); + goto JudyBranchB; + + case cJU_JPBRANCH_B3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + Digit = JU_DIGITATSTATE(Index, 3); + goto JudyBranchB; + + +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + Digit = JU_DIGITATSTATE(Index, 4); + goto JudyBranchB; + + case cJU_JPBRANCH_B5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + Digit = JU_DIGITATSTATE(Index, 5); + goto JudyBranchB; + + case cJU_JPBRANCH_B6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + Digit = JU_DIGITATSTATE(Index, 6); + goto JudyBranchB; + + case cJU_JPBRANCH_B7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Digit = JU_DIGITATSTATE(Index, 7); + goto JudyBranchB; + +#endif // JU_64BIT + + case cJU_JPBRANCH_B: + { + Pjbb_t Pjbb; + Word_t subexp; // in bitmap, 0..7. + BITMAPB_t BitMap; // for one subexpanse. + BITMAPB_t BitMask; // bit in BitMap for Indexs Digit. + + Digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + +// Common code for all BranchBs; come here with Digit set: + +JudyBranchB: + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjbb = P_JBB(Pjp->jp_Addr); + subexp = Digit / cJU_BITSPERSUBEXPB; + + BitMap = JU_JBB_BITMAP(Pjbb, subexp); + Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp)); + + BitMask = JU_BITPOSMASKB(Digit); + +// No JP in subexpanse for Index => Index not found: + + if (! (BitMap & BitMask)) break; + +// Count JPs in the subexpanse below the one for Index: + + Pjp += j__udyCountBitsB(BitMap & (BitMask - 1)); + + goto ContinueWalk; + + } // case cJU_JPBRANCH_B* + + +// **************************************************************************** +// JPBRANCH_U*: +// +// Notice the reverse order of the cases, and falling through to the next case, +// for performance. + + case cJU_JPBRANCH_U: + + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, cJU_ROOTSTATE); + +// If not a BranchU, traverse; otherwise fall into the next case, which makes +// this very fast code for a large Judy array (mainly BranchUs), especially +// when branches are already in the cache, such as for prev/next: + +#ifndef JU_64BIT + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U3) goto ContinueWalk; +#else + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U7) goto ContinueWalk; +#endif + +#ifdef JU_64BIT + case cJU_JPBRANCH_U7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 7); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U6) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 6); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U5) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 5); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U4) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 4); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U3) goto ContinueWalk; + // and fall through. + +#endif // JU_64BIT + + case cJU_JPBRANCH_U3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 3); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U2) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 2); + +// Note: BranchU2 is a special case that must continue traversal to a leaf, +// immed, full, or null type: + + goto ContinueWalk; + + +// **************************************************************************** +// JPLEAF*: +// +// Note: Here the calls of JU_DCDNOTMATCHINDEX() are necessary and check +// whether Index is out of the expanse of a narrow pointer. + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + + case cJU_JPLEAF1: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf1(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF1VALUEAREA(Pjll, Pop1) + posidx));) + } + +#endif // (JUDYL || (! JU_64BIT)) + + case cJU_JPLEAF2: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf2(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF2VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF3: + { + int posidx; // signed offset in leaf. + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf3(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF3VALUEAREA(Pjll, Pop1) + posidx));) + } +#ifdef JU_64BIT + case cJU_JPLEAF4: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf4(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF4VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF5: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf5(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF5VALUEAREA(Pjll, Pop1) + posidx));) + } + + case cJU_JPLEAF6: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf6(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF6VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF7: + { + int posidx; // signed offset in leaf. + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf7(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF7VALUEAREA(Pjll, Pop1) + posidx));) + } +#endif // JU_64BIT + + +// **************************************************************************** +// JPLEAF_B1: + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; +#ifdef JUDYL + int posidx; + Word_t subexp; // in bitmap, 0..7. + BITMAPL_t BitMap; // for one subexpanse. + BITMAPL_t BitMask; // bit in BitMap for Indexs Digit. + Pjv_t Pjv; +#endif + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + Pjlb = P_JLB(Pjp->jp_Addr); + +#ifdef JUDY1 + +// Simply check if Indexs bit is set in the bitmap: + + if (JU_BITMAPTESTL(Pjlb, Index)) return(1); + break; + +#else // JUDYL + +// JudyL is much more complicated because of value area subarrays: + + Digit = JU_DIGITATSTATE(Index, 1); + subexp = Digit / cJU_BITSPERSUBEXPL; + BitMap = JU_JLB_BITMAP(Pjlb, subexp); + BitMask = JU_BITPOSMASKL(Digit); + +// No value in subexpanse for Index => Index not found: + + if (! (BitMap & BitMask)) break; + +// Count value areas in the subexpanse below the one for Index: + + Pjv = P_JV(JL_JLB_PVALUE(Pjlb, subexp)); + assert(Pjv != (Pjv_t) NULL); + posidx = j__udyCountBitsL(BitMap & (BitMask - 1)); + + return((PPvoid_t) (Pjv + posidx)); + +#endif // JUDYL + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 + +// **************************************************************************** +// JPFULLPOPU1: +// +// If the Index is in the expanse, it is necessarily valid (found). + + case cJ1_JPFULLPOPU1: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + return(1); + +#ifdef notdef // for future enhancements +#ifdef JU_64BIT + +// Note: Need ? if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + case cJ1_JPFULLPOPU1m15: + if (Pjp->jp_1Index[14] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m14: + if (Pjp->jp_1Index[13] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m13: + if (Pjp->jp_1Index[12] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m12: + if (Pjp->jp_1Index[11] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m11: + if (Pjp->jp_1Index[10] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m10: + if (Pjp->jp_1Index[9] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m9: + if (Pjp->jp_1Index[8] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m8: + if (Pjp->jp_1Index[7] == (uint8_t)Index) break; +#endif + case cJ1_JPFULLPOPU1m7: + if (Pjp->jp_1Index[6] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m6: + if (Pjp->jp_1Index[5] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m5: + if (Pjp->jp_1Index[4] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m4: + if (Pjp->jp_1Index[3] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m3: + if (Pjp->jp_1Index[2] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m2: + if (Pjp->jp_1Index[1] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m1: + if (Pjp->jp_1Index[0] == (uint8_t)Index) break; + + return(1); // found, not in exclusion list + +#endif // JUDY1 +#endif // notdef + +// **************************************************************************** +// JPIMMED*: +// +// Note that the contents of jp_DcdPopO are different for cJU_JPIMMED_*_01: + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + if (JU_JPDCDPOP0(Pjp) != JU_TRIMTODCDSIZE(Index)) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) &(Pjp->jp_Addr));) // immediate value area. + + +// Macros to make code more readable and avoid dup errors + +#ifdef JUDY1 + +#define CHECKINDEXNATIVE(LEAF_T, PJP, IDX, INDEX) \ +if (((LEAF_T *)((PJP)->jp_1Index))[(IDX) - 1] == (LEAF_T)(INDEX)) \ + return(1) + +#define CHECKLEAFNONNAT(LFBTS, PJP, INDEX, IDX, COPY) \ +{ \ + Word_t i_ndex; \ + uint8_t *a_ddr; \ + a_ddr = (PJP)->jp_1Index + (((IDX) - 1) * (LFBTS)); \ + COPY(i_ndex, a_ddr); \ + if (i_ndex == JU_LEASTBYTES((INDEX), (LFBTS))) \ + return(1); \ +} +#endif + +#ifdef JUDYL + +#define CHECKINDEXNATIVE(LEAF_T, PJP, IDX, INDEX) \ +if (((LEAF_T *)((PJP)->jp_LIndex))[(IDX) - 1] == (LEAF_T)(INDEX)) \ + return((PPvoid_t)(P_JV((PJP)->jp_Addr) + (IDX) - 1)) + +#define CHECKLEAFNONNAT(LFBTS, PJP, INDEX, IDX, COPY) \ +{ \ + Word_t i_ndex; \ + uint8_t *a_ddr; \ + a_ddr = (PJP)->jp_LIndex + (((IDX) - 1) * (LFBTS)); \ + COPY(i_ndex, a_ddr); \ + if (i_ndex == JU_LEASTBYTES((INDEX), (LFBTS))) \ + return((PPvoid_t)(P_JV((PJP)->jp_Addr) + (IDX) - 1)); \ +} +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_15: CHECKINDEXNATIVE(uint8_t, Pjp, 15, Index); + case cJ1_JPIMMED_1_14: CHECKINDEXNATIVE(uint8_t, Pjp, 14, Index); + case cJ1_JPIMMED_1_13: CHECKINDEXNATIVE(uint8_t, Pjp, 13, Index); + case cJ1_JPIMMED_1_12: CHECKINDEXNATIVE(uint8_t, Pjp, 12, Index); + case cJ1_JPIMMED_1_11: CHECKINDEXNATIVE(uint8_t, Pjp, 11, Index); + case cJ1_JPIMMED_1_10: CHECKINDEXNATIVE(uint8_t, Pjp, 10, Index); + case cJ1_JPIMMED_1_09: CHECKINDEXNATIVE(uint8_t, Pjp, 9, Index); + case cJ1_JPIMMED_1_08: CHECKINDEXNATIVE(uint8_t, Pjp, 8, Index); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_07: CHECKINDEXNATIVE(uint8_t, Pjp, 7, Index); + case cJU_JPIMMED_1_06: CHECKINDEXNATIVE(uint8_t, Pjp, 6, Index); + case cJU_JPIMMED_1_05: CHECKINDEXNATIVE(uint8_t, Pjp, 5, Index); + case cJU_JPIMMED_1_04: CHECKINDEXNATIVE(uint8_t, Pjp, 4, Index); +#endif + case cJU_JPIMMED_1_03: CHECKINDEXNATIVE(uint8_t, Pjp, 3, Index); + case cJU_JPIMMED_1_02: CHECKINDEXNATIVE(uint8_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint8_t, Pjp, 1, Index); + break; + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_07: CHECKINDEXNATIVE(uint16_t, Pjp, 7, Index); + case cJ1_JPIMMED_2_06: CHECKINDEXNATIVE(uint16_t, Pjp, 6, Index); + case cJ1_JPIMMED_2_05: CHECKINDEXNATIVE(uint16_t, Pjp, 5, Index); + case cJ1_JPIMMED_2_04: CHECKINDEXNATIVE(uint16_t, Pjp, 4, Index); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_03: CHECKINDEXNATIVE(uint16_t, Pjp, 3, Index); + case cJU_JPIMMED_2_02: CHECKINDEXNATIVE(uint16_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint16_t, Pjp, 1, Index); + break; +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_05: + CHECKLEAFNONNAT(3, Pjp, Index, 5, JU_COPY3_PINDEX_TO_LONG); + case cJ1_JPIMMED_3_04: + CHECKLEAFNONNAT(3, Pjp, Index, 4, JU_COPY3_PINDEX_TO_LONG); + case cJ1_JPIMMED_3_03: + CHECKLEAFNONNAT(3, Pjp, Index, 3, JU_COPY3_PINDEX_TO_LONG); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: + CHECKLEAFNONNAT(3, Pjp, Index, 2, JU_COPY3_PINDEX_TO_LONG); + CHECKLEAFNONNAT(3, Pjp, Index, 1, JU_COPY3_PINDEX_TO_LONG); + break; +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + + case cJ1_JPIMMED_4_03: CHECKINDEXNATIVE(uint32_t, Pjp, 3, Index); + case cJ1_JPIMMED_4_02: CHECKINDEXNATIVE(uint32_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint32_t, Pjp, 1, Index); + break; + + case cJ1_JPIMMED_5_03: + CHECKLEAFNONNAT(5, Pjp, Index, 3, JU_COPY5_PINDEX_TO_LONG); + case cJ1_JPIMMED_5_02: + CHECKLEAFNONNAT(5, Pjp, Index, 2, JU_COPY5_PINDEX_TO_LONG); + CHECKLEAFNONNAT(5, Pjp, Index, 1, JU_COPY5_PINDEX_TO_LONG); + break; + + case cJ1_JPIMMED_6_02: + CHECKLEAFNONNAT(6, Pjp, Index, 2, JU_COPY6_PINDEX_TO_LONG); + CHECKLEAFNONNAT(6, Pjp, Index, 1, JU_COPY6_PINDEX_TO_LONG); + break; + + case cJ1_JPIMMED_7_02: + CHECKLEAFNONNAT(7, Pjp, Index, 2, JU_COPY7_PINDEX_TO_LONG); + CHECKLEAFNONNAT(7, Pjp, Index, 1, JU_COPY7_PINDEX_TO_LONG); + break; + +#endif // (JUDY1 && JU_64BIT) + + +// **************************************************************************** +// INVALID JP TYPE: + + default: + +ReturnCorrupt: + +#ifdef JUDYGETINLINE // Pjpm is known to be non-null: + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); +#else + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); +#endif + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // switch on JP type + +JUDY1CODE(return(0);) +JUDYLCODE(return((PPvoid_t) NULL);) + +} // Judy1Test() / JudyLGet() + + +#ifndef JUDYGETINLINE // only compile the following function once: +#ifdef DEBUG + +// **************************************************************************** +// J U D Y C H E C K P O P +// +// Given a pointer to a Judy array, traverse the entire array to ensure +// population counts add up correctly. This can catch various coding errors. +// +// Since walking the entire tree is probably time-consuming, enable this +// function by setting env parameter $CHECKPOP to first call at which to start +// checking. Note: This function is called both from insert and delete code. +// +// Note: Even though this function does nothing useful for LEAFW leaves, its +// good practice to call it anyway, and cheap too. +// +// TBD: This is a debug-only check function similar to JudyCheckSorted(), but +// since it walks the tree it is Judy1/JudyL-specific and must live in a source +// file that is built both ways. +// +// TBD: As feared, enabling this code for every insert/delete makes Judy +// deathly slow, even for a small tree (10K indexes). Its not so bad if +// present but disabled (<1% slowdown measured). Still, should it be ifdefd +// other than DEBUG and/or called less often? +// +// TBD: Should this "population checker" be expanded to a comprehensive tree +// checker? It currently detects invalid LEAFW/JP types as well as inconsistent +// pop1s. Other possible checks, all based on essentially redundant data in +// the Judy tree, include: +// +// - Zero LS bits in jp_Addr field. +// +// - Correct Dcd bits. +// +// - Consistent JP types (always descending down the tree). +// +// - Sorted linear lists in BranchLs and leaves (using JudyCheckSorted(), but +// ideally that function is already called wherever appropriate after any +// linear list is modified). +// +// - Any others possible? + +#include <stdlib.h> // for getenv() and atol(). + +static Word_t JudyCheckPopSM(Pjp_t Pjp, Word_t RootPop1); + +FUNCTION void JudyCheckPop( + Pvoid_t PArray) +{ +static bool_t checked = FALSE; // already checked env parameter. +static bool_t enabled = FALSE; // env parameter set. +static bool_t active = FALSE; // calls >= callsmin. +static Word_t callsmin; // start point from $CHECKPOP. +static Word_t calls = 0; // times called so far. + + +// CHECK FOR EXTERNAL ENABLING: + + if (! checked) // only check once. + { + char * value; // for getenv(). + + checked = TRUE; + + if ((value = getenv("CHECKPOP")) == (char *) NULL) + { +#ifdef notdef +// Take this out because nightly tests want to be flavor-independent; its not +// OK to emit special non-error output from the debug flavor: + + (void) puts("JudyCheckPop() present but not enabled by " + "$CHECKPOP env parameter; set it to the number of " + "calls at which to begin checking"); +#endif + return; + } + + callsmin = atol(value); // note: non-number evaluates to 0. + enabled = TRUE; + + (void) printf("JudyCheckPop() present and enabled; callsmin = " + "%lu\n", callsmin); + } + else if (! enabled) return; + +// Previously or just now enabled; check if non-active or newly active: + + if (! active) + { + if (++calls < callsmin) return; + + (void) printf("JudyCheckPop() activated at call %lu\n", calls); + active = TRUE; + } + +// IGNORE LEAFW AT TOP OF TREE: + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + return; + +// Check JPM pop0 against tree, recursively: +// +// Note: The traversal code in JudyCheckPopSM() is simplest when the case +// statement for each JP type compares the pop1 for that JP to its subtree (if +// any) after traversing the subtree (thats the hard part) and adding up +// actual pop1s. A top branchs JP in the JPM does not have room for a +// full-word pop1, so pass it in as a special case. + + { + Pjpm_t Pjpm = P_JPM(PArray); + (void) JudyCheckPopSM(&(Pjpm->jpm_JP), Pjpm->jpm_Pop0 + 1); + return; + } + +} // JudyCheckPop() + + +// **************************************************************************** +// J U D Y C H E C K P O P S M +// +// Recursive state machine (subroutine) for JudyCheckPop(): Given a Pjp (other +// than JPNULL*; caller should shortcut) and the root population for top-level +// branches, check the subtrees actual pop1 against its nominal value, and +// return the total pop1 for the subtree. +// +// Note: Expect RootPop1 to be ignored at lower levels, so pass down 0, which +// should pop an assertion if this expectation is violated. + +FUNCTION static Word_t JudyCheckPopSM( + Pjp_t Pjp, // top of subtree. + Word_t RootPop1) // whole array, for top-level branches only. +{ + Word_t pop1_jp; // nominal population from the JP. + Word_t pop1 = 0; // actual population at this level. + Word_t offset; // in a branch. + +#define PREPBRANCH(cPopBytes,Next) \ + pop1_jp = JU_JPBRANCH_POP0(Pjp, cPopBytes) + 1; goto Next + +assert((((Word_t) (Pjp->jp_Addr)) & 7) == 3); + switch (JU_JPTYPE(Pjp)) + { + + case cJU_JPBRANCH_L2: PREPBRANCH(2, BranchL); + case cJU_JPBRANCH_L3: PREPBRANCH(3, BranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: PREPBRANCH(4, BranchL); + case cJU_JPBRANCH_L5: PREPBRANCH(5, BranchL); + case cJU_JPBRANCH_L6: PREPBRANCH(6, BranchL); + case cJU_JPBRANCH_L7: PREPBRANCH(7, BranchL); +#endif + case cJU_JPBRANCH_L: pop1_jp = RootPop1; + { + Pjbl_t Pjbl; +BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + + for (offset = 0; offset < (Pjbl->jbl_NumJPs); ++offset) + pop1 += JudyCheckPopSM((Pjbl->jbl_jp) + offset, 0); + + assert(pop1_jp == pop1); + return(pop1); + } + + case cJU_JPBRANCH_B2: PREPBRANCH(2, BranchB); + case cJU_JPBRANCH_B3: PREPBRANCH(3, BranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: PREPBRANCH(4, BranchB); + case cJU_JPBRANCH_B5: PREPBRANCH(5, BranchB); + case cJU_JPBRANCH_B6: PREPBRANCH(6, BranchB); + case cJU_JPBRANCH_B7: PREPBRANCH(7, BranchB); +#endif + case cJU_JPBRANCH_B: pop1_jp = RootPop1; + { + Word_t subexp; + Word_t jpcount; + Pjbb_t Pjbb; +BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + + for (offset = 0; offset < jpcount; ++offset) + { + pop1 += JudyCheckPopSM(P_JP(JU_JBB_PJP(Pjbb, subexp)) + + offset, 0); + } + } + + assert(pop1_jp == pop1); + return(pop1); + } + + case cJU_JPBRANCH_U2: PREPBRANCH(2, BranchU); + case cJU_JPBRANCH_U3: PREPBRANCH(3, BranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: PREPBRANCH(4, BranchU); + case cJU_JPBRANCH_U5: PREPBRANCH(5, BranchU); + case cJU_JPBRANCH_U6: PREPBRANCH(6, BranchU); + case cJU_JPBRANCH_U7: PREPBRANCH(7, BranchU); +#endif + case cJU_JPBRANCH_U: pop1_jp = RootPop1; + { + Pjbu_t Pjbu; +BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + + for (offset = 0; offset < cJU_BRANCHUNUMJPS; ++offset) + { + if (((Pjbu->jbu_jp[offset].jp_Type) >= cJU_JPNULL1) + && ((Pjbu->jbu_jp[offset].jp_Type) <= cJU_JPNULLMAX)) + { + continue; // skip null JP to save time. + } + + pop1 += JudyCheckPopSM((Pjbu->jbu_jp) + offset, 0); + } + + assert(pop1_jp == pop1); + return(pop1); + } + + +// -- Cases below here terminate and do not recurse. -- +// +// For all of these cases except JPLEAF_B1, there is no way to check the JPs +// pop1 against the object itself; just return the pop1; but for linear leaves, +// a bounds check is possible. + +#define CHECKLEAF(MaxPop1) \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + assert(pop1 >= 1); \ + assert(pop1 <= (MaxPop1)); \ + return(pop1) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKLEAF(cJU_LEAF1_MAXPOP1); +#endif + case cJU_JPLEAF2: CHECKLEAF(cJU_LEAF2_MAXPOP1); + case cJU_JPLEAF3: CHECKLEAF(cJU_LEAF3_MAXPOP1); +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKLEAF(cJU_LEAF4_MAXPOP1); + case cJU_JPLEAF5: CHECKLEAF(cJU_LEAF5_MAXPOP1); + case cJU_JPLEAF6: CHECKLEAF(cJU_LEAF6_MAXPOP1); + case cJU_JPLEAF7: CHECKLEAF(cJU_LEAF7_MAXPOP1); +#endif + + case cJU_JPLEAF_B1: + { + Word_t subexp; + Pjlb_t Pjlb; + + pop1_jp = JU_JPLEAF_POP0(Pjp) + 1; + + Pjlb = P_JLB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + pop1 += j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + + assert(pop1_jp == pop1); + return(pop1); + } + + JUDY1CODE(case cJ1_JPFULLPOPU1: return(cJU_JPFULLPOPU1_POP0);) + + case cJU_JPIMMED_1_01: return(1); + case cJU_JPIMMED_2_01: return(1); + case cJU_JPIMMED_3_01: return(1); +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: return(1); + case cJU_JPIMMED_5_01: return(1); + case cJU_JPIMMED_6_01: return(1); + case cJU_JPIMMED_7_01: return(1); +#endif + + case cJU_JPIMMED_1_02: return(2); + case cJU_JPIMMED_1_03: return(3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: return(4); + case cJU_JPIMMED_1_05: return(5); + case cJU_JPIMMED_1_06: return(6); + case cJU_JPIMMED_1_07: return(7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: return(8); + case cJ1_JPIMMED_1_09: return(9); + case cJ1_JPIMMED_1_10: return(10); + case cJ1_JPIMMED_1_11: return(11); + case cJ1_JPIMMED_1_12: return(12); + case cJ1_JPIMMED_1_13: return(13); + case cJ1_JPIMMED_1_14: return(14); + case cJ1_JPIMMED_1_15: return(15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: return(2); + case cJU_JPIMMED_2_03: return(3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: return(4); + case cJ1_JPIMMED_2_05: return(5); + case cJ1_JPIMMED_2_06: return(6); + case cJ1_JPIMMED_2_07: return(7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: return(2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: return(3); + case cJ1_JPIMMED_3_04: return(4); + case cJ1_JPIMMED_3_05: return(5); + + case cJ1_JPIMMED_4_02: return(2); + case cJ1_JPIMMED_4_03: return(3); + case cJ1_JPIMMED_5_02: return(2); + case cJ1_JPIMMED_5_03: return(3); + case cJ1_JPIMMED_6_02: return(2); + case cJ1_JPIMMED_7_02: return(2); +#endif + + } // switch (JU_JPTYPE(Pjp)) + + assert(FALSE); // unrecognized JP type => corruption. + return(0); // to make some compilers happy. + +} // JudyCheckPopSM() + +#endif // DEBUG +#endif // ! JUDYGETINLINE diff --git a/libnetdata/libjudy/src/JudyL/JudyLIns.c b/libnetdata/libjudy/src/JudyL/JudyLIns.c new file mode 100644 index 000000000..f96df4101 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLIns.c @@ -0,0 +1,1873 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.116 $ $Source: /judy/src/JudyCommon/JudyIns.c $ +// +// Judy1Set() and JudyLIns() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// TBD: Should some of the assertions here be converted to product code that +// returns JU_ERRNO_CORRUPT? + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +// Note: Call JudyCheckPop() even before "already inserted" returns, to catch +// population errors; see fix in 4.84: + +DBGCODE(extern void JudyCheckPop(Pvoid_t PArray);) +DBGCODE(extern void JudyCheckSorted(Pjll_t Pjll, Word_t Pop1, long IndexSize);) + +#ifdef TRACEJP +#include "JudyPrintJP.c" +#endif + + +// These are defined to generic values in JudyCommon/JudyPrivateTypes.h: +// +// TBD: These should be exported from a header file, but perhaps not, as they +// are only used here, and exported from Judy*Decascade, which is a separate +// file for profiling reasons (to prevent inlining), but which potentially +// could be merged with this file, either in SoftCM or at compile-time. + +#ifdef JUDY1 +extern int j__udy1CreateBranchB(Pjp_t, Pjp_t, uint8_t *, Word_t, Pvoid_t); +extern int j__udy1CreateBranchU(Pjp_t, Pvoid_t); + +#ifndef JU_64BIT +extern int j__udy1Cascade1(Pjp_t, Pvoid_t); +#endif +extern int j__udy1Cascade2(Pjp_t, Pvoid_t); +extern int j__udy1Cascade3(Pjp_t, Pvoid_t); +#ifdef JU_64BIT +extern int j__udy1Cascade4(Pjp_t, Pvoid_t); +extern int j__udy1Cascade5(Pjp_t, Pvoid_t); +extern int j__udy1Cascade6(Pjp_t, Pvoid_t); +extern int j__udy1Cascade7(Pjp_t, Pvoid_t); +#endif +extern int j__udy1CascadeL(Pjp_t, Pvoid_t); + +extern int j__udy1InsertBranch(Pjp_t Pjp, Word_t Index, Word_t Btype, Pjpm_t); + +#else // JUDYL + +extern int j__udyLCreateBranchB(Pjp_t, Pjp_t, uint8_t *, Word_t, Pvoid_t); +extern int j__udyLCreateBranchU(Pjp_t, Pvoid_t); + +extern int j__udyLCascade1(Pjp_t, Pvoid_t); +extern int j__udyLCascade2(Pjp_t, Pvoid_t); +extern int j__udyLCascade3(Pjp_t, Pvoid_t); +#ifdef JU_64BIT +extern int j__udyLCascade4(Pjp_t, Pvoid_t); +extern int j__udyLCascade5(Pjp_t, Pvoid_t); +extern int j__udyLCascade6(Pjp_t, Pvoid_t); +extern int j__udyLCascade7(Pjp_t, Pvoid_t); +#endif +extern int j__udyLCascadeL(Pjp_t, Pvoid_t); + +extern int j__udyLInsertBranch(Pjp_t Pjp, Word_t Index, Word_t Btype, Pjpm_t); +#endif + + +// **************************************************************************** +// MACROS FOR COMMON CODE: +// +// Check if Index is an outlier to (that is, not a member of) this expanse: +// +// An outlier is an Index in-the-expanse of the slot containing the pointer, +// but not-in-the-expanse of the "narrow" pointer in that slot. (This means +// the Dcd part of the Index differs from the equivalent part of jp_DcdPopO.) +// Therefore, the remedy is to put a cJU_JPBRANCH_L* between the narrow pointer +// and the object to which it points, and add the outlier Index as an Immediate +// in the cJU_JPBRANCH_L*. The "trick" is placing the cJU_JPBRANCH_L* at a +// Level that is as low as possible. This is determined by counting the digits +// in the existing narrow pointer that are the same as the digits in the new +// Index (see j__udyInsertBranch()). +// +// Note: At some high Levels, cJU_DCDMASK() is all zeros => dead code; assume +// the compiler optimizes this out. + +#define JU_CHECK_IF_OUTLIER(Pjp, Index, cLevel, Pjpm) \ + if (JU_DCDNOTMATCHINDEX(Index, Pjp, cLevel)) \ + return(j__udyInsertBranch(Pjp, Index, cLevel, Pjpm)) + +// Check if an Index is already in a leaf or immediate, after calling +// j__udySearchLeaf*() to set Offset: +// +// A non-negative Offset means the Index already exists, so return 0; otherwise +// complement Offset to proceed. + +#ifdef JUDY1 +#define Pjv ignore // placeholder. +#define JU_CHECK_IF_EXISTS(Offset,ignore,Pjpm) \ + { \ + if ((Offset) >= 0) return(0); \ + (Offset) = ~(Offset); \ + } +#else +// For JudyL, also set the value area pointer in the Pjpm: + +#define JU_CHECK_IF_EXISTS(Offset,Pjv,Pjpm) \ + { \ + if ((Offset) >= 0) \ + { \ + (Pjpm)->jpm_PValue = (Pjv) + (Offset); \ + return(0); \ + } \ + (Offset) = ~(Offset); \ + } +#endif + + +// **************************************************************************** +// __ J U D Y I N S W A L K +// +// Walk the Judy tree to do a set/insert. This is only called internally, and +// recursively. Unlike Judy1Test() and JudyLGet(), the extra time required for +// recursion should be negligible compared with the total. +// +// Return -1 for error (details in JPM), 0 for Index already inserted, 1 for +// new Index inserted. + +FUNCTION static int j__udyInsWalk( + Pjp_t Pjp, // current JP to descend. + Word_t Index, // to insert. + Pjpm_t Pjpm) // for returning info to top Level. +{ + uint8_t digit; // from Index, current offset into a branch. + jp_t newJP; // for creating a new Immed JP. + Word_t exppop1; // expanse (leaf) population. + int retcode; // return codes: -1, 0, 1. + +#ifdef SUBEXPCOUNTS +// Pointer to BranchB/U subexpanse counter: +// +// Note: Very important for performance reasons (avoids cache fills). + + PWord_t PSubExp = (PWord_t) NULL; +#endif + +ContinueInsWalk: // for modifying state without recursing. + +#ifdef TRACEJP + JudyPrintJP(Pjp, "i", __LINE__); +#endif + + switch (JU_JPTYPE(Pjp)) // entry: Pjp, Index. + { + + +// **************************************************************************** +// JPNULL*: +// +// Convert JP in place from current null type to cJU_JPIMMED_*_01 by +// calculating new JP type. + + case cJU_JPNULL1: + case cJU_JPNULL2: + case cJU_JPNULL3: +#ifdef JU_64BIT + case cJU_JPNULL4: + case cJU_JPNULL5: + case cJU_JPNULL6: + case cJU_JPNULL7: +#endif + assert((Pjp->jp_Addr) == 0); + JU_JPSETADT(Pjp, 0, Index, JU_JPTYPE(Pjp) + cJU_JPIMMED_1_01 - cJU_JPNULL1); +#ifdef JUDYL + // value area is first word of new Immed_01 JP: + Pjpm->jpm_PValue = (Pjv_t) (&(Pjp->jp_Addr)); +#endif + return(1); + + +// **************************************************************************** +// JPBRANCH_L*: +// +// If the new Index is not an outlier to the branchs expanse, and the branch +// should not be converted to uncompressed, extract the digit and record the +// Immediate type to create for a new Immed JP, before going to common code. +// +// Note: JU_CHECK_IF_OUTLIER() is a no-op for BranchB3[7] on 32[64]-bit. + +#define JU_BRANCH_OUTLIER(DIGIT,POP1,cLEVEL,PJP,INDEX,PJPM) \ + JU_CHECK_IF_OUTLIER(PJP, INDEX, cLEVEL, PJPM); \ + (DIGIT) = JU_DIGITATSTATE(INDEX, cLEVEL); \ + (POP1) = JU_JPBRANCH_POP0(PJP, cLEVEL) + + case cJU_JPBRANCH_L2: + JU_BRANCH_OUTLIER(digit, exppop1, 2, Pjp, Index, Pjpm); + goto JudyBranchL; + + case cJU_JPBRANCH_L3: + JU_BRANCH_OUTLIER(digit, exppop1, 3, Pjp, Index, Pjpm); + goto JudyBranchL; + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + JU_BRANCH_OUTLIER(digit, exppop1, 4, Pjp, Index, Pjpm); + goto JudyBranchL; + + case cJU_JPBRANCH_L5: + JU_BRANCH_OUTLIER(digit, exppop1, 5, Pjp, Index, Pjpm); + goto JudyBranchL; + + case cJU_JPBRANCH_L6: + JU_BRANCH_OUTLIER(digit, exppop1, 6, Pjp, Index, Pjpm); + goto JudyBranchL; + + case cJU_JPBRANCH_L7: + JU_BRANCH_OUTLIER(digit, exppop1, 7, Pjp, Index, Pjpm); + goto JudyBranchL; +#endif + +// Similar to common code above, but no outlier check is needed, and the Immed +// type depends on the word size: + + case cJU_JPBRANCH_L: + { + Pjbl_t PjblRaw; // pointer to old linear branch. + Pjbl_t Pjbl; + Pjbu_t PjbuRaw; // pointer to new uncompressed branch. + Pjbu_t Pjbu; + Word_t numJPs; // number of JPs = populated expanses. + int offset; // in branch. + + digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + exppop1 = Pjpm->jpm_Pop0; + + // fall through: + +// COMMON CODE FOR LINEAR BRANCHES: +// +// Come here with digit and exppop1 already set. + +JudyBranchL: + PjblRaw = (Pjbl_t) (Pjp->jp_Addr); + Pjbl = P_JBL(PjblRaw); + +// If population under this branch greater than: + + if (exppop1 > JU_BRANCHL_MAX_POP) + goto ConvertBranchLtoU; + + numJPs = Pjbl->jbl_NumJPs; + + if ((numJPs == 0) || (numJPs > cJU_BRANCHLMAXJPS)) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); + return(-1); + } + +// Search for a match to the digit: + + offset = j__udySearchLeaf1((Pjll_t) (Pjbl->jbl_Expanse), numJPs, + digit); + +// If Index is found, offset is into an array of 1..cJU_BRANCHLMAXJPS JPs: + + if (offset >= 0) + { + Pjp = (Pjbl->jbl_jp) + offset; // address of next JP. + break; // continue walk. + } + +// Expanse is missing (not populated) for the passed Index, so insert an Immed +// -- if theres room: + + if (numJPs < cJU_BRANCHLMAXJPS) + { + offset = ~offset; // insertion offset. + + JU_JPSETADT(&newJP, 0, Index, + JU_JPTYPE(Pjp) + cJU_JPIMMED_1_01-cJU_JPBRANCH_L2); + + JU_INSERTINPLACE(Pjbl->jbl_Expanse, numJPs, offset, digit); + JU_INSERTINPLACE(Pjbl->jbl_jp, numJPs, offset, newJP); + + DBGCODE(JudyCheckSorted((Pjll_t) (Pjbl->jbl_Expanse), + numJPs + 1, /* IndexSize = */ 1);) + ++(Pjbl->jbl_NumJPs); +#ifdef JUDYL + // value area is first word of new Immed 01 JP: + Pjpm->jpm_PValue = (Pjv_t) ((Pjbl->jbl_jp) + offset); +#endif + return(1); + } + + +// MAXED OUT LINEAR BRANCH, CONVERT TO A BITMAP BRANCH, THEN INSERT: +// +// Copy the linear branch to a bitmap branch. +// +// TBD: Consider renaming j__udyCreateBranchB() to j__udyConvertBranchLtoB(). + + assert((numJPs) <= cJU_BRANCHLMAXJPS); + + if (j__udyCreateBranchB(Pjp, Pjbl->jbl_jp, Pjbl->jbl_Expanse, + numJPs, Pjpm) == -1) + { + return(-1); + } + +// Convert jp_Type from linear branch to equivalent bitmap branch: + + Pjp->jp_Type += cJU_JPBRANCH_B - cJU_JPBRANCH_L; + + j__udyFreeJBL(PjblRaw, Pjpm); // free old BranchL. + +// Having changed branch types, now do the insert in the new branch type: + + goto ContinueInsWalk; + + +// OPPORTUNISTICALLY CONVERT FROM BRANCHL TO BRANCHU: +// +// Memory efficiency is no object because the branchs pop1 is large enough, so +// speed up array access. Come here with PjblRaw set. Note: This is goto +// code because the previous block used to fall through into it as well, but no +// longer. + +ConvertBranchLtoU: + +// Allocate memory for an uncompressed branch: + + if ((PjbuRaw = j__udyAllocJBU(Pjpm)) == (Pjbu_t) NULL) + return(-1); + Pjbu = P_JBU(PjbuRaw); + +// Set the proper NULL type for most of the uncompressed branchs JPs: + + JU_JPSETADT(&newJP, 0, 0, + JU_JPTYPE(Pjp) - cJU_JPBRANCH_L2 + cJU_JPNULL1); + +// Initialize: Pre-set uncompressed branch to mostly JPNULL*s: + + for (numJPs = 0; numJPs < cJU_BRANCHUNUMJPS; ++numJPs) + Pjbu->jbu_jp[numJPs] = newJP; + +// Copy JPs from linear branch to uncompressed branch: + + { +#ifdef SUBEXPCOUNTS + Word_t popmask = cJU_POP0MASK(JU_JPTYPE(Pjp)) + - cJU_JPBRANCH_L2 - 2; + + for (numJPs = 0; numJPs < cJU_NUMSUBEXPU; ++numJPs) + Pjbu->jbu_subPop1[numJPs] = 0; +#endif + for (numJPs = 0; numJPs < Pjbl->jbl_NumJPs; ++numJPs) + { + Pjp_t Pjp1 = &(Pjbl->jbl_jp[numJPs]); + offset = Pjbl->jbl_Expanse[numJPs]; + Pjbu->jbu_jp[offset] = *Pjp1; +#ifdef SUBEXPCOUNTS + Pjbu->jbu_subPop1[offset/cJU_NUMSUBEXPU] += + JU_JPDCDPOP0(Pjp1) & popmask + 1; +#endif + } + } + j__udyFreeJBL(PjblRaw, Pjpm); // free old BranchL. + +// Plug new values into parent JP: + + Pjp->jp_Addr = (Word_t) PjbuRaw; + Pjp->jp_Type += cJU_JPBRANCH_U - cJU_JPBRANCH_L; // to BranchU. + +// Save global population of last BranchU conversion: + + Pjpm->jpm_LastUPop0 = Pjpm->jpm_Pop0; + goto ContinueInsWalk; + + } // case cJU_JPBRANCH_L. + + +// **************************************************************************** +// JPBRANCH_B*: +// +// If the new Index is not an outlier to the branchs expanse, extract the +// digit and record the Immediate type to create for a new Immed JP, before +// going to common code. +// +// Note: JU_CHECK_IF_OUTLIER() is a no-op for BranchB3[7] on 32[64]-bit. + + case cJU_JPBRANCH_B2: + JU_BRANCH_OUTLIER(digit, exppop1, 2, Pjp, Index, Pjpm); + goto JudyBranchB; + + case cJU_JPBRANCH_B3: + JU_BRANCH_OUTLIER(digit, exppop1, 3, Pjp, Index, Pjpm); + goto JudyBranchB; + +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + JU_BRANCH_OUTLIER(digit, exppop1, 4, Pjp, Index, Pjpm); + goto JudyBranchB; + + case cJU_JPBRANCH_B5: + JU_BRANCH_OUTLIER(digit, exppop1, 5, Pjp, Index, Pjpm); + goto JudyBranchB; + + case cJU_JPBRANCH_B6: + JU_BRANCH_OUTLIER(digit, exppop1, 6, Pjp, Index, Pjpm); + goto JudyBranchB; + + case cJU_JPBRANCH_B7: + JU_BRANCH_OUTLIER(digit, exppop1, 7, Pjp, Index, Pjpm); + goto JudyBranchB; +#endif + + case cJU_JPBRANCH_B: + { + Pjbb_t Pjbb; // pointer to bitmap branch. + Pjbb_t PjbbRaw; // pointer to bitmap branch. + Pjp_t Pjp2Raw; // 1 of N arrays of JPs. + Pjp_t Pjp2; // 1 of N arrays of JPs. + Word_t subexp; // 1 of N subexpanses in bitmap. + BITMAPB_t bitmap; // for one subexpanse. + BITMAPB_t bitmask; // bit set for Indexs digit. + Word_t numJPs; // number of JPs = populated expanses. + int offset; // in bitmap branch. + +// Similar to common code above, but no outlier check is needed, and the Immed +// type depends on the word size: + + digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + exppop1 = Pjpm->jpm_Pop0; + + // fall through: + + +// COMMON CODE FOR BITMAP BRANCHES: +// +// Come here with digit and exppop1 already set. + +JudyBranchB: + +// If population increment is greater than.. (300): + + if ((Pjpm->jpm_Pop0 - Pjpm->jpm_LastUPop0) > JU_BTOU_POP_INCREMENT) + { + +// If total population of array is greater than.. (750): + + if (Pjpm->jpm_Pop0 > JU_BRANCHB_MAX_POP) + { + +// If population under the branch is greater than.. (135): + + if (exppop1 > JU_BRANCHB_MIN_POP) + { + if (j__udyCreateBranchU(Pjp, Pjpm) == -1) return(-1); + +// Save global population of last BranchU conversion: + + Pjpm->jpm_LastUPop0 = Pjpm->jpm_Pop0; + + goto ContinueInsWalk; + } + } + } + +// CONTINUE TO USE BRANCHB: +// +// Get pointer to bitmap branch (JBB): + + PjbbRaw = (Pjbb_t) (Pjp->jp_Addr); + Pjbb = P_JBB(PjbbRaw); + +// Form the Int32 offset, and Bit offset values: +// +// 8 bit Decode | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | +// |SubExpanse | Bit offset | +// +// Get the 1 of 8 expanses from digit, Bits 5..7 = 1 of 8, and get the 32-bit +// word that may have a bit set: + + subexp = digit / cJU_BITSPERSUBEXPB; + bitmap = JU_JBB_BITMAP(Pjbb, subexp); + + Pjp2Raw = JU_JBB_PJP(Pjbb, subexp); + Pjp2 = P_JP(Pjp2Raw); + +// Get the bit position that represents the desired expanse, and get the offset +// into the array of JPs for the JP that matches the bit. + + bitmask = JU_BITPOSMASKB(digit); + offset = j__udyCountBitsB(bitmap & (bitmask - 1)); + +// If JP is already in this expanse, get Pjp and continue the walk: + + if (bitmap & bitmask) + { +#ifdef SUBEXPCOUNTS + PSubExp = &(Pjbb->jbb_Counts[subexp]); // ptr to subexp counts. +#endif + Pjp = Pjp2 + offset; + break; // continue walk. + } + + +// ADD NEW EXPANSE FOR NEW INDEX: +// +// The new expanse always an cJU_JPIMMED_*_01 containing just the new Index, so +// finish setting up an Immed JP. + + JU_JPSETADT(&newJP, 0, Index, + JU_JPTYPE(Pjp) + cJU_JPIMMED_1_01-cJU_JPBRANCH_B2); + +// Get 1 of the 8 JP arrays and calculate number of JPs in subexpanse array: + + Pjp2Raw = JU_JBB_PJP(Pjbb, subexp); + Pjp2 = P_JP(Pjp2Raw); + numJPs = j__udyCountBitsB(bitmap); + +// Expand branch JP subarray in-place: + + if (JU_BRANCHBJPGROWINPLACE(numJPs)) + { + assert(numJPs > 0); + JU_INSERTINPLACE(Pjp2, numJPs, offset, newJP); +#ifdef JUDYL + // value area is first word of new Immed 01 JP: + Pjpm->jpm_PValue = (Pjv_t) (Pjp2 + offset); +#endif + } + +// No room, allocate a bigger bitmap branch JP subarray: + + else + { + Pjp_t PjpnewRaw; + Pjp_t Pjpnew; + + if ((PjpnewRaw = j__udyAllocJBBJP(numJPs + 1, Pjpm)) == 0) + return(-1); + Pjpnew = P_JP(PjpnewRaw); + +// If there was an old JP array, then copy it, insert the new Immed JP, and +// free the old array: + + if (numJPs) + { + JU_INSERTCOPY(Pjpnew, Pjp2, numJPs, offset, newJP); + j__udyFreeJBBJP(Pjp2Raw, numJPs, Pjpm); +#ifdef JUDYL + // value area is first word of new Immed 01 JP: + Pjpm->jpm_PValue = (Pjv_t) (Pjpnew + offset); +#endif + } + +// New JP subarray; point to cJU_JPIMMED_*_01 and place it: + + else + { + assert(JU_JBB_PJP(Pjbb, subexp) == (Pjp_t) NULL); + Pjp = Pjpnew; + *Pjp = newJP; // copy to new memory. +#ifdef JUDYL + // value area is first word of new Immed 01 JP: + Pjpm->jpm_PValue = (Pjv_t) (&(Pjp->jp_Addr)); +#endif + } + +// Place new JP subarray in BranchB: + + JU_JBB_PJP(Pjbb, subexp) = PjpnewRaw; + + } // else + +// Set the new Indexs bit: + + JU_JBB_BITMAP(Pjbb, subexp) |= bitmask; + + return(1); + + } // case + + +// **************************************************************************** +// JPBRANCH_U*: +// +// Just drop through the JP for the correct digit. If the JP turns out to be a +// JPNULL*, thats OK, the memory is already allocated, and the next walk +// simply places an Immed in it. +// +#ifdef SUBEXPCOUNTS +#define JU_GETSUBEXP(PSubExp,Pjbu,Digit) \ + (PSubExp) = &((Pjbu)->jbu_subPop1[(Digit) / cJU_NUMSUBEXPU]) +#else +#define JU_GETSUBEXP(PSubExp,Pjbu,Digit) // null. +#endif + +#define JU_JBU_PJP_SUBEXP(Pjp,PSubExp,Index,Level) \ + { \ + uint8_t digit = JU_DIGITATSTATE(Index, Level); \ + Pjbu_t P_jbu = P_JBU((Pjp)->jp_Addr); \ + (Pjp) = &(P_jbu->jbu_jp[digit]); \ + JU_GETSUBEXP(PSubExp, P_jbu, digit); \ + } + + case cJU_JPBRANCH_U2: + JU_CHECK_IF_OUTLIER(Pjp, Index, 2, Pjpm); + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 2); + break; + +#ifdef JU_64BIT + case cJU_JPBRANCH_U3: + JU_CHECK_IF_OUTLIER(Pjp, Index, 3, Pjpm); + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 3); + break; + + case cJU_JPBRANCH_U4: + JU_CHECK_IF_OUTLIER(Pjp, Index, 4, Pjpm); + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 4); + break; + + case cJU_JPBRANCH_U5: + JU_CHECK_IF_OUTLIER(Pjp, Index, 5, Pjpm); + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 5); + break; + + case cJU_JPBRANCH_U6: + JU_CHECK_IF_OUTLIER(Pjp, Index, 6, Pjpm); + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 6); + break; + + case cJU_JPBRANCH_U7: + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 7); +#else + case cJU_JPBRANCH_U3: + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, 3); +#endif + break; + + case cJU_JPBRANCH_U: + JU_JBU_PJP_SUBEXP(Pjp, PSubExp, Index, cJU_ROOTSTATE); + break; + + +// **************************************************************************** +// JPLEAF*: +// +// COMMON CODE FRAGMENTS TO MINIMIZE REDUNDANCY BELOW: +// +// These are necessary to support performance by function and loop unrolling +// while avoiding huge amounts of nearly identical code. +// +// Prepare to handle a linear leaf: Check for an outlier; set pop1 and pointer +// to leaf: + +#ifdef JUDY1 +#define JU_LEAFVALUE(Pjv) // null. +#define JU_LEAFPREPVALUE(Pjv, ValueArea) // null. +#else +#define JU_LEAFVALUE(Pjv) Pjv_t Pjv +#define JU_LEAFPREPVALUE(Pjv, ValueArea) (Pjv) = ValueArea(Pleaf, exppop1) +#endif + +#define JU_LEAFPREP(cIS,Type,MaxPop1,ValueArea) \ + Pjll_t PjllRaw; \ + Type Pleaf; /* specific type */ \ + int offset; \ + JU_LEAFVALUE(Pjv); \ + \ + JU_CHECK_IF_OUTLIER(Pjp, Index, cIS, Pjpm); \ + \ + exppop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + assert(exppop1 <= (MaxPop1)); \ + PjllRaw = (Pjll_t) (Pjp->jp_Addr); \ + Pleaf = (Type) P_JLL(PjllRaw); \ + JU_LEAFPREPVALUE(Pjv, ValueArea) + +// Add to, or grow, a linear leaf: Find Index position; if the Index is +// absent, if theres room in the leaf, insert the Index [and value of 0] in +// place, otherwise grow the leaf: +// +// Note: These insertions always take place with whole words, using +// JU_INSERTINPLACE() or JU_INSERTCOPY(). + +#ifdef JUDY1 +#define JU_LEAFGROWVALUEADD(Pjv,ExpPop1,Offset) // null. +#else +#define JU_LEAFGROWVALUEADD(Pjv,ExpPop1,Offset) \ + JU_INSERTINPLACE(Pjv, ExpPop1, Offset, 0); \ + Pjpm->jpm_PValue = (Pjv) + (Offset) +#endif + +#ifdef JUDY1 +#define JU_LEAFGROWVALUENEW(ValueArea,Pjv,ExpPop1,Offset) // null. +#else +#define JU_LEAFGROWVALUENEW(ValueArea,Pjv,ExpPop1,Offset) \ + { \ + Pjv_t Pjvnew = ValueArea(Pleafnew, (ExpPop1) + 1); \ + JU_INSERTCOPY(Pjvnew, Pjv, ExpPop1, Offset, 0); \ + Pjpm->jpm_PValue = (Pjvnew) + (Offset); \ + } +#endif + +#define JU_LEAFGROW(cIS,Type,MaxPop1,Search,ValueArea,GrowInPlace, \ + InsertInPlace,InsertCopy,Alloc,Free) \ + \ + offset = Search(Pleaf, exppop1, Index); \ + JU_CHECK_IF_EXISTS(offset, Pjv, Pjpm); \ + \ + if (GrowInPlace(exppop1)) /* add to current leaf */ \ + { \ + InsertInPlace(Pleaf, exppop1, offset, Index); \ + JU_LEAFGROWVALUEADD(Pjv, exppop1, offset); \ + DBGCODE(JudyCheckSorted((Pjll_t) Pleaf, exppop1 + 1, cIS);) \ + return(1); \ + } \ + \ + if (exppop1 < (MaxPop1)) /* grow to new leaf */ \ + { \ + Pjll_t PjllnewRaw; \ + Type Pleafnew; \ + if ((PjllnewRaw = Alloc(exppop1 + 1, Pjpm)) == 0) return(-1); \ + Pleafnew = (Type) P_JLL(PjllnewRaw); \ + InsertCopy(Pleafnew, Pleaf, exppop1, offset, Index); \ + JU_LEAFGROWVALUENEW(ValueArea, Pjv, exppop1, offset); \ + DBGCODE(JudyCheckSorted((Pjll_t) Pleafnew, exppop1 + 1, cIS);) \ + Free(PjllRaw, exppop1, Pjpm); \ + (Pjp->jp_Addr) = (Word_t) PjllnewRaw; \ + return(1); \ + } \ + assert(exppop1 == (MaxPop1)) + +// Handle linear leaf overflow (cascade): Splay or compress into smaller +// leaves: + +#define JU_LEAFCASCADE(MaxPop1,Cascade,Free) \ + if (Cascade(Pjp, Pjpm) == -1) return(-1); \ + Free(PjllRaw, MaxPop1, Pjpm); \ + goto ContinueInsWalk + +// Wrapper around all of the above: + +#define JU_LEAFSET(cIS,Type,MaxPop1,Search,GrowInPlace,InsertInPlace, \ + InsertCopy,Cascade,Alloc,Free,ValueArea) \ + { \ + JU_LEAFPREP(cIS,Type,MaxPop1,ValueArea); \ + JU_LEAFGROW(cIS,Type,MaxPop1,Search,ValueArea,GrowInPlace, \ + InsertInPlace,InsertCopy,Alloc,Free); \ + JU_LEAFCASCADE(MaxPop1,Cascade,Free); \ + } + +// END OF MACROS; LEAFL CASES START HERE: +// +// 64-bit Judy1 does not have 1-byte leaves: + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + + case cJU_JPLEAF1: + + JU_LEAFSET(1, uint8_t *, cJU_LEAF1_MAXPOP1, j__udySearchLeaf1, + JU_LEAF1GROWINPLACE, JU_INSERTINPLACE, JU_INSERTCOPY, + j__udyCascade1, j__udyAllocJLL1, j__udyFreeJLL1, + JL_LEAF1VALUEAREA); + +#endif // (JUDYL || ! JU_64BIT) + + case cJU_JPLEAF2: + + JU_LEAFSET(2, uint16_t *, cJU_LEAF2_MAXPOP1, j__udySearchLeaf2, + JU_LEAF2GROWINPLACE, JU_INSERTINPLACE, JU_INSERTCOPY, + j__udyCascade2, j__udyAllocJLL2, j__udyFreeJLL2, + JL_LEAF2VALUEAREA); + + case cJU_JPLEAF3: + + JU_LEAFSET(3, uint8_t *, cJU_LEAF3_MAXPOP1, j__udySearchLeaf3, + JU_LEAF3GROWINPLACE, JU_INSERTINPLACE3, JU_INSERTCOPY3, + j__udyCascade3, j__udyAllocJLL3, j__udyFreeJLL3, + JL_LEAF3VALUEAREA); + +#ifdef JU_64BIT + case cJU_JPLEAF4: + + JU_LEAFSET(4, uint32_t *, cJU_LEAF4_MAXPOP1, j__udySearchLeaf4, + JU_LEAF4GROWINPLACE, JU_INSERTINPLACE, JU_INSERTCOPY, + j__udyCascade4, j__udyAllocJLL4, j__udyFreeJLL4, + JL_LEAF4VALUEAREA); + + case cJU_JPLEAF5: + + JU_LEAFSET(5, uint8_t *, cJU_LEAF5_MAXPOP1, j__udySearchLeaf5, + JU_LEAF5GROWINPLACE, JU_INSERTINPLACE5, JU_INSERTCOPY5, + j__udyCascade5, j__udyAllocJLL5, j__udyFreeJLL5, + JL_LEAF5VALUEAREA); + + case cJU_JPLEAF6: + + JU_LEAFSET(6, uint8_t *, cJU_LEAF6_MAXPOP1, j__udySearchLeaf6, + JU_LEAF6GROWINPLACE, JU_INSERTINPLACE6, JU_INSERTCOPY6, + j__udyCascade6, j__udyAllocJLL6, j__udyFreeJLL6, + JL_LEAF6VALUEAREA); + + case cJU_JPLEAF7: + + JU_LEAFSET(7, uint8_t *, cJU_LEAF7_MAXPOP1, j__udySearchLeaf7, + JU_LEAF7GROWINPLACE, JU_INSERTINPLACE7, JU_INSERTCOPY7, + j__udyCascade7, j__udyAllocJLL7, j__udyFreeJLL7, + JL_LEAF7VALUEAREA); +#endif // JU_64BIT + + +// **************************************************************************** +// JPLEAF_B1: +// +// 8 bit Decode | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | +// |SubExpanse | Bit offset | +// +// Note: For JudyL, values are stored in 8 subexpanses, each a linear word +// array of up to 32 values each. + + case cJU_JPLEAF_B1: + { +#ifdef JUDYL + Pjv_t PjvRaw; // pointer to value part of the leaf. + Pjv_t Pjv; // pointer to value part of the leaf. + Pjv_t PjvnewRaw; // new value area. + Pjv_t Pjvnew; // new value area. + Word_t subexp; // 1 of 8 subexpanses in bitmap. + Pjlb_t Pjlb; // pointer to bitmap part of the leaf. + BITMAPL_t bitmap; // for one subexpanse. + BITMAPL_t bitmask; // bit set for Indexs digit. + int offset; // of index in value area. +#endif + + JU_CHECK_IF_OUTLIER(Pjp, Index, 1, Pjpm); + +#ifdef JUDY1 + +// If Index (bit) is already set, return now: + + if (JU_BITMAPTESTL(P_JLB(Pjp->jp_Addr), Index)) return(0); + +// If bitmap is not full, set the new Indexs bit; otherwise convert to a Full: + + if ((exppop1 = JU_JPLEAF_POP0(Pjp) + 1) + < cJU_JPFULLPOPU1_POP0) + { + JU_BITMAPSETL(P_JLB(Pjp->jp_Addr), Index); + } + else + { + j__udyFreeJLB1((Pjlb_t) (Pjp->jp_Addr), Pjpm); // free LeafB1. + Pjp->jp_Type = cJ1_JPFULLPOPU1; + Pjp->jp_Addr = 0; + } + +#else // JUDYL + +// This is very different from Judy1 because of the need to return a value area +// even for an existing Index, or manage the value area for a new Index, and +// because JudyL has no Full type: + +// Get last byte to decode from Index, and pointer to bitmap leaf: + + digit = JU_DIGITATSTATE(Index, 1); + Pjlb = P_JLB(Pjp->jp_Addr); + +// Prepare additional values: + + subexp = digit / cJU_BITSPERSUBEXPL; // which subexpanse. + bitmap = JU_JLB_BITMAP(Pjlb, subexp); // subexps 32-bit map. + PjvRaw = JL_JLB_PVALUE(Pjlb, subexp); // corresponding values. + Pjv = P_JV(PjvRaw); // corresponding values. + bitmask = JU_BITPOSMASKL(digit); // mask for Index. + offset = j__udyCountBitsL(bitmap & (bitmask - 1)); // of Index. + +// If Index already exists, get value pointer and exit: + + if (bitmap & bitmask) + { + assert(Pjv); + Pjpm->jpm_PValue = Pjv + offset; // existing value. + return(0); + } + +// Get the total bits set = expanse population of Value area: + + exppop1 = j__udyCountBitsL(bitmap); + +// If the value area can grow in place, do it: + + if (JL_LEAFVGROWINPLACE(exppop1)) + { + JU_INSERTINPLACE(Pjv, exppop1, offset, 0); + JU_JLB_BITMAP(Pjlb, subexp) |= bitmask; // set Indexs bit. + Pjpm->jpm_PValue = Pjv + offset; // new value area. + return(1); + } + +// Increase size of value area: + + if ((PjvnewRaw = j__udyLAllocJV(exppop1 + 1, Pjpm)) + == (Pjv_t) NULL) return(-1); + Pjvnew = P_JV(PjvnewRaw); + + if (exppop1) // have existing value area. + { + assert(Pjv); + JU_INSERTCOPY(Pjvnew, Pjv, exppop1, offset, 0); + Pjpm->jpm_PValue = Pjvnew + offset; + j__udyLFreeJV(PjvRaw, exppop1, Pjpm); // free old values. + } + else // first index, new value area: + { + Pjpm->jpm_PValue = Pjvnew; + *(Pjpm->jpm_PValue) = 0; + } + +// Set bit for new Index and place new leaf value area in bitmap: + + JU_JLB_BITMAP(Pjlb, subexp) |= bitmask; + JL_JLB_PVALUE(Pjlb, subexp) = PjvnewRaw; + +#endif // JUDYL + + return(1); + + } // case + + +#ifdef JUDY1 +// **************************************************************************** +// JPFULLPOPU1: +// +// If Index is not an outlier, then by definition its already set. + + case cJ1_JPFULLPOPU1: + + JU_CHECK_IF_OUTLIER(Pjp, Index, 1, Pjpm); + return(0); +#endif + + +// **************************************************************************** +// JPIMMED*: +// +// This is some of the most complex code in Judy considering Judy1 versus JudyL +// and 32-bit versus 64-bit variations. The following comments attempt to make +// this clearer. +// +// Of the 2 words in a JP, for immediate indexes Judy1 can use 2 words - 1 byte +// = 7 [15] bytes, but JudyL can only use 1 word - 1 byte = 3 [7] bytes because +// the other word is needed for a value area or a pointer to a value area. +// +// For both Judy1 and JudyL, cJU_JPIMMED_*_01 indexes are in word 2; otherwise +// for Judy1 only, a list of 2 or more indexes starts in word 1. JudyL keeps +// the list in word 2 because word 1 is a pointer (to a LeafV, that is, a leaf +// containing only values). Furthermore, cJU_JPIMMED_*_01 indexes are stored +// all-but-first-byte in jp_DcdPopO, not just the Index Sizes bytes. +// +// TBD: This can be confusing because Doug didnt use data structures for it. +// Instead he often directly accesses Pjp for the first word and jp_DcdPopO for +// the second word. It would be nice to use data structs, starting with +// jp_1Index and jp_LIndex where possible. +// +// Maximum Immed JP types for Judy1/JudyL, depending on Index Size (cIS): +// +// 32-bit 64-bit +// +// bytes: 7/ 3 15/ 7 (Judy1/JudyL) +// +// cIS +// 1_ 07/03 15/07 (as in: cJ1_JPIMMED_1_07) +// 2_ 03/01 07/03 +// 3_ 02/01 05/02 +// 4_ 03/01 +// 5_ 03/01 +// 6_ 02/01 +// 7_ 02/01 +// +// State transitions while inserting an Index, matching the above table: +// (Yes, this is very terse... Study it and it will make sense.) +// (Note, parts of this diagram are repeated below for quick reference.) +// +// +-- reformat JP here for Judy1 only, from word-2 to word-1 +// | +// | JUDY1 || JU_64BIT JUDY1 && JU_64BIT +// V +// 1_01 => 1_02 => 1_03 => [ 1_04 => ... => 1_07 => [ 1_08..15 => ]] Leaf1 (*) +// 2_01 => [ 2_02 => 2_03 => [ 2_04..07 => ]] Leaf2 +// 3_01 => [ 3_02 => [ 3_03..05 => ]] Leaf3 +// JU_64BIT only: +// 4_01 => [[ 4_02..03 => ]] Leaf4 +// 5_01 => [[ 5_02..03 => ]] Leaf5 +// 6_01 => [[ 6_02 => ]] Leaf6 +// 7_01 => [[ 7_02 => ]] Leaf7 +// +// (*) For Judy1 & 64-bit, go directly from cJU_JPIMMED_1_15 to a LeafB1; skip +// Leaf1, as described in Judy1.h regarding cJ1_JPLEAF1. + + +// COMMON CODE FRAGMENTS TO MINIMIZE REDUNDANCY BELOW: +// +// These are necessary to support performance by function and loop unrolling +// while avoiding huge amounts of nearly identical code. +// +// The differences between Judy1 and JudyL with respect to value area handling +// are just too large for completely common code between them... Oh well, some +// big ifdefs follow. However, even in the following ifdefd code, use cJU_*, +// JU_*, and Judy*() instead of cJ1_* / cJL_*, J1_* / JL_*, and +// Judy1*()/JudyL*(), for minimum diffs. +// +// Handle growth of cJU_JPIMMED_*_01 to cJU_JPIMMED_*_02, for an even or odd +// Index Size (cIS), given oldIndex, Index, and Pjll in the context: +// +// Put oldIndex and Index in their proper order. For odd indexes, must copy +// bytes. + +#ifdef JUDY1 + +#define JU_IMMSET_01_COPY_EVEN(ignore1,ignore2) \ + if (oldIndex < Index) { Pjll[0] = oldIndex; Pjll[1] = Index; } \ + else { Pjll[0] = Index; Pjll[1] = oldIndex; } + +#define JU_IMMSET_01_COPY_ODD(cIS,CopyWord) \ + if (oldIndex < Index) \ + { \ + CopyWord(Pjll + 0, oldIndex); \ + CopyWord(Pjll + (cIS), Index); \ + } \ + else \ + { \ + CopyWord(Pjll + 0, Index); \ + CopyWord(Pjll + (cIS), oldIndex); \ + } + +// The "real" *_01 Copy macro: +// +// Trim the high byte off Index, look for a match with the old Index, and if +// none, insert the new Index in the leaf in the correct place, given Pjp and +// Index in the context. +// +// Note: A single immediate index lives in the jp_DcdPopO field, but two or +// more reside starting at Pjp->jp_1Index. + +#define JU_IMMSET_01_COPY(cIS,LeafType,NewJPType,Copy,CopyWord) \ + { \ + LeafType Pjll; \ + Word_t oldIndex = JU_JPDCDPOP0(Pjp); \ + \ + Index = JU_TRIMTODCDSIZE(Index); \ + if (oldIndex == Index) return(0); \ + \ + Pjll = (LeafType) (Pjp->jp_1Index); \ + Copy(cIS,CopyWord); \ + DBGCODE(JudyCheckSorted(Pjll, 2, cIS);) \ + \ + Pjp->jp_Type = (NewJPType); \ + return(1); \ + } + +#else // JUDYL + +// Variations to also handle value areas; see comments above: +// +// For JudyL, Pjv (start of value area) and oldValue are also in the context; +// leave Pjv set to the value area for Index. + +#define JU_IMMSET_01_COPY_EVEN(cIS,CopyWord) \ + if (oldIndex < Index) \ + { \ + Pjll[0] = oldIndex; \ + Pjv [0] = oldValue; \ + Pjll[1] = Index; \ + ++Pjv; \ + } \ + else \ + { \ + Pjll[0] = Index; \ + Pjll[1] = oldIndex; \ + Pjv [1] = oldValue; \ + } + +#define JU_IMMSET_01_COPY_ODD(cIS,CopyWord) \ + if (oldIndex < Index) \ + { \ + CopyWord(Pjll + 0, oldIndex); \ + CopyWord(Pjll + (cIS), Index); \ + Pjv[0] = oldValue; \ + ++Pjv; \ + } \ + else \ + { \ + CopyWord(Pjll + 0, Index); \ + CopyWord(Pjll + (cIS), oldIndex); \ + Pjv[1] = oldValue; \ + } + +// The old value area is in the first word (*Pjp), and Pjv and Pjpm are also in +// the context. Also, unlike Judy1, indexes remain in word 2 (jp_LIndex), +// meaning insert-in-place rather than copy. +// +// Return jpm_PValue pointing to Indexs value area. If Index is new, allocate +// a 2-value-leaf and attach it to the JP. + +#define JU_IMMSET_01_COPY(cIS,LeafType,NewJPType,Copy,CopyWord) \ + { \ + LeafType Pjll; \ + Word_t oldIndex = JU_JPDCDPOP0(Pjp); \ + Word_t oldValue; \ + Pjv_t PjvRaw; \ + Pjv_t Pjv; \ + \ + Index = JU_TRIMTODCDSIZE(Index); \ + \ + if (oldIndex == Index) \ + { \ + Pjpm->jpm_PValue = (Pjv_t) Pjp; \ + return(0); \ + } \ + \ + if ((PjvRaw = j__udyLAllocJV(2, Pjpm)) == (Pjv_t) NULL) \ + return(-1); \ + Pjv = P_JV(PjvRaw); \ + \ + oldValue = Pjp->jp_Addr; \ + (Pjp->jp_Addr) = (Word_t) PjvRaw; \ + Pjll = (LeafType) (Pjp->jp_LIndex); \ + \ + Copy(cIS,CopyWord); \ + DBGCODE(JudyCheckSorted(Pjll, 2, cIS);) \ + \ + Pjp->jp_Type = (NewJPType); \ + *Pjv = 0; \ + Pjpm->jpm_PValue = Pjv; \ + return(1); \ + } + +// The following is a unique mix of JU_IMMSET_01() and JU_IMMSETCASCADE() for +// going from cJU_JPIMMED_*_01 directly to a cJU_JPLEAF* for JudyL: +// +// If Index is not already set, allocate a leaf, copy the old and new indexes +// into it, clear and return the new value area, and modify the current JP. +// Note that jp_DcdPop is set to a pop0 of 0 for now, and incremented later. + + +#define JU_IMMSET_01_CASCADE(cIS,LeafType,NewJPType,ValueArea, \ + Copy,CopyWord,Alloc) \ + { \ + Word_t D_P0; \ + LeafType PjllRaw; \ + LeafType Pjll; \ + Word_t oldIndex = JU_JPDCDPOP0(Pjp); \ + Word_t oldValue; \ + Pjv_t Pjv; \ + \ + Index = JU_TRIMTODCDSIZE(Index); \ + \ + if (oldIndex == Index) \ + { \ + Pjpm->jpm_PValue = (Pjv_t) (&(Pjp->jp_Addr)); \ + return(0); \ + } \ + \ + if ((PjllRaw = (LeafType) Alloc(2, Pjpm)) == (LeafType) NULL) \ + return(-1); \ + Pjll = (LeafType) P_JLL(PjllRaw); \ + Pjv = ValueArea(Pjll, 2); \ + \ + oldValue = Pjp->jp_Addr; \ + \ + Copy(cIS,CopyWord); \ + DBGCODE(JudyCheckSorted(Pjll, 2, cIS);) \ + \ + *Pjv = 0; \ + Pjpm->jpm_PValue = Pjv; \ + D_P0 = Index & cJU_DCDMASK(cIS); /* pop0 = 0 */ \ + JU_JPSETADT(Pjp, (Word_t)PjllRaw, D_P0, NewJPType); \ + \ + return(1); \ + } + +#endif // JUDYL + +// Handle growth of cJU_JPIMMED_*_[02..15]: + +#ifdef JUDY1 + +// Insert an Index into an immediate JP that has room for more, if the Index is +// not already present; given Pjp, Index, exppop1, Pjv, and Pjpm in the +// context: +// +// Note: Use this only when the JP format doesnt change, that is, going from +// cJU_JPIMMED_X_0Y to cJU_JPIMMED_X_0Z, where X >= 2 and Y+1 = Z. +// +// Note: Incrementing jp_Type is how to increase the Index population. + +#define JU_IMMSETINPLACE(cIS,LeafType,BaseJPType_02,Search,InsertInPlace) \ + { \ + LeafType Pjll; \ + int offset; \ + \ + exppop1 = JU_JPTYPE(Pjp) - (BaseJPType_02) + 2; \ + offset = Search((Pjll_t) (Pjp->jp_1Index), exppop1, Index); \ + \ + JU_CHECK_IF_EXISTS(offset, ignore, Pjpm); \ + \ + Pjll = (LeafType) (Pjp->jp_1Index); \ + InsertInPlace(Pjll, exppop1, offset, Index); \ + DBGCODE(JudyCheckSorted(Pjll, exppop1 + 1, cIS);) \ + ++(Pjp->jp_Type); \ + return(1); \ + } + +// Insert an Index into an immediate JP that has no room for more: +// +// If the Index is not already present, do a cascade (to a leaf); given Pjp, +// Index, Pjv, and Pjpm in the context. + + +#define JU_IMMSETCASCADE(cIS,OldPop1,LeafType,NewJPType, \ + ignore,Search,InsertCopy,Alloc) \ + { \ + Word_t D_P0; \ + Pjll_t PjllRaw; \ + Pjll_t Pjll; \ + int offset; \ + \ + offset = Search((Pjll_t) (Pjp->jp_1Index), (OldPop1), Index); \ + JU_CHECK_IF_EXISTS(offset, ignore, Pjpm); \ + \ + if ((PjllRaw = Alloc((OldPop1) + 1, Pjpm)) == 0) return(-1); \ + Pjll = P_JLL(PjllRaw); \ + \ + InsertCopy((LeafType) Pjll, (LeafType) (Pjp->jp_1Index), \ + OldPop1, offset, Index); \ + DBGCODE(JudyCheckSorted(Pjll, (OldPop1) + 1, cIS);) \ + \ + D_P0 = (Index & cJU_DCDMASK(cIS)) + (OldPop1) - 1; \ + JU_JPSETADT(Pjp, (Word_t)PjllRaw, D_P0, NewJPType); \ + return(1); \ + } + +#else // JUDYL + +// Variations to also handle value areas; see comments above: +// +// For JudyL, Pjv (start of value area) is also in the context. +// +// TBD: This code makes a true but weak assumption that a JudyL 32-bit 2-index +// value area must be copied to a new 3-index value area. AND it doesnt know +// anything about JudyL 64-bit cases (cJU_JPIMMED_1_0[3-7] only) where the +// value area can grow in place! However, this should not break it, just slow +// it down. + +#define JU_IMMSETINPLACE(cIS,LeafType,BaseJPType_02,Search,InsertInPlace) \ + { \ + LeafType Pleaf; \ + int offset; \ + Pjv_t PjvRaw; \ + Pjv_t Pjv; \ + Pjv_t PjvnewRaw; \ + Pjv_t Pjvnew; \ + \ + exppop1 = JU_JPTYPE(Pjp) - (BaseJPType_02) + 2; \ + offset = Search((Pjll_t) (Pjp->jp_LIndex), exppop1, Index); \ + PjvRaw = (Pjv_t) (Pjp->jp_Addr); \ + Pjv = P_JV(PjvRaw); \ + \ + JU_CHECK_IF_EXISTS(offset, Pjv, Pjpm); \ + \ + if ((PjvnewRaw = j__udyLAllocJV(exppop1 + 1, Pjpm)) \ + == (Pjv_t) NULL) return(-1); \ + Pjvnew = P_JV(PjvnewRaw); \ + \ + Pleaf = (LeafType) (Pjp->jp_LIndex); \ + \ + InsertInPlace(Pleaf, exppop1, offset, Index); \ + /* see TBD above about this: */ \ + JU_INSERTCOPY(Pjvnew, Pjv, exppop1, offset, 0); \ + DBGCODE(JudyCheckSorted(Pleaf, exppop1 + 1, cIS);) \ + j__udyLFreeJV(PjvRaw, exppop1, Pjpm); \ + Pjp->jp_Addr = (Word_t) PjvnewRaw; \ + Pjpm->jpm_PValue = Pjvnew + offset; \ + \ + ++(Pjp->jp_Type); \ + return(1); \ + } + +#define JU_IMMSETCASCADE(cIS,OldPop1,LeafType,NewJPType, \ + ValueArea,Search,InsertCopy,Alloc) \ + { \ + Word_t D_P0; \ + Pjll_t PjllRaw; \ + Pjll_t Pjll; \ + int offset; \ + Pjv_t PjvRaw; \ + Pjv_t Pjv; \ + Pjv_t Pjvnew; \ + \ + PjvRaw = (Pjv_t) (Pjp->jp_Addr); \ + Pjv = P_JV(PjvRaw); \ + offset = Search((Pjll_t) (Pjp->jp_LIndex), (OldPop1), Index); \ + JU_CHECK_IF_EXISTS(offset, Pjv, Pjpm); \ + \ + if ((PjllRaw = Alloc((OldPop1) + 1, Pjpm)) == 0) \ + return(-1); \ + Pjll = P_JLL(PjllRaw); \ + InsertCopy((LeafType) Pjll, (LeafType) (Pjp->jp_LIndex), \ + OldPop1, offset, Index); \ + DBGCODE(JudyCheckSorted(Pjll, (OldPop1) + 1, cIS);) \ + \ + Pjvnew = ValueArea(Pjll, (OldPop1) + 1); \ + JU_INSERTCOPY(Pjvnew, Pjv, OldPop1, offset, 0); \ + j__udyLFreeJV(PjvRaw, (OldPop1), Pjpm); \ + Pjpm->jpm_PValue = Pjvnew + offset; \ + \ + D_P0 = (Index & cJU_DCDMASK(cIS)) + (OldPop1) - 1; \ + JU_JPSETADT(Pjp, (Word_t)PjllRaw, D_P0, NewJPType); \ + return(1); \ + } + +#endif // JUDYL + +// Common convenience/shorthand wrappers around JU_IMMSET_01_COPY() for +// even/odd index sizes: + +#define JU_IMMSET_01( cIS, LeafType, NewJPType) \ + JU_IMMSET_01_COPY(cIS, LeafType, NewJPType, JU_IMMSET_01_COPY_EVEN, \ + ignore) + +#define JU_IMMSET_01_ODD( cIS, NewJPType, CopyWord) \ + JU_IMMSET_01_COPY(cIS, uint8_t *, NewJPType, JU_IMMSET_01_COPY_ODD, \ + CopyWord) + + +// END OF MACROS; IMMED CASES START HERE: + +// cJU_JPIMMED_*_01 cases: +// +// 1_01 always leads to 1_02: +// +// (1_01 => 1_02 => 1_03 => [ 1_04 => ... => 1_07 => [ 1_08..15 => ]] LeafL) + + case cJU_JPIMMED_1_01: JU_IMMSET_01(1, uint8_t *, cJU_JPIMMED_1_02); + +// 2_01 leads to 2_02, and 3_01 leads to 3_02, except for JudyL 32-bit, where +// they lead to a leaf: +// +// (2_01 => [ 2_02 => 2_03 => [ 2_04..07 => ]] LeafL) +// (3_01 => [ 3_02 => [ 3_03..05 => ]] LeafL) + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_01: JU_IMMSET_01(2, uint16_t *, cJU_JPIMMED_2_02); + case cJU_JPIMMED_3_01: JU_IMMSET_01_ODD (3, cJU_JPIMMED_3_02, + JU_COPY3_LONG_TO_PINDEX); +#else + case cJU_JPIMMED_2_01: + JU_IMMSET_01_CASCADE(2, uint16_t *, cJU_JPLEAF2, JL_LEAF2VALUEAREA, + JU_IMMSET_01_COPY_EVEN, ignore, + j__udyAllocJLL2); + case cJU_JPIMMED_3_01: + JU_IMMSET_01_CASCADE(3, uint8_t *, cJU_JPLEAF3, JL_LEAF3VALUEAREA, + JU_IMMSET_01_COPY_ODD, + JU_COPY3_LONG_TO_PINDEX, j__udyAllocJLL3); +#endif + +#ifdef JU_64BIT + +// [4-7]_01 lead to [4-7]_02 for Judy1, and to leaves for JudyL: +// +// (4_01 => [[ 4_02..03 => ]] LeafL) +// (5_01 => [[ 5_02..03 => ]] LeafL) +// (6_01 => [[ 6_02 => ]] LeafL) +// (7_01 => [[ 7_02 => ]] LeafL) + +#ifdef JUDY1 + case cJU_JPIMMED_4_01: JU_IMMSET_01(4, uint32_t *, cJ1_JPIMMED_4_02); + case cJU_JPIMMED_5_01: JU_IMMSET_01_ODD(5, cJ1_JPIMMED_5_02, + JU_COPY5_LONG_TO_PINDEX); + case cJU_JPIMMED_6_01: JU_IMMSET_01_ODD(6, cJ1_JPIMMED_6_02, + JU_COPY6_LONG_TO_PINDEX); + case cJU_JPIMMED_7_01: JU_IMMSET_01_ODD(7, cJ1_JPIMMED_7_02, + JU_COPY7_LONG_TO_PINDEX); +#else // JUDYL + case cJU_JPIMMED_4_01: + JU_IMMSET_01_CASCADE(4, uint32_t *, cJU_JPLEAF4, JL_LEAF4VALUEAREA, + JU_IMMSET_01_COPY_EVEN, ignore, + j__udyAllocJLL4); + case cJU_JPIMMED_5_01: + JU_IMMSET_01_CASCADE(5, uint8_t *, cJU_JPLEAF5, JL_LEAF5VALUEAREA, + JU_IMMSET_01_COPY_ODD, + JU_COPY5_LONG_TO_PINDEX, j__udyAllocJLL5); + case cJU_JPIMMED_6_01: + JU_IMMSET_01_CASCADE(6, uint8_t *, cJU_JPLEAF6, JL_LEAF6VALUEAREA, + JU_IMMSET_01_COPY_ODD, + JU_COPY6_LONG_TO_PINDEX, j__udyAllocJLL6); + case cJU_JPIMMED_7_01: + JU_IMMSET_01_CASCADE(7, uint8_t *, cJU_JPLEAF7, JL_LEAF7VALUEAREA, + JU_IMMSET_01_COPY_ODD, + JU_COPY7_LONG_TO_PINDEX, j__udyAllocJLL7); +#endif // JUDYL +#endif // JU_64BIT + +// cJU_JPIMMED_1_* cases that can grow in place: +// +// (1_01 => 1_02 => 1_03 => [ 1_04 => ... => 1_07 => [ 1_08..15 => ]] LeafL) + + case cJU_JPIMMED_1_02: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_03: + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJU_JPIMMED_1_07: + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: +#endif + JU_IMMSETINPLACE(1, uint8_t *, cJU_JPIMMED_1_02, j__udySearchLeaf1, + JU_INSERTINPLACE); + +// cJU_JPIMMED_1_* cases that must cascade: +// +// (1_01 => 1_02 => 1_03 => [ 1_04 => ... => 1_07 => [ 1_08..15 => ]] LeafL) + +#if (defined(JUDYL) && (! defined(JU_64BIT))) + case cJU_JPIMMED_1_03: + JU_IMMSETCASCADE(1, 3, uint8_t *, cJU_JPLEAF1, JL_LEAF1VALUEAREA, + j__udySearchLeaf1, JU_INSERTCOPY, + j__udyAllocJLL1); +#endif +#if (defined(JUDY1) && (! defined(JU_64BIT))) + case cJU_JPIMMED_1_07: + JU_IMMSETCASCADE(1, 7, uint8_t *, cJU_JPLEAF1, ignore, + j__udySearchLeaf1, JU_INSERTCOPY, + j__udyAllocJLL1); + +#endif +#if (defined(JUDYL) && defined(JU_64BIT)) + case cJU_JPIMMED_1_07: + JU_IMMSETCASCADE(1, 7, uint8_t *, cJU_JPLEAF1, JL_LEAF1VALUEAREA, + j__udySearchLeaf1, JU_INSERTCOPY, + j__udyAllocJLL1); + +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) +// Special case, as described above, go directly from Immed to LeafB1: + + case cJ1_JPIMMED_1_15: + { + Word_t DcdP0; + int offset; + Pjlb_t PjlbRaw; + Pjlb_t Pjlb; + + offset = j__udySearchLeaf1((Pjll_t) Pjp->jp_1Index, 15, Index); + + JU_CHECK_IF_EXISTS(offset, ignore, Pjpm); + +// Create a bitmap leaf (special case for Judy1 64-bit only, see usage): Set +// new Index in bitmap, copy an Immed1_15 to the bitmap, and set the parent JP +// EXCEPT jp_DcdPopO, leaving any followup to the caller: + + if ((PjlbRaw = j__udyAllocJLB1(Pjpm)) == (Pjlb_t) NULL) + return(-1); + Pjlb = P_JLB(PjlbRaw); + + JU_BITMAPSETL(Pjlb, Index); + + for (offset = 0; offset < 15; ++offset) + JU_BITMAPSETL(Pjlb, Pjp->jp_1Index[offset]); + +// Set jp_DcdPopO including the current pop0; incremented later: + DcdP0 = (Index & cJU_DCDMASK(1)) + 15 - 1; + JU_JPSETADT(Pjp, (Word_t)PjlbRaw, DcdP0, cJU_JPLEAF_B1); + + return(1); + } +#endif + +// cJU_JPIMMED_[2..7]_[02..15] cases that grow in place or cascade: +// +// (2_01 => [ 2_02 => 2_03 => [ 2_04..07 => ]] LeafL) + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJU_JPIMMED_2_03: + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + JU_IMMSETINPLACE(2, uint16_t *, cJU_JPIMMED_2_02, j__udySearchLeaf2, + JU_INSERTINPLACE); +#endif + +#undef OLDPOP1 +#if ((defined(JUDY1) && (! defined(JU_64BIT))) || (defined(JUDYL) && defined(JU_64BIT))) + case cJU_JPIMMED_2_03: +#define OLDPOP1 3 +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_07: +#define OLDPOP1 7 +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + JU_IMMSETCASCADE(2, OLDPOP1, uint16_t *, cJU_JPLEAF2, + JL_LEAF2VALUEAREA, j__udySearchLeaf2, + JU_INSERTCOPY, j__udyAllocJLL2); +#endif + +// (3_01 => [ 3_02 => [ 3_03..05 => ]] LeafL) + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJU_JPIMMED_3_02: + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + + JU_IMMSETINPLACE(3, uint8_t *, cJU_JPIMMED_3_02, j__udySearchLeaf3, + JU_INSERTINPLACE3); +#endif + +#undef OLDPOP1 +#if ((defined(JUDY1) && (! defined(JU_64BIT))) || (defined(JUDYL) && defined(JU_64BIT))) + case cJU_JPIMMED_3_02: +#define OLDPOP1 2 +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_05: +#define OLDPOP1 5 +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + JU_IMMSETCASCADE(3, OLDPOP1, uint8_t *, cJU_JPLEAF3, + JL_LEAF3VALUEAREA, j__udySearchLeaf3, + JU_INSERTCOPY3, j__udyAllocJLL3); +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + +// (4_01 => [[ 4_02..03 => ]] LeafL) + + case cJ1_JPIMMED_4_02: + + JU_IMMSETINPLACE(4, uint32_t *, cJ1_JPIMMED_4_02, j__udySearchLeaf4, + JU_INSERTINPLACE); + + case cJ1_JPIMMED_4_03: + + JU_IMMSETCASCADE(4, 3, uint32_t *, cJU_JPLEAF4, ignore, + j__udySearchLeaf4, JU_INSERTCOPY, + j__udyAllocJLL4); + +// (5_01 => [[ 5_02..03 => ]] LeafL) + + case cJ1_JPIMMED_5_02: + + JU_IMMSETINPLACE(5, uint8_t *, cJ1_JPIMMED_5_02, j__udySearchLeaf5, + JU_INSERTINPLACE5); + + case cJ1_JPIMMED_5_03: + + JU_IMMSETCASCADE(5, 3, uint8_t *, cJU_JPLEAF5, ignore, + j__udySearchLeaf5, JU_INSERTCOPY5, + j__udyAllocJLL5); + +// (6_01 => [[ 6_02 => ]] LeafL) + + case cJ1_JPIMMED_6_02: + + JU_IMMSETCASCADE(6, 2, uint8_t *, cJU_JPLEAF6, ignore, + j__udySearchLeaf6, JU_INSERTCOPY6, + j__udyAllocJLL6); + +// (7_01 => [[ 7_02 => ]] LeafL) + + case cJ1_JPIMMED_7_02: + + JU_IMMSETCASCADE(7, 2, uint8_t *, cJU_JPLEAF7, ignore, + j__udySearchLeaf7, JU_INSERTCOPY7, + j__udyAllocJLL7); + +#endif // (JUDY1 && JU_64BIT) + + +// **************************************************************************** +// INVALID JP TYPE: + + default: JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); return(-1); + + } // switch on JP type + + { + +#ifdef SUBEXPCOUNTS + +// This code might seem strange here. However it saves some memory read time +// during insert (~70nS) because a pipelined processor does not need to "stall" +// waiting for the memory read to complete. Hope the compiler is not too smart +// or dumb and moves the code down to where it looks like it belongs (below a +// few lines). + + Word_t SubExpCount = 0; // current subexpanse counter. + + if (PSubExp != (PWord_t) NULL) // only if BranchB/U. + SubExpCount = PSubExp[0]; +#endif + +// PROCESS JP -- RECURSIVELY: +// +// For non-Immed JP types, if successful, post-increment the population count +// at this Level. + + retcode = j__udyInsWalk(Pjp, Index, Pjpm); + +// Successful insert, increment JP and subexpanse count: + + if ((JU_JPTYPE(Pjp) < cJU_JPIMMED_1_01) && (retcode == 1)) + { + jp_t JP; + Word_t DcdP0; +#ifdef SUBEXPCOUNTS + +// Note: Pjp must be a pointer to a BranchB/U: + + if (PSubExp != (PWord_t) NULL) PSubExp[0] = SubExpCount + 1; +#endif + + JP = *Pjp; + DcdP0 = JU_JPDCDPOP0(Pjp) + 1; + JU_JPSETADT(Pjp, JP.jp_Addr, DcdP0, JU_JPTYPE(&JP)); + } + } + return(retcode); + +} // j__udyInsWalk() + + +// **************************************************************************** +// J U D Y 1 S E T +// J U D Y L I N S +// +// Main entry point. See the manual entry for details. + +#ifdef JUDY1 +FUNCTION int Judy1Set +#else +FUNCTION PPvoid_t JudyLIns +#endif + ( + PPvoid_t PPArray, // in which to insert. + Word_t Index, // to insert. + PJError_t PJError // optional, for returning error info. + ) +{ +#ifdef JUDY1 +#define Pjv ignore // placeholders for macros. +#define Pjvnew ignore +#else + Pjv_t Pjv; // value area in old leaf. + Pjv_t Pjvnew; // value area in new leaf. +#endif + Pjpm_t Pjpm; // array-global info. + int offset; // position in which to store new Index. + Pjlw_t Pjlw; + + +// CHECK FOR NULL POINTER (error by caller): + + if (PPArray == (PPvoid_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPPARRAY); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjlw = P_JLW(*PPArray); // first word of leaf. + +// **************************************************************************** +// PROCESS TOP LEVEL "JRP" BRANCHES AND LEAVES: + +// **************************************************************************** +// JRPNULL (EMPTY ARRAY): BUILD A LEAFW WITH ONE INDEX: + +// if a valid empty array (null pointer), so create an array of population == 1: + + if (Pjlw == (Pjlw_t)NULL) + { + Pjlw_t Pjlwnew; + + Pjlwnew = j__udyAllocJLW(1); + JUDY1CODE(JU_CHECKALLOC(Pjlw_t, Pjlwnew, JERRI );) + JUDYLCODE(JU_CHECKALLOC(Pjlw_t, Pjlwnew, PPJERR);) + + Pjlwnew[0] = 1 - 1; // pop0 = 0. + Pjlwnew[1] = Index; + + *PPArray = (Pvoid_t) Pjlwnew; + DBGCODE(JudyCheckPop(*PPArray);) + + JUDY1CODE(return(1); ) + JUDYLCODE(Pjlwnew[2] = 0; ) // value area. + JUDYLCODE(return((PPvoid_t) (Pjlwnew + 2)); ) + + } // NULL JRP + +// **************************************************************************** +// LEAFW, OTHER SIZE: + + if (JU_LEAFW_POP0(*PPArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlwnew; + Word_t pop1; + + Pjlw = P_JLW(*PPArray); // first word of leaf. + pop1 = Pjlw[0] + 1; + +#ifdef JUDYL + Pjv = JL_LEAFWVALUEAREA(Pjlw, pop1); +#endif + offset = j__udySearchLeafW(Pjlw + 1, pop1, Index); + + if (offset >= 0) // index is already valid: + { + DBGCODE(JudyCheckPop(*PPArray);) + JUDY1CODE(return(0); ) + JUDYLCODE(return((PPvoid_t) (Pjv + offset)); ) + } + + offset = ~offset; + +// Insert index in cases where no new memory is needed: + + if (JU_LEAFWGROWINPLACE(pop1)) + { + ++Pjlw[0]; // increase population. + + JU_INSERTINPLACE(Pjlw + 1, pop1, offset, Index); +#ifdef JUDYL + JU_INSERTINPLACE(Pjv, pop1, offset, 0); +#endif + DBGCODE(JudyCheckPop(*PPArray);) + DBGCODE(JudyCheckSorted(Pjlw + 1, pop1 + 1, cJU_ROOTSTATE);) + + JUDY1CODE(return(1); ) + JUDYLCODE(return((PPvoid_t) (Pjv + offset)); ) + } + +// Insert index into a new, larger leaf: + + if (pop1 < cJU_LEAFW_MAXPOP1) // can grow to a larger leaf. + { + Pjlwnew = j__udyAllocJLW(pop1 + 1); + JUDY1CODE(JU_CHECKALLOC(Pjlw_t, Pjlwnew, JERRI );) + JUDYLCODE(JU_CHECKALLOC(Pjlw_t, Pjlwnew, PPJERR);) + + Pjlwnew[0] = pop1; // set pop0 in new leaf. + + JU_INSERTCOPY(Pjlwnew + 1, Pjlw + 1, pop1, offset, Index); +#ifdef JUDYL + Pjvnew = JL_LEAFWVALUEAREA(Pjlwnew, pop1 + 1); + JU_INSERTCOPY(Pjvnew, Pjv, pop1, offset, 0); +#endif + DBGCODE(JudyCheckSorted(Pjlwnew + 1, pop1 + 1, cJU_ROOTSTATE);) + + j__udyFreeJLW(Pjlw, pop1, NULL); + + *PPArray = (Pvoid_t) Pjlwnew; + DBGCODE(JudyCheckPop(*PPArray);) + + JUDY1CODE(return(1); ) + JUDYLCODE(return((PPvoid_t) (Pjvnew + offset)); ) + } + + assert(pop1 == cJU_LEAFW_MAXPOP1); + +// Leaf at max size => cannot insert new index, so cascade instead: +// +// Upon cascading from a LEAFW leaf to the first branch, must allocate and +// initialize a JPM. + + Pjpm = j__udyAllocJPM(); + JUDY1CODE(JU_CHECKALLOC(Pjpm_t, Pjpm, JERRI );) + JUDYLCODE(JU_CHECKALLOC(Pjpm_t, Pjpm, PPJERR);) + + (Pjpm->jpm_Pop0) = cJU_LEAFW_MAXPOP1 - 1; + (Pjpm->jpm_JP.jp_Addr) = (Word_t) Pjlw; + + if (j__udyCascadeL(&(Pjpm->jpm_JP), Pjpm) == -1) + { + JU_COPY_ERRNO(PJError, Pjpm); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +// Note: No need to pass Pjpm for memory decrement; LEAFW memory is never +// counted in a JPM at all: + + j__udyFreeJLW(Pjlw, cJU_LEAFW_MAXPOP1, NULL); + *PPArray = (Pvoid_t) Pjpm; + + } // JU_LEAFW + +// **************************************************************************** +// BRANCH: + + { + int retcode; // really only needed for Judy1, but free for JudyL. + + Pjpm = P_JPM(*PPArray); + retcode = j__udyInsWalk(&(Pjpm->jpm_JP), Index, Pjpm); + + if (retcode == -1) + { + JU_COPY_ERRNO(PJError, Pjpm); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + if (retcode == 1) ++(Pjpm->jpm_Pop0); // incr total array popu. + + assert(((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_L) + || ((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_B) + || ((Pjpm->jpm_JP.jp_Type) == cJU_JPBRANCH_U)); + DBGCODE(JudyCheckPop(*PPArray);) + +#ifdef JUDY1 + assert((retcode == 0) || (retcode == 1)); + return(retcode); // == JU_RET_*_JPM(). +#else + assert(Pjpm->jpm_PValue != (Pjv_t) NULL); + return((PPvoid_t) Pjpm->jpm_PValue); +#endif + } + /*NOTREACHED*/ + +} // Judy1Set() / JudyLIns() diff --git a/libnetdata/libjudy/src/JudyL/JudyLInsArray.c b/libnetdata/libjudy/src/JudyL/JudyLInsArray.c new file mode 100644 index 000000000..f8e361f27 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLInsArray.c @@ -0,0 +1,1178 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// TBD: It would probably be faster for the caller if the JudyL version took +// PIndex as an interleaved array of indexes and values rather than just +// indexes with a separate values array (PValue), especially considering +// indexes and values are copied here with for-loops anyway and not the +// equivalent of memcpy(). All code could be revised to simply count by two +// words for JudyL? Supports "streaming" the data to/from disk better later? +// In which case get rid of JU_ERRNO_NULLPVALUE, no longer needed, and simplify +// the API to this code. +// _________________ + +// @(#) $Revision: 4.21 $ $Source: /judy/src/JudyCommon/JudyInsArray.c $ +// +// Judy1SetArray() and JudyLInsArray() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +DBGCODE(extern void JudyCheckPop(Pvoid_t PArray);) + + +// IMMED AND LEAF SIZE AND BRANCH TYPE ARRAYS: +// +// These support fast and easy lookup by level. + +static uint8_t immed_maxpop1[] = { + 0, + cJU_IMMED1_MAXPOP1, + cJU_IMMED2_MAXPOP1, + cJU_IMMED3_MAXPOP1, +#ifdef JU_64BIT + cJU_IMMED4_MAXPOP1, + cJU_IMMED5_MAXPOP1, + cJU_IMMED6_MAXPOP1, + cJU_IMMED7_MAXPOP1, +#endif + // note: There are no IMMEDs for whole words. +}; + +static uint8_t leaf_maxpop1[] = { + 0, +#if (defined(JUDYL) || (! defined(JU_64BIT))) + cJU_LEAF1_MAXPOP1, +#else + 0, // 64-bit Judy1 has no Leaf1. +#endif + cJU_LEAF2_MAXPOP1, + cJU_LEAF3_MAXPOP1, +#ifdef JU_64BIT + cJU_LEAF4_MAXPOP1, + cJU_LEAF5_MAXPOP1, + cJU_LEAF6_MAXPOP1, + cJU_LEAF7_MAXPOP1, +#endif + // note: Root-level leaves are handled differently. +}; + +static uint8_t branchL_JPtype[] = { + 0, + 0, + cJU_JPBRANCH_L2, + cJU_JPBRANCH_L3, +#ifdef JU_64BIT + cJU_JPBRANCH_L4, + cJU_JPBRANCH_L5, + cJU_JPBRANCH_L6, + cJU_JPBRANCH_L7, +#endif + cJU_JPBRANCH_L, +}; + +static uint8_t branchB_JPtype[] = { + 0, + 0, + cJU_JPBRANCH_B2, + cJU_JPBRANCH_B3, +#ifdef JU_64BIT + cJU_JPBRANCH_B4, + cJU_JPBRANCH_B5, + cJU_JPBRANCH_B6, + cJU_JPBRANCH_B7, +#endif + cJU_JPBRANCH_B, +}; + +static uint8_t branchU_JPtype[] = { + 0, + 0, + cJU_JPBRANCH_U2, + cJU_JPBRANCH_U3, +#ifdef JU_64BIT + cJU_JPBRANCH_U4, + cJU_JPBRANCH_U5, + cJU_JPBRANCH_U6, + cJU_JPBRANCH_U7, +#endif + cJU_JPBRANCH_U, +}; + +// Subexpanse masks are similer to JU_DCDMASK() but without the need to clear +// the first digits bits. Avoid doing variable shifts by precomputing a +// lookup array. + +static Word_t subexp_mask[] = { + 0, + ~cJU_POP0MASK(1), + ~cJU_POP0MASK(2), + ~cJU_POP0MASK(3), +#ifdef JU_64BIT + ~cJU_POP0MASK(4), + ~cJU_POP0MASK(5), + ~cJU_POP0MASK(6), + ~cJU_POP0MASK(7), +#endif +}; + + +// FUNCTION PROTOTYPES: + +static bool_t j__udyInsArray(Pjp_t PjpParent, int Level, PWord_t PPop1, + PWord_t PIndex, +#ifdef JUDYL + Pjv_t PValue, +#endif + Pjpm_t Pjpm); + + +// **************************************************************************** +// J U D Y 1 S E T A R R A Y +// J U D Y L I N S A R R A Y +// +// Main entry point. See the manual entry for external overview. +// +// TBD: Until thats written, note that the function returns 1 for success or +// JERRI for serious error, including insufficient memory to build whole array; +// use Judy*Count() to see how many were stored, the first N of the total +// Count. Also, since it takes Count == Pop1, it cannot handle a full array. +// Also, "sorted" means ascending without duplicates, otherwise you get the +// "unsorted" error. +// +// The purpose of these functions is to allow rapid construction of a large +// Judy array given a sorted list of indexes (and for JudyL, corresponding +// values). At least one customer saw this as useful, and probably it would +// also be useful as a sufficient workaround for fast(er) unload/reload to/from +// disk. +// +// This code is written recursively for simplicity, until/unless someone +// decides to make it faster and more complex. Hopefully recursion is fast +// enough simply because the function is so much faster than a series of +// Set/Ins calls. + +#ifdef JUDY1 +FUNCTION int Judy1SetArray +#else +FUNCTION int JudyLInsArray +#endif + ( + PPvoid_t PPArray, // in which to insert, initially empty. + Word_t Count, // number of indexes (and values) to insert. +const Word_t * const PIndex, // list of indexes to insert. +#ifdef JUDYL +const Word_t * const PValue, // list of corresponding values. +#endif + PJError_t PJError // optional, for returning error info. + ) +{ + Pjlw_t Pjlw; // new root-level leaf. + Pjlw_t Pjlwindex; // first index in root-level leaf. + int offset; // in PIndex. + + +// CHECK FOR NULL OR NON-NULL POINTER (error by caller): + + if (PPArray == (PPvoid_t) NULL) + { JU_SET_ERRNO(PJError, JU_ERRNO_NULLPPARRAY); return(JERRI); } + + if (*PPArray != (Pvoid_t) NULL) + { JU_SET_ERRNO(PJError, JU_ERRNO_NONNULLPARRAY); return(JERRI); } + + if (PIndex == (PWord_t) NULL) + { JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); return(JERRI); } + +#ifdef JUDYL + if (PValue == (PWord_t) NULL) + { JU_SET_ERRNO(PJError, JU_ERRNO_NULLPVALUE); return(JERRI); } +#endif + + +// HANDLE LARGE COUNT (= POP1) (typical case): +// +// Allocate and initialize a JPM, set the root pointer to point to it, and then +// build the tree underneath it. + +// Common code for unusual error handling when no JPM available: + + if (Count > cJU_LEAFW_MAXPOP1) // too big for root-level leaf. + { + Pjpm_t Pjpm; // new, to allocate. + +// Allocate JPM: + + Pjpm = j__udyAllocJPM(); + JU_CHECKALLOC(Pjpm_t, Pjpm, JERRI); + *PPArray = (Pvoid_t) Pjpm; + +// Set some JPM fields: + + (Pjpm->jpm_Pop0) = Count - 1; + // note: (Pjpm->jpm_TotalMemWords) is now initialized. + +// Build Judy tree: +// +// In case of error save the final Count, possibly modified, unless modified to +// 0, in which case free the JPM itself: + + if (! j__udyInsArray(&(Pjpm->jpm_JP), cJU_ROOTSTATE, &Count, + (PWord_t) PIndex, +#ifdef JUDYL + (Pjv_t) PValue, +#endif + Pjpm)) + { + JU_COPY_ERRNO(PJError, Pjpm); + + if (Count) // partial success, adjust pop0: + { + (Pjpm->jpm_Pop0) = Count - 1; + } + else // total failure, free JPM: + { + j__udyFreeJPM(Pjpm, (Pjpm_t) NULL); + *PPArray = (Pvoid_t) NULL; + } + + DBGCODE(JudyCheckPop(*PPArray);) + return(JERRI); + } + + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + + } // large count + + +// HANDLE SMALL COUNT (= POP1): +// +// First ensure indexes are in sorted order: + + for (offset = 1; offset < Count; ++offset) + { + if (PIndex[offset - 1] >= PIndex[offset]) + { JU_SET_ERRNO(PJError, JU_ERRNO_UNSORTED); return(JERRI); } + } + + if (Count == 0) return(1); // *PPArray remains null. + + { + Pjlw = j__udyAllocJLW(Count + 1); + JU_CHECKALLOC(Pjlw_t, Pjlw, JERRI); + *PPArray = (Pvoid_t) Pjlw; + Pjlw[0] = Count - 1; // set pop0. + Pjlwindex = Pjlw + 1; + } + +// Copy whole-word indexes (and values) to the root-level leaf: + + JU_COPYMEM(Pjlwindex, PIndex, Count); +JUDYLCODE(JU_COPYMEM(JL_LEAFWVALUEAREA(Pjlw, Count), PValue, Count)); + + DBGCODE(JudyCheckPop(*PPArray);) + return(1); + +} // Judy1SetArray() / JudyLInsArray() + + +// **************************************************************************** +// __ J U D Y I N S A R R A Y +// +// Given: +// +// - a pointer to a JP +// +// - the JPs level in the tree, that is, the number of digits left to decode +// in the indexes under the JP (one less than the level of the JPM or branch +// in which the JP resides); cJU_ROOTSTATE on first entry (when JP is the one +// in the JPM), down to 1 for a Leaf1, LeafB1, or FullPop +// +// - a pointer to the number of indexes (and corresponding values) to store in +// this subtree, to modify in case of partial success +// +// - a list of indexes (and for JudyL, corresponding values) to store in this +// subtree +// +// - a JPM for tracking memory usage and returning errors +// +// Recursively build a subtree (immediate indexes, leaf, or branch with +// subtrees) and modify the JP accordingly. On the way down, build a BranchU +// (only) for any expanse with *PPop1 too high for a leaf; on the way out, +// convert the BranchU to a BranchL or BranchB if appropriate. Keep memory +// statistics in the JPM. +// +// Return TRUE for success, or FALSE with error information set in the JPM in +// case of error, in which case leave a partially constructed but healthy tree, +// and modify parent population counts on the way out. +// +// Note: Each call of this function makes all modifications to the PjpParent +// it receives; neither the parent nor child calls do this. + +FUNCTION static bool_t j__udyInsArray( + Pjp_t PjpParent, // parent JP in/under which to store. + int Level, // initial digits remaining to decode. + PWord_t PPop1, // number of indexes to store. + PWord_t PIndex, // list of indexes to store. +#ifdef JUDYL + Pjv_t PValue, // list of corresponding values. +#endif + Pjpm_t Pjpm) // for memory and errors. +{ + Pjp_t Pjp; // lower-level JP. + Word_t Pjbany; // any type of branch. + int levelsub; // actual, of Pjps node, <= Level. + Word_t pop1 = *PPop1; // fast local value. + Word_t pop1sub; // population of one subexpanse. + uint8_t JPtype; // current JP type. + uint8_t JPtype_null; // precomputed value for new branch. + jp_t JPnull; // precomputed for speed. + Pjbu_t PjbuRaw; // constructed BranchU. + Pjbu_t Pjbu; + int digit; // in BranchU. + Word_t digitmask; // for a digit in a BranchU. + Word_t digitshifted; // shifted to correct offset. + Word_t digitshincr; // increment for digitshifted. + int offset; // in PIndex, or a bitmap subexpanse. + int numJPs; // number non-null in a BranchU. + bool_t retval; // to return from this func. +JUDYLCODE(Pjv_t PjvRaw); // destination value area. +JUDYLCODE(Pjv_t Pjv); + + +// MACROS FOR COMMON CODE: +// +// Note: These use function and local parameters from the context. +// Note: Assume newly allocated memory is zeroed. + +// Indicate whether a sorted list of indexes in PIndex, based on the first and +// last indexes in the list using pop1, are in the same subexpanse between +// Level and L_evel: +// +// This can be confusing! Note that SAMESUBEXP(L) == TRUE means the indexes +// are the same through level L + 1, and it says nothing about level L and +// lower; they might be the same or they might differ. +// +// Note: In principle SAMESUBEXP needs a mask for the digits from Level, +// inclusive, to L_evel, exclusive. But in practice, since the indexes are all +// known to be identical above Level, it just uses a mask for the digits +// through L_evel + 1; see subexp_mask[]. + +#define SAMESUBEXP(L_evel) \ + (! ((PIndex[0] ^ PIndex[pop1 - 1]) & subexp_mask[L_evel])) + +// Set PjpParent to a null JP appropriate for the level of the node to which it +// points, which is 1 less than the level of the node in which the JP resides, +// which is by definition Level: +// +// Note: This can set the JPMs JP to an invalid jp_Type, but it doesnt +// matter because the JPM is deleted by the caller. + +#define SETJPNULL_PARENT \ + JU_JPSETADT(PjpParent, 0, 0, cJU_JPNULL1 + Level - 1); + +// Variation to set a specified JP (in a branch being built) to a precomputed +// null JP: + +#define SETJPNULL(Pjp) *(Pjp) = JPnull + +// Handle complete (as opposed to partial) memory allocation failure: Set the +// parent JP to an appropriate null type (to leave a consistent tree), zero the +// callers population count, and return FALSE: +// +// Note: At Level == cJU_ROOTSTATE this sets the JPMs JPs jp_Type to a bogus +// value, but it doesnt matter because the JPM should be deleted by the +// caller. + +#define NOMEM { SETJPNULL_PARENT; *PPop1 = 0; return(FALSE); } + +// Allocate a Leaf1-N and save the address in Pjll; in case of failure, NOMEM: + +#define ALLOCLEAF(AllocLeaf) \ + if ((PjllRaw = AllocLeaf(pop1, Pjpm)) == (Pjll_t) NULL) NOMEM; \ + Pjll = P_JLL(PjllRaw); + +// Copy indexes smaller than words (and values which are whole words) from +// given arrays to immediate indexes or a leaf: +// +// TBD: These macros overlap with some of the code in JudyCascade.c; do some +// merging? That file has functions while these are macros. + +#define COPYTOLEAF_EVEN_SUB(Pjll,LeafType) \ + { \ + LeafType * P_leaf = (LeafType *) (Pjll); \ + Word_t p_op1 = pop1; \ + PWord_t P_Index = PIndex; \ + \ + assert(pop1 > 0); \ + \ + do { *P_leaf++ = *P_Index++; /* truncates */\ + } while (--(p_op1)); \ + } + +#define COPYTOLEAF_ODD_SUB(cLevel,Pjll,Copy) \ + { \ + uint8_t * P_leaf = (uint8_t *) (Pjll); \ + Word_t p_op1 = pop1; \ + PWord_t P_Index = PIndex; \ + \ + assert(pop1 > 0); \ + \ + do { \ + Copy(P_leaf, *P_Index); \ + P_leaf += (cLevel); ++P_Index; \ + } while (--(p_op1)); \ + } + +#ifdef JUDY1 + +#define COPYTOLEAF_EVEN(Pjll,LeafType) COPYTOLEAF_EVEN_SUB(Pjll,LeafType) +#define COPYTOLEAF_ODD(cLevel,Pjll,Copy) COPYTOLEAF_ODD_SUB(cLevel,Pjll,Copy) + +#else // JUDYL adds copying of values: + +#define COPYTOLEAF_EVEN(Pjll,LeafType) \ + { \ + COPYTOLEAF_EVEN_SUB(Pjll,LeafType) \ + JU_COPYMEM(Pjv, PValue, pop1); \ + } + +#define COPYTOLEAF_ODD(cLevel,Pjll,Copy) \ + { \ + COPYTOLEAF_ODD_SUB( cLevel,Pjll,Copy) \ + JU_COPYMEM(Pjv, PValue, pop1); \ + } + +#endif + +// Set the JP type for an immediate index, where BaseJPType is JPIMMED_*_02: + +#define SETIMMTYPE(BaseJPType) (PjpParent->jp_Type) = (BaseJPType) + pop1 - 2 + +// Allocate and populate a Leaf1-N: +// +// Build MAKELEAF_EVEN() and MAKELEAF_ODD() using macros for common code. + +#define MAKELEAF_SUB1(AllocLeaf,ValueArea,LeafType) \ + ALLOCLEAF(AllocLeaf); \ + JUDYLCODE(Pjv = ValueArea(Pjll, pop1)) + + +#define MAKELEAF_SUB2(cLevel,JPType) \ +{ \ + Word_t D_cdP0; \ + assert(pop1 - 1 <= cJU_POP0MASK(cLevel)); \ + D_cdP0 = (*PIndex & cJU_DCDMASK(cLevel)) | (pop1 - 1); \ + JU_JPSETADT(PjpParent, (Word_t)PjllRaw, D_cdP0, JPType); \ +} + + +#define MAKELEAF_EVEN(cLevel,JPType,AllocLeaf,ValueArea,LeafType) \ + MAKELEAF_SUB1(AllocLeaf,ValueArea,LeafType); \ + COPYTOLEAF_EVEN(Pjll, LeafType); \ + MAKELEAF_SUB2(cLevel, JPType) + +#define MAKELEAF_ODD(cLevel,JPType,AllocLeaf,ValueArea,Copy) \ + MAKELEAF_SUB1(AllocLeaf,ValueArea,LeafType); \ + COPYTOLEAF_ODD(cLevel, Pjll, Copy); \ + MAKELEAF_SUB2(cLevel, JPType) + +// Ensure that the indexes to be stored in immediate indexes or a leaf are +// sorted: +// +// This check is pure overhead, but required in order to protect the Judy array +// against caller error, to avoid a later corruption or core dump from a +// seemingly valid Judy array. Do this check piecemeal at the leaf level while +// the indexes are already in the cache. Higher-level order-checking occurs +// while building branches. +// +// Note: Any sorting error in the expanse of a single immediate indexes JP or +// a leaf => save no indexes in that expanse. + +#define CHECKLEAFORDER \ + { \ + for (offset = 1; offset < pop1; ++offset) \ + { \ + if (PIndex[offset - 1] >= PIndex[offset]) \ + { \ + SETJPNULL_PARENT; \ + *PPop1 = 0; \ + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_UNSORTED); \ + return(FALSE); \ + } \ + } \ + } + + +// ------ START OF CODE ------ + + assert( Level >= 1); + assert( Level <= cJU_ROOTSTATE); + assert((Level < cJU_ROOTSTATE) || (pop1 > cJU_LEAFW_MAXPOP1)); + + +// CHECK FOR TOP LEVEL: +// +// Special case: If at the top level (PjpParent is in the JPM), a top-level +// branch must be created, even if its a BranchL with just one JP. (The JPM +// cannot point to a leaf because the leaf would have to be a lower-level, +// higher-capacity leaf under a narrow pointer (otherwise a root-level leaf +// would suffice), and the JPMs JP cant handle a narrow pointer because the +// jp_DcdPopO field isnt big enough.) Otherwise continue to check for a pop1 +// small enough to support immediate indexes or a leaf before giving up and +// making a lower-level branch. + + if (Level == cJU_ROOTSTATE) + { + levelsub = cJU_ROOTSTATE; + goto BuildBranch2; + } + assert(Level < cJU_ROOTSTATE); + + +// SKIP JPIMMED_*_01: +// +// Immeds with pop1 == 1 should be handled in-line during branch construction. + + assert(pop1 > 1); + + +// BUILD JPIMMED_*_02+: +// +// The starting address of the indexes depends on Judy1 or JudyL; also, JudyL +// includes a pointer to a values-only leaf. + + if (pop1 <= immed_maxpop1[Level]) // note: always < root level. + { + JUDY1CODE(uint8_t * Pjll = (uint8_t *) (PjpParent->jp_1Index);) + JUDYLCODE(uint8_t * Pjll = (uint8_t *) (PjpParent->jp_LIndex);) + + CHECKLEAFORDER; // indexes to be stored are sorted. + +#ifdef JUDYL + if ((PjvRaw = j__udyLAllocJV(pop1, Pjpm)) == (Pjv_t) NULL) + NOMEM; + (PjpParent->jp_Addr) = (Word_t) PjvRaw; + Pjv = P_JV(PjvRaw); +#endif + + switch (Level) + { + case 1: COPYTOLEAF_EVEN(Pjll, uint8_t); + SETIMMTYPE(cJU_JPIMMED_1_02); + break; +#if (defined(JUDY1) || defined(JU_64BIT)) + case 2: COPYTOLEAF_EVEN(Pjll, uint16_t); + SETIMMTYPE(cJU_JPIMMED_2_02); + break; + case 3: COPYTOLEAF_ODD(3, Pjll, JU_COPY3_LONG_TO_PINDEX); + SETIMMTYPE(cJU_JPIMMED_3_02); + break; +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case 4: COPYTOLEAF_EVEN(Pjll, uint32_t); + SETIMMTYPE(cJ1_JPIMMED_4_02); + break; + case 5: COPYTOLEAF_ODD(5, Pjll, JU_COPY5_LONG_TO_PINDEX); + SETIMMTYPE(cJ1_JPIMMED_5_02); + break; + case 6: COPYTOLEAF_ODD(6, Pjll, JU_COPY6_LONG_TO_PINDEX); + SETIMMTYPE(cJ1_JPIMMED_6_02); + break; + case 7: COPYTOLEAF_ODD(7, Pjll, JU_COPY7_LONG_TO_PINDEX); + SETIMMTYPE(cJ1_JPIMMED_7_02); + break; +#endif + default: assert(FALSE); // should be impossible. + } + + return(TRUE); // note: no children => no *PPop1 mods. + + } // JPIMMED_*_02+ + + +// BUILD JPLEAF*: +// +// This code is a little tricky. The method is: For each level starting at +// the present Level down through levelsub = 1, and then as a special case for +// LeafB1 and FullPop (which are also at levelsub = 1 but have different +// capacity, see later), check if pop1 fits in a leaf (using leaf_maxpop1[]) +// at that level. If so, except for Level == levelsub, check if all of the +// current indexes to be stored are in the same (narrow) subexpanse, that is, +// the digits from Level to levelsub + 1, inclusive, are identical between the +// first and last index in the (sorted) list (in PIndex). If this condition is +// satisfied at any level, build a leaf at that level (under a narrow pointer +// if Level > levelsub). +// +// Note: Doing the search in this order results in storing the indexes in +// "least compressed form." + + for (levelsub = Level; levelsub >= 1; --levelsub) + { + Pjll_t PjllRaw; + Pjll_t Pjll; + +// Check if pop1 is too large to fit in a leaf at levelsub; if so, try the next +// lower level: + + if (pop1 > leaf_maxpop1[levelsub]) continue; + +// If pop1 fits in a leaf at levelsub, but levelsub is lower than Level, must +// also check whether all the indexes in the expanse to store can in fact be +// placed under a narrow pointer; if not, a leaf cannot be used, at this or any +// lower level (levelsub): + + if ((levelsub < Level) && (! SAMESUBEXP(levelsub))) + goto BuildBranch; // cant use a narrow, need a branch. + +// Ensure valid pop1 and all indexes are in fact common through Level: + + assert(pop1 <= cJU_POP0MASK(Level) + 1); + assert(! ((PIndex[0] ^ PIndex[pop1 - 1]) & cJU_DCDMASK(Level))); + + CHECKLEAFORDER; // indexes to be stored are sorted. + +// Build correct type of leaf: +// +// Note: The jp_DcdPopO and jp_Type assignments in MAKELEAF_* happen correctly +// for the levelsub (not Level) of the new leaf, even if its under a narrow +// pointer. + + switch (levelsub) + { +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case 1: MAKELEAF_EVEN(1, cJU_JPLEAF1, j__udyAllocJLL1, + JL_LEAF1VALUEAREA, uint8_t); + break; +#endif + case 2: MAKELEAF_EVEN(2, cJU_JPLEAF2, j__udyAllocJLL2, + JL_LEAF2VALUEAREA, uint16_t); + break; + case 3: MAKELEAF_ODD( 3, cJU_JPLEAF3, j__udyAllocJLL3, + JL_LEAF3VALUEAREA, JU_COPY3_LONG_TO_PINDEX); + break; +#ifdef JU_64BIT + case 4: MAKELEAF_EVEN(4, cJU_JPLEAF4, j__udyAllocJLL4, + JL_LEAF4VALUEAREA, uint32_t); + break; + case 5: MAKELEAF_ODD( 5, cJU_JPLEAF5, j__udyAllocJLL5, + JL_LEAF5VALUEAREA, JU_COPY5_LONG_TO_PINDEX); + break; + case 6: MAKELEAF_ODD( 6, cJU_JPLEAF6, j__udyAllocJLL6, + JL_LEAF6VALUEAREA, JU_COPY6_LONG_TO_PINDEX); + break; + case 7: MAKELEAF_ODD( 7, cJU_JPLEAF7, j__udyAllocJLL7, + JL_LEAF7VALUEAREA, JU_COPY7_LONG_TO_PINDEX); + break; +#endif + default: assert(FALSE); // should be impossible. + } + + return(TRUE); // note: no children => no *PPop1 mods. + + } // JPLEAF* + + +// BUILD JPLEAF_B1 OR JPFULLPOPU1: +// +// See above about JPLEAF*. If pop1 doesnt fit in any level of linear leaf, +// it might still fit in a LeafB1 or FullPop, perhaps under a narrow pointer. + + if ((Level == 1) || SAMESUBEXP(1)) // same until last digit. + { + Pjlb_t PjlbRaw; // for bitmap leaf. + Pjlb_t Pjlb; + + assert(pop1 <= cJU_JPFULLPOPU1_POP0 + 1); + CHECKLEAFORDER; // indexes to be stored are sorted. + +#ifdef JUDY1 + +// JPFULLPOPU1: + + if (pop1 == cJU_JPFULLPOPU1_POP0 + 1) + { + Word_t Addr = PjpParent->jp_Addr; + Word_t DcdP0 = (*PIndex & cJU_DCDMASK(1)) + | cJU_JPFULLPOPU1_POP0; + JU_JPSETADT(PjpParent, Addr, DcdP0, cJ1_JPFULLPOPU1); + + return(TRUE); + } +#endif + +// JPLEAF_B1: + + if ((PjlbRaw = j__udyAllocJLB1(Pjpm)) == (Pjlb_t) NULL) + NOMEM; + Pjlb = P_JLB(PjlbRaw); + + for (offset = 0; offset < pop1; ++offset) + JU_BITMAPSETL(Pjlb, PIndex[offset]); + + retval = TRUE; // default. + +#ifdef JUDYL + +// Build subexpanse values-only leaves (LeafVs) under LeafB1: + + for (offset = 0; offset < cJU_NUMSUBEXPL; ++offset) + { + if (! (pop1sub = j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, offset)))) + continue; // skip empty subexpanse. + +// Allocate one LeafV = JP subarray; if out of memory, clear bitmaps for higher +// subexpanses and adjust *PPop1: + + if ((PjvRaw = j__udyLAllocJV(pop1sub, Pjpm)) + == (Pjv_t) NULL) + { + for (/* null */; offset < cJU_NUMSUBEXPL; ++offset) + { + *PPop1 -= j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, offset)); + JU_JLB_BITMAP(Pjlb, offset) = 0; + } + + retval = FALSE; + break; + } + +// Populate values-only leaf and save the pointer to it: + + Pjv = P_JV(PjvRaw); + JU_COPYMEM(Pjv, PValue, pop1sub); + JL_JLB_PVALUE(Pjlb, offset) = PjvRaw; // first-tier pointer. + PValue += pop1sub; + + } // for each subexpanse + +#endif // JUDYL + +// Attach new LeafB1 to parent JP; note use of *PPop1 possibly < pop1: + + JU_JPSETADT(PjpParent, (Word_t) PjlbRaw, + (*PIndex & cJU_DCDMASK(1)) | (*PPop1 - 1), cJU_JPLEAF_B1); + + return(retval); + + } // JPLEAF_B1 or JPFULLPOPU1 + + +// BUILD JPBRANCH_U*: +// +// Arriving at BuildBranch means Level < top level but the pop1 is too large +// for immediate indexes or a leaf, even under a narrow pointer, including a +// LeafB1 or FullPop at level 1. This implies SAMESUBEXP(1) == FALSE, that is, +// the indexes to be stored "branch" at level 2 or higher. + +BuildBranch: // come here directly if a leaf wont work. + + assert(Level >= 2); + assert(Level < cJU_ROOTSTATE); + assert(! SAMESUBEXP(1)); // sanity check, see above. + +// Determine the appropriate level for a new branch node; see if a narrow +// pointer can be used: +// +// This can be confusing. The branch is required at the lowest level L where +// the indexes to store are not in the same subexpanse at level L-1. Work down +// from Level to tree level 3, which is 1 above the lowest tree level = 2 at +// which a branch can be used. Theres no need to check SAMESUBEXP at level 2 +// because its known to be false at level 2-1 = 1. +// +// Note: Unlike for a leaf node, a narrow pointer is always used for a branch +// if possible, that is, maximum compression is always used, except at the top +// level of the tree, where a JPM cannot support a narrow pointer, meaning a +// top BranchL can have a single JP (fanout = 1); but that case jumps directly +// to BuildBranch2. +// +// Note: For 32-bit systems the only usable values for a narrow pointer are +// Level = 3 and levelsub = 2; 64-bit systems have many more choices; but +// hopefully this for-loop is fast enough even on a 32-bit system. +// +// TBD: If not fast enough, #ifdef JU_64BIT and handle the 32-bit case faster. + + for (levelsub = Level; levelsub >= 3; --levelsub) // see above. + if (! SAMESUBEXP(levelsub - 1)) // at limit of narrow pointer. + break; // put branch at levelsub. + +BuildBranch2: // come here directly for Level = levelsub = cJU_ROOTSTATE. + + assert(levelsub >= 2); + assert(levelsub <= Level); + +// Initially build a BranchU: +// +// Always start with a BranchU because the number of populated subexpanses is +// not yet known. Use digitmask, digitshifted, and digitshincr to avoid +// expensive variable shifts within JU_DIGITATSTATE within the loop. +// +// TBD: The use of digitmask, etc. results in more increment operations per +// loop, is there an even faster way? +// +// TBD: Would it pay to pre-count the populated JPs (subexpanses) and +// pre-compress the branch, that is, build a BranchL or BranchB immediately, +// also taking account of opportunistic uncompression rules? Probably not +// because at high levels of the tree there might be huge numbers of indexes +// (hence cache lines) to scan in the PIndex array to determine the fanout +// (number of JPs) needed. + + if ((PjbuRaw = j__udyAllocJBU(Pjpm)) == (Pjbu_t) NULL) NOMEM; + Pjbu = P_JBU(PjbuRaw); + + JPtype_null = cJU_JPNULL1 + levelsub - 2; // in new BranchU. + JU_JPSETADT(&JPnull, 0, 0, JPtype_null); + + Pjp = Pjbu->jbu_jp; // for convenience in loop. + numJPs = 0; // non-null in the BranchU. + digitmask = cJU_MASKATSTATE(levelsub); // see above. + digitshincr = 1UL << (cJU_BITSPERBYTE * (levelsub - 1)); + retval = TRUE; + +// Scan and populate JPs (subexpanses): +// +// Look for all indexes matching each digit in the BranchU (at the correct +// levelsub), and meanwhile notice any sorting error. Increment PIndex (and +// PValue) and reduce pop1 for each subexpanse handled successfully. + + for (digit = digitshifted = 0; + digit < cJU_BRANCHUNUMJPS; + ++digit, digitshifted += digitshincr, ++Pjp) + { + DBGCODE(Word_t pop1subprev;) + assert(pop1 != 0); // end of indexes is handled elsewhere. + +// Count indexes in digits subexpanse: + + for (pop1sub = 0; pop1sub < pop1; ++pop1sub) + if (digitshifted != (PIndex[pop1sub] & digitmask)) break; + +// Empty subexpanse (typical, performance path) or sorting error (rare): + + if (pop1sub == 0) + { + if (digitshifted < (PIndex[0] & digitmask)) + { SETJPNULL(Pjp); continue; } // empty subexpanse. + + assert(pop1 < *PPop1); // did save >= 1 index and decr pop1. + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_UNSORTED); + goto AbandonBranch; + } + +// Non-empty subexpanse: +// +// First shortcut by handling pop1sub == 1 (JPIMMED_*_01) inline locally. + + if (pop1sub == 1) // note: can be at root level. + { + Word_t Addr = 0; + JUDYLCODE(Addr = (Word_t) (*PValue++);) + JU_JPSETADT(Pjp, Addr, *PIndex, cJU_JPIMMED_1_01 + levelsub -2); + + ++numJPs; + + if (--pop1) { ++PIndex; continue; } // more indexes to store. + + ++digit; ++Pjp; // skip JP just saved. + goto ClearBranch; // save time. + } + +// Recurse to populate one digits (subexpanses) JP; if successful, skip +// indexes (and values) just stored (performance path), except when expanse is +// completely stored: + + DBGCODE(pop1subprev = pop1sub;) + + if (j__udyInsArray(Pjp, levelsub - 1, &pop1sub, (PWord_t) PIndex, +#ifdef JUDYL + (Pjv_t) PValue, +#endif + Pjpm)) + { // complete success. + ++numJPs; + assert(pop1subprev == pop1sub); + assert(pop1 >= pop1sub); + + if ((pop1 -= pop1sub) != 0) // more indexes to store: + { + PIndex += pop1sub; // skip indexes just stored. + JUDYLCODE(PValue += pop1sub;) + continue; + } + // else leave PIndex in BranchUs expanse. + +// No more indexes to store in BranchUs expanse: + + ++digit; ++Pjp; // skip JP just saved. + goto ClearBranch; // save time. + } + +// Handle any error at a lower level of recursion: +// +// In case of partial success, pop1sub != 0, but it was reduced from the value +// passed to j__udyInsArray(); skip this JP later during ClearBranch. + + assert(pop1subprev > pop1sub); // check j__udyInsArray(). + assert(pop1 > pop1sub); // check j__udyInsArray(). + + if (pop1sub) // partial success. + { ++digit; ++Pjp; ++numJPs; } // skip JP just saved. + + pop1 -= pop1sub; // deduct saved indexes if any. + +// Same-level sorting error, or any lower-level error; abandon the rest of the +// branch: +// +// Arrive here with pop1 = remaining unsaved indexes (always non-zero). Adjust +// the *PPop1 value to record and return, modify retval, and use ClearBranch to +// finish up. + +AbandonBranch: + assert(pop1 != 0); // more to store, see above. + assert(pop1 <= *PPop1); // sanity check. + + *PPop1 -= pop1; // deduct unsaved indexes. + pop1 = 0; // to avoid error later. + retval = FALSE; + +// Error (rare), or end of indexes while traversing new BranchU (performance +// path); either way, mark the remaining JPs, if any, in the BranchU as nulls +// and exit the loop: +// +// Arrive here with digit and Pjp set to the first JP to set to null. + +ClearBranch: + for (/* null */; digit < cJU_BRANCHUNUMJPS; ++digit, ++Pjp) + SETJPNULL(Pjp); + break; // saves one more compare. + + } // for each digit + + +// FINISH JPBRANCH_U*: +// +// Arrive here with a BranchU built under Pjbu, numJPs set, and either: retval +// == TRUE and *PPop1 unmodified, or else retval == FALSE, *PPop1 set to the +// actual number of indexes saved (possibly 0 for complete failure at a lower +// level upon the first call of j__udyInsArray()), and the Judy error set in +// Pjpm. Either way, PIndex points to an index within the expanse just +// handled. + + Pjbany = (Word_t) PjbuRaw; // default = use this BranchU. + JPtype = branchU_JPtype[levelsub]; + +// Check for complete failure above: + + assert((! retval) || *PPop1); // sanity check. + + if ((! retval) && (*PPop1 == 0)) // nothing stored, full failure. + { + j__udyFreeJBU(PjbuRaw, Pjpm); + SETJPNULL_PARENT; + return(FALSE); + } + +// Complete or partial success so far; watch for sorting error after the +// maximum digit (255) in the BranchU, which is indicated by having more +// indexes to store in the BranchUs expanse: +// +// For example, if an index to store has a digit of 255 at levelsub, followed +// by an index with a digit of 254, the for-loop above runs out of digits +// without reducing pop1 to 0. + + if (pop1 != 0) + { + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_UNSORTED); + *PPop1 -= pop1; // deduct unsaved indexes. + retval = FALSE; + } + assert(*PPop1 != 0); // branch (still) cannot be empty. + + +// OPTIONALLY COMPRESS JPBRANCH_U*: +// +// See if the BranchU should be compressed to a BranchL or BranchB; if so, do +// that and free the BranchU; otherwise just use the existing BranchU. Follow +// the same rules as in JudyIns.c (version 4.95): Only check local population +// (cJU_OPP_UNCOMP_POP0) for BranchL, and only check global memory efficiency +// (JU_OPP_UNCOMPRESS) for BranchB. TBD: Have the rules changed? +// +// Note: Because of differing order of operations, the latter compression +// might not result in the same set of branch nodes as a series of sequential +// insertions. +// +// Note: Allocating a BranchU only to sometimes convert it to a BranchL or +// BranchB is unfortunate, but attempting to work with a temporary BranchU on +// the stack and then allocate and keep it as a BranchU in many cases is worse +// in terms of error handling. + + +// COMPRESS JPBRANCH_U* TO JPBRANCH_L*: + + if (numJPs <= cJU_BRANCHLMAXJPS) // JPs fit in a BranchL. + { + Pjbl_t PjblRaw = (Pjbl_t) NULL; // new BranchL; init for cc. + Pjbl_t Pjbl; + + if ((*PPop1 > JU_BRANCHL_MAX_POP) // pop too high. + || ((PjblRaw = j__udyAllocJBL(Pjpm)) == (Pjbl_t) NULL)) + { // cant alloc BranchL. + goto SetParent; // just keep BranchU. + } + + Pjbl = P_JBL(PjblRaw); + +// Copy BranchU JPs to BranchL: + + (Pjbl->jbl_NumJPs) = numJPs; + offset = 0; + + for (digit = 0; digit < cJU_BRANCHUNUMJPS; ++digit) + { + if ((((Pjbu->jbu_jp) + digit)->jp_Type) == JPtype_null) + continue; + + (Pjbl->jbl_Expanse[offset ]) = digit; + (Pjbl->jbl_jp [offset++]) = Pjbu->jbu_jp[digit]; + } + assert(offset == numJPs); // found same number. + +// Free the BranchU and prepare to use the new BranchL instead: + + j__udyFreeJBU(PjbuRaw, Pjpm); + + Pjbany = (Word_t) PjblRaw; + JPtype = branchL_JPtype[levelsub]; + + } // compress to BranchL + + +// COMPRESS JPBRANCH_U* TO JPBRANCH_B*: +// +// If unable to allocate the BranchB or any JP subarray, free all related +// memory and just keep the BranchU. +// +// Note: This use of JU_OPP_UNCOMPRESS is a bit conservative because the +// BranchU is already allocated while the (presumably smaller) BranchB is not, +// the opposite of how its used in single-insert code. + + else + { + Pjbb_t PjbbRaw = (Pjbb_t) NULL; // new BranchB; init for cc. + Pjbb_t Pjbb; + Pjp_t Pjp2; // in BranchU. + + if ((*PPop1 > JU_BRANCHB_MAX_POP) // pop too high. + || ((PjbbRaw = j__udyAllocJBB(Pjpm)) == (Pjbb_t) NULL)) + { // cant alloc BranchB. + goto SetParent; // just keep BranchU. + } + + Pjbb = P_JBB(PjbbRaw); + +// Set bits in bitmap for populated subexpanses: + + Pjp2 = Pjbu->jbu_jp; + + for (digit = 0; digit < cJU_BRANCHUNUMJPS; ++digit) + if ((((Pjbu->jbu_jp) + digit)->jp_Type) != JPtype_null) + JU_BITMAPSETB(Pjbb, digit); + +// Copy non-null JPs to BranchB JP subarrays: + + for (offset = 0; offset < cJU_NUMSUBEXPB; ++offset) + { + Pjp_t PjparrayRaw; + Pjp_t Pjparray; + + if (! (numJPs = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, offset)))) + continue; // skip empty subexpanse. + +// If unable to allocate a JP subarray, free all BranchB memory so far and +// continue to use the BranchU: + + if ((PjparrayRaw = j__udyAllocJBBJP(numJPs, Pjpm)) + == (Pjp_t) NULL) + { + while (offset-- > 0) + { + if (JU_JBB_PJP(Pjbb, offset) == (Pjp_t) NULL) continue; + + j__udyFreeJBBJP(JU_JBB_PJP(Pjbb, offset), + j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, offset)), + Pjpm); + } + j__udyFreeJBB(PjbbRaw, Pjpm); + goto SetParent; // keep BranchU. + } + +// Set one JP subarray pointer and copy the subexpanses JPs to the subarray: +// +// Scan the BranchU for non-null JPs until numJPs JPs are copied. + + JU_JBB_PJP(Pjbb, offset) = PjparrayRaw; + Pjparray = P_JP(PjparrayRaw); + + while (numJPs-- > 0) + { + while ((Pjp2->jp_Type) == JPtype_null) + { + ++Pjp2; + assert(Pjp2 < (Pjbu->jbu_jp) + cJU_BRANCHUNUMJPS); + } + *Pjparray++ = *Pjp2++; + } + } // for each subexpanse + +// Free the BranchU and prepare to use the new BranchB instead: + + j__udyFreeJBU(PjbuRaw, Pjpm); + + Pjbany = (Word_t) PjbbRaw; + JPtype = branchB_JPtype[levelsub]; + + } // compress to BranchB + + +// COMPLETE OR PARTIAL SUCCESS: +// +// Attach new branch (under Pjp, with JPtype) to parent JP; note use of *PPop1, +// possibly reduced due to partial failure. + +SetParent: + (PjpParent->jp_Addr) = Pjbany; + (PjpParent->jp_Type) = JPtype; + + if (Level < cJU_ROOTSTATE) // PjpParent not in JPM: + { + Word_t DcdP0 = (*PIndex & cJU_DCDMASK(levelsub)) | (*PPop1 - 1); + + JU_JPSETADT(PjpParent ,Pjbany, DcdP0, JPtype); + } + + return(retval); + +} // j__udyInsArray() diff --git a/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c b/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c new file mode 100644 index 000000000..cfa16bd6d --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c @@ -0,0 +1,135 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.17 $ $Source: /judy/src/JudyCommon/JudyInsertBranch.c $ + +// BranchL insertion functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +extern int j__udyCreateBranchL(Pjp_t, Pjp_t, uint8_t *, Word_t, Pvoid_t); + + +// **************************************************************************** +// __ J U D Y I N S E R T B R A N C H +// +// Insert 2-element BranchL in between Pjp and Pjp->jp_Addr. +// +// Return -1 if out of memory, otherwise return 1. + +FUNCTION int j__udyInsertBranch( + Pjp_t Pjp, // JP containing narrow pointer. + Word_t Index, // outlier to Pjp. + Word_t BranchLevel, // of what JP points to, mapped from JP type. + Pjpm_t Pjpm) // for global accounting. +{ + jp_t JP2 [2]; + jp_t JP; + Pjp_t PjpNull; + Word_t XorExp; + Word_t Inew, Iold; + Word_t DCDMask; // initially for original BranchLevel. + int Ret; + uint8_t Exp2[2]; + uint8_t DecodeByteN, DecodeByteO; + +// Get the current mask for the DCD digits: + + DCDMask = cJU_DCDMASK(BranchLevel); + +// Obtain Dcd bits that differ between Index and JP, shifted so the +// digit for BranchLevel is the LSB: + + XorExp = ((Index ^ JU_JPDCDPOP0(Pjp)) & (cJU_ALLONES >> cJU_BITSPERBYTE)) + >> (BranchLevel * cJU_BITSPERBYTE); + assert(XorExp); // Index must be an outlier. + +// Count levels between object under narrow pointer and the level at which +// the outlier diverges from it, which is always at least initial +// BranchLevel + 1, to end up with the level (JP type) at which to insert +// the new intervening BranchL: + + do { ++BranchLevel; } while ((XorExp >>= cJU_BITSPERBYTE)); + assert((BranchLevel > 1) && (BranchLevel < cJU_ROOTSTATE)); + +// Get the MSB (highest digit) that differs between the old expanse and +// the new Index to insert: + + DecodeByteO = JU_DIGITATSTATE(JU_JPDCDPOP0(Pjp), BranchLevel); + DecodeByteN = JU_DIGITATSTATE(Index, BranchLevel); + + assert(DecodeByteO != DecodeByteN); + +// Determine sorted order for old expanse and new Index digits: + + if (DecodeByteN > DecodeByteO) { Iold = 0; Inew = 1; } + else { Iold = 1; Inew = 0; } + +// Copy old JP into staging area for new Branch + JP2 [Iold] = *Pjp; + Exp2[Iold] = DecodeByteO; + Exp2[Inew] = DecodeByteN; + +// Create a 2 Expanse Linear branch +// +// Note: Pjp->jp_Addr is set by j__udyCreateBranchL() + + Ret = j__udyCreateBranchL(Pjp, JP2, Exp2, 2, Pjpm); + if (Ret == -1) return(-1); + +// Get Pjp to the NULL of where to do insert + PjpNull = ((P_JBL(Pjp->jp_Addr))->jbl_jp) + Inew; + +// Convert to a cJU_JPIMMED_*_01 at the correct level: +// Build JP and set type below to: cJU_JPIMMED_X_01 + JU_JPSETADT(PjpNull, 0, Index, cJU_JPIMMED_1_01 - 2 + BranchLevel); + +// Return pointer to Value area in cJU_JPIMMED_X_01 + JUDYLCODE(Pjpm->jpm_PValue = (Pjv_t) PjpNull;) + +// The old JP now points to a BranchL that is at higher level. Therefore +// it contains excess DCD bits (in the least significant position) that +// must be removed (zeroed); that is, they become part of the Pop0 +// subfield. Note that the remaining (lower) bytes in the Pop0 field do +// not change. +// +// Take from the old DCDMask, which went "down" to a lower BranchLevel, +// and zero any high bits that are still in the mask at the new, higher +// BranchLevel; then use this mask to zero the bits in jp_DcdPopO: + +// Set old JP to a BranchL at correct level + + Pjp->jp_Type = cJU_JPBRANCH_L2 - 2 + BranchLevel; + DCDMask ^= cJU_DCDMASK(BranchLevel); + DCDMask = ~DCDMask & JU_JPDCDPOP0(Pjp); + JP = *Pjp; + JU_JPSETADT(Pjp, JP.jp_Addr, DCDMask, JP.jp_Type); + + return(1); + +} // j__udyInsertBranch() diff --git a/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c b/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c new file mode 100644 index 000000000..9a7d02f21 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c @@ -0,0 +1,782 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.45 $ $Source: /judy/src/JudyCommon/JudyMallocIF.c $ +// +// Judy malloc/free interface functions for Judy1 and JudyL. +// +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DTRACEMI (Malloc Interface) to turn on tracing of malloc/free +// calls at the interface level. (See also TRACEMF in lower-level code.) +// Use -DTRACEMI2 for a terser format suitable for trace analysis. +// +// There can be malloc namespace bits in the LSBs of "raw" addresses from most, +// but not all, of the j__udy*Alloc*() functions; see also JudyPrivate.h. To +// test the Judy code, compile this file with -DMALLOCBITS and use debug flavor +// only (for assertions). This test ensures that (a) all callers properly mask +// the namespace bits out before dereferencing a pointer (or else a core dump +// occurs), and (b) all callers send "raw" (unmasked) addresses to +// j__udy*Free*() calls. +// +// Note: Currently -DDEBUG turns on MALLOCBITS automatically. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +// Set "hidden" global j__uMaxWords to the maximum number of words to allocate +// to any one array (large enough to have a JPM, otherwise j__uMaxWords is +// ignored), to trigger a fake malloc error when the number is exceeded. Note, +// this code is always executed, not #ifdefd, because its virtually free. +// +// Note: To keep the MALLOC macro faster and simpler, set j__uMaxWords to +// MAXINT, not zero, by default. + +Word_t j__uMaxWords = ~0UL; + +// This macro hides the faking of a malloc failure: +// +// Note: To keep this fast, just compare WordsPrev to j__uMaxWords without the +// complexity of first adding WordsNow, meaning the trigger point is not +// exactly where you might assume, but it shouldnt matter. + +#define MALLOC(MallocFunc,WordsPrev,WordsNow) \ + (((WordsPrev) > j__uMaxWords) ? 0UL : MallocFunc(WordsNow)) + +// Clear words starting at address: +// +// Note: Only use this for objects that care; in other cases, it doesnt +// matter if the objects memory is pre-zeroed. + +#define ZEROWORDS(Addr,Words) \ + { \ + Word_t Words__ = (Words); \ + PWord_t Addr__ = (PWord_t) (Addr); \ + while (Words__--) *Addr__++ = 0UL; \ + } + +#ifdef TRACEMI + +// TRACING SUPPORT: +// +// Note: For TRACEMI, use a format for address printing compatible with other +// tracing facilities; in particular, %x not %lx, to truncate the "noisy" high +// part on 64-bit systems. +// +// TBD: The trace macros need fixing for alternate address types. +// +// Note: TRACEMI2 supports trace analysis no matter the underlying malloc/free +// engine used. + +#include <stdio.h> + +static Word_t j__udyMemSequence = 0L; // event sequence number. + +#define TRACE_ALLOC5(a,b,c,d,e) (void) printf(a, (b), c, d) +#define TRACE_FREE5( a,b,c,d,e) (void) printf(a, (b), c, d) +#define TRACE_ALLOC6(a,b,c,d,e,f) (void) printf(a, (b), c, d, e) +#define TRACE_FREE6( a,b,c,d,e,f) (void) printf(a, (b), c, d, e) + +#else + +#ifdef TRACEMI2 + +#include <stdio.h> + +#define b_pw cJU_BYTESPERWORD + +#define TRACE_ALLOC5(a,b,c,d,e) \ + (void) printf("a %lx %lx %lx\n", (b), (d) * b_pw, e) +#define TRACE_FREE5( a,b,c,d,e) \ + (void) printf("f %lx %lx %lx\n", (b), (d) * b_pw, e) +#define TRACE_ALLOC6(a,b,c,d,e,f) \ + (void) printf("a %lx %lx %lx\n", (b), (e) * b_pw, f) +#define TRACE_FREE6( a,b,c,d,e,f) \ + (void) printf("f %lx %lx %lx\n", (b), (e) * b_pw, f) + +static Word_t j__udyMemSequence = 0L; // event sequence number. + +#else + +#define TRACE_ALLOC5(a,b,c,d,e) // null. +#define TRACE_FREE5( a,b,c,d,e) // null. +#define TRACE_ALLOC6(a,b,c,d,e,f) // null. +#define TRACE_FREE6( a,b,c,d,e,f) // null. + +#endif // ! TRACEMI2 +#endif // ! TRACEMI + + +// MALLOC NAMESPACE SUPPORT: + +#if (defined(DEBUG) && (! defined(MALLOCBITS))) // for now, DEBUG => MALLOCBITS: +#define MALLOCBITS 1 +#endif + +#ifdef MALLOCBITS +#define MALLOCBITS_VALUE 0x3 // bit pattern to use. +#define MALLOCBITS_MASK 0x7 // note: matches mask__ in JudyPrivate.h. + +#define MALLOCBITS_SET( Type,Addr) \ + ((Addr) = (Type) ((Word_t) (Addr) | MALLOCBITS_VALUE)) +#define MALLOCBITS_TEST(Type,Addr) \ + assert((((Word_t) (Addr)) & MALLOCBITS_MASK) == MALLOCBITS_VALUE); \ + ((Addr) = (Type) ((Word_t) (Addr) & ~MALLOCBITS_VALUE)) +#else +#define MALLOCBITS_SET( Type,Addr) // null. +#define MALLOCBITS_TEST(Type,Addr) // null. +#endif + + +// SAVE ERROR INFORMATION IN A Pjpm: +// +// "Small" (invalid) Addr values are used to distinguish overrun and no-mem +// errors. (TBD, non-zero invalid values are no longer returned from +// lower-level functions, that is, JU_ERRNO_OVERRUN is no longer detected.) + +#define J__UDYSETALLOCERROR(Addr) \ + { \ + JU_ERRID(Pjpm) = __LINE__; \ + if ((Word_t) (Addr) > 0) JU_ERRNO(Pjpm) = JU_ERRNO_OVERRUN; \ + else JU_ERRNO(Pjpm) = JU_ERRNO_NOMEM; \ + return(0); \ + } + + +// **************************************************************************** +// ALLOCATION FUNCTIONS: +// +// To help the compiler catch coding errors, each function returns a specific +// object type. +// +// Note: Only j__udyAllocJPM() and j__udyAllocJLW() return multiple values <= +// sizeof(Word_t) to indicate the type of memory allocation failure. Other +// allocation functions convert this failure to a JU_ERRNO. + + +// Note: Unlike other j__udyAlloc*() functions, Pjpms are returned non-raw, +// that is, without malloc namespace or root pointer type bits: + +FUNCTION Pjpm_t j__udyAllocJPM(void) +{ + Word_t Words = (sizeof(jpm_t) + cJU_BYTESPERWORD - 1) / cJU_BYTESPERWORD; + Pjpm_t Pjpm = (Pjpm_t) MALLOC(JudyMalloc, Words, Words); + + assert((Words * cJU_BYTESPERWORD) == sizeof(jpm_t)); + + if ((Word_t) Pjpm > sizeof(Word_t)) + { + ZEROWORDS(Pjpm, Words); + Pjpm->jpm_TotalMemWords = Words; + } + + TRACE_ALLOC5("0x%x %8lu = j__udyAllocJPM(), Words = %lu\n", + Pjpm, j__udyMemSequence++, Words, cJU_LEAFW_MAXPOP1 + 1); + // MALLOCBITS_SET(Pjpm_t, Pjpm); // see above. + return(Pjpm); + +} // j__udyAllocJPM() + + +FUNCTION Pjbl_t j__udyAllocJBL(Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbl_t) / cJU_BYTESPERWORD; + Pjbl_t PjblRaw = (Pjbl_t) MALLOC(JudyMallocVirtual, + Pjpm->jpm_TotalMemWords, Words); + + assert((Words * cJU_BYTESPERWORD) == sizeof(jbl_t)); + + if ((Word_t) PjblRaw > sizeof(Word_t)) + { + ZEROWORDS(P_JBL(PjblRaw), Words); + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjblRaw); } + + TRACE_ALLOC5("0x%x %8lu = j__udyAllocJBL(), Words = %lu\n", PjblRaw, + j__udyMemSequence++, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjbl_t, PjblRaw); + return(PjblRaw); + +} // j__udyAllocJBL() + + +FUNCTION Pjbb_t j__udyAllocJBB(Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbb_t) / cJU_BYTESPERWORD; + Pjbb_t PjbbRaw = (Pjbb_t) MALLOC(JudyMallocVirtual, + Pjpm->jpm_TotalMemWords, Words); + + assert((Words * cJU_BYTESPERWORD) == sizeof(jbb_t)); + + if ((Word_t) PjbbRaw > sizeof(Word_t)) + { + ZEROWORDS(P_JBB(PjbbRaw), Words); + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjbbRaw); } + + TRACE_ALLOC5("0x%x %8lu = j__udyAllocJBB(), Words = %lu\n", PjbbRaw, + j__udyMemSequence++, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjbb_t, PjbbRaw); + return(PjbbRaw); + +} // j__udyAllocJBB() + + +FUNCTION Pjp_t j__udyAllocJBBJP(Word_t NumJPs, Pjpm_t Pjpm) +{ + Word_t Words = JU_BRANCHJP_NUMJPSTOWORDS(NumJPs); + Pjp_t PjpRaw; + + PjpRaw = (Pjp_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjpRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjpRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJBBJP(%lu), Words = %lu\n", PjpRaw, + j__udyMemSequence++, NumJPs, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjp_t, PjpRaw); + return(PjpRaw); + +} // j__udyAllocJBBJP() + + +FUNCTION Pjbu_t j__udyAllocJBU(Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbu_t) / cJU_BYTESPERWORD; + Pjbu_t PjbuRaw = (Pjbu_t) MALLOC(JudyMallocVirtual, + Pjpm->jpm_TotalMemWords, Words); + + assert((Words * cJU_BYTESPERWORD) == sizeof(jbu_t)); + + if ((Word_t) PjbuRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjbuRaw); } + + TRACE_ALLOC5("0x%x %8lu = j__udyAllocJBU(), Words = %lu\n", PjbuRaw, + j__udyMemSequence++, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjbu_t, PjbuRaw); + return(PjbuRaw); + +} // j__udyAllocJBU() + + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + +FUNCTION Pjll_t j__udyAllocJLL1(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF1POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL1(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL1() + +#endif // (JUDYL || (! JU_64BIT)) + + +FUNCTION Pjll_t j__udyAllocJLL2(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF2POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL2(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL2() + + +FUNCTION Pjll_t j__udyAllocJLL3(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF3POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL3(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL3() + + +#ifdef JU_64BIT + +FUNCTION Pjll_t j__udyAllocJLL4(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF4POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL4(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL4() + + +FUNCTION Pjll_t j__udyAllocJLL5(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF5POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL5(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL5() + + +FUNCTION Pjll_t j__udyAllocJLL6(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF6POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL6(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL6() + + +FUNCTION Pjll_t j__udyAllocJLL7(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF7POPTOWORDS(Pop1); + Pjll_t PjllRaw; + + PjllRaw = (Pjll_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjllRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjllRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLL7(%lu), Words = %lu\n", PjllRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjll_t, PjllRaw); + return(PjllRaw); + +} // j__udyAllocJLL7() + +#endif // JU_64BIT + + +// Note: Root-level leaf addresses are always whole words (Pjlw_t), and unlike +// other j__udyAlloc*() functions, they are returned non-raw, that is, without +// malloc namespace or root pointer type bits (the latter are added later by +// the caller): + +FUNCTION Pjlw_t j__udyAllocJLW(Word_t Pop1) +{ + Word_t Words = JU_LEAFWPOPTOWORDS(Pop1); + Pjlw_t Pjlw = (Pjlw_t) MALLOC(JudyMalloc, Words, Words); + + TRACE_ALLOC6("0x%x %8lu = j__udyAllocJLW(%lu), Words = %lu\n", Pjlw, + j__udyMemSequence++, Pop1, Words, Pop1); + // MALLOCBITS_SET(Pjlw_t, Pjlw); // see above. + return(Pjlw); + +} // j__udyAllocJLW() + + +FUNCTION Pjlb_t j__udyAllocJLB1(Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jlb_t) / cJU_BYTESPERWORD; + Pjlb_t PjlbRaw; + + PjlbRaw = (Pjlb_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + assert((Words * cJU_BYTESPERWORD) == sizeof(jlb_t)); + + if ((Word_t) PjlbRaw > sizeof(Word_t)) + { + ZEROWORDS(P_JLB(PjlbRaw), Words); + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjlbRaw); } + + TRACE_ALLOC5("0x%x %8lu = j__udyAllocJLB1(), Words = %lu\n", PjlbRaw, + j__udyMemSequence++, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjlb_t, PjlbRaw); + return(PjlbRaw); + +} // j__udyAllocJLB1() + + +#ifdef JUDYL + +FUNCTION Pjv_t j__udyLAllocJV(Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JL_LEAFVPOPTOWORDS(Pop1); + Pjv_t PjvRaw; + + PjvRaw = (Pjv_t) MALLOC(JudyMalloc, Pjpm->jpm_TotalMemWords, Words); + + if ((Word_t) PjvRaw > sizeof(Word_t)) + { + Pjpm->jpm_TotalMemWords += Words; + } + else { J__UDYSETALLOCERROR(PjvRaw); } + + TRACE_ALLOC6("0x%x %8lu = j__udyLAllocJV(%lu), Words = %lu\n", PjvRaw, + j__udyMemSequence++, Pop1, Words, (Pjpm->jpm_Pop0) + 2); + MALLOCBITS_SET(Pjv_t, PjvRaw); + return(PjvRaw); + +} // j__udyLAllocJV() + +#endif // JUDYL + + +// **************************************************************************** +// FREE FUNCTIONS: +// +// To help the compiler catch coding errors, each function takes a specific +// object type to free. + + +// Note: j__udyFreeJPM() receives a root pointer with NO root pointer type +// bits present, that is, they must be stripped by the caller using P_JPM(): + +FUNCTION void j__udyFreeJPM(Pjpm_t PjpmFree, Pjpm_t PjpmStats) +{ + Word_t Words = (sizeof(jpm_t) + cJU_BYTESPERWORD - 1) / cJU_BYTESPERWORD; + + // MALLOCBITS_TEST(Pjpm_t, PjpmFree); // see above. + JudyFree((Pvoid_t) PjpmFree, Words); + + if (PjpmStats != (Pjpm_t) NULL) PjpmStats->jpm_TotalMemWords -= Words; + +// Note: Log PjpmFree->jpm_Pop0, similar to other j__udyFree*() functions, not +// an assumed value of cJU_LEAFW_MAXPOP1, for when the caller is +// Judy*FreeArray(), jpm_Pop0 is set to 0, and the population after the free +// really will be 0, not cJU_LEAFW_MAXPOP1. + + TRACE_FREE6("0x%x %8lu = j__udyFreeJPM(%lu), Words = %lu\n", PjpmFree, + j__udyMemSequence++, Words, Words, PjpmFree->jpm_Pop0); + + +} // j__udyFreeJPM() + + +FUNCTION void j__udyFreeJBL(Pjbl_t Pjbl, Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbl_t) / cJU_BYTESPERWORD; + + MALLOCBITS_TEST(Pjbl_t, Pjbl); + JudyFreeVirtual((Pvoid_t) Pjbl, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE5("0x%x %8lu = j__udyFreeJBL(), Words = %lu\n", Pjbl, + j__udyMemSequence++, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJBL() + + +FUNCTION void j__udyFreeJBB(Pjbb_t Pjbb, Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbb_t) / cJU_BYTESPERWORD; + + MALLOCBITS_TEST(Pjbb_t, Pjbb); + JudyFreeVirtual((Pvoid_t) Pjbb, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE5("0x%x %8lu = j__udyFreeJBB(), Words = %lu\n", Pjbb, + j__udyMemSequence++, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJBB() + + +FUNCTION void j__udyFreeJBBJP(Pjp_t Pjp, Word_t NumJPs, Pjpm_t Pjpm) +{ + Word_t Words = JU_BRANCHJP_NUMJPSTOWORDS(NumJPs); + + MALLOCBITS_TEST(Pjp_t, Pjp); + JudyFree((Pvoid_t) Pjp, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJBBJP(%lu), Words = %lu\n", Pjp, + j__udyMemSequence++, NumJPs, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJBBJP() + + +FUNCTION void j__udyFreeJBU(Pjbu_t Pjbu, Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jbu_t) / cJU_BYTESPERWORD; + + MALLOCBITS_TEST(Pjbu_t, Pjbu); + JudyFreeVirtual((Pvoid_t) Pjbu, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE5("0x%x %8lu = j__udyFreeJBU(), Words = %lu\n", Pjbu, + j__udyMemSequence++, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJBU() + + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + +FUNCTION void j__udyFreeJLL1(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF1POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL1(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL1() + +#endif // (JUDYL || (! JU_64BIT)) + + +FUNCTION void j__udyFreeJLL2(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF2POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL2(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL2() + + +FUNCTION void j__udyFreeJLL3(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF3POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL3(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL3() + + +#ifdef JU_64BIT + +FUNCTION void j__udyFreeJLL4(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF4POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL4(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL4() + + +FUNCTION void j__udyFreeJLL5(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF5POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL5(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL5() + + +FUNCTION void j__udyFreeJLL6(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF6POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL6(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL6() + + +FUNCTION void j__udyFreeJLL7(Pjll_t Pjll, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAF7POPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjll_t, Pjll); + JudyFree((Pvoid_t) Pjll, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLL7(%lu), Words = %lu\n", Pjll, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLL7() + +#endif // JU_64BIT + + +// Note: j__udyFreeJLW() receives a root pointer with NO root pointer type +// bits present, that is, they are stripped by P_JLW(): + +FUNCTION void j__udyFreeJLW(Pjlw_t Pjlw, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JU_LEAFWPOPTOWORDS(Pop1); + + // MALLOCBITS_TEST(Pjlw_t, Pjlw); // see above. + JudyFree((Pvoid_t) Pjlw, Words); + + if (Pjpm) Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyFreeJLW(%lu), Words = %lu\n", Pjlw, + j__udyMemSequence++, Pop1, Words, Pop1 - 1); + + +} // j__udyFreeJLW() + + +FUNCTION void j__udyFreeJLB1(Pjlb_t Pjlb, Pjpm_t Pjpm) +{ + Word_t Words = sizeof(jlb_t) / cJU_BYTESPERWORD; + + MALLOCBITS_TEST(Pjlb_t, Pjlb); + JudyFree((Pvoid_t) Pjlb, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE5("0x%x %8lu = j__udyFreeJLB1(), Words = %lu\n", Pjlb, + j__udyMemSequence++, Words, Pjpm->jpm_Pop0); + + +} // j__udyFreeJLB1() + + +#ifdef JUDYL + +FUNCTION void j__udyLFreeJV(Pjv_t Pjv, Word_t Pop1, Pjpm_t Pjpm) +{ + Word_t Words = JL_LEAFVPOPTOWORDS(Pop1); + + MALLOCBITS_TEST(Pjv_t, Pjv); + JudyFree((Pvoid_t) Pjv, Words); + + Pjpm->jpm_TotalMemWords -= Words; + + TRACE_FREE6("0x%x %8lu = j__udyLFreeJV(%lu), Words = %lu\n", Pjv, + j__udyMemSequence++, Pop1, Words, Pjpm->jpm_Pop0); + + +} // j__udyLFreeJV() + +#endif // JUDYL diff --git a/libnetdata/libjudy/src/JudyL/JudyLMemActive.c b/libnetdata/libjudy/src/JudyL/JudyLMemActive.c new file mode 100644 index 000000000..fb58d0e25 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLMemActive.c @@ -0,0 +1,259 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.7 $ $Source: /judy/src/JudyCommon/JudyMemActive.c $ +// +// Return number of bytes of memory used to support a Judy1/L array. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +FUNCTION static Word_t j__udyGetMemActive(Pjp_t); + + +// **************************************************************************** +// J U D Y 1 M E M A C T I V E +// J U D Y L M E M A C T I V E + +#ifdef JUDY1 +FUNCTION Word_t Judy1MemActive +#else +FUNCTION Word_t JudyLMemActive +#endif + ( + Pcvoid_t PArray // from which to retrieve. + ) +{ + if (PArray == (Pcvoid_t)NULL) return(0); + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + Word_t Words = Pjlw[0] + 1; // population. +#ifdef JUDY1 + return((Words + 1) * sizeof(Word_t)); +#else + return(((Words * 2) + 1) * sizeof(Word_t)); +#endif + } + else + { + Pjpm_t Pjpm = P_JPM(PArray); + return(j__udyGetMemActive(&Pjpm->jpm_JP) + sizeof(jpm_t)); + } + +} // JudyMemActive() + + +// **************************************************************************** +// __ J U D Y G E T M E M A C T I V E + +FUNCTION static Word_t j__udyGetMemActive( + Pjp_t Pjp) // top of subtree. +{ + Word_t offset; // in a branch. + Word_t Bytes = 0; // actual bytes used at this level. + Word_t IdxSz; // bytes per index in leaves + + switch (JU_JPTYPE(Pjp)) + { + + case cJU_JPBRANCH_L2: + case cJU_JPBRANCH_L3: +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + case cJU_JPBRANCH_L5: + case cJU_JPBRANCH_L6: + case cJU_JPBRANCH_L7: +#endif + case cJU_JPBRANCH_L: + { + Pjbl_t Pjbl = P_JBL(Pjp->jp_Addr); + + for (offset = 0; offset < (Pjbl->jbl_NumJPs); ++offset) + Bytes += j__udyGetMemActive((Pjbl->jbl_jp) + offset); + + return(Bytes + sizeof(jbl_t)); + } + + case cJU_JPBRANCH_B2: + case cJU_JPBRANCH_B3: +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + case cJU_JPBRANCH_B5: + case cJU_JPBRANCH_B6: + case cJU_JPBRANCH_B7: +#endif + case cJU_JPBRANCH_B: + { + Word_t subexp; + Word_t jpcount; + Pjbb_t Pjbb = P_JBB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + Bytes += jpcount * sizeof(jp_t); + + for (offset = 0; offset < jpcount; ++offset) + { + Bytes += j__udyGetMemActive(P_JP(JU_JBB_PJP(Pjbb, subexp)) + + offset); + } + } + + return(Bytes + sizeof(jbb_t)); + } + + case cJU_JPBRANCH_U2: + case cJU_JPBRANCH_U3: +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: + case cJU_JPBRANCH_U5: + case cJU_JPBRANCH_U6: + case cJU_JPBRANCH_U7: +#endif + case cJU_JPBRANCH_U: + { + Pjbu_t Pjbu = P_JBU(Pjp->jp_Addr); + + for (offset = 0; offset < cJU_BRANCHUNUMJPS; ++offset) + { + if (((Pjbu->jbu_jp[offset].jp_Type) >= cJU_JPNULL1) + && ((Pjbu->jbu_jp[offset].jp_Type) <= cJU_JPNULLMAX)) + { + continue; // skip null JP to save time. + } + + Bytes += j__udyGetMemActive(Pjbu->jbu_jp + offset); + } + + return(Bytes + sizeof(jbu_t)); + } + + +// -- Cases below here terminate and do not recurse. -- + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: IdxSz = 1; goto LeafWords; +#endif + case cJU_JPLEAF2: IdxSz = 2; goto LeafWords; + case cJU_JPLEAF3: IdxSz = 3; goto LeafWords; +#ifdef JU_64BIT + case cJU_JPLEAF4: IdxSz = 4; goto LeafWords; + case cJU_JPLEAF5: IdxSz = 5; goto LeafWords; + case cJU_JPLEAF6: IdxSz = 6; goto LeafWords; + case cJU_JPLEAF7: IdxSz = 7; goto LeafWords; +#endif +LeafWords: + +#ifdef JUDY1 + return(IdxSz * (JU_JPLEAF_POP0(Pjp) + 1)); +#else + return((IdxSz + sizeof(Word_t)) + * (JU_JPLEAF_POP0(Pjp) + 1)); +#endif + case cJU_JPLEAF_B1: + { +#ifdef JUDY1 + return(sizeof(jlb_t)); +#else + Bytes = (JU_JPLEAF_POP0(Pjp) + 1) * sizeof(Word_t); + + return(Bytes + sizeof(jlb_t)); +#endif + } + + JUDY1CODE(case cJ1_JPFULLPOPU1: return(0);) + +#ifdef JUDY1 +#define J__Mpy 0 +#else +#define J__Mpy sizeof(Word_t) +#endif + + case cJU_JPIMMED_1_01: return(0); + case cJU_JPIMMED_2_01: return(0); + case cJU_JPIMMED_3_01: return(0); +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: return(0); + case cJU_JPIMMED_5_01: return(0); + case cJU_JPIMMED_6_01: return(0); + case cJU_JPIMMED_7_01: return(0); +#endif + + case cJU_JPIMMED_1_02: return(J__Mpy * 2); + case cJU_JPIMMED_1_03: return(J__Mpy * 3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: return(J__Mpy * 4); + case cJU_JPIMMED_1_05: return(J__Mpy * 5); + case cJU_JPIMMED_1_06: return(J__Mpy * 6); + case cJU_JPIMMED_1_07: return(J__Mpy * 7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: return(0); + case cJ1_JPIMMED_1_09: return(0); + case cJ1_JPIMMED_1_10: return(0); + case cJ1_JPIMMED_1_11: return(0); + case cJ1_JPIMMED_1_12: return(0); + case cJ1_JPIMMED_1_13: return(0); + case cJ1_JPIMMED_1_14: return(0); + case cJ1_JPIMMED_1_15: return(0); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: return(J__Mpy * 2); + case cJU_JPIMMED_2_03: return(J__Mpy * 3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: return(0); + case cJ1_JPIMMED_2_05: return(0); + case cJ1_JPIMMED_2_06: return(0); + case cJ1_JPIMMED_2_07: return(0); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: return(J__Mpy * 2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: return(0); + case cJ1_JPIMMED_3_04: return(0); + case cJ1_JPIMMED_3_05: return(0); + + case cJ1_JPIMMED_4_02: return(0); + case cJ1_JPIMMED_4_03: return(0); + case cJ1_JPIMMED_5_02: return(0); + case cJ1_JPIMMED_5_03: return(0); + case cJ1_JPIMMED_6_02: return(0); + case cJ1_JPIMMED_7_02: return(0); +#endif + + } // switch (JU_JPTYPE(Pjp)) + + return(0); // to make some compilers happy. + +} // j__udyGetMemActive() diff --git a/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c b/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c new file mode 100644 index 000000000..81e3a79ce --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c @@ -0,0 +1,61 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.5 $ $Source: /judy/src/JudyCommon/JudyMemUsed.c $ +// +// Return number of bytes of memory used to support a Judy1/L array. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +#ifdef JUDY1 +FUNCTION Word_t Judy1MemUsed +#else // JUDYL +FUNCTION Word_t JudyLMemUsed +#endif + ( + Pcvoid_t PArray // from which to retrieve. + ) +{ + Word_t Words = 0; + + if (PArray == (Pcvoid_t) NULL) return(0); + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + Words = JU_LEAFWPOPTOWORDS(Pjlw[0] + 1); // based on pop1. + } + else + { + Pjpm_t Pjpm = P_JPM(PArray); + Words = Pjpm->jpm_TotalMemWords; + } + + return(Words * sizeof(Word_t)); // convert to bytes. + +} // Judy1MemUsed() / JudyLMemUsed() diff --git a/libnetdata/libjudy/src/JudyL/JudyLNext.c b/libnetdata/libjudy/src/JudyL/JudyLNext.c new file mode 100644 index 000000000..4bcdccf10 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLNext.c @@ -0,0 +1,1890 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.54 $ $Source: /judy/src/JudyCommon/JudyPrevNext.c $ +// +// Judy*Prev() and Judy*Next() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DJUDYNEXT for the Judy*Next() function; otherwise defaults to +// Judy*Prev(). + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifndef JUDYNEXT +#ifndef JUDYPREV +#define JUDYPREV 1 // neither set => use default. +#endif +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + + +// **************************************************************************** +// J U D Y 1 P R E V +// J U D Y 1 N E X T +// J U D Y L P R E V +// J U D Y L N E X T +// +// See the manual entry for the API. +// +// OVERVIEW OF Judy*Prev(): +// +// Use a reentrant switch statement (state machine, SM1 = "get") to decode the +// callers *PIndex-1, starting with the (PArray), through branches, if +// any, down to an immediate or a leaf. Look for *PIndex-1 in that leaf, and +// if found, return it. +// +// A dead end is either a branch that does not contain a JP for the appropriate +// digit in *PIndex-1, or a leaf that does not contain the undecoded digits of +// *PIndex-1. Upon reaching a dead end, backtrack through the leaf/branches +// that were just traversed, using a list (history) of parent JPs that is built +// while going forward in SM1Get. Start with the current leaf or branch. In a +// backtracked leaf, look for an Index less than *PIndex-1. In each +// backtracked branch, look "sideways" for the next JP, if any, lower than the +// one for the digit (from *PIndex-1) that was previously decoded. While +// backtracking, if a leaf has no previous Index or a branch has no lower JP, +// go to its parent branch in turn. Upon reaching the JRP, return failure, "no +// previous Index". The backtrack process is sufficiently different from +// SM1Get to merit its own separate reentrant switch statement (SM2 = +// "backtrack"). +// +// While backtracking, upon finding a lower JP in a branch, there is certain to +// be a "prev" Index under that JP (unless the Judy array is corrupt). +// Traverse forward again, this time taking the last (highest, right-most) JP +// in each branch, and the last (highest) Index upon reaching an immediate or a +// leaf. This traversal is sufficiently different from SM1Get and SM2Backtrack +// to merit its own separate reentrant switch statement (SM3 = "findlimit"). +// +// "Decode" bytes in JPs complicate this process a little. In SM1Get, when a +// JP is a narrow pointer, that is, when states are skipped (so the skipped +// digits are stored in jp_DcdPopO), compare the relevant digits to the same +// digits in *PIndex-1. If they are EQUAL, proceed in SM1Get as before. If +// jp_DcdPopOs digits are GREATER, treat the JP as a dead end and proceed in +// SM2Backtrack. If jp_DcdPopOs digits are LESS, treat the JP as if it had +// just been found during a backtrack and proceed directly in SM3Findlimit. +// +// Note that Decode bytes can be ignored in SM3Findlimit; they dont matter. +// Also note that in practice the Decode bytes are routinely compared with +// *PIndex-1 because thats simpler and no slower than first testing for +// narrowness. +// +// Decode bytes also make it unnecessary to construct the Index to return (the +// revised *PIndex) during the search. This step is deferred until finding an +// Index during backtrack or findlimit, before returning it. The first digit +// of *PIndex is derived (saved) based on which JP is used in a JRP branch. +// The remaining digits are obtained from the jp_DcdPopO field in the JP (if +// any) above the immediate or leaf containing the found (prev) Index, plus the +// remaining digit(s) in the immediate or leaf itself. In the case of a LEAFW, +// the Index to return is found directly in the leaf. +// +// Note: Theoretically, as described above, upon reaching a dead end, SM1Get +// passes control to SM2Backtrack to look sideways, even in a leaf. Actually +// its a little more efficient for the SM1Get leaf cases to shortcut this and +// take care of the sideways searches themselves. Hence the history list only +// contains branch JPs, and SM2Backtrack only handles branches. In fact, even +// the branch handling cases in SM1Get do some shortcutting (sideways +// searching) to avoid pushing history and calling SM2Backtrack unnecessarily. +// +// Upon reaching an Index to return after backtracking, *PIndex must be +// modified to the found Index. In principle this could be done by building +// the Index from a saved rootdigit (in the top branch) plus the Dcd bytes from +// the parent JP plus the appropriate Index bytes from the leaf. However, +// Immediates are difficult because their parent JPs lack one (last) digit. So +// instead just build the *PIndex to return "top down" while backtracking and +// findlimiting. +// +// This function is written iteratively for speed, rather than recursively. +// +// CAVEATS: +// +// Why use a backtrack list (history stack), since it has finite size? The +// size is small for Judy on both 32-bit and 64-bit systems, and a list (really +// just an array) is fast to maintain and use. Other alternatives include +// doing a lookahead (lookaside) in each branch while traversing forward +// (decoding), and restarting from the top upon a dead end. +// +// A lookahead means noting the last branch traversed which contained a +// non-null JP lower than the one specified by a digit in *PIndex-1, and +// returning to that point for SM3Findlimit. This seems like a good idea, and +// should be pretty cheap for linear and bitmap branches, but it could result +// in up to 31 unnecessary additional cache line fills (in extreme cases) for +// every uncompressed branch traversed. We have considered means of attaching +// to or hiding within an uncompressed branch (in null JPs) a "cache line map" +// or other structure, such as an offset to the next non-null JP, that would +// speed this up, but it seems unnecessary merely to avoid having a +// finite-length list (array). (If JudySL is ever made "native", the finite +// list length will be an issue.) +// +// Restarting at the top of the Judy array after a dead end requires a careful +// modification of *PIndex-1 to decrement the digit for the parent branch and +// set the remaining lower digits to all 1s. This must be repeated each time a +// parent branch contains another dead end, so even though it should all happen +// in cache, the CPU time can be excessive. (For JudySL or an equivalent +// "infinitely deep" Judy array, consider a hybrid of a large, finite, +// "circular" list and a restart-at-top when the list is backtracked to +// exhaustion.) +// +// Why search for *PIndex-1 instead of *PIndex during SM1Get? In rare +// instances this prevents an unnecessary decode down the wrong path followed +// by a backtrack; its pretty cheap to set up initially; and it means the +// SM1Get machine can simply return if/when it finds that Index. +// +// TBD: Wed like to enhance this function to make successive searches faster. +// This would require saving some previous state, including the previous Index +// returned, and in which leaf it was found. If the next call is for the same +// Index and the array has not been modified, start at the same leaf. This +// should be much easier to implement since this is iterative rather than +// recursive code. +// +// VARIATIONS FOR Judy*Next(): +// +// The Judy*Next() code is nearly a perfect mirror of the Judy*Prev() code. +// See the Judy*Prev() overview comments, and mentally switch the following: +// +// - "*PIndex-1" => "*PIndex+1" +// - "less than" => "greater than" +// - "lower" => "higher" +// - "lowest" => "highest" +// - "next-left" => "next-right" +// - "right-most" => "left-most" +// +// Note: SM3Findlimit could be called SM3Findmax/SM3Findmin, but a common name +// for both Prev and Next means many fewer ifdefs in this code. +// +// TBD: Currently this code traverses a JP whether its expanse is partially or +// completely full (populated). For Judy1 (only), since there is no value area +// needed, consider shortcutting to a "success" return upon encountering a full +// JP in SM1Get (or even SM3Findlimit?) A full JP looks like this: +// +// (((JU_JPDCDPOP0(Pjp) ^ cJU_ALLONES) & cJU_POP0MASK(cLevel)) == 0) + +#ifdef JUDY1 +#ifdef JUDYPREV +FUNCTION int Judy1Prev +#else +FUNCTION int Judy1Next +#endif +#else +#ifdef JUDYPREV +FUNCTION PPvoid_t JudyLPrev +#else +FUNCTION PPvoid_t JudyLNext +#endif +#endif + ( + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError // optional, for returning error info. + ) +{ + Pjp_t Pjp, Pjp2; // current JPs. + Pjbl_t Pjbl; // Pjp->jp_Addr masked and cast to types: + Pjbb_t Pjbb; + Pjbu_t Pjbu; + +// Note: The following initialization is not strictly required but it makes +// gcc -Wall happy because there is an "impossible" path from Immed handling to +// SM1LeafLImm code that looks like Pjll might be used before set: + + Pjll_t Pjll = (Pjll_t) NULL; + Word_t state; // current state in SM. + Word_t digit; // next digit to decode from Index. + +// Note: The following initialization is not strictly required but it makes +// gcc -Wall happy because there is an "impossible" path from Immed handling to +// SM1LeafLImm code (for JudyL & JudyPrev only) that looks like pop1 might be +// used before set: + +#if (defined(JUDYL) && defined(JUDYPREV)) + Word_t pop1 = 0; // in a leaf. +#else + Word_t pop1; // in a leaf. +#endif + int offset; // linear branch/leaf, from j__udySearchLeaf*(). + int subexp; // subexpanse in a bitmap branch. + Word_t bitposmask; // bit in bitmap for Index. + +// History for SM2Backtrack: +// +// For a given histnum, APjphist[histnum] is a parent JP that points to a +// branch, and Aoffhist[histnum] is the offset of the NEXT JP in the branch to +// which the parent JP points. The meaning of Aoffhist[histnum] depends on the +// type of branch to which the parent JP points: +// +// Linear: Offset of the next JP in the JP list. +// +// Bitmap: Which subexpanse, plus the offset of the next JP in the +// subexpanses JP list (to avoid bit-counting again), plus for Judy*Next(), +// hidden one byte to the left, which digit, because Judy*Next() also needs +// this. +// +// Uncompressed: Digit, which is actually the offset of the JP in the branch. +// +// Note: Only branch JPs are stored in APjphist[] because, as explained +// earlier, SM1Get shortcuts sideways searches in leaves (and even in branches +// in some cases), so SM2Backtrack only handles branches. + +#define HISTNUMMAX cJU_ROOTSTATE // maximum branches traversable. + Pjp_t APjphist[HISTNUMMAX]; // list of branch JPs traversed. + int Aoffhist[HISTNUMMAX]; // list of next JP offsets; see above. + int histnum = 0; // number of JPs now in list. + + +// ---------------------------------------------------------------------------- +// M A C R O S +// +// These are intended to make the code a bit more readable and less redundant. + + +// "PUSH" AND "POP" Pjp AND offset ON HISTORY STACKS: +// +// Note: Ensure a corrupt Judy array does not overflow *hist[]. Meanwhile, +// underflowing *hist[] simply means theres no more room to backtrack => +// "no previous/next Index". + +#define HISTPUSH(Pjp,Offset) \ + APjphist[histnum] = (Pjp); \ + Aoffhist[histnum] = (Offset); \ + \ + if (++histnum >= HISTNUMMAX) \ + { \ + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT) \ + JUDY1CODE(return(JERRI );) \ + JUDYLCODE(return(PPJERR);) \ + } + +#define HISTPOP(Pjp,Offset) \ + if ((histnum--) < 1) JU_RET_NOTFOUND; \ + (Pjp) = APjphist[histnum]; \ + (Offset) = Aoffhist[histnum] + +// How to pack/unpack Aoffhist[] values for bitmap branches: + +#ifdef JUDYPREV + +#define HISTPUSHBOFF(Subexp,Offset,Digit) \ + (((Subexp) * cJU_BITSPERSUBEXPB) | (Offset)) + +#define HISTPOPBOFF(Subexp,Offset,Digit) \ + (Subexp) = (Offset) / cJU_BITSPERSUBEXPB; \ + (Offset) %= cJU_BITSPERSUBEXPB +#else + +#define HISTPUSHBOFF(Subexp,Offset,Digit) \ + (((Digit) << cJU_BITSPERBYTE) \ + | ((Subexp) * cJU_BITSPERSUBEXPB) | (Offset)) + +#define HISTPOPBOFF(Subexp,Offset,Digit) \ + (Digit) = (Offset) >> cJU_BITSPERBYTE; \ + (Subexp) = ((Offset) & JU_LEASTBYTESMASK(1)) / cJU_BITSPERSUBEXPB; \ + (Offset) %= cJU_BITSPERSUBEXPB +#endif + + +// CHECK FOR NULL JP: + +#define JPNULL(Type) (((Type) >= cJU_JPNULL1) && ((Type) <= cJU_JPNULLMAX)) + + +// SEARCH A BITMAP: +// +// This is a weak analog of j__udySearchLeaf*() for bitmaps. Return the actual +// or next-left position, base 0, of Digit in the single uint32_t bitmap, also +// given a Bitposmask for Digit. +// +// Unlike j__udySearchLeaf*(), the offset is not returned bit-complemented if +// Digits bit is unset, because the caller can check the bitmap themselves to +// determine that. Also, if Digits bit is unset, the returned offset is to +// the next-left JP (including -1), not to the "ideal" position for the Index = +// next-right JP. +// +// Shortcut and skip calling j__udyCountBits*() if the bitmap is full, in which +// case (Digit % cJU_BITSPERSUBEXP*) itself is the base-0 offset. +// +// TBD for Judy*Next(): Should this return next-right instead of next-left? +// That is, +1 from current value? Maybe not, if Digits bit IS set, +1 would +// be wrong. + +#define SEARCHBITMAPB(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPB) ? (Digit % cJU_BITSPERSUBEXPB) : \ + j__udyCountBitsB((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#define SEARCHBITMAPL(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPL) ? (Digit % cJU_BITSPERSUBEXPL) : \ + j__udyCountBitsL((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#ifdef JUDYPREV +// Equivalent to search for the highest offset in Bitmap: + +#define SEARCHBITMAPMAXB(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPB) ? cJU_BITSPERSUBEXPB - 1 : \ + j__udyCountBitsB(Bitmap) - 1) + +#define SEARCHBITMAPMAXL(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPL) ? cJU_BITSPERSUBEXPL - 1 : \ + j__udyCountBitsL(Bitmap) - 1) +#endif + + +// CHECK DECODE BYTES: +// +// Check Decode bytes in a JP against the equivalent portion of *PIndex. If +// *PIndex is lower (for Judy*Prev()) or higher (for Judy*Next()), this JP is a +// dead end (the same as if it had been absent in a linear or bitmap branch or +// null in an uncompressed branch), enter SM2Backtrack; otherwise enter +// SM3Findlimit to find the highest/lowest Index under this JP, as if the code +// had already backtracked to this JP. + +#ifdef JUDYPREV +#define CDcmp__ < +#else +#define CDcmp__ > +#endif + +#define CHECKDCD(cState) \ + if (JU_DCDNOTMATCHINDEX(*PIndex, Pjp, cState)) \ + { \ + if ((*PIndex & cJU_DCDMASK(cState)) \ + CDcmp__(JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(cState))) \ + { \ + goto SM2Backtrack; \ + } \ + goto SM3Findlimit; \ + } + + +// PREPARE TO HANDLE A LEAFW OR JRP BRANCH IN SM1: +// +// Extract a state-dependent digit from Index in a "constant" way, then jump to +// common code for multiple cases. + +#define SM1PREPB(cState,Next) \ + state = (cState); \ + digit = JU_DIGITATSTATE(*PIndex, cState); \ + goto Next + + +// PREPARE TO HANDLE A LEAFW OR JRP BRANCH IN SM3: +// +// Optionally save Dcd bytes into *PIndex, then save state and jump to common +// code for multiple cases. + +#define SM3PREPB_DCD(cState,Next) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + SM3PREPB(cState,Next) + +#define SM3PREPB(cState,Next) state = (cState); goto Next + + +// ---------------------------------------------------------------------------- +// CHECK FOR SHORTCUTS: +// +// Error out if PIndex is null. Execute JU_RET_NOTFOUND if the Judy array is +// empty or *PIndex is already the minimum/maximum Index possible. +// +// Note: As documented, in case of failure *PIndex may be modified. + + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +#ifdef JUDYPREV + if ((PArray == (Pvoid_t) NULL) || ((*PIndex)-- == 0)) +#else + if ((PArray == (Pvoid_t) NULL) || ((*PIndex)++ == cJU_ALLONES)) +#endif + JU_RET_NOTFOUND; + + +// HANDLE JRP: +// +// Before even entering SM1Get, check the JRP type. For JRP branches, traverse +// the JPM; handle LEAFW leaves directly; but look for the most common cases +// first. + +// ROOT-STATE LEAF that starts with a Pop0 word; just look within the leaf: +// +// If *PIndex is in the leaf, return it; otherwise return the Index, if any, +// below where it would belong. + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + pop1 = Pjlw[0] + 1; + + if ((offset = j__udySearchLeafW(Pjlw + 1, pop1, *PIndex)) + >= 0) // Index is present. + { + assert(offset < pop1); // in expected range. + JU_RET_FOUND_LEAFW(Pjlw, pop1, offset); // *PIndex is set. + } + +#ifdef JUDYPREV + if ((offset = ~offset) == 0) // no next-left Index. +#else + if ((offset = ~offset) >= pop1) // no next-right Index. +#endif + JU_RET_NOTFOUND; + + assert(offset <= pop1); // valid result. + +#ifdef JUDYPREV + *PIndex = Pjlw[offset--]; // next-left Index, base 1. +#else + *PIndex = Pjlw[offset + 1]; // next-right Index, base 1. +#endif + JU_RET_FOUND_LEAFW(Pjlw, pop1, offset); // base 0. + + } + else // JRP BRANCH + { + Pjpm_t Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); + +// goto SM1Get; + } + +// ============================================================================ +// STATE MACHINE 1 -- GET INDEX: +// +// Search for *PIndex (already decremented/incremented so as to be inclusive). +// If found, return it. Otherwise in theory hand off to SM2Backtrack or +// SM3Findlimit, but in practice "shortcut" by first sideways searching the +// current branch or leaf upon hitting a dead end. During sideways search, +// modify *PIndex to a new path taken. +// +// ENTRY: Pjp points to next JP to interpret, whose Decode bytes have not yet +// been checked. This JP is not yet listed in history. +// +// Note: Check Decode bytes at the start of each loop, not after looking up a +// new JP, so its easy to do constant shifts/masks, although this requires +// cautious handling of Pjp, offset, and *hist[] for correct entry to +// SM2Backtrack. +// +// EXIT: Return, or branch to SM2Backtrack or SM3Findlimit with correct +// interface, as described elsewhere. +// +// WARNING: For run-time efficiency the following cases replicate code with +// varying constants, rather than using common code with variable values! + +SM1Get: // return here for next branch/leaf. + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_L2: CHECKDCD(2); SM1PREPB(2, SM1BranchL); + case cJU_JPBRANCH_L3: CHECKDCD(3); SM1PREPB(3, SM1BranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: CHECKDCD(4); SM1PREPB(4, SM1BranchL); + case cJU_JPBRANCH_L5: CHECKDCD(5); SM1PREPB(5, SM1BranchL); + case cJU_JPBRANCH_L6: CHECKDCD(6); SM1PREPB(6, SM1BranchL); + case cJU_JPBRANCH_L7: CHECKDCD(7); SM1PREPB(7, SM1BranchL); +#endif + case cJU_JPBRANCH_L: SM1PREPB(cJU_ROOTSTATE, SM1BranchL); + +// Common code (state-independent) for all cases of linear branches: + +SM1BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +// Found JP matching current digit in *PIndex; record parent JP and the next +// JPs offset, and iterate to the next JP: + + if ((offset = j__udySearchLeaf1((Pjll_t) (Pjbl->jbl_Expanse), + Pjbl->jbl_NumJPs, digit)) >= 0) + { + HISTPUSH(Pjp, offset); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM1Get; + } + +// Dead end, no JP in BranchL for next digit in *PIndex: +// +// Get the ideal location of digits JP, and if theres no next-left/right JP +// in the BranchL, shortcut and start backtracking one level up; ignore the +// current Pjp because it points to a BranchL with no next-left/right JP. + +#ifdef JUDYPREV + if ((offset = (~offset) - 1) < 0) // no next-left JP in BranchL. +#else + if ((offset = (~offset)) >= Pjbl->jbl_NumJPs) // no next-right. +#endif + goto SM2Backtrack; + +// Theres a next-left/right JP in the current BranchL; save its digit in +// *PIndex and shortcut to SM3Findlimit: + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Check Decode bytes, if any, in the current JP, then look for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_B2: CHECKDCD(2); SM1PREPB(2, SM1BranchB); + case cJU_JPBRANCH_B3: CHECKDCD(3); SM1PREPB(3, SM1BranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: CHECKDCD(4); SM1PREPB(4, SM1BranchB); + case cJU_JPBRANCH_B5: CHECKDCD(5); SM1PREPB(5, SM1BranchB); + case cJU_JPBRANCH_B6: CHECKDCD(6); SM1PREPB(6, SM1BranchB); + case cJU_JPBRANCH_B7: CHECKDCD(7); SM1PREPB(7, SM1BranchB); +#endif + case cJU_JPBRANCH_B: SM1PREPB(cJU_ROOTSTATE, SM1BranchB); + +// Common code (state-independent) for all cases of bitmap branches: + +SM1BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + +// Locate the digits JP in the subexpanse list, if present, otherwise the +// offset of the next-left JP, if any: + + subexp = digit / cJU_BITSPERSUBEXPB; + assert(subexp < cJU_NUMSUBEXPB); // falls in expected range. + bitposmask = JU_BITPOSMASKB(digit); + offset = SEARCHBITMAPB(JU_JBB_BITMAP(Pjbb, subexp), digit, + bitposmask); + // right range: + assert((offset >= -1) && (offset < (int) cJU_BITSPERSUBEXPB)); + +// Found JP matching current digit in *PIndex: +// +// Record the parent JP and the next JPs offset; and iterate to the next JP. + +// if (JU_BITMAPTESTB(Pjbb, digit)) // slower. + if (JU_JBB_BITMAP(Pjbb, subexp) & bitposmask) // faster. + { + // not negative since at least one bit is set: + assert(offset >= 0); + + HISTPUSH(Pjp, HISTPUSHBOFF(subexp, offset, digit)); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM1Get; // iterate to next JP. + } + +// Dead end, no JP in BranchB for next digit in *PIndex: +// +// If theres a next-left/right JP in the current BranchB, shortcut to +// SM3Findlimit. Note: offset is already set to the correct value for the +// next-left/right JP. + +#ifdef JUDYPREV + if (offset >= 0) // next-left JP is in this subexpanse. + goto SM1BranchBFindlimit; + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JBB_BITMAP(Pjbb, subexp) & JU_MASKHIGHEREXC(bitposmask)) + { + ++offset; // next-left => next-right. + goto SM1BranchBFindlimit; + } + + while (++subexp < cJU_NUMSUBEXPB) // search next-right subexps. +#endif + { + if (! JU_JBB_PJP(Pjbb, subexp)) continue; // empty subexpanse. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + offset = 0; +#endif + +// Save the next-left/right JPs digit in *PIndex: + +SM1BranchBFindlimit: + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), + offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchB: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a BranchB with no next-left/right JP. + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Check Decode bytes, if any, in the current JP, then look for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_U2: CHECKDCD(2); SM1PREPB(2, SM1BranchU); + case cJU_JPBRANCH_U3: CHECKDCD(3); SM1PREPB(3, SM1BranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: CHECKDCD(4); SM1PREPB(4, SM1BranchU); + case cJU_JPBRANCH_U5: CHECKDCD(5); SM1PREPB(5, SM1BranchU); + case cJU_JPBRANCH_U6: CHECKDCD(6); SM1PREPB(6, SM1BranchU); + case cJU_JPBRANCH_U7: CHECKDCD(7); SM1PREPB(7, SM1BranchU); +#endif + case cJU_JPBRANCH_U: SM1PREPB(cJU_ROOTSTATE, SM1BranchU); + +// Common code (state-independent) for all cases of uncompressed branches: + +SM1BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + Pjp2 = (Pjbu->jbu_jp) + digit; + +// Found JP matching current digit in *PIndex: +// +// Record the parent JP and the next JPs digit, and iterate to the next JP. +// +// TBD: Instead of this, just goto SM1Get, and add cJU_JPNULL* cases to the +// SM1Get state machine? Then backtrack? However, it means you cant detect +// an inappropriate cJU_JPNULL*, when it occurs in other than a BranchU, and +// return JU_RET_CORRUPT. + + if (! JPNULL(JU_JPTYPE(Pjp2))) // digit has a JP. + { + HISTPUSH(Pjp, digit); + Pjp = Pjp2; + goto SM1Get; + } + +// Dead end, no JP in BranchU for next digit in *PIndex: +// +// Search for a next-left/right JP in the current BranchU, and if one is found, +// save its digit in *PIndex and shortcut to SM3Findlimit: + +#ifdef JUDYPREV + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + while (digit < cJU_BRANCHUNUMJPS - 1) + { + Pjp = (Pjbu->jbu_jp) + (++digit); +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchU: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a BranchU with no next-left/right JP. + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for +// *PIndex. + +#define SM1LEAFL(Func) \ + Pjll = P_JLL(Pjp->jp_Addr); \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + offset = Func(Pjll, pop1, *PIndex); \ + goto SM1LeafLImm + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKDCD(1); SM1LEAFL(j__udySearchLeaf1); +#endif + case cJU_JPLEAF2: CHECKDCD(2); SM1LEAFL(j__udySearchLeaf2); + case cJU_JPLEAF3: CHECKDCD(3); SM1LEAFL(j__udySearchLeaf3); + +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKDCD(4); SM1LEAFL(j__udySearchLeaf4); + case cJU_JPLEAF5: CHECKDCD(5); SM1LEAFL(j__udySearchLeaf5); + case cJU_JPLEAF6: CHECKDCD(6); SM1LEAFL(j__udySearchLeaf6); + case cJU_JPLEAF7: CHECKDCD(7); SM1LEAFL(j__udySearchLeaf7); +#endif + +// Common code (state-independent) for all cases of linear leaves and +// immediates: + +SM1LeafLImm: + if (offset >= 0) // *PIndex is in LeafL / Immed. +#ifdef JUDY1 + JU_RET_FOUND; +#else + { // JudyL is trickier... + switch (JU_JPTYPE(Pjp)) + { +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + case cJU_JPLEAF2: JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + case cJU_JPLEAF3: JU_RET_FOUND_LEAF3(Pjll, pop1, offset); +#ifdef JU_64BIT + case cJU_JPLEAF4: JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + case cJU_JPLEAF5: JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + case cJU_JPLEAF6: JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + case cJU_JPLEAF7: JU_RET_FOUND_LEAF7(Pjll, pop1, offset); +#endif + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + JU_RET_FOUND_IMM_01(Pjp); + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#ifdef JU_64BIT + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: + case cJU_JPIMMED_3_02: +#endif + JU_RET_FOUND_IMM(Pjp, offset); + } + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // impossible? + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // found *PIndex + +#endif // JUDYL + +// Dead end, no Index in LeafL / Immed for remaining digit(s) in *PIndex: +// +// Get the ideal location of Index, and if theres no next-left/right Index in +// the LeafL / Immed, shortcut and start backtracking one level up; ignore the +// current Pjp because it points to a LeafL / Immed with no next-left/right +// Index. + +#ifdef JUDYPREV + if ((offset = (~offset) - 1) < 0) // no next-left Index. +#else + if ((offset = (~offset)) >= pop1) // no next-right Index. +#endif + goto SM2Backtrack; + +// Theres a next-left/right Index in the current LeafL / Immed; shortcut by +// copying its digit(s) to *PIndex and returning it. +// +// Unfortunately this is pretty hairy, especially avoiding endian issues. +// +// The cJU_JPLEAF* cases are very similar to same-index-size cJU_JPIMMED* cases +// for *_02 and above, but must return differently, at least for JudyL, so +// spell them out separately here at the cost of a little redundant code for +// Judy1. + + switch (JU_JPTYPE(Pjp)) + { +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + JU_SETDIGIT1(*PIndex, ((uint8_t *) Pjll)[offset]); + JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + + case cJU_JPLEAF2: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + + case cJU_JPLEAF3: + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + +#ifdef JU_64BIT + case cJU_JPLEAF4: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + + case cJU_JPLEAF5: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + } + + case cJU_JPLEAF6: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + } + + case cJU_JPLEAF7: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_LEAF7(Pjll, pop1, offset); + } + +#endif // JU_64BIT + +#define SET_01(cState) JU_SETDIGITS(*PIndex, JU_JPDCDPOP0(Pjp), cState) + + case cJU_JPIMMED_1_01: SET_01(1); goto SM1Imm_01; + case cJU_JPIMMED_2_01: SET_01(2); goto SM1Imm_01; + case cJU_JPIMMED_3_01: SET_01(3); goto SM1Imm_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: SET_01(4); goto SM1Imm_01; + case cJU_JPIMMED_5_01: SET_01(5); goto SM1Imm_01; + case cJU_JPIMMED_6_01: SET_01(6); goto SM1Imm_01; + case cJU_JPIMMED_7_01: SET_01(7); goto SM1Imm_01; +#endif +SM1Imm_01: JU_RET_FOUND_IMM_01(Pjp); + +// Shorthand for where to find start of Index bytes array: + +#ifdef JUDY1 +#define PJI (Pjp->jp_1Index) +#else +#define PJI (Pjp->jp_LIndex) +#endif + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + JU_SETDIGIT1(*PIndex, ((uint8_t *) PJI)[offset]); + JU_RET_FOUND_IMM(Pjp, offset); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: + case cJ1_JPIMMED_4_03: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); + + case cJ1_JPIMMED_5_02: + case cJ1_JPIMMED_5_03: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_6_02: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_7_02: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + +#endif // (JUDY1 && JU_64BIT) + + } // switch for not-found *PIndex + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // impossible? + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Check Decode bytes, if any, in the current JP, then look in the leaf for +// *PIndex. + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; + CHECKDCD(1); + + Pjlb = P_JLB(Pjp->jp_Addr); + digit = JU_DIGITATSTATE(*PIndex, 1); + subexp = JU_SUBEXPL(digit); + bitposmask = JU_BITPOSMASKL(digit); + assert(subexp < cJU_NUMSUBEXPL); // falls in expected range. + +// *PIndex exists in LeafB1: + +// if (JU_BITMAPTESTL(Pjlb, digit)) // slower. + if (JU_JLB_BITMAP(Pjlb, subexp) & bitposmask) // faster. + { +#ifdef JUDYL // needs offset at this point: + offset = SEARCHBITMAPL(JU_JLB_BITMAP(Pjlb, subexp), digit, bitposmask); +#endif + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + } + +// Dead end, no Index in LeafB1 for remaining digit in *PIndex: +// +// If theres a next-left/right Index in the current LeafB1, which for +// Judy*Next() is true if any bits are set for higher Indexes, shortcut by +// returning it. Note: For Judy*Prev(), offset is set here to the correct +// value for the next-left JP. + + offset = SEARCHBITMAPL(JU_JLB_BITMAP(Pjlb, subexp), digit, + bitposmask); + // right range: + assert((offset >= -1) && (offset < (int) cJU_BITSPERSUBEXPL)); + +#ifdef JUDYPREV + if (offset >= 0) // next-left JP is in this subexpanse. + goto SM1LeafB1Findlimit; + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JLB_BITMAP(Pjlb, subexp) & JU_MASKHIGHEREXC(bitposmask)) + { + ++offset; // next-left => next-right. + goto SM1LeafB1Findlimit; + } + + while (++subexp < cJU_NUMSUBEXPL) // search next-right subexps. +#endif + { + if (! JU_JLB_BITMAP(Pjlb, subexp)) continue; // empty subexp. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXL(JU_JLB_BITMAP(Pjlb, subexp)); + // expected range: + assert((offset >= 0) && (offset < (int) cJU_BITSPERSUBEXPL)); +#else + offset = 0; +#endif + +// Save the next-left/right Indexess digit in *PIndex: + +SM1LeafB1Findlimit: + JU_BITMAPDIGITL(digit, subexp, JU_JLB_BITMAP(Pjlb, subexp), offset); + JU_SETDIGIT1(*PIndex, digit); + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + } + +// Theres no next-left/right Index in the LeafB1: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a LeafB1 with no next-left/right Index. + + goto SM2Backtrack; + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// If the Decode bytes match, *PIndex is found (without modification). + + case cJ1_JPFULLPOPU1: + + CHECKDCD(1); + JU_RET_FOUND_FULLPOPU1; +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: + +#ifdef JUDYPREV +#define SM1IMM_SETPOP1(cPop1) +#else +#define SM1IMM_SETPOP1(cPop1) pop1 = (cPop1) +#endif + +#define SM1IMM(Func,cPop1) \ + SM1IMM_SETPOP1(cPop1); \ + offset = Func((Pjll_t) (PJI), cPop1, *PIndex); \ + goto SM1LeafLImm + +// Special case for Pop1 = 1 Immediate JPs: +// +// If *PIndex is in the immediate, offset is 0, otherwise the binary NOT of the +// offset where it belongs, 0 or 1, same as from the search functions. + +#ifdef JUDYPREV +#define SM1IMM_01_SETPOP1 +#else +#define SM1IMM_01_SETPOP1 pop1 = 1 +#endif + +#define SM1IMM_01 \ + SM1IMM_01_SETPOP1; \ + offset = ((JU_JPDCDPOP0(Pjp) < JU_TRIMTODCDSIZE(*PIndex)) ? ~1 : \ + (JU_JPDCDPOP0(Pjp) == JU_TRIMTODCDSIZE(*PIndex)) ? 0 : \ + ~0); \ + goto SM1LeafLImm + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + SM1IMM_01; + +// TBD: Doug says it would be OK to have fewer calls and calculate arg 2, here +// and in Judy*Count() also. + + case cJU_JPIMMED_1_02: SM1IMM(j__udySearchLeaf1, 2); + case cJU_JPIMMED_1_03: SM1IMM(j__udySearchLeaf1, 3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: SM1IMM(j__udySearchLeaf1, 4); + case cJU_JPIMMED_1_05: SM1IMM(j__udySearchLeaf1, 5); + case cJU_JPIMMED_1_06: SM1IMM(j__udySearchLeaf1, 6); + case cJU_JPIMMED_1_07: SM1IMM(j__udySearchLeaf1, 7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: SM1IMM(j__udySearchLeaf1, 8); + case cJ1_JPIMMED_1_09: SM1IMM(j__udySearchLeaf1, 9); + case cJ1_JPIMMED_1_10: SM1IMM(j__udySearchLeaf1, 10); + case cJ1_JPIMMED_1_11: SM1IMM(j__udySearchLeaf1, 11); + case cJ1_JPIMMED_1_12: SM1IMM(j__udySearchLeaf1, 12); + case cJ1_JPIMMED_1_13: SM1IMM(j__udySearchLeaf1, 13); + case cJ1_JPIMMED_1_14: SM1IMM(j__udySearchLeaf1, 14); + case cJ1_JPIMMED_1_15: SM1IMM(j__udySearchLeaf1, 15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: SM1IMM(j__udySearchLeaf2, 2); + case cJU_JPIMMED_2_03: SM1IMM(j__udySearchLeaf2, 3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: SM1IMM(j__udySearchLeaf2, 4); + case cJ1_JPIMMED_2_05: SM1IMM(j__udySearchLeaf2, 5); + case cJ1_JPIMMED_2_06: SM1IMM(j__udySearchLeaf2, 6); + case cJ1_JPIMMED_2_07: SM1IMM(j__udySearchLeaf2, 7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: SM1IMM(j__udySearchLeaf3, 2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: SM1IMM(j__udySearchLeaf3, 3); + case cJ1_JPIMMED_3_04: SM1IMM(j__udySearchLeaf3, 4); + case cJ1_JPIMMED_3_05: SM1IMM(j__udySearchLeaf3, 5); + + case cJ1_JPIMMED_4_02: SM1IMM(j__udySearchLeaf4, 2); + case cJ1_JPIMMED_4_03: SM1IMM(j__udySearchLeaf4, 3); + + case cJ1_JPIMMED_5_02: SM1IMM(j__udySearchLeaf5, 2); + case cJ1_JPIMMED_5_03: SM1IMM(j__udySearchLeaf5, 3); + + case cJ1_JPIMMED_6_02: SM1IMM(j__udySearchLeaf6, 2); + + case cJ1_JPIMMED_7_02: SM1IMM(j__udySearchLeaf7, 2); +#endif + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM1Get switch. + + /*NOTREACHED*/ + + +// ============================================================================ +// STATE MACHINE 2 -- BACKTRACK BRANCH TO PREVIOUS JP: +// +// Look for the next-left/right JP in a branch, backing up the history list as +// necessary. Upon finding a next-left/right JP, modify the corresponding +// digit in *PIndex before passing control to SM3Findlimit. +// +// Note: As described earlier, only branch JPs are expected here; other types +// fall into the default case. +// +// Note: If a found JP contains needed Dcd bytes, thats OK, theyre copied to +// *PIndex in SM3Findlimit. +// +// TBD: This code has a lot in common with similar code in the shortcut cases +// in SM1Get. Can combine this code somehow? +// +// ENTRY: List, possibly empty, of JPs and offsets in APjphist[] and +// Aoffhist[]; see earlier comments. +// +// EXIT: Execute JU_RET_NOTFOUND if no previous/next JP; otherwise jump to +// SM3Findlimit to resume a new but different downward search. + +SM2Backtrack: // come or return here for first/next sideways search. + + HISTPOP(Pjp, offset); + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: + + case cJU_JPBRANCH_L2: state = 2; goto SM2BranchL; + case cJU_JPBRANCH_L3: state = 3; goto SM2BranchL; +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: state = 4; goto SM2BranchL; + case cJU_JPBRANCH_L5: state = 5; goto SM2BranchL; + case cJU_JPBRANCH_L6: state = 6; goto SM2BranchL; + case cJU_JPBRANCH_L7: state = 7; goto SM2BranchL; +#endif + case cJU_JPBRANCH_L: state = cJU_ROOTSTATE; goto SM2BranchL; + +SM2BranchL: +#ifdef JUDYPREV + if (--offset < 0) goto SM2Backtrack; // no next-left JP in BranchL. +#endif + Pjbl = P_JBL(Pjp->jp_Addr); +#ifdef JUDYNEXT + if (++offset >= (Pjbl->jbl_NumJPs)) goto SM2Backtrack; + // no next-right JP in BranchL. +#endif + +// Theres a next-left/right JP in the current BranchL; save its digit in +// *PIndex and continue with SM3Findlimit: + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: + + case cJU_JPBRANCH_B2: state = 2; goto SM2BranchB; + case cJU_JPBRANCH_B3: state = 3; goto SM2BranchB; +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: state = 4; goto SM2BranchB; + case cJU_JPBRANCH_B5: state = 5; goto SM2BranchB; + case cJU_JPBRANCH_B6: state = 6; goto SM2BranchB; + case cJU_JPBRANCH_B7: state = 7; goto SM2BranchB; +#endif + case cJU_JPBRANCH_B: state = cJU_ROOTSTATE; goto SM2BranchB; + +SM2BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + HISTPOPBOFF(subexp, offset, digit); // unpack values. + +// If theres a next-left/right JP in the current BranchB, which for +// Judy*Next() is true if any bits are set for higher Indexes, continue to +// SM3Findlimit: +// +// Note: offset is set to the JP previously traversed; go one to the +// left/right. + +#ifdef JUDYPREV + if (offset > 0) // next-left JP is in this subexpanse. + { + --offset; + goto SM2BranchBFindlimit; + } + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JBB_BITMAP(Pjbb, subexp) + & JU_MASKHIGHEREXC(JU_BITPOSMASKB(digit))) + { + ++offset; // next-left => next-right. + goto SM2BranchBFindlimit; + } + + while (++subexp < cJU_NUMSUBEXPB) // search next-right subexps. +#endif + { + if (! JU_JBB_PJP(Pjbb, subexp)) continue; // empty subexpanse. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + offset = 0; +#endif + +// Save the next-left/right JPs digit in *PIndex: + +SM2BranchBFindlimit: + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), + offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchB: + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: + + case cJU_JPBRANCH_U2: state = 2; goto SM2BranchU; + case cJU_JPBRANCH_U3: state = 3; goto SM2BranchU; +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: state = 4; goto SM2BranchU; + case cJU_JPBRANCH_U5: state = 5; goto SM2BranchU; + case cJU_JPBRANCH_U6: state = 6; goto SM2BranchU; + case cJU_JPBRANCH_U7: state = 7; goto SM2BranchU; +#endif + case cJU_JPBRANCH_U: state = cJU_ROOTSTATE; goto SM2BranchU; + +SM2BranchU: + +// Search for a next-left/right JP in the current BranchU, and if one is found, +// save its digit in *PIndex and continue to SM3Findlimit: + + Pjbu = P_JBU(Pjp->jp_Addr); + digit = offset; + +#ifdef JUDYPREV + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + while (digit < cJU_BRANCHUNUMJPS - 1) + { + Pjp = (Pjbu->jbu_jp) + (++digit); +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchU: + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM2Backtrack switch. + + /*NOTREACHED*/ + + +// ============================================================================ +// STATE MACHINE 3 -- FIND LIMIT JP/INDEX: +// +// Look for the highest/lowest (right/left-most) JP in each branch and the +// highest/lowest Index in a leaf or immediate, and return it. While +// traversing, modify appropriate digit(s) in *PIndex to reflect the path +// taken, including Dcd bytes in each JP (which could hold critical missing +// digits for skipped branches). +// +// ENTRY: Pjp set to a JP under which to find max/min JPs (if a branch JP) or +// a max/min Index and return (if a leaf or immediate JP). +// +// EXIT: Execute JU_RET_FOUND* upon reaching a leaf or immediate. Should be +// impossible to fail, unless the Judy array is corrupt. + +SM3Findlimit: // come or return here for first/next branch/leaf. + + switch (JU_JPTYPE(Pjp)) + { +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Simply use the highest/lowest (right/left-most) JP in the BranchL, but first +// copy the Dcd bytes to *PIndex if there are any (only if state < +// cJU_ROOTSTATE - 1). + + case cJU_JPBRANCH_L2: SM3PREPB_DCD(2, SM3BranchL); +#ifndef JU_64BIT + case cJU_JPBRANCH_L3: SM3PREPB( 3, SM3BranchL); +#else + case cJU_JPBRANCH_L3: SM3PREPB_DCD(3, SM3BranchL); + case cJU_JPBRANCH_L4: SM3PREPB_DCD(4, SM3BranchL); + case cJU_JPBRANCH_L5: SM3PREPB_DCD(5, SM3BranchL); + case cJU_JPBRANCH_L6: SM3PREPB_DCD(6, SM3BranchL); + case cJU_JPBRANCH_L7: SM3PREPB( 7, SM3BranchL); +#endif + case cJU_JPBRANCH_L: SM3PREPB( cJU_ROOTSTATE, SM3BranchL); + +SM3BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +#ifdef JUDYPREV + if ((offset = (Pjbl->jbl_NumJPs) - 1) < 0) +#else + offset = 0; if ((Pjbl->jbl_NumJPs) == 0) +#endif + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Look for the highest/lowest (right/left-most) non-null subexpanse, then use +// the highest/lowest JP in that subexpanse, but first copy Dcd bytes, if there +// are any (only if state < cJU_ROOTSTATE - 1), to *PIndex. + + case cJU_JPBRANCH_B2: SM3PREPB_DCD(2, SM3BranchB); +#ifndef JU_64BIT + case cJU_JPBRANCH_B3: SM3PREPB( 3, SM3BranchB); +#else + case cJU_JPBRANCH_B3: SM3PREPB_DCD(3, SM3BranchB); + case cJU_JPBRANCH_B4: SM3PREPB_DCD(4, SM3BranchB); + case cJU_JPBRANCH_B5: SM3PREPB_DCD(5, SM3BranchB); + case cJU_JPBRANCH_B6: SM3PREPB_DCD(6, SM3BranchB); + case cJU_JPBRANCH_B7: SM3PREPB( 7, SM3BranchB); +#endif + case cJU_JPBRANCH_B: SM3PREPB( cJU_ROOTSTATE, SM3BranchB); + +SM3BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); +#ifdef JUDYPREV + subexp = cJU_NUMSUBEXPB; + + while (! (JU_JBB_BITMAP(Pjbb, --subexp))) // find non-empty subexp. + { + if (subexp <= 0) // wholly empty bitmap. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + subexp = -1; + + while (! (JU_JBB_BITMAP(Pjbb, ++subexp))) // find non-empty subexp. + { + if (subexp >= cJU_NUMSUBEXPB - 1) // didnt find one. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = 0; +#endif + + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Look for the highest/lowest (right/left-most) non-null JP, and use it, but +// first copy Dcd bytes to *PIndex if there are any (only if state < +// cJU_ROOTSTATE - 1). + + case cJU_JPBRANCH_U2: SM3PREPB_DCD(2, SM3BranchU); +#ifndef JU_64BIT + case cJU_JPBRANCH_U3: SM3PREPB( 3, SM3BranchU); +#else + case cJU_JPBRANCH_U3: SM3PREPB_DCD(3, SM3BranchU); + case cJU_JPBRANCH_U4: SM3PREPB_DCD(4, SM3BranchU); + case cJU_JPBRANCH_U5: SM3PREPB_DCD(5, SM3BranchU); + case cJU_JPBRANCH_U6: SM3PREPB_DCD(6, SM3BranchU); + case cJU_JPBRANCH_U7: SM3PREPB( 7, SM3BranchU); +#endif + case cJU_JPBRANCH_U: SM3PREPB( cJU_ROOTSTATE, SM3BranchU); + +SM3BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); +#ifdef JUDYPREV + digit = cJU_BRANCHUNUMJPS; + + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + + for (digit = 0; digit < cJU_BRANCHUNUMJPS; ++digit) + { + Pjp = (Pjbu->jbu_jp) + digit; +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// No non-null JPs in BranchU: + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Simply use the highest/lowest (right/left-most) Index in the LeafL, but the +// details vary depending on leaf Index Size. First copy Dcd bytes, if there +// are any (only if state < cJU_ROOTSTATE - 1), to *PIndex. + +#define SM3LEAFLDCD(cState) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + SM3LEAFLNODCD + +#ifdef JUDY1 +#define SM3LEAFL_SETPOP1 // not needed in any cases. +#else +#define SM3LEAFL_SETPOP1 pop1 = JU_JPLEAF_POP0(Pjp) + 1 +#endif + +#ifdef JUDYPREV +#define SM3LEAFLNODCD \ + Pjll = P_JLL(Pjp->jp_Addr); \ + SM3LEAFL_SETPOP1; \ + offset = JU_JPLEAF_POP0(Pjp); assert(offset >= 0) +#else +#define SM3LEAFLNODCD \ + Pjll = P_JLL(Pjp->jp_Addr); \ + SM3LEAFL_SETPOP1; \ + offset = 0; assert(JU_JPLEAF_POP0(Pjp) >= 0); +#endif + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + SM3LEAFLDCD(1); + JU_SETDIGIT1(*PIndex, ((uint8_t *) Pjll)[offset]); + JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + + case cJU_JPLEAF2: + + SM3LEAFLDCD(2); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + +#ifndef JU_64BIT + case cJU_JPLEAF3: + { + Word_t lsb; + SM3LEAFLNODCD; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + +#else + case cJU_JPLEAF3: + { + Word_t lsb; + SM3LEAFLDCD(3); + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + + case cJU_JPLEAF4: + + SM3LEAFLDCD(4); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + + case cJU_JPLEAF5: + { + Word_t lsb; + SM3LEAFLDCD(5); + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + } + + case cJU_JPLEAF6: + { + Word_t lsb; + SM3LEAFLDCD(6); + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + } + + case cJU_JPLEAF7: + { + Word_t lsb; + SM3LEAFLNODCD; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_LEAF7(Pjll, pop1, offset); + } +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Look for the highest/lowest (right/left-most) non-null subexpanse, then use +// the highest/lowest Index in that subexpanse, but first copy Dcd bytes +// (always present since state 1 < cJU_ROOTSTATE) to *PIndex. + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; + + JU_SETDCD(*PIndex, Pjp, 1); + + Pjlb = P_JLB(Pjp->jp_Addr); +#ifdef JUDYPREV + subexp = cJU_NUMSUBEXPL; + + while (! JU_JLB_BITMAP(Pjlb, --subexp)) // find non-empty subexp. + { + if (subexp <= 0) // wholly empty bitmap. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + +// TBD: Might it be faster to just use a variant of BITMAPDIGIT*() that yields +// the digit for the right-most Index with a bit set? + + offset = SEARCHBITMAPMAXL(JU_JLB_BITMAP(Pjlb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPL)); +#else + subexp = -1; + + while (! JU_JLB_BITMAP(Pjlb, ++subexp)) // find non-empty subexp. + { + if (subexp >= cJU_NUMSUBEXPL - 1) // didnt find one. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = 0; +#endif + + JU_BITMAPDIGITL(digit, subexp, JU_JLB_BITMAP(Pjlb, subexp), offset); + JU_SETDIGIT1(*PIndex, digit); + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// Copy Dcd bytes to *PIndex (always present since state 1 < cJU_ROOTSTATE), +// then set the highest/lowest possible digit as the LSB in *PIndex. + + case cJ1_JPFULLPOPU1: + + JU_SETDCD( *PIndex, Pjp, 1); +#ifdef JUDYPREV + JU_SETDIGIT1(*PIndex, cJU_BITSPERBITMAP - 1); +#else + JU_SETDIGIT1(*PIndex, 0); +#endif + JU_RET_FOUND_FULLPOPU1; +#endif // JUDY1 + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: +// +// Simply use the highest/lowest (right/left-most) Index in the Imm, but the +// details vary depending on leaf Index Size and pop1. Note: There are no Dcd +// bytes in an Immediate JP, but in a cJU_JPIMMED_*_01 JP, the field holds the +// least bytes of the immediate Index. + + case cJU_JPIMMED_1_01: SET_01(1); goto SM3Imm_01; + case cJU_JPIMMED_2_01: SET_01(2); goto SM3Imm_01; + case cJU_JPIMMED_3_01: SET_01(3); goto SM3Imm_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: SET_01(4); goto SM3Imm_01; + case cJU_JPIMMED_5_01: SET_01(5); goto SM3Imm_01; + case cJU_JPIMMED_6_01: SET_01(6); goto SM3Imm_01; + case cJU_JPIMMED_7_01: SET_01(7); goto SM3Imm_01; +#endif +SM3Imm_01: JU_RET_FOUND_IMM_01(Pjp); + +#ifdef JUDYPREV +#define SM3IMM_OFFSET(cPop1) (cPop1) - 1 // highest. +#else +#define SM3IMM_OFFSET(cPop1) 0 // lowest. +#endif + +#define SM3IMM(cPop1,Next) \ + offset = SM3IMM_OFFSET(cPop1); \ + goto Next + + case cJU_JPIMMED_1_02: SM3IMM( 2, SM3Imm1); + case cJU_JPIMMED_1_03: SM3IMM( 3, SM3Imm1); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: SM3IMM( 4, SM3Imm1); + case cJU_JPIMMED_1_05: SM3IMM( 5, SM3Imm1); + case cJU_JPIMMED_1_06: SM3IMM( 6, SM3Imm1); + case cJU_JPIMMED_1_07: SM3IMM( 7, SM3Imm1); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: SM3IMM( 8, SM3Imm1); + case cJ1_JPIMMED_1_09: SM3IMM( 9, SM3Imm1); + case cJ1_JPIMMED_1_10: SM3IMM(10, SM3Imm1); + case cJ1_JPIMMED_1_11: SM3IMM(11, SM3Imm1); + case cJ1_JPIMMED_1_12: SM3IMM(12, SM3Imm1); + case cJ1_JPIMMED_1_13: SM3IMM(13, SM3Imm1); + case cJ1_JPIMMED_1_14: SM3IMM(14, SM3Imm1); + case cJ1_JPIMMED_1_15: SM3IMM(15, SM3Imm1); +#endif + +SM3Imm1: JU_SETDIGIT1(*PIndex, ((uint8_t *) PJI)[offset]); + JU_RET_FOUND_IMM(Pjp, offset); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: SM3IMM(2, SM3Imm2); + case cJU_JPIMMED_2_03: SM3IMM(3, SM3Imm2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: SM3IMM(4, SM3Imm2); + case cJ1_JPIMMED_2_05: SM3IMM(5, SM3Imm2); + case cJ1_JPIMMED_2_06: SM3IMM(6, SM3Imm2); + case cJ1_JPIMMED_2_07: SM3IMM(7, SM3Imm2); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +SM3Imm2: *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: SM3IMM(2, SM3Imm3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: SM3IMM(3, SM3Imm3); + case cJ1_JPIMMED_3_04: SM3IMM(4, SM3Imm3); + case cJ1_JPIMMED_3_05: SM3IMM(5, SM3Imm3); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +SM3Imm3: + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: SM3IMM(2, SM3Imm4); + case cJ1_JPIMMED_4_03: SM3IMM(3, SM3Imm4); + +SM3Imm4: *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); + + case cJ1_JPIMMED_5_02: SM3IMM(2, SM3Imm5); + case cJ1_JPIMMED_5_03: SM3IMM(3, SM3Imm5); + +SM3Imm5: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_6_02: SM3IMM(2, SM3Imm6); + +SM3Imm6: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_7_02: SM3IMM(2, SM3Imm7); + +SM3Imm7: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif // (JUDY1 && JU_64BIT) + + +// ---------------------------------------------------------------------------- +// OTHER CASES: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM3Findlimit switch. + + /*NOTREACHED*/ + +} // Judy1Prev() / Judy1Next() / JudyLPrev() / JudyLNext() diff --git a/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c b/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c new file mode 100644 index 000000000..4da43565d --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c @@ -0,0 +1,1390 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.32 $ $Source: /judy/src/JudyCommon/JudyPrevNextEmpty.c $ +// +// Judy*PrevEmpty() and Judy*NextEmpty() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DJUDYNEXT for the Judy*NextEmpty() function; otherwise +// defaults to Judy*PrevEmpty(). +// +// Compile with -DTRACEJPSE to trace JP traversals. +// +// This file is separate from JudyPrevNext.c because it differs too greatly for +// ifdefs. This might be a bit surprising, but there are two reasons: +// +// - First, down in the details, searching for an empty index (SearchEmpty) is +// remarkably asymmetric with searching for a valid index (SearchValid), +// mainly with respect to: No return of a value area for JudyL; partially- +// full versus totally-full JPs; and handling of narrow pointers. +// +// - Second, we chose to implement SearchEmpty without a backtrack stack or +// backtrack engine, partly as an experiment, and partly because we think +// restarting from the top of the tree is less likely for SearchEmpty than +// for SearchValid, because empty indexes are more likely than valid indexes. +// +// A word about naming: A prior version of this feature (see 4.13) was named +// Judy*Free(), but there were concerns about that being read as a verb rather +// than an adjective. After prolonged debate and based on user input, we +// changed "Free" to "Empty". + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifndef JUDYNEXT +#ifndef JUDYPREV +#define JUDYPREV 1 // neither set => use default. +#endif +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +#ifdef TRACEJPSE +#include "JudyPrintJP.c" +#endif + + +// **************************************************************************** +// J U D Y 1 P R E V E M P T Y +// J U D Y 1 N E X T E M P T Y +// J U D Y L P R E V E M P T Y +// J U D Y L N E X T E M P T Y +// +// See the manual entry for the API. +// +// OVERVIEW OF Judy*PrevEmpty() / Judy*NextEmpty(): +// +// See also for comparison the equivalent comments in JudyPrevNext.c. +// +// Take the callers *PIndex and subtract/add 1, but watch out for +// underflow/overflow, which means "no previous/next empty index found." Use a +// reentrant switch statement (state machine, see SMGetRestart and +// SMGetContinue) to decode Index, starting with the JRP (PArray), through a +// JPM and branches, if any, down to an immediate or a leaf. Look for Index in +// that immediate or leaf, and if not found (invalid index), return success +// (Index is empty). +// +// This search can result in a dead end where taking a different path is +// required. There are four kinds of dead ends: +// +// BRANCH PRIMARY dead end: Encountering a fully-populated JP for the +// appropriate digit in Index. Search sideways in the branch for the +// previous/next absent/null/non-full JP, and if one is found, set Index to the +// highest/lowest index possible in that JPs expanse. Then if the JP is an +// absent or null JP, return success; otherwise for a non-full JP, traverse +// through the partially populated JP. +// +// BRANCH SECONDARY dead end: Reaching the end of a branch during a sideways +// search after a branch primary dead end. Set Index to the lowest/highest +// index possible in the whole branchs expanse (one higher/lower than the +// previous/next branchs expanse), then restart at the top of the tree, which +// includes pre-decrementing/incrementing Index (again) and watching for +// underflow/overflow (again). +// +// LEAF PRIMARY dead end: Finding a valid (non-empty) index in an immediate or +// leaf matching Index. Search sideways in the immediate/leaf for the +// previous/next empty index; if found, set *PIndex to match and return success. +// +// LEAF SECONDARY dead end: Reaching the end of an immediate or leaf during a +// sideways search after a leaf primary dead end. Just as for a branch +// secondary dead end, restart at the top of the tree with Index set to the +// lowest/highest index possible in the whole immediate/leafs expanse. +// TBD: If leaf secondary dead end occurs, could shortcut and treat it as a +// branch primary dead end; but this would require remembering the parent +// branchs type and offset (a "one-deep stack"), and also wrestling with +// narrow pointers, at least for leaves (but not for immediates). +// +// Note some ASYMMETRIES between SearchValid and SearchEmpty: +// +// - The SearchValid code, upon descending through a narrow pointer, if Index +// is outside the expanse of the subsidiary node (effectively a secondary +// dead end), must decide whether to backtrack or findlimit. But the +// SearchEmpty code simply returns success (Index is empty). +// +// - Similarly, the SearchValid code, upon finding no previous/next index in +// the expanse of a narrow pointer (again, a secondary dead end), can simply +// start to backtrack at the parent JP. But the SearchEmpty code would have +// to first determine whether or not the parent JPs narrow expanse contains +// a previous/next empty index outside the subexpanse. Rather than keeping a +// parent state stack and backtracking this way, upon a secondary dead end, +// the SearchEmpty code simply restarts at the top of the tree, whether or +// not a narrow pointer is involved. Again, see the equivalent comments in +// JudyPrevNext.c for comparison. +// +// This function is written iteratively for speed, rather than recursively. +// +// TBD: Wed like to enhance this function to make successive searches faster. +// This would require saving some previous state, including the previous Index +// returned, and in which leaf it was found. If the next call is for the same +// Index and the array has not been modified, start at the same leaf. This +// should be much easier to implement since this is iterative rather than +// recursive code. + +#ifdef JUDY1 +#ifdef JUDYPREV +FUNCTION int Judy1PrevEmpty +#else +FUNCTION int Judy1NextEmpty +#endif +#else +#ifdef JUDYPREV +FUNCTION int JudyLPrevEmpty +#else +FUNCTION int JudyLNextEmpty +#endif +#endif + ( + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError // optional, for returning error info. + ) +{ + Word_t Index; // fast copy, in a register. + Pjp_t Pjp; // current JP. + Pjbl_t Pjbl; // Pjp->jp_Addr masked and cast to types: + Pjbb_t Pjbb; + Pjbu_t Pjbu; + Pjlb_t Pjlb; + PWord_t Pword; // alternate name for use by GET* macros. + + Word_t digit; // next digit to decode from Index. + Word_t digits; // current state in SM = digits left to decode. + Word_t pop0; // in a leaf. + Word_t pop0mask; // precalculated to avoid variable shifts. + long offset; // within a branch or leaf (can be large). + int subexp; // subexpanse in a bitmap branch. + BITMAPB_t bitposmaskB; // bit in bitmap for bitmap branch. + BITMAPL_t bitposmaskL; // bit in bitmap for bitmap leaf. + Word_t possfullJP1; // JP types for possibly full subexpanses: + Word_t possfullJP2; + Word_t possfullJP3; + + +// ---------------------------------------------------------------------------- +// M A C R O S +// +// These are intended to make the code a bit more readable and less redundant. + + +// CHECK FOR NULL JP: +// +// TBD: In principle this can be reduced (here and in other *.c files) to just +// the latter clause since no Type should ever be below cJU_JPNULL1, but in +// fact some root pointer types can be lower, so for safety do both checks. + +#define JPNULL(Type) (((Type) >= cJU_JPNULL1) && ((Type) <= cJU_JPNULLMAX)) + + +// CHECK FOR A FULL JP: +// +// Given a JP, indicate if it is fully populated. Use digits, pop0mask, and +// possfullJP1..3 in the context. +// +// This is a difficult problem because it requires checking the Pop0 bits for +// all-ones, but the number of bytes depends on the JP type, which is not +// directly related to the parent branchs type or level -- the JPs child +// could be under a narrow pointer (hence not full). The simple answer +// requires switching on or otherwise calculating the JP type, which could be +// slow. Instead, in SMPREPB* precalculate pop0mask and also record in +// possfullJP1..3 the child JP (branch) types that could possibly be full (one +// level down), and use them here. For level-2 branches (with digits == 2), +// the test for a full child depends on Judy1/JudyL. +// +// Note: This cannot be applied to the JP in a JPM because it doesnt have +// enough pop0 digits. +// +// TBD: JPFULL_BRANCH diligently checks for BranchL or BranchB, where neither +// of those can ever be full as it turns out. Could just check for a BranchU +// at the right level. Also, pop0mask might be overkill, its not used much, +// so perhaps just call cJU_POP0MASK(digits - 1) here? +// +// First, JPFULL_BRANCH checks for a full expanse for a JP whose child can be a +// branch, that is, a JP in a branch at level 3 or higher: + +#define JPFULL_BRANCH(Pjp) \ + ((((JU_JPDCDPOP0(Pjp) ^ cJU_ALLONES) & pop0mask) == 0) \ + && ((JU_JPTYPE(Pjp) == possfullJP1) \ + || (JU_JPTYPE(Pjp) == possfullJP2) \ + || (JU_JPTYPE(Pjp) == possfullJP3))) + +#ifdef JUDY1 +#define JPFULL(Pjp) \ + ((digits == 2) ? \ + (JU_JPTYPE(Pjp) == cJ1_JPFULLPOPU1) : JPFULL_BRANCH(Pjp)) +#else +#define JPFULL(Pjp) \ + ((digits == 2) ? \ + (JU_JPTYPE(Pjp) == cJU_JPLEAF_B1) \ + && (((JU_JPDCDPOP0(Pjp) & cJU_POP0MASK(1)) == cJU_POP0MASK(1))) : \ + JPFULL_BRANCH(Pjp)) +#endif + + +// RETURN SUCCESS: +// +// This hides the need to set *PIndex back to the local value of Index -- use a +// local value for faster operation. Note that the callers *PIndex is ALWAYS +// modified upon success, at least decremented/incremented. + +#define RET_SUCCESS { *PIndex = Index; return(1); } + + +// RETURN A CORRUPTION: + +#define RET_CORRUPT { JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); return(JERRI); } + + +// SEARCH A BITMAP BRANCH: +// +// This is a weak analog of j__udySearchLeaf*() for bitmap branches. Return +// the actual or next-left position, base 0, of Digit in a BITMAPB_t bitmap +// (subexpanse of a full bitmap), also given a Bitposmask for Digit. The +// position is the offset within the set bits. +// +// Unlike j__udySearchLeaf*(), the offset is not returned bit-complemented if +// Digits bit is unset, because the caller can check the bitmap themselves to +// determine that. Also, if Digits bit is unset, the returned offset is to +// the next-left JP or index (including -1), not to the "ideal" position for +// the index = next-right JP or index. +// +// Shortcut and skip calling j__udyCountBitsB() if the bitmap is full, in which +// case (Digit % cJU_BITSPERSUBEXPB) itself is the base-0 offset. + +#define SEARCHBITMAPB(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPB) ? (Digit % cJU_BITSPERSUBEXPB) : \ + j__udyCountBitsB((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#ifdef JUDYPREV +// Equivalent to search for the highest offset in Bitmap, that is, one less +// than the number of bits set: + +#define SEARCHBITMAPMAXB(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPB) ? cJU_BITSPERSUBEXPB - 1 : \ + j__udyCountBitsB(Bitmap) - 1) +#endif + + +// CHECK DECODE BYTES: +// +// Check Decode bytes in a JP against the equivalent portion of Index. If they +// dont match, Index is outside the subexpanse of a narrow pointer, hence is +// empty. + +#define CHECKDCD(cDigits) \ + if (JU_DCDNOTMATCHINDEX(Index, Pjp, cDigits)) RET_SUCCESS + + +// REVISE REMAINDER OF INDEX: +// +// Put one digit in place in Index and clear/set the lower digits, if any, so +// the resulting Index is at the start/end of an expanse, or just clear/set the +// least digits. +// +// Actually, to make simple use of JU_LEASTBYTESMASK, first clear/set all least +// digits of Index including the digit to be overridden, then set the value of +// that one digit. If Digits == 1 the first operation is redundant, but either +// very fast or even removed by the optimizer. + +#define CLEARLEASTDIGITS(Digits) Index &= ~JU_LEASTBYTESMASK(Digits) +#define SETLEASTDIGITS( Digits) Index |= JU_LEASTBYTESMASK(Digits) + +#define CLEARLEASTDIGITS_D(Digit,Digits) \ + { \ + CLEARLEASTDIGITS(Digits); \ + JU_SETDIGIT(Index, Digit, Digits); \ + } + +#define SETLEASTDIGITS_D(Digit,Digits) \ + { \ + SETLEASTDIGITS(Digits); \ + JU_SETDIGIT(Index, Digit, Digits); \ + } + + +// SET REMAINDER OF INDEX AND THEN RETURN OR CONTINUE: + +#define SET_AND_RETURN(OpLeastDigits,Digit,Digits) \ + { \ + OpLeastDigits(Digit, Digits); \ + RET_SUCCESS; \ + } + +#define SET_AND_CONTINUE(OpLeastDigits,Digit,Digits) \ + { \ + OpLeastDigits(Digit, Digits); \ + goto SMGetContinue; \ + } + + +// PREPARE TO HANDLE A LEAFW OR JP BRANCH IN THE STATE MACHINE: +// +// Extract a state-dependent digit from Index in a "constant" way, then jump to +// common code for multiple cases. +// +// TBD: Should this macro do more, such as preparing variable-shift masks for +// use in CLEARLEASTDIGITS and SETLEASTDIGITS? + +#define SMPREPB(cDigits,Next,PossFullJP1,PossFullJP2,PossFullJP3) \ + digits = (cDigits); \ + digit = JU_DIGITATSTATE(Index, cDigits); \ + pop0mask = cJU_POP0MASK((cDigits) - 1); /* for branchs JPs */ \ + possfullJP1 = (PossFullJP1); \ + possfullJP2 = (PossFullJP2); \ + possfullJP3 = (PossFullJP3); \ + goto Next + +// Variations for specific-level branches and for shorthands: +// +// Note: SMPREPB2 need not initialize possfullJP* because JPFULL does not use +// them for digits == 2, but gcc -Wall isnt quite smart enough to see this, so +// waste a bit of time and space to get rid of the warning: + +#define SMPREPB2(Next) \ + digits = 2; \ + digit = JU_DIGITATSTATE(Index, 2); \ + pop0mask = cJU_POP0MASK(1); /* for branchs JPs */ \ + possfullJP1 = possfullJP2 = possfullJP3 = 0; \ + goto Next + +#define SMPREPB3(Next) SMPREPB(3, Next, cJU_JPBRANCH_L2, \ + cJU_JPBRANCH_B2, \ + cJU_JPBRANCH_U2) +#ifndef JU_64BIT +#define SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L3, \ + cJU_JPBRANCH_B3, \ + cJU_JPBRANCH_U3) +#else +#define SMPREPB4(Next) SMPREPB(4, Next, cJU_JPBRANCH_L3, \ + cJU_JPBRANCH_B3, \ + cJU_JPBRANCH_U3) +#define SMPREPB5(Next) SMPREPB(5, Next, cJU_JPBRANCH_L4, \ + cJU_JPBRANCH_B4, \ + cJU_JPBRANCH_U4) +#define SMPREPB6(Next) SMPREPB(6, Next, cJU_JPBRANCH_L5, \ + cJU_JPBRANCH_B5, \ + cJU_JPBRANCH_U5) +#define SMPREPB7(Next) SMPREPB(7, Next, cJU_JPBRANCH_L6, \ + cJU_JPBRANCH_B6, \ + cJU_JPBRANCH_U6) +#define SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L7, \ + cJU_JPBRANCH_B7, \ + cJU_JPBRANCH_U7) +#endif + + +// RESTART AFTER SECONDARY DEAD END: +// +// Set Index to the first/last index in the branch or leaf subexpanse and start +// over at the top of the tree. + +#ifdef JUDYPREV +#define SMRESTART(Digits) { CLEARLEASTDIGITS(Digits); goto SMGetRestart; } +#else +#define SMRESTART(Digits) { SETLEASTDIGITS( Digits); goto SMGetRestart; } +#endif + + +// CHECK EDGE OF LEAFS EXPANSE: +// +// Given the LSBs of the lowest/highest valid index in a leaf (or equivalently +// in an immediate JP), the level (index size) of the leaf, and the full index +// to return (as Index in the context) already set to the full index matching +// the lowest/highest one, determine if there is an empty index in the leafs +// expanse below/above the lowest/highest index, which is true if the +// lowest/highest index is not at the "edge" of the leafs expanse based on its +// LSBs. If so, return Index decremented/incremented; otherwise restart at the +// top of the tree. +// +// Note: In many cases Index is already at the right spot and calling +// SMRESTART instead of just going directly to SMGetRestart is a bit of +// overkill. +// +// Note: Variable shift occurs if Digits is not a constant. + +#ifdef JUDYPREV +#define LEAF_EDGE(MinIndex,Digits) \ + { \ + if (MinIndex) { --Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#else +#define LEAF_EDGE(MaxIndex,Digits) \ + { \ + if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits)) \ + { ++Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#endif + +// Same as above except Index is not already set to match the lowest/highest +// index, so do that before decrementing/incrementing it: + +#ifdef JUDYPREV +#define LEAF_EDGE_SET(MinIndex,Digits) \ + { \ + if (MinIndex) \ + { JU_SETDIGITS(Index, MinIndex, Digits); --Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#else +#define LEAF_EDGE_SET(MaxIndex,Digits) \ + { \ + if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits)) \ + { JU_SETDIGITS(Index, MaxIndex, Digits); ++Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#endif + + +// FIND A HOLE (EMPTY INDEX) IN AN IMMEDIATE OR LEAF: +// +// Given an index location in a leaf (or equivalently an immediate JP) known to +// contain a usable hole (an empty index less/greater than Index), and the LSBs +// of a minimum/maximum index to locate, find the previous/next empty index and +// return it. +// +// Note: "Even" index sizes (1,2,4[,8] bytes) have corresponding native C +// types; "odd" index sizes dont, but they are not represented here because +// they are handled completely differently; see elsewhere. + +#ifdef JUDYPREV + +#define LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB) \ + { \ + while (*(Pjll) > (IndexLSB)) --(Pjll); /* too high */ \ + if (*(Pjll) < (IndexLSB)) RET_SUCCESS /* Index is empty */ \ + while (*(--(Pjll)) == --(IndexLSB)) /* null, find a hole */;\ + JU_SETDIGITS(Index, IndexLSB, cDigits); \ + RET_SUCCESS; \ + } +#else +#define LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB) \ + { \ + while (*(Pjll) < (IndexLSB)) ++(Pjll); /* too low */ \ + if (*(Pjll) > (IndexLSB)) RET_SUCCESS /* Index is empty */ \ + while (*(++(Pjll)) == ++(IndexLSB)) /* null, find a hole */;\ + JU_SETDIGITS(Index, IndexLSB, cDigits); \ + RET_SUCCESS; \ + } +#endif + + +// SEARCH FOR AN EMPTY INDEX IN AN IMMEDIATE OR LEAF: +// +// Given a pointer to the first index in a leaf (or equivalently an immediate +// JP), the population of the leaf, and a first empty Index to find (inclusive, +// as Index in the context), where Index is known to fall within the expanse of +// the leaf to search, efficiently find the previous/next empty index in the +// leaf, if any. For simplicity the following overview is stated in terms of +// Judy*NextEmpty() only, but the same concepts apply symmetrically for +// Judy*PrevEmpty(). Also, in each case the comparisons are for the LSBs of +// Index and leaf indexes, according to the leafs level. +// +// 1. If Index is GREATER than the last (highest) index in the leaf +// (maxindex), return success, Index is empty. (Remember, Index is known +// to be in the leafs expanse.) +// +// 2. If Index is EQUAL to maxindex: If maxindex is not at the edge of the +// leafs expanse, increment Index and return success, there is an empty +// Index one higher than any in the leaf; otherwise restart with Index +// reset to the upper edge of the leafs expanse. Note: This might cause +// an extra cache line fill, but this is OK for repeatedly-called search +// code, and it saves CPU time. +// +// 3. If Index is LESS than maxindex, check for "dense to end of leaf": +// Subtract Index from maxindex, and back up that many slots in the leaf. +// If the resulting offset is not before the start of the leaf then compare +// the index at this offset (baseindex) with Index: +// +// 3a. If GREATER, the leaf must be corrupt, since indexes are sorted and +// there are no duplicates. +// +// 3b. If EQUAL, the leaf is "dense" from Index to maxindex, meaning there is +// no reason to search it. "Slide right" to the high end of the leaf +// (modify Index to maxindex) and continue with step 2 above. +// +// 3c. If LESS, continue with step 4. +// +// 4. If the offset based on maxindex minus Index falls BEFORE the start of +// the leaf, or if, per 3c above, baseindex is LESS than Index, the leaf is +// guaranteed "not dense to the end" and a usable empty Index must exist. +// This supports a more efficient search loop. Start at the FIRST index in +// the leaf, or one BEYOND baseindex, respectively, and search the leaf as +// follows, comparing each current index (currindex) with Index: +// +// 4a. If LESS, keep going to next index. Note: This is certain to terminate +// because maxindex is known to be greater than Index, hence the loop can +// be small and fast. +// +// 4b. If EQUAL, loop and increment Index until finding currindex greater than +// Index, and return success with the modified Index. +// +// 4c. If GREATER, return success, Index (unmodified) is empty. +// +// Note: These are macros rather than functions for speed. + +#ifdef JUDYPREV + +#define JSLE_EVEN(Addr,Pop0,cDigits,LeafType) \ + { \ + LeafType * PjllLSB = (LeafType *) (Addr); \ + LeafType IndexLSB = Index; /* auto-masking */ \ + \ + /* Index before or at start of leaf: */ \ + \ + if (*PjllLSB >= IndexLSB) /* no need to search */ \ + { \ + if (*PjllLSB > IndexLSB) RET_SUCCESS; /* Index empty */ \ + LEAF_EDGE(*PjllLSB, cDigits); \ + } \ + \ + /* Index in or after leaf: */ \ + \ + offset = IndexLSB - *PjllLSB; /* tentative offset */ \ + if (offset <= (Pop0)) /* can check density */ \ + { \ + PjllLSB += offset; /* move to slot */ \ + \ + if (*PjllLSB <= IndexLSB) /* dense or corrupt */ \ + { \ + if (*PjllLSB == IndexLSB) /* dense, check edge */ \ + LEAF_EDGE_SET(PjllLSB[-offset], cDigits); \ + RET_CORRUPT; \ + } \ + --PjllLSB; /* not dense, start at previous */ \ + } \ + else PjllLSB = ((LeafType *) (Addr)) + (Pop0); /* start at max */ \ + \ + LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB); \ + } + +// JSLE_ODD is completely different from JSLE_EVEN because its important to +// minimize copying odd indexes to compare them (see 4.14). Furthermore, a +// very complex version (4.17, but abandoned before fully debugged) that +// avoided calling j__udySearchLeaf*() ran twice as fast as 4.14, but still +// half as fast as SearchValid. Doug suggested that to minimize complexity and +// share common code we should use j__udySearchLeaf*() for the initial search +// to establish if Index is empty, which should be common. If Index is valid +// in a leaf or immediate indexes, odds are good that an empty Index is nearby, +// so for simplicity just use a *COPY* function to linearly search the +// remainder. +// +// TBD: Pathological case? Average performance should be good, but worst-case +// might suffer. When Search says the initial Index is valid, so a linear +// copy-and-compare is begun, if the caller builds fairly large leaves with +// dense clusters AND frequently does a SearchEmpty at one end of such a +// cluster, performance wont be very good. Might a dense-check help? This +// means checking offset against the index at offset, and then against the +// first/last index in the leaf. We doubt the pathological case will appear +// much in real applications because they will probably alternate SearchValid +// and SearchEmpty calls. + +#define JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy) \ + { \ + Word_t IndexLSB; /* least bytes only */ \ + Word_t IndexFound; /* in leaf */ \ + \ + if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0) \ + RET_SUCCESS; /* Index is empty */ \ + \ + IndexLSB = JU_LEASTBYTES(Index, cDigits); \ + offset *= (cDigits); \ + \ + while ((offset -= (cDigits)) >= 0) \ + { /* skip until empty or start */ \ + Copy(IndexFound, ((uint8_t *) (Pjll)) + offset); \ + if (IndexFound != (--IndexLSB)) /* found an empty */ \ + { JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; }\ + } \ + LEAF_EDGE_SET(IndexLSB, cDigits); \ + } + +#else // JUDYNEXT + +#define JSLE_EVEN(Addr,Pop0,cDigits,LeafType) \ + { \ + LeafType * PjllLSB = ((LeafType *) (Addr)) + (Pop0); \ + LeafType IndexLSB = Index; /* auto-masking */ \ + \ + /* Index at or after end of leaf: */ \ + \ + if (*PjllLSB <= IndexLSB) /* no need to search */ \ + { \ + if (*PjllLSB < IndexLSB) RET_SUCCESS; /* Index empty */\ + LEAF_EDGE(*PjllLSB, cDigits); \ + } \ + \ + /* Index before or in leaf: */ \ + \ + offset = *PjllLSB - IndexLSB; /* tentative offset */ \ + if (offset <= (Pop0)) /* can check density */ \ + { \ + PjllLSB -= offset; /* move to slot */ \ + \ + if (*PjllLSB >= IndexLSB) /* dense or corrupt */ \ + { \ + if (*PjllLSB == IndexLSB) /* dense, check edge */ \ + LEAF_EDGE_SET(PjllLSB[offset], cDigits); \ + RET_CORRUPT; \ + } \ + ++PjllLSB; /* not dense, start at next */ \ + } \ + else PjllLSB = (LeafType *) (Addr); /* start at minimum */ \ + \ + LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB); \ + } + +#define JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy) \ + { \ + Word_t IndexLSB; /* least bytes only */ \ + Word_t IndexFound; /* in leaf */ \ + int offsetmax; /* in bytes */ \ + \ + if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0) \ + RET_SUCCESS; /* Index is empty */ \ + \ + IndexLSB = JU_LEASTBYTES(Index, cDigits); \ + offset *= (cDigits); \ + offsetmax = (Pop0) * (cDigits); /* single multiply */ \ + \ + while ((offset += (cDigits)) <= offsetmax) \ + { /* skip until empty or end */ \ + Copy(IndexFound, ((uint8_t *) (Pjll)) + offset); \ + if (IndexFound != (++IndexLSB)) /* found an empty */ \ + { JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; } \ + } \ + LEAF_EDGE_SET(IndexLSB, cDigits); \ + } + +#endif // JUDYNEXT + +// Note: Immediate indexes never fill a single index group, so for odd index +// sizes, save time by calling JSLE_ODD_IMM instead of JSLE_ODD. + +#define j__udySearchLeafEmpty1(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 1, uint8_t) + +#define j__udySearchLeafEmpty2(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 2, uint16_t) + +#define j__udySearchLeafEmpty3(Addr,Pop0) \ + JSLE_ODD(3, Addr, Pop0, j__udySearchLeaf3, JU_COPY3_PINDEX_TO_LONG) + +#ifndef JU_64BIT + +#define j__udySearchLeafEmptyL(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 4, Word_t) + +#else + +#define j__udySearchLeafEmpty4(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 4, uint32_t) + +#define j__udySearchLeafEmpty5(Addr,Pop0) \ + JSLE_ODD(5, Addr, Pop0, j__udySearchLeaf5, JU_COPY5_PINDEX_TO_LONG) + +#define j__udySearchLeafEmpty6(Addr,Pop0) \ + JSLE_ODD(6, Addr, Pop0, j__udySearchLeaf6, JU_COPY6_PINDEX_TO_LONG) + +#define j__udySearchLeafEmpty7(Addr,Pop0) \ + JSLE_ODD(7, Addr, Pop0, j__udySearchLeaf7, JU_COPY7_PINDEX_TO_LONG) + +#define j__udySearchLeafEmptyL(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 8, Word_t) + +#endif // JU_64BIT + + +// ---------------------------------------------------------------------------- +// START OF CODE: +// +// CHECK FOR SHORTCUTS: +// +// Error out if PIndex is null. + + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + return(JERRI); + } + + Index = *PIndex; // fast local copy. + +// Set and pre-decrement/increment Index, watching for underflow/overflow: +// +// An out-of-bounds Index means failure: No previous/next empty index. + +SMGetRestart: // return here with revised Index. + +#ifdef JUDYPREV + if (Index-- == 0) return(0); +#else + if (++Index == 0) return(0); +#endif + +// An empty array with an in-bounds (not underflowed/overflowed) Index means +// success: +// +// Note: This check is redundant after restarting at SMGetRestart, but should +// take insignificant time. + + if (PArray == (Pvoid_t) NULL) RET_SUCCESS; + +// ---------------------------------------------------------------------------- +// ROOT-LEVEL LEAF that starts with a Pop0 word; just look within the leaf: +// +// If Index is not in the leaf, return success; otherwise return the first +// empty Index, if any, below/above where it would belong. + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + pop0 = Pjlw[0]; + +#ifdef JUDY1 + if (pop0 == 0) // special case. + { +#ifdef JUDYPREV + if ((Index != Pjlw[1]) || (Index-- != 0)) RET_SUCCESS; +#else + if ((Index != Pjlw[1]) || (++Index != 0)) RET_SUCCESS; +#endif + return(0); // no previous/next empty index. + } +#endif // JUDY1 + + j__udySearchLeafEmptyL(Pjlw + 1, pop0); + +// No return -- thanks ALAN + + } + else + +// ---------------------------------------------------------------------------- +// HANDLE JRP Branch: +// +// For JRP branches, traverse the JPM; handle LEAFW +// directly; but look for the most common cases first. + + { + Pjpm_t Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); + +// goto SMGetContinue; + } + + +// ============================================================================ +// STATE MACHINE -- GET INDEX: +// +// Search for Index (already decremented/incremented so as to be an inclusive +// search). If not found (empty index), return success. Otherwise do a +// previous/next search, and if successful modify Index to the empty index +// found. See function header comments. +// +// ENTRY: Pjp points to next JP to interpret, whose Decode bytes have not yet +// been checked. +// +// Note: Check Decode bytes at the start of each loop, not after looking up a +// new JP, so its easy to do constant shifts/masks. +// +// EXIT: Return, or branch to SMGetRestart with modified Index, or branch to +// SMGetContinue with a modified Pjp, as described elsewhere. +// +// WARNING: For run-time efficiency the following cases replicate code with +// varying constants, rather than using common code with variable values! + +SMGetContinue: // return here for next branch/leaf. + +#ifdef TRACEJPSE + JudyPrintJP(Pjp, "sf", __LINE__); +#endif + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_L2: CHECKDCD(2); SMPREPB2(SMBranchL); + case cJU_JPBRANCH_L3: CHECKDCD(3); SMPREPB3(SMBranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: CHECKDCD(4); SMPREPB4(SMBranchL); + case cJU_JPBRANCH_L5: CHECKDCD(5); SMPREPB5(SMBranchL); + case cJU_JPBRANCH_L6: CHECKDCD(6); SMPREPB6(SMBranchL); + case cJU_JPBRANCH_L7: CHECKDCD(7); SMPREPB7(SMBranchL); +#endif + case cJU_JPBRANCH_L: SMPREPBL(SMBranchL); + +// Common code (state-independent) for all cases of linear branches: + +SMBranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +// First, check if Indexs expanse (digit) is below/above the first/last +// populated expanse in the BranchL, in which case Index is empty; otherwise +// find the offset of the lowest/highest populated expanse at or above/below +// digit, if any: +// +// Note: The for-loop is guaranteed to exit eventually because the first/last +// expanse is known to be a terminator. +// +// Note: Cannot use j__udySearchLeaf*Empty1() here because it only applies to +// leaves and does not know about partial versus full JPs, unlike the use of +// j__udySearchLeaf1() for BranchLs in SearchValid code. Also, since linear +// leaf expanse lists are small, dont waste time calling j__udySearchLeaf1(), +// just scan the expanse list. + +#ifdef JUDYPREV + if ((Pjbl->jbl_Expanse[0]) > digit) RET_SUCCESS; + + for (offset = (Pjbl->jbl_NumJPs) - 1; /* null */; --offset) +#else + if ((Pjbl->jbl_Expanse[(Pjbl->jbl_NumJPs) - 1]) < digit) + RET_SUCCESS; + + for (offset = 0; /* null */; ++offset) +#endif + { + +// Too low/high, keep going; or too high/low, meaning the loop passed a hole +// and the initial Index is empty: + +#ifdef JUDYPREV + if ((Pjbl->jbl_Expanse[offset]) > digit) continue; + if ((Pjbl->jbl_Expanse[offset]) < digit) RET_SUCCESS; +#else + if ((Pjbl->jbl_Expanse[offset]) < digit) continue; + if ((Pjbl->jbl_Expanse[offset]) > digit) RET_SUCCESS; +#endif + +// Found expanse matching digit; if its not full, traverse through it: + + if (! JPFULL((Pjbl->jbl_jp) + offset)) + { + Pjp = (Pjbl->jbl_jp) + offset; + goto SMGetContinue; + } + +// Common code: While searching for a lower/higher hole or a non-full JP, upon +// finding a lower/higher hole, adjust Index using the revised digit and +// return; or upon finding a consecutive lower/higher expanse, if the expanses +// JP is non-full, modify Index and traverse through the JP: + +#define BRANCHL_CHECK(OpIncDec,OpLeastDigits,Digit,Digits) \ + { \ + if ((Pjbl->jbl_Expanse[offset]) != OpIncDec digit) \ + SET_AND_RETURN(OpLeastDigits, Digit, Digits); \ + \ + if (! JPFULL((Pjbl->jbl_jp) + offset)) \ + { \ + Pjp = (Pjbl->jbl_jp) + offset; \ + SET_AND_CONTINUE(OpLeastDigits, Digit, Digits); \ + } \ + } + +// BranchL primary dead end: Expanse matching Index/digit is full (rare except +// for dense/sequential indexes): +// +// Search for a lower/higher hole, a non-full JP, or the end of the expanse +// list, while decrementing/incrementing digit. + +#ifdef JUDYPREV + while (--offset >= 0) + BRANCHL_CHECK(--, SETLEASTDIGITS_D, digit, digits) +#else + while (++offset < Pjbl->jbl_NumJPs) + BRANCHL_CHECK(++, CLEARLEASTDIGITS_D, digit, digits) +#endif + +// Passed end of BranchL expanse list after finding a matching but full +// expanse: +// +// Digit now matches the lowest/highest expanse, which is a full expanse; if +// digit is at the end of BranchLs expanse (no hole before/after), break out +// of the loop; otherwise modify Index to the next lower/higher digit and +// return success: + +#ifdef JUDYPREV + if (digit == 0) break; + --digit; SET_AND_RETURN(SETLEASTDIGITS_D, digit, digits); +#else + if (digit == JU_LEASTBYTES(cJU_ALLONES, 1)) break; + ++digit; SET_AND_RETURN(CLEARLEASTDIGITS_D, digit, digits); +#endif + } // for-loop + +// BranchL secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_B2: CHECKDCD(2); SMPREPB2(SMBranchB); + case cJU_JPBRANCH_B3: CHECKDCD(3); SMPREPB3(SMBranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: CHECKDCD(4); SMPREPB4(SMBranchB); + case cJU_JPBRANCH_B5: CHECKDCD(5); SMPREPB5(SMBranchB); + case cJU_JPBRANCH_B6: CHECKDCD(6); SMPREPB6(SMBranchB); + case cJU_JPBRANCH_B7: CHECKDCD(7); SMPREPB7(SMBranchB); +#endif + case cJU_JPBRANCH_B: SMPREPBL(SMBranchB); + +// Common code (state-independent) for all cases of bitmap branches: + +SMBranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + +// Locate the digits JP in the subexpanse list, if present: + + subexp = digit / cJU_BITSPERSUBEXPB; + assert(subexp < cJU_NUMSUBEXPB); // falls in expected range. + bitposmaskB = JU_BITPOSMASKB(digit); + +// Absent JP = no JP matches current digit in Index: + +// if (! JU_BITMAPTESTB(Pjbb, digit)) // slower. + if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB)) // faster. + RET_SUCCESS; + +// Non-full JP matches current digit in Index: +// +// Iterate to the subsidiary non-full JP. + + offset = SEARCHBITMAPB(JU_JBB_BITMAP(Pjbb, subexp), digit, + bitposmaskB); + // not negative since at least one bit is set: + assert(offset >= 0); + assert(offset < (int) cJU_BITSPERSUBEXPB); + +// Watch for null JP subarray pointer with non-null bitmap (a corruption): + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) + == (Pjp_t) NULL) RET_CORRUPT; + + Pjp += offset; + if (! JPFULL(Pjp)) goto SMGetContinue; + +// BranchB primary dead end: +// +// Upon hitting a full JP in a BranchB for the next digit in Index, search +// sideways for a previous/next absent JP (unset bit) or non-full JP (set bit +// with non-full JP); first in the current bitmap subexpanse, then in +// lower/higher subexpanses. Upon entry, Pjp points to a known-unusable JP, +// ready to decrement/increment. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. +// +// TBD: For speed, shift bitposmaskB instead of using JU_BITMAPTESTB or +// JU_BITPOSMASKB, but this shift has knowledge of bit order that really should +// be encapsulated in a header file. + +#define BRANCHB_CHECKBIT(OpLeastDigits) \ + if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB)) /* absent JP */ \ + SET_AND_RETURN(OpLeastDigits, digit, digits) + +#define BRANCHB_CHECKJPFULL(OpLeastDigits) \ + if (! JPFULL(Pjp)) \ + SET_AND_CONTINUE(OpLeastDigits, digit, digits) + +#define BRANCHB_STARTSUBEXP(OpLeastDigits) \ + if (! JU_JBB_BITMAP(Pjbb, subexp)) /* empty subexpanse, shortcut */ \ + SET_AND_RETURN(OpLeastDigits, digit, digits) \ + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) RET_CORRUPT + +#ifdef JUDYPREV + + --digit; // skip initial digit. + bitposmaskB >>= 1; // see TBD above. + +BranchBNextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskB) // more bits to check in subexp. + { + BRANCHB_CHECKBIT(SETLEASTDIGITS_D); + --Pjp; // previous in subarray. + BRANCHB_CHECKJPFULL(SETLEASTDIGITS_D); + assert(digit >= 0); + --digit; + bitposmaskB >>= 1; + } + + if (subexp-- > 0) // more subexpanses. + { + BRANCHB_STARTSUBEXP(SETLEASTDIGITS_D); + Pjp += SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)) + 1; + bitposmaskB = (1U << (cJU_BITSPERSUBEXPB - 1)); + goto BranchBNextSubexp; + } + +#else // JUDYNEXT + + ++digit; // skip initial digit. + bitposmaskB <<= 1; // note: BITMAPB_t. + +BranchBNextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskB) // more bits to check in subexp. + { + BRANCHB_CHECKBIT(CLEARLEASTDIGITS_D); + ++Pjp; // previous in subarray. + BRANCHB_CHECKJPFULL(CLEARLEASTDIGITS_D); + assert(digit < cJU_SUBEXPPERSTATE); + ++digit; + bitposmaskB <<= 1; // note: BITMAPB_t. + } + + if (++subexp < cJU_NUMSUBEXPB) // more subexpanses. + { + BRANCHB_STARTSUBEXP(CLEARLEASTDIGITS_D); + --Pjp; // pre-decrement. + bitposmaskB = 1; + goto BranchBNextSubexp; + } + +#endif // JUDYNEXT + +// BranchB secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_U2: CHECKDCD(2); SMPREPB2(SMBranchU); + case cJU_JPBRANCH_U3: CHECKDCD(3); SMPREPB3(SMBranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: CHECKDCD(4); SMPREPB4(SMBranchU); + case cJU_JPBRANCH_U5: CHECKDCD(5); SMPREPB5(SMBranchU); + case cJU_JPBRANCH_U6: CHECKDCD(6); SMPREPB6(SMBranchU); + case cJU_JPBRANCH_U7: CHECKDCD(7); SMPREPB7(SMBranchU); +#endif + case cJU_JPBRANCH_U: SMPREPBL(SMBranchU); + +// Common code (state-independent) for all cases of uncompressed branches: + +SMBranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + Pjp = (Pjbu->jbu_jp) + digit; + +// Absent JP = null JP for current digit in Index: + + if (JPNULL(JU_JPTYPE(Pjp))) RET_SUCCESS; + +// Non-full JP matches current digit in Index: +// +// Iterate to the subsidiary JP. + + if (! JPFULL(Pjp)) goto SMGetContinue; + +// BranchU primary dead end: +// +// Upon hitting a full JP in a BranchU for the next digit in Index, search +// sideways for a previous/next null or non-full JP. BRANCHU_CHECKJP() is +// shorthand for common code. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. + +#define BRANCHU_CHECKJP(OpIncDec,OpLeastDigits) \ + { \ + OpIncDec Pjp; \ + \ + if (JPNULL(JU_JPTYPE(Pjp))) \ + SET_AND_RETURN(OpLeastDigits, digit, digits) \ + \ + if (! JPFULL(Pjp)) \ + SET_AND_CONTINUE(OpLeastDigits, digit, digits) \ + } + +#ifdef JUDYPREV + while (digit-- > 0) + BRANCHU_CHECKJP(--, SETLEASTDIGITS_D); +#else + while (++digit < cJU_BRANCHUNUMJPS) + BRANCHU_CHECKJP(++, CLEARLEASTDIGITS_D); +#endif + +// BranchU secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for the +// previous/next empty index starting at Index. Primary leaf dead end is +// hidden within j__udySearchLeaf*Empty*(). In case of secondary leaf dead +// end, restart at the top of the tree. +// +// Note: Pword is the name known to GET*; think of it as Pjlw. + +#define SMLEAFL(cDigits,Func) \ + Pword = (PWord_t) P_JLW(Pjp->jp_Addr); \ + pop0 = JU_JPLEAF_POP0(Pjp); \ + Func(Pword, pop0) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKDCD(1); SMLEAFL(1, j__udySearchLeafEmpty1); +#endif + case cJU_JPLEAF2: CHECKDCD(2); SMLEAFL(2, j__udySearchLeafEmpty2); + case cJU_JPLEAF3: CHECKDCD(3); SMLEAFL(3, j__udySearchLeafEmpty3); + +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKDCD(4); SMLEAFL(4, j__udySearchLeafEmpty4); + case cJU_JPLEAF5: CHECKDCD(5); SMLEAFL(5, j__udySearchLeafEmpty5); + case cJU_JPLEAF6: CHECKDCD(6); SMLEAFL(6, j__udySearchLeafEmpty6); + case cJU_JPLEAF7: CHECKDCD(7); SMLEAFL(7, j__udySearchLeafEmpty7); +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for the +// previous/next empty index starting at Index. + + case cJU_JPLEAF_B1: + + CHECKDCD(1); + + Pjlb = P_JLB(Pjp->jp_Addr); + digit = JU_DIGITATSTATE(Index, 1); + subexp = digit / cJU_BITSPERSUBEXPL; + bitposmaskL = JU_BITPOSMASKL(digit); + assert(subexp < cJU_NUMSUBEXPL); // falls in expected range. + +// Absent index = no index matches current digit in Index: + +// if (! JU_BITMAPTESTL(Pjlb, digit)) // slower. + if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL)) // faster. + RET_SUCCESS; + +// LeafB1 primary dead end: +// +// Upon hitting a valid (non-empty) index in a LeafB1 for the last digit in +// Index, search sideways for a previous/next absent index, first in the +// current bitmap subexpanse, then in lower/higher subexpanses. +// LEAFB1_CHECKBIT() is shorthand for common code to handle one bit in one +// bitmap subexpanse. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. +// +// TBD: For speed, shift bitposmaskL instead of using JU_BITMAPTESTL or +// JU_BITPOSMASKL, but this shift has knowledge of bit order that really should +// be encapsulated in a header file. + +#define LEAFB1_CHECKBIT(OpLeastDigits) \ + if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL)) \ + SET_AND_RETURN(OpLeastDigits, digit, 1) + +#define LEAFB1_STARTSUBEXP(OpLeastDigits) \ + if (! JU_JLB_BITMAP(Pjlb, subexp)) /* empty subexp */ \ + SET_AND_RETURN(OpLeastDigits, digit, 1) + +#ifdef JUDYPREV + + --digit; // skip initial digit. + bitposmaskL >>= 1; // see TBD above. + +LeafB1NextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskL) // more bits to check in subexp. + { + LEAFB1_CHECKBIT(SETLEASTDIGITS_D); + assert(digit >= 0); + --digit; + bitposmaskL >>= 1; + } + + if (subexp-- > 0) // more subexpanses. + { + LEAFB1_STARTSUBEXP(SETLEASTDIGITS_D); + bitposmaskL = (1UL << (cJU_BITSPERSUBEXPL - 1)); + goto LeafB1NextSubexp; + } + +#else // JUDYNEXT + + ++digit; // skip initial digit. + bitposmaskL <<= 1; // note: BITMAPL_t. + +LeafB1NextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskL) // more bits to check in subexp. + { + LEAFB1_CHECKBIT(CLEARLEASTDIGITS_D); + assert(digit < cJU_SUBEXPPERSTATE); + ++digit; + bitposmaskL <<= 1; // note: BITMAPL_t. + } + + if (++subexp < cJU_NUMSUBEXPL) // more subexpanses. + { + LEAFB1_STARTSUBEXP(CLEARLEASTDIGITS_D); + bitposmaskL = 1; + goto LeafB1NextSubexp; + } + +#endif // JUDYNEXT + +// LeafB1 secondary dead end, no empty index: + + SMRESTART(1); + + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// If the Decode bytes do not match, Index is empty (without modification); +// otherwise restart. + + case cJ1_JPFULLPOPU1: + + CHECKDCD(1); + SMRESTART(1); +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: +// +// Pop1 = 1 Immediate JPs: +// +// If Index is not in the immediate JP, return success; otherwise check if +// there is an empty index below/above the immediate JPs index, and if so, +// return success with modified Index, else restart. +// +// Note: Doug says its fast enough to calculate the index size (digits) in +// the following; no need to set it separately for each case. + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + if (JU_JPDCDPOP0(Pjp) != JU_TRIMTODCDSIZE(Index)) RET_SUCCESS; + digits = JU_JPTYPE(Pjp) - cJU_JPIMMED_1_01 + 1; + LEAF_EDGE(JU_LEASTBYTES(JU_JPDCDPOP0(Pjp), digits), digits); + +// Immediate JPs with Pop1 > 1: + +#define IMM_MULTI(Func,BaseJPType) \ + JUDY1CODE(Pword = (PWord_t) (Pjp->jp_1Index);) \ + JUDYLCODE(Pword = (PWord_t) (Pjp->jp_LIndex);) \ + Func(Pword, JU_JPTYPE(Pjp) - (BaseJPType) + 1) + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + IMM_MULTI(j__udySearchLeafEmpty1, cJU_JPIMMED_1_02); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + IMM_MULTI(j__udySearchLeafEmpty2, cJU_JPIMMED_2_02); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + IMM_MULTI(j__udySearchLeafEmpty3, cJU_JPIMMED_3_02); +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: + case cJ1_JPIMMED_4_03: + IMM_MULTI(j__udySearchLeafEmpty4, cJ1_JPIMMED_4_02); + + case cJ1_JPIMMED_5_02: + case cJ1_JPIMMED_5_03: + IMM_MULTI(j__udySearchLeafEmpty5, cJ1_JPIMMED_5_02); + + case cJ1_JPIMMED_6_02: + IMM_MULTI(j__udySearchLeafEmpty6, cJ1_JPIMMED_6_02); + + case cJ1_JPIMMED_7_02: + IMM_MULTI(j__udySearchLeafEmpty7, cJ1_JPIMMED_7_02); +#endif + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: RET_CORRUPT; + + } // SMGet switch. + +} // Judy1PrevEmpty() / Judy1NextEmpty() / JudyLPrevEmpty() / JudyLNextEmpty() diff --git a/libnetdata/libjudy/src/JudyL/JudyLPrev.c b/libnetdata/libjudy/src/JudyL/JudyLPrev.c new file mode 100644 index 000000000..4bcdccf10 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLPrev.c @@ -0,0 +1,1890 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.54 $ $Source: /judy/src/JudyCommon/JudyPrevNext.c $ +// +// Judy*Prev() and Judy*Next() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DJUDYNEXT for the Judy*Next() function; otherwise defaults to +// Judy*Prev(). + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifndef JUDYNEXT +#ifndef JUDYPREV +#define JUDYPREV 1 // neither set => use default. +#endif +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + + +// **************************************************************************** +// J U D Y 1 P R E V +// J U D Y 1 N E X T +// J U D Y L P R E V +// J U D Y L N E X T +// +// See the manual entry for the API. +// +// OVERVIEW OF Judy*Prev(): +// +// Use a reentrant switch statement (state machine, SM1 = "get") to decode the +// callers *PIndex-1, starting with the (PArray), through branches, if +// any, down to an immediate or a leaf. Look for *PIndex-1 in that leaf, and +// if found, return it. +// +// A dead end is either a branch that does not contain a JP for the appropriate +// digit in *PIndex-1, or a leaf that does not contain the undecoded digits of +// *PIndex-1. Upon reaching a dead end, backtrack through the leaf/branches +// that were just traversed, using a list (history) of parent JPs that is built +// while going forward in SM1Get. Start with the current leaf or branch. In a +// backtracked leaf, look for an Index less than *PIndex-1. In each +// backtracked branch, look "sideways" for the next JP, if any, lower than the +// one for the digit (from *PIndex-1) that was previously decoded. While +// backtracking, if a leaf has no previous Index or a branch has no lower JP, +// go to its parent branch in turn. Upon reaching the JRP, return failure, "no +// previous Index". The backtrack process is sufficiently different from +// SM1Get to merit its own separate reentrant switch statement (SM2 = +// "backtrack"). +// +// While backtracking, upon finding a lower JP in a branch, there is certain to +// be a "prev" Index under that JP (unless the Judy array is corrupt). +// Traverse forward again, this time taking the last (highest, right-most) JP +// in each branch, and the last (highest) Index upon reaching an immediate or a +// leaf. This traversal is sufficiently different from SM1Get and SM2Backtrack +// to merit its own separate reentrant switch statement (SM3 = "findlimit"). +// +// "Decode" bytes in JPs complicate this process a little. In SM1Get, when a +// JP is a narrow pointer, that is, when states are skipped (so the skipped +// digits are stored in jp_DcdPopO), compare the relevant digits to the same +// digits in *PIndex-1. If they are EQUAL, proceed in SM1Get as before. If +// jp_DcdPopOs digits are GREATER, treat the JP as a dead end and proceed in +// SM2Backtrack. If jp_DcdPopOs digits are LESS, treat the JP as if it had +// just been found during a backtrack and proceed directly in SM3Findlimit. +// +// Note that Decode bytes can be ignored in SM3Findlimit; they dont matter. +// Also note that in practice the Decode bytes are routinely compared with +// *PIndex-1 because thats simpler and no slower than first testing for +// narrowness. +// +// Decode bytes also make it unnecessary to construct the Index to return (the +// revised *PIndex) during the search. This step is deferred until finding an +// Index during backtrack or findlimit, before returning it. The first digit +// of *PIndex is derived (saved) based on which JP is used in a JRP branch. +// The remaining digits are obtained from the jp_DcdPopO field in the JP (if +// any) above the immediate or leaf containing the found (prev) Index, plus the +// remaining digit(s) in the immediate or leaf itself. In the case of a LEAFW, +// the Index to return is found directly in the leaf. +// +// Note: Theoretically, as described above, upon reaching a dead end, SM1Get +// passes control to SM2Backtrack to look sideways, even in a leaf. Actually +// its a little more efficient for the SM1Get leaf cases to shortcut this and +// take care of the sideways searches themselves. Hence the history list only +// contains branch JPs, and SM2Backtrack only handles branches. In fact, even +// the branch handling cases in SM1Get do some shortcutting (sideways +// searching) to avoid pushing history and calling SM2Backtrack unnecessarily. +// +// Upon reaching an Index to return after backtracking, *PIndex must be +// modified to the found Index. In principle this could be done by building +// the Index from a saved rootdigit (in the top branch) plus the Dcd bytes from +// the parent JP plus the appropriate Index bytes from the leaf. However, +// Immediates are difficult because their parent JPs lack one (last) digit. So +// instead just build the *PIndex to return "top down" while backtracking and +// findlimiting. +// +// This function is written iteratively for speed, rather than recursively. +// +// CAVEATS: +// +// Why use a backtrack list (history stack), since it has finite size? The +// size is small for Judy on both 32-bit and 64-bit systems, and a list (really +// just an array) is fast to maintain and use. Other alternatives include +// doing a lookahead (lookaside) in each branch while traversing forward +// (decoding), and restarting from the top upon a dead end. +// +// A lookahead means noting the last branch traversed which contained a +// non-null JP lower than the one specified by a digit in *PIndex-1, and +// returning to that point for SM3Findlimit. This seems like a good idea, and +// should be pretty cheap for linear and bitmap branches, but it could result +// in up to 31 unnecessary additional cache line fills (in extreme cases) for +// every uncompressed branch traversed. We have considered means of attaching +// to or hiding within an uncompressed branch (in null JPs) a "cache line map" +// or other structure, such as an offset to the next non-null JP, that would +// speed this up, but it seems unnecessary merely to avoid having a +// finite-length list (array). (If JudySL is ever made "native", the finite +// list length will be an issue.) +// +// Restarting at the top of the Judy array after a dead end requires a careful +// modification of *PIndex-1 to decrement the digit for the parent branch and +// set the remaining lower digits to all 1s. This must be repeated each time a +// parent branch contains another dead end, so even though it should all happen +// in cache, the CPU time can be excessive. (For JudySL or an equivalent +// "infinitely deep" Judy array, consider a hybrid of a large, finite, +// "circular" list and a restart-at-top when the list is backtracked to +// exhaustion.) +// +// Why search for *PIndex-1 instead of *PIndex during SM1Get? In rare +// instances this prevents an unnecessary decode down the wrong path followed +// by a backtrack; its pretty cheap to set up initially; and it means the +// SM1Get machine can simply return if/when it finds that Index. +// +// TBD: Wed like to enhance this function to make successive searches faster. +// This would require saving some previous state, including the previous Index +// returned, and in which leaf it was found. If the next call is for the same +// Index and the array has not been modified, start at the same leaf. This +// should be much easier to implement since this is iterative rather than +// recursive code. +// +// VARIATIONS FOR Judy*Next(): +// +// The Judy*Next() code is nearly a perfect mirror of the Judy*Prev() code. +// See the Judy*Prev() overview comments, and mentally switch the following: +// +// - "*PIndex-1" => "*PIndex+1" +// - "less than" => "greater than" +// - "lower" => "higher" +// - "lowest" => "highest" +// - "next-left" => "next-right" +// - "right-most" => "left-most" +// +// Note: SM3Findlimit could be called SM3Findmax/SM3Findmin, but a common name +// for both Prev and Next means many fewer ifdefs in this code. +// +// TBD: Currently this code traverses a JP whether its expanse is partially or +// completely full (populated). For Judy1 (only), since there is no value area +// needed, consider shortcutting to a "success" return upon encountering a full +// JP in SM1Get (or even SM3Findlimit?) A full JP looks like this: +// +// (((JU_JPDCDPOP0(Pjp) ^ cJU_ALLONES) & cJU_POP0MASK(cLevel)) == 0) + +#ifdef JUDY1 +#ifdef JUDYPREV +FUNCTION int Judy1Prev +#else +FUNCTION int Judy1Next +#endif +#else +#ifdef JUDYPREV +FUNCTION PPvoid_t JudyLPrev +#else +FUNCTION PPvoid_t JudyLNext +#endif +#endif + ( + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError // optional, for returning error info. + ) +{ + Pjp_t Pjp, Pjp2; // current JPs. + Pjbl_t Pjbl; // Pjp->jp_Addr masked and cast to types: + Pjbb_t Pjbb; + Pjbu_t Pjbu; + +// Note: The following initialization is not strictly required but it makes +// gcc -Wall happy because there is an "impossible" path from Immed handling to +// SM1LeafLImm code that looks like Pjll might be used before set: + + Pjll_t Pjll = (Pjll_t) NULL; + Word_t state; // current state in SM. + Word_t digit; // next digit to decode from Index. + +// Note: The following initialization is not strictly required but it makes +// gcc -Wall happy because there is an "impossible" path from Immed handling to +// SM1LeafLImm code (for JudyL & JudyPrev only) that looks like pop1 might be +// used before set: + +#if (defined(JUDYL) && defined(JUDYPREV)) + Word_t pop1 = 0; // in a leaf. +#else + Word_t pop1; // in a leaf. +#endif + int offset; // linear branch/leaf, from j__udySearchLeaf*(). + int subexp; // subexpanse in a bitmap branch. + Word_t bitposmask; // bit in bitmap for Index. + +// History for SM2Backtrack: +// +// For a given histnum, APjphist[histnum] is a parent JP that points to a +// branch, and Aoffhist[histnum] is the offset of the NEXT JP in the branch to +// which the parent JP points. The meaning of Aoffhist[histnum] depends on the +// type of branch to which the parent JP points: +// +// Linear: Offset of the next JP in the JP list. +// +// Bitmap: Which subexpanse, plus the offset of the next JP in the +// subexpanses JP list (to avoid bit-counting again), plus for Judy*Next(), +// hidden one byte to the left, which digit, because Judy*Next() also needs +// this. +// +// Uncompressed: Digit, which is actually the offset of the JP in the branch. +// +// Note: Only branch JPs are stored in APjphist[] because, as explained +// earlier, SM1Get shortcuts sideways searches in leaves (and even in branches +// in some cases), so SM2Backtrack only handles branches. + +#define HISTNUMMAX cJU_ROOTSTATE // maximum branches traversable. + Pjp_t APjphist[HISTNUMMAX]; // list of branch JPs traversed. + int Aoffhist[HISTNUMMAX]; // list of next JP offsets; see above. + int histnum = 0; // number of JPs now in list. + + +// ---------------------------------------------------------------------------- +// M A C R O S +// +// These are intended to make the code a bit more readable and less redundant. + + +// "PUSH" AND "POP" Pjp AND offset ON HISTORY STACKS: +// +// Note: Ensure a corrupt Judy array does not overflow *hist[]. Meanwhile, +// underflowing *hist[] simply means theres no more room to backtrack => +// "no previous/next Index". + +#define HISTPUSH(Pjp,Offset) \ + APjphist[histnum] = (Pjp); \ + Aoffhist[histnum] = (Offset); \ + \ + if (++histnum >= HISTNUMMAX) \ + { \ + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT) \ + JUDY1CODE(return(JERRI );) \ + JUDYLCODE(return(PPJERR);) \ + } + +#define HISTPOP(Pjp,Offset) \ + if ((histnum--) < 1) JU_RET_NOTFOUND; \ + (Pjp) = APjphist[histnum]; \ + (Offset) = Aoffhist[histnum] + +// How to pack/unpack Aoffhist[] values for bitmap branches: + +#ifdef JUDYPREV + +#define HISTPUSHBOFF(Subexp,Offset,Digit) \ + (((Subexp) * cJU_BITSPERSUBEXPB) | (Offset)) + +#define HISTPOPBOFF(Subexp,Offset,Digit) \ + (Subexp) = (Offset) / cJU_BITSPERSUBEXPB; \ + (Offset) %= cJU_BITSPERSUBEXPB +#else + +#define HISTPUSHBOFF(Subexp,Offset,Digit) \ + (((Digit) << cJU_BITSPERBYTE) \ + | ((Subexp) * cJU_BITSPERSUBEXPB) | (Offset)) + +#define HISTPOPBOFF(Subexp,Offset,Digit) \ + (Digit) = (Offset) >> cJU_BITSPERBYTE; \ + (Subexp) = ((Offset) & JU_LEASTBYTESMASK(1)) / cJU_BITSPERSUBEXPB; \ + (Offset) %= cJU_BITSPERSUBEXPB +#endif + + +// CHECK FOR NULL JP: + +#define JPNULL(Type) (((Type) >= cJU_JPNULL1) && ((Type) <= cJU_JPNULLMAX)) + + +// SEARCH A BITMAP: +// +// This is a weak analog of j__udySearchLeaf*() for bitmaps. Return the actual +// or next-left position, base 0, of Digit in the single uint32_t bitmap, also +// given a Bitposmask for Digit. +// +// Unlike j__udySearchLeaf*(), the offset is not returned bit-complemented if +// Digits bit is unset, because the caller can check the bitmap themselves to +// determine that. Also, if Digits bit is unset, the returned offset is to +// the next-left JP (including -1), not to the "ideal" position for the Index = +// next-right JP. +// +// Shortcut and skip calling j__udyCountBits*() if the bitmap is full, in which +// case (Digit % cJU_BITSPERSUBEXP*) itself is the base-0 offset. +// +// TBD for Judy*Next(): Should this return next-right instead of next-left? +// That is, +1 from current value? Maybe not, if Digits bit IS set, +1 would +// be wrong. + +#define SEARCHBITMAPB(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPB) ? (Digit % cJU_BITSPERSUBEXPB) : \ + j__udyCountBitsB((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#define SEARCHBITMAPL(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPL) ? (Digit % cJU_BITSPERSUBEXPL) : \ + j__udyCountBitsL((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#ifdef JUDYPREV +// Equivalent to search for the highest offset in Bitmap: + +#define SEARCHBITMAPMAXB(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPB) ? cJU_BITSPERSUBEXPB - 1 : \ + j__udyCountBitsB(Bitmap) - 1) + +#define SEARCHBITMAPMAXL(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPL) ? cJU_BITSPERSUBEXPL - 1 : \ + j__udyCountBitsL(Bitmap) - 1) +#endif + + +// CHECK DECODE BYTES: +// +// Check Decode bytes in a JP against the equivalent portion of *PIndex. If +// *PIndex is lower (for Judy*Prev()) or higher (for Judy*Next()), this JP is a +// dead end (the same as if it had been absent in a linear or bitmap branch or +// null in an uncompressed branch), enter SM2Backtrack; otherwise enter +// SM3Findlimit to find the highest/lowest Index under this JP, as if the code +// had already backtracked to this JP. + +#ifdef JUDYPREV +#define CDcmp__ < +#else +#define CDcmp__ > +#endif + +#define CHECKDCD(cState) \ + if (JU_DCDNOTMATCHINDEX(*PIndex, Pjp, cState)) \ + { \ + if ((*PIndex & cJU_DCDMASK(cState)) \ + CDcmp__(JU_JPDCDPOP0(Pjp) & cJU_DCDMASK(cState))) \ + { \ + goto SM2Backtrack; \ + } \ + goto SM3Findlimit; \ + } + + +// PREPARE TO HANDLE A LEAFW OR JRP BRANCH IN SM1: +// +// Extract a state-dependent digit from Index in a "constant" way, then jump to +// common code for multiple cases. + +#define SM1PREPB(cState,Next) \ + state = (cState); \ + digit = JU_DIGITATSTATE(*PIndex, cState); \ + goto Next + + +// PREPARE TO HANDLE A LEAFW OR JRP BRANCH IN SM3: +// +// Optionally save Dcd bytes into *PIndex, then save state and jump to common +// code for multiple cases. + +#define SM3PREPB_DCD(cState,Next) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + SM3PREPB(cState,Next) + +#define SM3PREPB(cState,Next) state = (cState); goto Next + + +// ---------------------------------------------------------------------------- +// CHECK FOR SHORTCUTS: +// +// Error out if PIndex is null. Execute JU_RET_NOTFOUND if the Judy array is +// empty or *PIndex is already the minimum/maximum Index possible. +// +// Note: As documented, in case of failure *PIndex may be modified. + + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + +#ifdef JUDYPREV + if ((PArray == (Pvoid_t) NULL) || ((*PIndex)-- == 0)) +#else + if ((PArray == (Pvoid_t) NULL) || ((*PIndex)++ == cJU_ALLONES)) +#endif + JU_RET_NOTFOUND; + + +// HANDLE JRP: +// +// Before even entering SM1Get, check the JRP type. For JRP branches, traverse +// the JPM; handle LEAFW leaves directly; but look for the most common cases +// first. + +// ROOT-STATE LEAF that starts with a Pop0 word; just look within the leaf: +// +// If *PIndex is in the leaf, return it; otherwise return the Index, if any, +// below where it would belong. + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + pop1 = Pjlw[0] + 1; + + if ((offset = j__udySearchLeafW(Pjlw + 1, pop1, *PIndex)) + >= 0) // Index is present. + { + assert(offset < pop1); // in expected range. + JU_RET_FOUND_LEAFW(Pjlw, pop1, offset); // *PIndex is set. + } + +#ifdef JUDYPREV + if ((offset = ~offset) == 0) // no next-left Index. +#else + if ((offset = ~offset) >= pop1) // no next-right Index. +#endif + JU_RET_NOTFOUND; + + assert(offset <= pop1); // valid result. + +#ifdef JUDYPREV + *PIndex = Pjlw[offset--]; // next-left Index, base 1. +#else + *PIndex = Pjlw[offset + 1]; // next-right Index, base 1. +#endif + JU_RET_FOUND_LEAFW(Pjlw, pop1, offset); // base 0. + + } + else // JRP BRANCH + { + Pjpm_t Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); + +// goto SM1Get; + } + +// ============================================================================ +// STATE MACHINE 1 -- GET INDEX: +// +// Search for *PIndex (already decremented/incremented so as to be inclusive). +// If found, return it. Otherwise in theory hand off to SM2Backtrack or +// SM3Findlimit, but in practice "shortcut" by first sideways searching the +// current branch or leaf upon hitting a dead end. During sideways search, +// modify *PIndex to a new path taken. +// +// ENTRY: Pjp points to next JP to interpret, whose Decode bytes have not yet +// been checked. This JP is not yet listed in history. +// +// Note: Check Decode bytes at the start of each loop, not after looking up a +// new JP, so its easy to do constant shifts/masks, although this requires +// cautious handling of Pjp, offset, and *hist[] for correct entry to +// SM2Backtrack. +// +// EXIT: Return, or branch to SM2Backtrack or SM3Findlimit with correct +// interface, as described elsewhere. +// +// WARNING: For run-time efficiency the following cases replicate code with +// varying constants, rather than using common code with variable values! + +SM1Get: // return here for next branch/leaf. + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_L2: CHECKDCD(2); SM1PREPB(2, SM1BranchL); + case cJU_JPBRANCH_L3: CHECKDCD(3); SM1PREPB(3, SM1BranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: CHECKDCD(4); SM1PREPB(4, SM1BranchL); + case cJU_JPBRANCH_L5: CHECKDCD(5); SM1PREPB(5, SM1BranchL); + case cJU_JPBRANCH_L6: CHECKDCD(6); SM1PREPB(6, SM1BranchL); + case cJU_JPBRANCH_L7: CHECKDCD(7); SM1PREPB(7, SM1BranchL); +#endif + case cJU_JPBRANCH_L: SM1PREPB(cJU_ROOTSTATE, SM1BranchL); + +// Common code (state-independent) for all cases of linear branches: + +SM1BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +// Found JP matching current digit in *PIndex; record parent JP and the next +// JPs offset, and iterate to the next JP: + + if ((offset = j__udySearchLeaf1((Pjll_t) (Pjbl->jbl_Expanse), + Pjbl->jbl_NumJPs, digit)) >= 0) + { + HISTPUSH(Pjp, offset); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM1Get; + } + +// Dead end, no JP in BranchL for next digit in *PIndex: +// +// Get the ideal location of digits JP, and if theres no next-left/right JP +// in the BranchL, shortcut and start backtracking one level up; ignore the +// current Pjp because it points to a BranchL with no next-left/right JP. + +#ifdef JUDYPREV + if ((offset = (~offset) - 1) < 0) // no next-left JP in BranchL. +#else + if ((offset = (~offset)) >= Pjbl->jbl_NumJPs) // no next-right. +#endif + goto SM2Backtrack; + +// Theres a next-left/right JP in the current BranchL; save its digit in +// *PIndex and shortcut to SM3Findlimit: + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Check Decode bytes, if any, in the current JP, then look for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_B2: CHECKDCD(2); SM1PREPB(2, SM1BranchB); + case cJU_JPBRANCH_B3: CHECKDCD(3); SM1PREPB(3, SM1BranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: CHECKDCD(4); SM1PREPB(4, SM1BranchB); + case cJU_JPBRANCH_B5: CHECKDCD(5); SM1PREPB(5, SM1BranchB); + case cJU_JPBRANCH_B6: CHECKDCD(6); SM1PREPB(6, SM1BranchB); + case cJU_JPBRANCH_B7: CHECKDCD(7); SM1PREPB(7, SM1BranchB); +#endif + case cJU_JPBRANCH_B: SM1PREPB(cJU_ROOTSTATE, SM1BranchB); + +// Common code (state-independent) for all cases of bitmap branches: + +SM1BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + +// Locate the digits JP in the subexpanse list, if present, otherwise the +// offset of the next-left JP, if any: + + subexp = digit / cJU_BITSPERSUBEXPB; + assert(subexp < cJU_NUMSUBEXPB); // falls in expected range. + bitposmask = JU_BITPOSMASKB(digit); + offset = SEARCHBITMAPB(JU_JBB_BITMAP(Pjbb, subexp), digit, + bitposmask); + // right range: + assert((offset >= -1) && (offset < (int) cJU_BITSPERSUBEXPB)); + +// Found JP matching current digit in *PIndex: +// +// Record the parent JP and the next JPs offset; and iterate to the next JP. + +// if (JU_BITMAPTESTB(Pjbb, digit)) // slower. + if (JU_JBB_BITMAP(Pjbb, subexp) & bitposmask) // faster. + { + // not negative since at least one bit is set: + assert(offset >= 0); + + HISTPUSH(Pjp, HISTPUSHBOFF(subexp, offset, digit)); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM1Get; // iterate to next JP. + } + +// Dead end, no JP in BranchB for next digit in *PIndex: +// +// If theres a next-left/right JP in the current BranchB, shortcut to +// SM3Findlimit. Note: offset is already set to the correct value for the +// next-left/right JP. + +#ifdef JUDYPREV + if (offset >= 0) // next-left JP is in this subexpanse. + goto SM1BranchBFindlimit; + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JBB_BITMAP(Pjbb, subexp) & JU_MASKHIGHEREXC(bitposmask)) + { + ++offset; // next-left => next-right. + goto SM1BranchBFindlimit; + } + + while (++subexp < cJU_NUMSUBEXPB) // search next-right subexps. +#endif + { + if (! JU_JBB_PJP(Pjbb, subexp)) continue; // empty subexpanse. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + offset = 0; +#endif + +// Save the next-left/right JPs digit in *PIndex: + +SM1BranchBFindlimit: + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), + offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchB: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a BranchB with no next-left/right JP. + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Check Decode bytes, if any, in the current JP, then look for a JP for the +// next digit in *PIndex. + + case cJU_JPBRANCH_U2: CHECKDCD(2); SM1PREPB(2, SM1BranchU); + case cJU_JPBRANCH_U3: CHECKDCD(3); SM1PREPB(3, SM1BranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: CHECKDCD(4); SM1PREPB(4, SM1BranchU); + case cJU_JPBRANCH_U5: CHECKDCD(5); SM1PREPB(5, SM1BranchU); + case cJU_JPBRANCH_U6: CHECKDCD(6); SM1PREPB(6, SM1BranchU); + case cJU_JPBRANCH_U7: CHECKDCD(7); SM1PREPB(7, SM1BranchU); +#endif + case cJU_JPBRANCH_U: SM1PREPB(cJU_ROOTSTATE, SM1BranchU); + +// Common code (state-independent) for all cases of uncompressed branches: + +SM1BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + Pjp2 = (Pjbu->jbu_jp) + digit; + +// Found JP matching current digit in *PIndex: +// +// Record the parent JP and the next JPs digit, and iterate to the next JP. +// +// TBD: Instead of this, just goto SM1Get, and add cJU_JPNULL* cases to the +// SM1Get state machine? Then backtrack? However, it means you cant detect +// an inappropriate cJU_JPNULL*, when it occurs in other than a BranchU, and +// return JU_RET_CORRUPT. + + if (! JPNULL(JU_JPTYPE(Pjp2))) // digit has a JP. + { + HISTPUSH(Pjp, digit); + Pjp = Pjp2; + goto SM1Get; + } + +// Dead end, no JP in BranchU for next digit in *PIndex: +// +// Search for a next-left/right JP in the current BranchU, and if one is found, +// save its digit in *PIndex and shortcut to SM3Findlimit: + +#ifdef JUDYPREV + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + while (digit < cJU_BRANCHUNUMJPS - 1) + { + Pjp = (Pjbu->jbu_jp) + (++digit); +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchU: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a BranchU with no next-left/right JP. + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for +// *PIndex. + +#define SM1LEAFL(Func) \ + Pjll = P_JLL(Pjp->jp_Addr); \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + offset = Func(Pjll, pop1, *PIndex); \ + goto SM1LeafLImm + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKDCD(1); SM1LEAFL(j__udySearchLeaf1); +#endif + case cJU_JPLEAF2: CHECKDCD(2); SM1LEAFL(j__udySearchLeaf2); + case cJU_JPLEAF3: CHECKDCD(3); SM1LEAFL(j__udySearchLeaf3); + +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKDCD(4); SM1LEAFL(j__udySearchLeaf4); + case cJU_JPLEAF5: CHECKDCD(5); SM1LEAFL(j__udySearchLeaf5); + case cJU_JPLEAF6: CHECKDCD(6); SM1LEAFL(j__udySearchLeaf6); + case cJU_JPLEAF7: CHECKDCD(7); SM1LEAFL(j__udySearchLeaf7); +#endif + +// Common code (state-independent) for all cases of linear leaves and +// immediates: + +SM1LeafLImm: + if (offset >= 0) // *PIndex is in LeafL / Immed. +#ifdef JUDY1 + JU_RET_FOUND; +#else + { // JudyL is trickier... + switch (JU_JPTYPE(Pjp)) + { +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + case cJU_JPLEAF2: JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + case cJU_JPLEAF3: JU_RET_FOUND_LEAF3(Pjll, pop1, offset); +#ifdef JU_64BIT + case cJU_JPLEAF4: JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + case cJU_JPLEAF5: JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + case cJU_JPLEAF6: JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + case cJU_JPLEAF7: JU_RET_FOUND_LEAF7(Pjll, pop1, offset); +#endif + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + JU_RET_FOUND_IMM_01(Pjp); + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#ifdef JU_64BIT + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: + case cJU_JPIMMED_3_02: +#endif + JU_RET_FOUND_IMM(Pjp, offset); + } + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // impossible? + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // found *PIndex + +#endif // JUDYL + +// Dead end, no Index in LeafL / Immed for remaining digit(s) in *PIndex: +// +// Get the ideal location of Index, and if theres no next-left/right Index in +// the LeafL / Immed, shortcut and start backtracking one level up; ignore the +// current Pjp because it points to a LeafL / Immed with no next-left/right +// Index. + +#ifdef JUDYPREV + if ((offset = (~offset) - 1) < 0) // no next-left Index. +#else + if ((offset = (~offset)) >= pop1) // no next-right Index. +#endif + goto SM2Backtrack; + +// Theres a next-left/right Index in the current LeafL / Immed; shortcut by +// copying its digit(s) to *PIndex and returning it. +// +// Unfortunately this is pretty hairy, especially avoiding endian issues. +// +// The cJU_JPLEAF* cases are very similar to same-index-size cJU_JPIMMED* cases +// for *_02 and above, but must return differently, at least for JudyL, so +// spell them out separately here at the cost of a little redundant code for +// Judy1. + + switch (JU_JPTYPE(Pjp)) + { +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + JU_SETDIGIT1(*PIndex, ((uint8_t *) Pjll)[offset]); + JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + + case cJU_JPLEAF2: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + + case cJU_JPLEAF3: + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + +#ifdef JU_64BIT + case cJU_JPLEAF4: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + + case cJU_JPLEAF5: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + } + + case cJU_JPLEAF6: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + } + + case cJU_JPLEAF7: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_LEAF7(Pjll, pop1, offset); + } + +#endif // JU_64BIT + +#define SET_01(cState) JU_SETDIGITS(*PIndex, JU_JPDCDPOP0(Pjp), cState) + + case cJU_JPIMMED_1_01: SET_01(1); goto SM1Imm_01; + case cJU_JPIMMED_2_01: SET_01(2); goto SM1Imm_01; + case cJU_JPIMMED_3_01: SET_01(3); goto SM1Imm_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: SET_01(4); goto SM1Imm_01; + case cJU_JPIMMED_5_01: SET_01(5); goto SM1Imm_01; + case cJU_JPIMMED_6_01: SET_01(6); goto SM1Imm_01; + case cJU_JPIMMED_7_01: SET_01(7); goto SM1Imm_01; +#endif +SM1Imm_01: JU_RET_FOUND_IMM_01(Pjp); + +// Shorthand for where to find start of Index bytes array: + +#ifdef JUDY1 +#define PJI (Pjp->jp_1Index) +#else +#define PJI (Pjp->jp_LIndex) +#endif + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + JU_SETDIGIT1(*PIndex, ((uint8_t *) PJI)[offset]); + JU_RET_FOUND_IMM(Pjp, offset); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: + case cJ1_JPIMMED_4_03: + + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); + + case cJ1_JPIMMED_5_02: + case cJ1_JPIMMED_5_03: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_6_02: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_7_02: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + +#endif // (JUDY1 && JU_64BIT) + + } // switch for not-found *PIndex + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); // impossible? + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Check Decode bytes, if any, in the current JP, then look in the leaf for +// *PIndex. + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; + CHECKDCD(1); + + Pjlb = P_JLB(Pjp->jp_Addr); + digit = JU_DIGITATSTATE(*PIndex, 1); + subexp = JU_SUBEXPL(digit); + bitposmask = JU_BITPOSMASKL(digit); + assert(subexp < cJU_NUMSUBEXPL); // falls in expected range. + +// *PIndex exists in LeafB1: + +// if (JU_BITMAPTESTL(Pjlb, digit)) // slower. + if (JU_JLB_BITMAP(Pjlb, subexp) & bitposmask) // faster. + { +#ifdef JUDYL // needs offset at this point: + offset = SEARCHBITMAPL(JU_JLB_BITMAP(Pjlb, subexp), digit, bitposmask); +#endif + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + } + +// Dead end, no Index in LeafB1 for remaining digit in *PIndex: +// +// If theres a next-left/right Index in the current LeafB1, which for +// Judy*Next() is true if any bits are set for higher Indexes, shortcut by +// returning it. Note: For Judy*Prev(), offset is set here to the correct +// value for the next-left JP. + + offset = SEARCHBITMAPL(JU_JLB_BITMAP(Pjlb, subexp), digit, + bitposmask); + // right range: + assert((offset >= -1) && (offset < (int) cJU_BITSPERSUBEXPL)); + +#ifdef JUDYPREV + if (offset >= 0) // next-left JP is in this subexpanse. + goto SM1LeafB1Findlimit; + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JLB_BITMAP(Pjlb, subexp) & JU_MASKHIGHEREXC(bitposmask)) + { + ++offset; // next-left => next-right. + goto SM1LeafB1Findlimit; + } + + while (++subexp < cJU_NUMSUBEXPL) // search next-right subexps. +#endif + { + if (! JU_JLB_BITMAP(Pjlb, subexp)) continue; // empty subexp. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXL(JU_JLB_BITMAP(Pjlb, subexp)); + // expected range: + assert((offset >= 0) && (offset < (int) cJU_BITSPERSUBEXPL)); +#else + offset = 0; +#endif + +// Save the next-left/right Indexess digit in *PIndex: + +SM1LeafB1Findlimit: + JU_BITMAPDIGITL(digit, subexp, JU_JLB_BITMAP(Pjlb, subexp), offset); + JU_SETDIGIT1(*PIndex, digit); + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + } + +// Theres no next-left/right Index in the LeafB1: +// +// Shortcut and start backtracking one level up; ignore the current Pjp because +// it points to a LeafB1 with no next-left/right Index. + + goto SM2Backtrack; + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// If the Decode bytes match, *PIndex is found (without modification). + + case cJ1_JPFULLPOPU1: + + CHECKDCD(1); + JU_RET_FOUND_FULLPOPU1; +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: + +#ifdef JUDYPREV +#define SM1IMM_SETPOP1(cPop1) +#else +#define SM1IMM_SETPOP1(cPop1) pop1 = (cPop1) +#endif + +#define SM1IMM(Func,cPop1) \ + SM1IMM_SETPOP1(cPop1); \ + offset = Func((Pjll_t) (PJI), cPop1, *PIndex); \ + goto SM1LeafLImm + +// Special case for Pop1 = 1 Immediate JPs: +// +// If *PIndex is in the immediate, offset is 0, otherwise the binary NOT of the +// offset where it belongs, 0 or 1, same as from the search functions. + +#ifdef JUDYPREV +#define SM1IMM_01_SETPOP1 +#else +#define SM1IMM_01_SETPOP1 pop1 = 1 +#endif + +#define SM1IMM_01 \ + SM1IMM_01_SETPOP1; \ + offset = ((JU_JPDCDPOP0(Pjp) < JU_TRIMTODCDSIZE(*PIndex)) ? ~1 : \ + (JU_JPDCDPOP0(Pjp) == JU_TRIMTODCDSIZE(*PIndex)) ? 0 : \ + ~0); \ + goto SM1LeafLImm + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + SM1IMM_01; + +// TBD: Doug says it would be OK to have fewer calls and calculate arg 2, here +// and in Judy*Count() also. + + case cJU_JPIMMED_1_02: SM1IMM(j__udySearchLeaf1, 2); + case cJU_JPIMMED_1_03: SM1IMM(j__udySearchLeaf1, 3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: SM1IMM(j__udySearchLeaf1, 4); + case cJU_JPIMMED_1_05: SM1IMM(j__udySearchLeaf1, 5); + case cJU_JPIMMED_1_06: SM1IMM(j__udySearchLeaf1, 6); + case cJU_JPIMMED_1_07: SM1IMM(j__udySearchLeaf1, 7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: SM1IMM(j__udySearchLeaf1, 8); + case cJ1_JPIMMED_1_09: SM1IMM(j__udySearchLeaf1, 9); + case cJ1_JPIMMED_1_10: SM1IMM(j__udySearchLeaf1, 10); + case cJ1_JPIMMED_1_11: SM1IMM(j__udySearchLeaf1, 11); + case cJ1_JPIMMED_1_12: SM1IMM(j__udySearchLeaf1, 12); + case cJ1_JPIMMED_1_13: SM1IMM(j__udySearchLeaf1, 13); + case cJ1_JPIMMED_1_14: SM1IMM(j__udySearchLeaf1, 14); + case cJ1_JPIMMED_1_15: SM1IMM(j__udySearchLeaf1, 15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: SM1IMM(j__udySearchLeaf2, 2); + case cJU_JPIMMED_2_03: SM1IMM(j__udySearchLeaf2, 3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: SM1IMM(j__udySearchLeaf2, 4); + case cJ1_JPIMMED_2_05: SM1IMM(j__udySearchLeaf2, 5); + case cJ1_JPIMMED_2_06: SM1IMM(j__udySearchLeaf2, 6); + case cJ1_JPIMMED_2_07: SM1IMM(j__udySearchLeaf2, 7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: SM1IMM(j__udySearchLeaf3, 2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: SM1IMM(j__udySearchLeaf3, 3); + case cJ1_JPIMMED_3_04: SM1IMM(j__udySearchLeaf3, 4); + case cJ1_JPIMMED_3_05: SM1IMM(j__udySearchLeaf3, 5); + + case cJ1_JPIMMED_4_02: SM1IMM(j__udySearchLeaf4, 2); + case cJ1_JPIMMED_4_03: SM1IMM(j__udySearchLeaf4, 3); + + case cJ1_JPIMMED_5_02: SM1IMM(j__udySearchLeaf5, 2); + case cJ1_JPIMMED_5_03: SM1IMM(j__udySearchLeaf5, 3); + + case cJ1_JPIMMED_6_02: SM1IMM(j__udySearchLeaf6, 2); + + case cJ1_JPIMMED_7_02: SM1IMM(j__udySearchLeaf7, 2); +#endif + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM1Get switch. + + /*NOTREACHED*/ + + +// ============================================================================ +// STATE MACHINE 2 -- BACKTRACK BRANCH TO PREVIOUS JP: +// +// Look for the next-left/right JP in a branch, backing up the history list as +// necessary. Upon finding a next-left/right JP, modify the corresponding +// digit in *PIndex before passing control to SM3Findlimit. +// +// Note: As described earlier, only branch JPs are expected here; other types +// fall into the default case. +// +// Note: If a found JP contains needed Dcd bytes, thats OK, theyre copied to +// *PIndex in SM3Findlimit. +// +// TBD: This code has a lot in common with similar code in the shortcut cases +// in SM1Get. Can combine this code somehow? +// +// ENTRY: List, possibly empty, of JPs and offsets in APjphist[] and +// Aoffhist[]; see earlier comments. +// +// EXIT: Execute JU_RET_NOTFOUND if no previous/next JP; otherwise jump to +// SM3Findlimit to resume a new but different downward search. + +SM2Backtrack: // come or return here for first/next sideways search. + + HISTPOP(Pjp, offset); + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: + + case cJU_JPBRANCH_L2: state = 2; goto SM2BranchL; + case cJU_JPBRANCH_L3: state = 3; goto SM2BranchL; +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: state = 4; goto SM2BranchL; + case cJU_JPBRANCH_L5: state = 5; goto SM2BranchL; + case cJU_JPBRANCH_L6: state = 6; goto SM2BranchL; + case cJU_JPBRANCH_L7: state = 7; goto SM2BranchL; +#endif + case cJU_JPBRANCH_L: state = cJU_ROOTSTATE; goto SM2BranchL; + +SM2BranchL: +#ifdef JUDYPREV + if (--offset < 0) goto SM2Backtrack; // no next-left JP in BranchL. +#endif + Pjbl = P_JBL(Pjp->jp_Addr); +#ifdef JUDYNEXT + if (++offset >= (Pjbl->jbl_NumJPs)) goto SM2Backtrack; + // no next-right JP in BranchL. +#endif + +// Theres a next-left/right JP in the current BranchL; save its digit in +// *PIndex and continue with SM3Findlimit: + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: + + case cJU_JPBRANCH_B2: state = 2; goto SM2BranchB; + case cJU_JPBRANCH_B3: state = 3; goto SM2BranchB; +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: state = 4; goto SM2BranchB; + case cJU_JPBRANCH_B5: state = 5; goto SM2BranchB; + case cJU_JPBRANCH_B6: state = 6; goto SM2BranchB; + case cJU_JPBRANCH_B7: state = 7; goto SM2BranchB; +#endif + case cJU_JPBRANCH_B: state = cJU_ROOTSTATE; goto SM2BranchB; + +SM2BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + HISTPOPBOFF(subexp, offset, digit); // unpack values. + +// If theres a next-left/right JP in the current BranchB, which for +// Judy*Next() is true if any bits are set for higher Indexes, continue to +// SM3Findlimit: +// +// Note: offset is set to the JP previously traversed; go one to the +// left/right. + +#ifdef JUDYPREV + if (offset > 0) // next-left JP is in this subexpanse. + { + --offset; + goto SM2BranchBFindlimit; + } + + while (--subexp >= 0) // search next-left subexpanses. +#else + if (JU_JBB_BITMAP(Pjbb, subexp) + & JU_MASKHIGHEREXC(JU_BITPOSMASKB(digit))) + { + ++offset; // next-left => next-right. + goto SM2BranchBFindlimit; + } + + while (++subexp < cJU_NUMSUBEXPB) // search next-right subexps. +#endif + { + if (! JU_JBB_PJP(Pjbb, subexp)) continue; // empty subexpanse. + +#ifdef JUDYPREV + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + offset = 0; +#endif + +// Save the next-left/right JPs digit in *PIndex: + +SM2BranchBFindlimit: + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), + offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchB: + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: + + case cJU_JPBRANCH_U2: state = 2; goto SM2BranchU; + case cJU_JPBRANCH_U3: state = 3; goto SM2BranchU; +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: state = 4; goto SM2BranchU; + case cJU_JPBRANCH_U5: state = 5; goto SM2BranchU; + case cJU_JPBRANCH_U6: state = 6; goto SM2BranchU; + case cJU_JPBRANCH_U7: state = 7; goto SM2BranchU; +#endif + case cJU_JPBRANCH_U: state = cJU_ROOTSTATE; goto SM2BranchU; + +SM2BranchU: + +// Search for a next-left/right JP in the current BranchU, and if one is found, +// save its digit in *PIndex and continue to SM3Findlimit: + + Pjbu = P_JBU(Pjp->jp_Addr); + digit = offset; + +#ifdef JUDYPREV + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + while (digit < cJU_BRANCHUNUMJPS - 1) + { + Pjp = (Pjbu->jbu_jp) + (++digit); +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// Theres no next-left/right JP in the BranchU: + + goto SM2Backtrack; + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM2Backtrack switch. + + /*NOTREACHED*/ + + +// ============================================================================ +// STATE MACHINE 3 -- FIND LIMIT JP/INDEX: +// +// Look for the highest/lowest (right/left-most) JP in each branch and the +// highest/lowest Index in a leaf or immediate, and return it. While +// traversing, modify appropriate digit(s) in *PIndex to reflect the path +// taken, including Dcd bytes in each JP (which could hold critical missing +// digits for skipped branches). +// +// ENTRY: Pjp set to a JP under which to find max/min JPs (if a branch JP) or +// a max/min Index and return (if a leaf or immediate JP). +// +// EXIT: Execute JU_RET_FOUND* upon reaching a leaf or immediate. Should be +// impossible to fail, unless the Judy array is corrupt. + +SM3Findlimit: // come or return here for first/next branch/leaf. + + switch (JU_JPTYPE(Pjp)) + { +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Simply use the highest/lowest (right/left-most) JP in the BranchL, but first +// copy the Dcd bytes to *PIndex if there are any (only if state < +// cJU_ROOTSTATE - 1). + + case cJU_JPBRANCH_L2: SM3PREPB_DCD(2, SM3BranchL); +#ifndef JU_64BIT + case cJU_JPBRANCH_L3: SM3PREPB( 3, SM3BranchL); +#else + case cJU_JPBRANCH_L3: SM3PREPB_DCD(3, SM3BranchL); + case cJU_JPBRANCH_L4: SM3PREPB_DCD(4, SM3BranchL); + case cJU_JPBRANCH_L5: SM3PREPB_DCD(5, SM3BranchL); + case cJU_JPBRANCH_L6: SM3PREPB_DCD(6, SM3BranchL); + case cJU_JPBRANCH_L7: SM3PREPB( 7, SM3BranchL); +#endif + case cJU_JPBRANCH_L: SM3PREPB( cJU_ROOTSTATE, SM3BranchL); + +SM3BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +#ifdef JUDYPREV + if ((offset = (Pjbl->jbl_NumJPs) - 1) < 0) +#else + offset = 0; if ((Pjbl->jbl_NumJPs) == 0) +#endif + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + JU_SETDIGIT(*PIndex, Pjbl->jbl_Expanse[offset], state); + Pjp = (Pjbl->jbl_jp) + offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Look for the highest/lowest (right/left-most) non-null subexpanse, then use +// the highest/lowest JP in that subexpanse, but first copy Dcd bytes, if there +// are any (only if state < cJU_ROOTSTATE - 1), to *PIndex. + + case cJU_JPBRANCH_B2: SM3PREPB_DCD(2, SM3BranchB); +#ifndef JU_64BIT + case cJU_JPBRANCH_B3: SM3PREPB( 3, SM3BranchB); +#else + case cJU_JPBRANCH_B3: SM3PREPB_DCD(3, SM3BranchB); + case cJU_JPBRANCH_B4: SM3PREPB_DCD(4, SM3BranchB); + case cJU_JPBRANCH_B5: SM3PREPB_DCD(5, SM3BranchB); + case cJU_JPBRANCH_B6: SM3PREPB_DCD(6, SM3BranchB); + case cJU_JPBRANCH_B7: SM3PREPB( 7, SM3BranchB); +#endif + case cJU_JPBRANCH_B: SM3PREPB( cJU_ROOTSTATE, SM3BranchB); + +SM3BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); +#ifdef JUDYPREV + subexp = cJU_NUMSUBEXPB; + + while (! (JU_JBB_BITMAP(Pjbb, --subexp))) // find non-empty subexp. + { + if (subexp <= 0) // wholly empty bitmap. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPB)); +#else + subexp = -1; + + while (! (JU_JBB_BITMAP(Pjbb, ++subexp))) // find non-empty subexp. + { + if (subexp >= cJU_NUMSUBEXPB - 1) // didnt find one. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = 0; +#endif + + JU_BITMAPDIGITB(digit, subexp, JU_JBB_BITMAP(Pjbb, subexp), offset); + JU_SETDIGIT(*PIndex, digit, state); + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + + Pjp += offset; + goto SM3Findlimit; + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Look for the highest/lowest (right/left-most) non-null JP, and use it, but +// first copy Dcd bytes to *PIndex if there are any (only if state < +// cJU_ROOTSTATE - 1). + + case cJU_JPBRANCH_U2: SM3PREPB_DCD(2, SM3BranchU); +#ifndef JU_64BIT + case cJU_JPBRANCH_U3: SM3PREPB( 3, SM3BranchU); +#else + case cJU_JPBRANCH_U3: SM3PREPB_DCD(3, SM3BranchU); + case cJU_JPBRANCH_U4: SM3PREPB_DCD(4, SM3BranchU); + case cJU_JPBRANCH_U5: SM3PREPB_DCD(5, SM3BranchU); + case cJU_JPBRANCH_U6: SM3PREPB_DCD(6, SM3BranchU); + case cJU_JPBRANCH_U7: SM3PREPB( 7, SM3BranchU); +#endif + case cJU_JPBRANCH_U: SM3PREPB( cJU_ROOTSTATE, SM3BranchU); + +SM3BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); +#ifdef JUDYPREV + digit = cJU_BRANCHUNUMJPS; + + while (digit >= 1) + { + Pjp = (Pjbu->jbu_jp) + (--digit); +#else + + for (digit = 0; digit < cJU_BRANCHUNUMJPS; ++digit) + { + Pjp = (Pjbu->jbu_jp) + digit; +#endif + if (JPNULL(JU_JPTYPE(Pjp))) continue; + + JU_SETDIGIT(*PIndex, digit, state); + goto SM3Findlimit; + } + +// No non-null JPs in BranchU: + + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Simply use the highest/lowest (right/left-most) Index in the LeafL, but the +// details vary depending on leaf Index Size. First copy Dcd bytes, if there +// are any (only if state < cJU_ROOTSTATE - 1), to *PIndex. + +#define SM3LEAFLDCD(cState) \ + JU_SETDCD(*PIndex, Pjp, cState); \ + SM3LEAFLNODCD + +#ifdef JUDY1 +#define SM3LEAFL_SETPOP1 // not needed in any cases. +#else +#define SM3LEAFL_SETPOP1 pop1 = JU_JPLEAF_POP0(Pjp) + 1 +#endif + +#ifdef JUDYPREV +#define SM3LEAFLNODCD \ + Pjll = P_JLL(Pjp->jp_Addr); \ + SM3LEAFL_SETPOP1; \ + offset = JU_JPLEAF_POP0(Pjp); assert(offset >= 0) +#else +#define SM3LEAFLNODCD \ + Pjll = P_JLL(Pjp->jp_Addr); \ + SM3LEAFL_SETPOP1; \ + offset = 0; assert(JU_JPLEAF_POP0(Pjp) >= 0); +#endif + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: + + SM3LEAFLDCD(1); + JU_SETDIGIT1(*PIndex, ((uint8_t *) Pjll)[offset]); + JU_RET_FOUND_LEAF1(Pjll, pop1, offset); +#endif + + case cJU_JPLEAF2: + + SM3LEAFLDCD(2); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF2(Pjll, pop1, offset); + +#ifndef JU_64BIT + case cJU_JPLEAF3: + { + Word_t lsb; + SM3LEAFLNODCD; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + +#else + case cJU_JPLEAF3: + { + Word_t lsb; + SM3LEAFLDCD(3); + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_LEAF3(Pjll, pop1, offset); + } + + case cJU_JPLEAF4: + + SM3LEAFLDCD(4); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) Pjll)[offset]; + JU_RET_FOUND_LEAF4(Pjll, pop1, offset); + + case cJU_JPLEAF5: + { + Word_t lsb; + SM3LEAFLDCD(5); + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_LEAF5(Pjll, pop1, offset); + } + + case cJU_JPLEAF6: + { + Word_t lsb; + SM3LEAFLDCD(6); + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_LEAF6(Pjll, pop1, offset); + } + + case cJU_JPLEAF7: + { + Word_t lsb; + SM3LEAFLNODCD; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) Pjll) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_LEAF7(Pjll, pop1, offset); + } +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Look for the highest/lowest (right/left-most) non-null subexpanse, then use +// the highest/lowest Index in that subexpanse, but first copy Dcd bytes +// (always present since state 1 < cJU_ROOTSTATE) to *PIndex. + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; + + JU_SETDCD(*PIndex, Pjp, 1); + + Pjlb = P_JLB(Pjp->jp_Addr); +#ifdef JUDYPREV + subexp = cJU_NUMSUBEXPL; + + while (! JU_JLB_BITMAP(Pjlb, --subexp)) // find non-empty subexp. + { + if (subexp <= 0) // wholly empty bitmap. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + +// TBD: Might it be faster to just use a variant of BITMAPDIGIT*() that yields +// the digit for the right-most Index with a bit set? + + offset = SEARCHBITMAPMAXL(JU_JLB_BITMAP(Pjlb, subexp)); + // expected range: + assert((offset >= 0) && (offset < cJU_BITSPERSUBEXPL)); +#else + subexp = -1; + + while (! JU_JLB_BITMAP(Pjlb, ++subexp)) // find non-empty subexp. + { + if (subexp >= cJU_NUMSUBEXPL - 1) // didnt find one. + { + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + } + } + + offset = 0; +#endif + + JU_BITMAPDIGITL(digit, subexp, JU_JLB_BITMAP(Pjlb, subexp), offset); + JU_SETDIGIT1(*PIndex, digit); + JU_RET_FOUND_LEAF_B1(Pjlb, subexp, offset); +// == return((PPvoid_t) (P_JV(JL_JLB_PVALUE(Pjlb, subexp)) + (offset))); + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// Copy Dcd bytes to *PIndex (always present since state 1 < cJU_ROOTSTATE), +// then set the highest/lowest possible digit as the LSB in *PIndex. + + case cJ1_JPFULLPOPU1: + + JU_SETDCD( *PIndex, Pjp, 1); +#ifdef JUDYPREV + JU_SETDIGIT1(*PIndex, cJU_BITSPERBITMAP - 1); +#else + JU_SETDIGIT1(*PIndex, 0); +#endif + JU_RET_FOUND_FULLPOPU1; +#endif // JUDY1 + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: +// +// Simply use the highest/lowest (right/left-most) Index in the Imm, but the +// details vary depending on leaf Index Size and pop1. Note: There are no Dcd +// bytes in an Immediate JP, but in a cJU_JPIMMED_*_01 JP, the field holds the +// least bytes of the immediate Index. + + case cJU_JPIMMED_1_01: SET_01(1); goto SM3Imm_01; + case cJU_JPIMMED_2_01: SET_01(2); goto SM3Imm_01; + case cJU_JPIMMED_3_01: SET_01(3); goto SM3Imm_01; +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: SET_01(4); goto SM3Imm_01; + case cJU_JPIMMED_5_01: SET_01(5); goto SM3Imm_01; + case cJU_JPIMMED_6_01: SET_01(6); goto SM3Imm_01; + case cJU_JPIMMED_7_01: SET_01(7); goto SM3Imm_01; +#endif +SM3Imm_01: JU_RET_FOUND_IMM_01(Pjp); + +#ifdef JUDYPREV +#define SM3IMM_OFFSET(cPop1) (cPop1) - 1 // highest. +#else +#define SM3IMM_OFFSET(cPop1) 0 // lowest. +#endif + +#define SM3IMM(cPop1,Next) \ + offset = SM3IMM_OFFSET(cPop1); \ + goto Next + + case cJU_JPIMMED_1_02: SM3IMM( 2, SM3Imm1); + case cJU_JPIMMED_1_03: SM3IMM( 3, SM3Imm1); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: SM3IMM( 4, SM3Imm1); + case cJU_JPIMMED_1_05: SM3IMM( 5, SM3Imm1); + case cJU_JPIMMED_1_06: SM3IMM( 6, SM3Imm1); + case cJU_JPIMMED_1_07: SM3IMM( 7, SM3Imm1); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: SM3IMM( 8, SM3Imm1); + case cJ1_JPIMMED_1_09: SM3IMM( 9, SM3Imm1); + case cJ1_JPIMMED_1_10: SM3IMM(10, SM3Imm1); + case cJ1_JPIMMED_1_11: SM3IMM(11, SM3Imm1); + case cJ1_JPIMMED_1_12: SM3IMM(12, SM3Imm1); + case cJ1_JPIMMED_1_13: SM3IMM(13, SM3Imm1); + case cJ1_JPIMMED_1_14: SM3IMM(14, SM3Imm1); + case cJ1_JPIMMED_1_15: SM3IMM(15, SM3Imm1); +#endif + +SM3Imm1: JU_SETDIGIT1(*PIndex, ((uint8_t *) PJI)[offset]); + JU_RET_FOUND_IMM(Pjp, offset); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: SM3IMM(2, SM3Imm2); + case cJU_JPIMMED_2_03: SM3IMM(3, SM3Imm2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: SM3IMM(4, SM3Imm2); + case cJ1_JPIMMED_2_05: SM3IMM(5, SM3Imm2); + case cJ1_JPIMMED_2_06: SM3IMM(6, SM3Imm2); + case cJ1_JPIMMED_2_07: SM3IMM(7, SM3Imm2); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +SM3Imm2: *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(2))) + | ((uint16_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: SM3IMM(2, SM3Imm3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: SM3IMM(3, SM3Imm3); + case cJ1_JPIMMED_3_04: SM3IMM(4, SM3Imm3); + case cJ1_JPIMMED_3_05: SM3IMM(5, SM3Imm3); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) +SM3Imm3: + { + Word_t lsb; + JU_COPY3_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (3 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(3))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: SM3IMM(2, SM3Imm4); + case cJ1_JPIMMED_4_03: SM3IMM(3, SM3Imm4); + +SM3Imm4: *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(4))) + | ((uint32_t *) PJI)[offset]; + JU_RET_FOUND_IMM(Pjp, offset); + + case cJ1_JPIMMED_5_02: SM3IMM(2, SM3Imm5); + case cJ1_JPIMMED_5_03: SM3IMM(3, SM3Imm5); + +SM3Imm5: + { + Word_t lsb; + JU_COPY5_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (5 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(5))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_6_02: SM3IMM(2, SM3Imm6); + +SM3Imm6: + { + Word_t lsb; + JU_COPY6_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (6 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(6))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } + + case cJ1_JPIMMED_7_02: SM3IMM(2, SM3Imm7); + +SM3Imm7: + { + Word_t lsb; + JU_COPY7_PINDEX_TO_LONG(lsb, ((uint8_t *) PJI) + (7 * offset)); + *PIndex = (*PIndex & (~JU_LEASTBYTESMASK(7))) | lsb; + JU_RET_FOUND_IMM(Pjp, offset); + } +#endif // (JUDY1 && JU_64BIT) + + +// ---------------------------------------------------------------------------- +// OTHER CASES: + + default: JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // SM3Findlimit switch. + + /*NOTREACHED*/ + +} // Judy1Prev() / Judy1Next() / JudyLPrev() / JudyLNext() diff --git a/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c b/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c new file mode 100644 index 000000000..4da43565d --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c @@ -0,0 +1,1390 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.32 $ $Source: /judy/src/JudyCommon/JudyPrevNextEmpty.c $ +// +// Judy*PrevEmpty() and Judy*NextEmpty() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. +// +// Compile with -DJUDYNEXT for the Judy*NextEmpty() function; otherwise +// defaults to Judy*PrevEmpty(). +// +// Compile with -DTRACEJPSE to trace JP traversals. +// +// This file is separate from JudyPrevNext.c because it differs too greatly for +// ifdefs. This might be a bit surprising, but there are two reasons: +// +// - First, down in the details, searching for an empty index (SearchEmpty) is +// remarkably asymmetric with searching for a valid index (SearchValid), +// mainly with respect to: No return of a value area for JudyL; partially- +// full versus totally-full JPs; and handling of narrow pointers. +// +// - Second, we chose to implement SearchEmpty without a backtrack stack or +// backtrack engine, partly as an experiment, and partly because we think +// restarting from the top of the tree is less likely for SearchEmpty than +// for SearchValid, because empty indexes are more likely than valid indexes. +// +// A word about naming: A prior version of this feature (see 4.13) was named +// Judy*Free(), but there were concerns about that being read as a verb rather +// than an adjective. After prolonged debate and based on user input, we +// changed "Free" to "Empty". + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifndef JUDYNEXT +#ifndef JUDYPREV +#define JUDYPREV 1 // neither set => use default. +#endif +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +#ifdef TRACEJPSE +#include "JudyPrintJP.c" +#endif + + +// **************************************************************************** +// J U D Y 1 P R E V E M P T Y +// J U D Y 1 N E X T E M P T Y +// J U D Y L P R E V E M P T Y +// J U D Y L N E X T E M P T Y +// +// See the manual entry for the API. +// +// OVERVIEW OF Judy*PrevEmpty() / Judy*NextEmpty(): +// +// See also for comparison the equivalent comments in JudyPrevNext.c. +// +// Take the callers *PIndex and subtract/add 1, but watch out for +// underflow/overflow, which means "no previous/next empty index found." Use a +// reentrant switch statement (state machine, see SMGetRestart and +// SMGetContinue) to decode Index, starting with the JRP (PArray), through a +// JPM and branches, if any, down to an immediate or a leaf. Look for Index in +// that immediate or leaf, and if not found (invalid index), return success +// (Index is empty). +// +// This search can result in a dead end where taking a different path is +// required. There are four kinds of dead ends: +// +// BRANCH PRIMARY dead end: Encountering a fully-populated JP for the +// appropriate digit in Index. Search sideways in the branch for the +// previous/next absent/null/non-full JP, and if one is found, set Index to the +// highest/lowest index possible in that JPs expanse. Then if the JP is an +// absent or null JP, return success; otherwise for a non-full JP, traverse +// through the partially populated JP. +// +// BRANCH SECONDARY dead end: Reaching the end of a branch during a sideways +// search after a branch primary dead end. Set Index to the lowest/highest +// index possible in the whole branchs expanse (one higher/lower than the +// previous/next branchs expanse), then restart at the top of the tree, which +// includes pre-decrementing/incrementing Index (again) and watching for +// underflow/overflow (again). +// +// LEAF PRIMARY dead end: Finding a valid (non-empty) index in an immediate or +// leaf matching Index. Search sideways in the immediate/leaf for the +// previous/next empty index; if found, set *PIndex to match and return success. +// +// LEAF SECONDARY dead end: Reaching the end of an immediate or leaf during a +// sideways search after a leaf primary dead end. Just as for a branch +// secondary dead end, restart at the top of the tree with Index set to the +// lowest/highest index possible in the whole immediate/leafs expanse. +// TBD: If leaf secondary dead end occurs, could shortcut and treat it as a +// branch primary dead end; but this would require remembering the parent +// branchs type and offset (a "one-deep stack"), and also wrestling with +// narrow pointers, at least for leaves (but not for immediates). +// +// Note some ASYMMETRIES between SearchValid and SearchEmpty: +// +// - The SearchValid code, upon descending through a narrow pointer, if Index +// is outside the expanse of the subsidiary node (effectively a secondary +// dead end), must decide whether to backtrack or findlimit. But the +// SearchEmpty code simply returns success (Index is empty). +// +// - Similarly, the SearchValid code, upon finding no previous/next index in +// the expanse of a narrow pointer (again, a secondary dead end), can simply +// start to backtrack at the parent JP. But the SearchEmpty code would have +// to first determine whether or not the parent JPs narrow expanse contains +// a previous/next empty index outside the subexpanse. Rather than keeping a +// parent state stack and backtracking this way, upon a secondary dead end, +// the SearchEmpty code simply restarts at the top of the tree, whether or +// not a narrow pointer is involved. Again, see the equivalent comments in +// JudyPrevNext.c for comparison. +// +// This function is written iteratively for speed, rather than recursively. +// +// TBD: Wed like to enhance this function to make successive searches faster. +// This would require saving some previous state, including the previous Index +// returned, and in which leaf it was found. If the next call is for the same +// Index and the array has not been modified, start at the same leaf. This +// should be much easier to implement since this is iterative rather than +// recursive code. + +#ifdef JUDY1 +#ifdef JUDYPREV +FUNCTION int Judy1PrevEmpty +#else +FUNCTION int Judy1NextEmpty +#endif +#else +#ifdef JUDYPREV +FUNCTION int JudyLPrevEmpty +#else +FUNCTION int JudyLNextEmpty +#endif +#endif + ( + Pcvoid_t PArray, // Judy array to search. + Word_t * PIndex, // starting point and result. + PJError_t PJError // optional, for returning error info. + ) +{ + Word_t Index; // fast copy, in a register. + Pjp_t Pjp; // current JP. + Pjbl_t Pjbl; // Pjp->jp_Addr masked and cast to types: + Pjbb_t Pjbb; + Pjbu_t Pjbu; + Pjlb_t Pjlb; + PWord_t Pword; // alternate name for use by GET* macros. + + Word_t digit; // next digit to decode from Index. + Word_t digits; // current state in SM = digits left to decode. + Word_t pop0; // in a leaf. + Word_t pop0mask; // precalculated to avoid variable shifts. + long offset; // within a branch or leaf (can be large). + int subexp; // subexpanse in a bitmap branch. + BITMAPB_t bitposmaskB; // bit in bitmap for bitmap branch. + BITMAPL_t bitposmaskL; // bit in bitmap for bitmap leaf. + Word_t possfullJP1; // JP types for possibly full subexpanses: + Word_t possfullJP2; + Word_t possfullJP3; + + +// ---------------------------------------------------------------------------- +// M A C R O S +// +// These are intended to make the code a bit more readable and less redundant. + + +// CHECK FOR NULL JP: +// +// TBD: In principle this can be reduced (here and in other *.c files) to just +// the latter clause since no Type should ever be below cJU_JPNULL1, but in +// fact some root pointer types can be lower, so for safety do both checks. + +#define JPNULL(Type) (((Type) >= cJU_JPNULL1) && ((Type) <= cJU_JPNULLMAX)) + + +// CHECK FOR A FULL JP: +// +// Given a JP, indicate if it is fully populated. Use digits, pop0mask, and +// possfullJP1..3 in the context. +// +// This is a difficult problem because it requires checking the Pop0 bits for +// all-ones, but the number of bytes depends on the JP type, which is not +// directly related to the parent branchs type or level -- the JPs child +// could be under a narrow pointer (hence not full). The simple answer +// requires switching on or otherwise calculating the JP type, which could be +// slow. Instead, in SMPREPB* precalculate pop0mask and also record in +// possfullJP1..3 the child JP (branch) types that could possibly be full (one +// level down), and use them here. For level-2 branches (with digits == 2), +// the test for a full child depends on Judy1/JudyL. +// +// Note: This cannot be applied to the JP in a JPM because it doesnt have +// enough pop0 digits. +// +// TBD: JPFULL_BRANCH diligently checks for BranchL or BranchB, where neither +// of those can ever be full as it turns out. Could just check for a BranchU +// at the right level. Also, pop0mask might be overkill, its not used much, +// so perhaps just call cJU_POP0MASK(digits - 1) here? +// +// First, JPFULL_BRANCH checks for a full expanse for a JP whose child can be a +// branch, that is, a JP in a branch at level 3 or higher: + +#define JPFULL_BRANCH(Pjp) \ + ((((JU_JPDCDPOP0(Pjp) ^ cJU_ALLONES) & pop0mask) == 0) \ + && ((JU_JPTYPE(Pjp) == possfullJP1) \ + || (JU_JPTYPE(Pjp) == possfullJP2) \ + || (JU_JPTYPE(Pjp) == possfullJP3))) + +#ifdef JUDY1 +#define JPFULL(Pjp) \ + ((digits == 2) ? \ + (JU_JPTYPE(Pjp) == cJ1_JPFULLPOPU1) : JPFULL_BRANCH(Pjp)) +#else +#define JPFULL(Pjp) \ + ((digits == 2) ? \ + (JU_JPTYPE(Pjp) == cJU_JPLEAF_B1) \ + && (((JU_JPDCDPOP0(Pjp) & cJU_POP0MASK(1)) == cJU_POP0MASK(1))) : \ + JPFULL_BRANCH(Pjp)) +#endif + + +// RETURN SUCCESS: +// +// This hides the need to set *PIndex back to the local value of Index -- use a +// local value for faster operation. Note that the callers *PIndex is ALWAYS +// modified upon success, at least decremented/incremented. + +#define RET_SUCCESS { *PIndex = Index; return(1); } + + +// RETURN A CORRUPTION: + +#define RET_CORRUPT { JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); return(JERRI); } + + +// SEARCH A BITMAP BRANCH: +// +// This is a weak analog of j__udySearchLeaf*() for bitmap branches. Return +// the actual or next-left position, base 0, of Digit in a BITMAPB_t bitmap +// (subexpanse of a full bitmap), also given a Bitposmask for Digit. The +// position is the offset within the set bits. +// +// Unlike j__udySearchLeaf*(), the offset is not returned bit-complemented if +// Digits bit is unset, because the caller can check the bitmap themselves to +// determine that. Also, if Digits bit is unset, the returned offset is to +// the next-left JP or index (including -1), not to the "ideal" position for +// the index = next-right JP or index. +// +// Shortcut and skip calling j__udyCountBitsB() if the bitmap is full, in which +// case (Digit % cJU_BITSPERSUBEXPB) itself is the base-0 offset. + +#define SEARCHBITMAPB(Bitmap,Digit,Bitposmask) \ + (((Bitmap) == cJU_FULLBITMAPB) ? (Digit % cJU_BITSPERSUBEXPB) : \ + j__udyCountBitsB((Bitmap) & JU_MASKLOWERINC(Bitposmask)) - 1) + +#ifdef JUDYPREV +// Equivalent to search for the highest offset in Bitmap, that is, one less +// than the number of bits set: + +#define SEARCHBITMAPMAXB(Bitmap) \ + (((Bitmap) == cJU_FULLBITMAPB) ? cJU_BITSPERSUBEXPB - 1 : \ + j__udyCountBitsB(Bitmap) - 1) +#endif + + +// CHECK DECODE BYTES: +// +// Check Decode bytes in a JP against the equivalent portion of Index. If they +// dont match, Index is outside the subexpanse of a narrow pointer, hence is +// empty. + +#define CHECKDCD(cDigits) \ + if (JU_DCDNOTMATCHINDEX(Index, Pjp, cDigits)) RET_SUCCESS + + +// REVISE REMAINDER OF INDEX: +// +// Put one digit in place in Index and clear/set the lower digits, if any, so +// the resulting Index is at the start/end of an expanse, or just clear/set the +// least digits. +// +// Actually, to make simple use of JU_LEASTBYTESMASK, first clear/set all least +// digits of Index including the digit to be overridden, then set the value of +// that one digit. If Digits == 1 the first operation is redundant, but either +// very fast or even removed by the optimizer. + +#define CLEARLEASTDIGITS(Digits) Index &= ~JU_LEASTBYTESMASK(Digits) +#define SETLEASTDIGITS( Digits) Index |= JU_LEASTBYTESMASK(Digits) + +#define CLEARLEASTDIGITS_D(Digit,Digits) \ + { \ + CLEARLEASTDIGITS(Digits); \ + JU_SETDIGIT(Index, Digit, Digits); \ + } + +#define SETLEASTDIGITS_D(Digit,Digits) \ + { \ + SETLEASTDIGITS(Digits); \ + JU_SETDIGIT(Index, Digit, Digits); \ + } + + +// SET REMAINDER OF INDEX AND THEN RETURN OR CONTINUE: + +#define SET_AND_RETURN(OpLeastDigits,Digit,Digits) \ + { \ + OpLeastDigits(Digit, Digits); \ + RET_SUCCESS; \ + } + +#define SET_AND_CONTINUE(OpLeastDigits,Digit,Digits) \ + { \ + OpLeastDigits(Digit, Digits); \ + goto SMGetContinue; \ + } + + +// PREPARE TO HANDLE A LEAFW OR JP BRANCH IN THE STATE MACHINE: +// +// Extract a state-dependent digit from Index in a "constant" way, then jump to +// common code for multiple cases. +// +// TBD: Should this macro do more, such as preparing variable-shift masks for +// use in CLEARLEASTDIGITS and SETLEASTDIGITS? + +#define SMPREPB(cDigits,Next,PossFullJP1,PossFullJP2,PossFullJP3) \ + digits = (cDigits); \ + digit = JU_DIGITATSTATE(Index, cDigits); \ + pop0mask = cJU_POP0MASK((cDigits) - 1); /* for branchs JPs */ \ + possfullJP1 = (PossFullJP1); \ + possfullJP2 = (PossFullJP2); \ + possfullJP3 = (PossFullJP3); \ + goto Next + +// Variations for specific-level branches and for shorthands: +// +// Note: SMPREPB2 need not initialize possfullJP* because JPFULL does not use +// them for digits == 2, but gcc -Wall isnt quite smart enough to see this, so +// waste a bit of time and space to get rid of the warning: + +#define SMPREPB2(Next) \ + digits = 2; \ + digit = JU_DIGITATSTATE(Index, 2); \ + pop0mask = cJU_POP0MASK(1); /* for branchs JPs */ \ + possfullJP1 = possfullJP2 = possfullJP3 = 0; \ + goto Next + +#define SMPREPB3(Next) SMPREPB(3, Next, cJU_JPBRANCH_L2, \ + cJU_JPBRANCH_B2, \ + cJU_JPBRANCH_U2) +#ifndef JU_64BIT +#define SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L3, \ + cJU_JPBRANCH_B3, \ + cJU_JPBRANCH_U3) +#else +#define SMPREPB4(Next) SMPREPB(4, Next, cJU_JPBRANCH_L3, \ + cJU_JPBRANCH_B3, \ + cJU_JPBRANCH_U3) +#define SMPREPB5(Next) SMPREPB(5, Next, cJU_JPBRANCH_L4, \ + cJU_JPBRANCH_B4, \ + cJU_JPBRANCH_U4) +#define SMPREPB6(Next) SMPREPB(6, Next, cJU_JPBRANCH_L5, \ + cJU_JPBRANCH_B5, \ + cJU_JPBRANCH_U5) +#define SMPREPB7(Next) SMPREPB(7, Next, cJU_JPBRANCH_L6, \ + cJU_JPBRANCH_B6, \ + cJU_JPBRANCH_U6) +#define SMPREPBL(Next) SMPREPB(cJU_ROOTSTATE, Next, cJU_JPBRANCH_L7, \ + cJU_JPBRANCH_B7, \ + cJU_JPBRANCH_U7) +#endif + + +// RESTART AFTER SECONDARY DEAD END: +// +// Set Index to the first/last index in the branch or leaf subexpanse and start +// over at the top of the tree. + +#ifdef JUDYPREV +#define SMRESTART(Digits) { CLEARLEASTDIGITS(Digits); goto SMGetRestart; } +#else +#define SMRESTART(Digits) { SETLEASTDIGITS( Digits); goto SMGetRestart; } +#endif + + +// CHECK EDGE OF LEAFS EXPANSE: +// +// Given the LSBs of the lowest/highest valid index in a leaf (or equivalently +// in an immediate JP), the level (index size) of the leaf, and the full index +// to return (as Index in the context) already set to the full index matching +// the lowest/highest one, determine if there is an empty index in the leafs +// expanse below/above the lowest/highest index, which is true if the +// lowest/highest index is not at the "edge" of the leafs expanse based on its +// LSBs. If so, return Index decremented/incremented; otherwise restart at the +// top of the tree. +// +// Note: In many cases Index is already at the right spot and calling +// SMRESTART instead of just going directly to SMGetRestart is a bit of +// overkill. +// +// Note: Variable shift occurs if Digits is not a constant. + +#ifdef JUDYPREV +#define LEAF_EDGE(MinIndex,Digits) \ + { \ + if (MinIndex) { --Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#else +#define LEAF_EDGE(MaxIndex,Digits) \ + { \ + if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits)) \ + { ++Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#endif + +// Same as above except Index is not already set to match the lowest/highest +// index, so do that before decrementing/incrementing it: + +#ifdef JUDYPREV +#define LEAF_EDGE_SET(MinIndex,Digits) \ + { \ + if (MinIndex) \ + { JU_SETDIGITS(Index, MinIndex, Digits); --Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#else +#define LEAF_EDGE_SET(MaxIndex,Digits) \ + { \ + if ((MaxIndex) != JU_LEASTBYTES(cJU_ALLONES, Digits)) \ + { JU_SETDIGITS(Index, MaxIndex, Digits); ++Index; RET_SUCCESS; } \ + SMRESTART(Digits); \ + } +#endif + + +// FIND A HOLE (EMPTY INDEX) IN AN IMMEDIATE OR LEAF: +// +// Given an index location in a leaf (or equivalently an immediate JP) known to +// contain a usable hole (an empty index less/greater than Index), and the LSBs +// of a minimum/maximum index to locate, find the previous/next empty index and +// return it. +// +// Note: "Even" index sizes (1,2,4[,8] bytes) have corresponding native C +// types; "odd" index sizes dont, but they are not represented here because +// they are handled completely differently; see elsewhere. + +#ifdef JUDYPREV + +#define LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB) \ + { \ + while (*(Pjll) > (IndexLSB)) --(Pjll); /* too high */ \ + if (*(Pjll) < (IndexLSB)) RET_SUCCESS /* Index is empty */ \ + while (*(--(Pjll)) == --(IndexLSB)) /* null, find a hole */;\ + JU_SETDIGITS(Index, IndexLSB, cDigits); \ + RET_SUCCESS; \ + } +#else +#define LEAF_HOLE_EVEN(cDigits,Pjll,IndexLSB) \ + { \ + while (*(Pjll) < (IndexLSB)) ++(Pjll); /* too low */ \ + if (*(Pjll) > (IndexLSB)) RET_SUCCESS /* Index is empty */ \ + while (*(++(Pjll)) == ++(IndexLSB)) /* null, find a hole */;\ + JU_SETDIGITS(Index, IndexLSB, cDigits); \ + RET_SUCCESS; \ + } +#endif + + +// SEARCH FOR AN EMPTY INDEX IN AN IMMEDIATE OR LEAF: +// +// Given a pointer to the first index in a leaf (or equivalently an immediate +// JP), the population of the leaf, and a first empty Index to find (inclusive, +// as Index in the context), where Index is known to fall within the expanse of +// the leaf to search, efficiently find the previous/next empty index in the +// leaf, if any. For simplicity the following overview is stated in terms of +// Judy*NextEmpty() only, but the same concepts apply symmetrically for +// Judy*PrevEmpty(). Also, in each case the comparisons are for the LSBs of +// Index and leaf indexes, according to the leafs level. +// +// 1. If Index is GREATER than the last (highest) index in the leaf +// (maxindex), return success, Index is empty. (Remember, Index is known +// to be in the leafs expanse.) +// +// 2. If Index is EQUAL to maxindex: If maxindex is not at the edge of the +// leafs expanse, increment Index and return success, there is an empty +// Index one higher than any in the leaf; otherwise restart with Index +// reset to the upper edge of the leafs expanse. Note: This might cause +// an extra cache line fill, but this is OK for repeatedly-called search +// code, and it saves CPU time. +// +// 3. If Index is LESS than maxindex, check for "dense to end of leaf": +// Subtract Index from maxindex, and back up that many slots in the leaf. +// If the resulting offset is not before the start of the leaf then compare +// the index at this offset (baseindex) with Index: +// +// 3a. If GREATER, the leaf must be corrupt, since indexes are sorted and +// there are no duplicates. +// +// 3b. If EQUAL, the leaf is "dense" from Index to maxindex, meaning there is +// no reason to search it. "Slide right" to the high end of the leaf +// (modify Index to maxindex) and continue with step 2 above. +// +// 3c. If LESS, continue with step 4. +// +// 4. If the offset based on maxindex minus Index falls BEFORE the start of +// the leaf, or if, per 3c above, baseindex is LESS than Index, the leaf is +// guaranteed "not dense to the end" and a usable empty Index must exist. +// This supports a more efficient search loop. Start at the FIRST index in +// the leaf, or one BEYOND baseindex, respectively, and search the leaf as +// follows, comparing each current index (currindex) with Index: +// +// 4a. If LESS, keep going to next index. Note: This is certain to terminate +// because maxindex is known to be greater than Index, hence the loop can +// be small and fast. +// +// 4b. If EQUAL, loop and increment Index until finding currindex greater than +// Index, and return success with the modified Index. +// +// 4c. If GREATER, return success, Index (unmodified) is empty. +// +// Note: These are macros rather than functions for speed. + +#ifdef JUDYPREV + +#define JSLE_EVEN(Addr,Pop0,cDigits,LeafType) \ + { \ + LeafType * PjllLSB = (LeafType *) (Addr); \ + LeafType IndexLSB = Index; /* auto-masking */ \ + \ + /* Index before or at start of leaf: */ \ + \ + if (*PjllLSB >= IndexLSB) /* no need to search */ \ + { \ + if (*PjllLSB > IndexLSB) RET_SUCCESS; /* Index empty */ \ + LEAF_EDGE(*PjllLSB, cDigits); \ + } \ + \ + /* Index in or after leaf: */ \ + \ + offset = IndexLSB - *PjllLSB; /* tentative offset */ \ + if (offset <= (Pop0)) /* can check density */ \ + { \ + PjllLSB += offset; /* move to slot */ \ + \ + if (*PjllLSB <= IndexLSB) /* dense or corrupt */ \ + { \ + if (*PjllLSB == IndexLSB) /* dense, check edge */ \ + LEAF_EDGE_SET(PjllLSB[-offset], cDigits); \ + RET_CORRUPT; \ + } \ + --PjllLSB; /* not dense, start at previous */ \ + } \ + else PjllLSB = ((LeafType *) (Addr)) + (Pop0); /* start at max */ \ + \ + LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB); \ + } + +// JSLE_ODD is completely different from JSLE_EVEN because its important to +// minimize copying odd indexes to compare them (see 4.14). Furthermore, a +// very complex version (4.17, but abandoned before fully debugged) that +// avoided calling j__udySearchLeaf*() ran twice as fast as 4.14, but still +// half as fast as SearchValid. Doug suggested that to minimize complexity and +// share common code we should use j__udySearchLeaf*() for the initial search +// to establish if Index is empty, which should be common. If Index is valid +// in a leaf or immediate indexes, odds are good that an empty Index is nearby, +// so for simplicity just use a *COPY* function to linearly search the +// remainder. +// +// TBD: Pathological case? Average performance should be good, but worst-case +// might suffer. When Search says the initial Index is valid, so a linear +// copy-and-compare is begun, if the caller builds fairly large leaves with +// dense clusters AND frequently does a SearchEmpty at one end of such a +// cluster, performance wont be very good. Might a dense-check help? This +// means checking offset against the index at offset, and then against the +// first/last index in the leaf. We doubt the pathological case will appear +// much in real applications because they will probably alternate SearchValid +// and SearchEmpty calls. + +#define JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy) \ + { \ + Word_t IndexLSB; /* least bytes only */ \ + Word_t IndexFound; /* in leaf */ \ + \ + if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0) \ + RET_SUCCESS; /* Index is empty */ \ + \ + IndexLSB = JU_LEASTBYTES(Index, cDigits); \ + offset *= (cDigits); \ + \ + while ((offset -= (cDigits)) >= 0) \ + { /* skip until empty or start */ \ + Copy(IndexFound, ((uint8_t *) (Pjll)) + offset); \ + if (IndexFound != (--IndexLSB)) /* found an empty */ \ + { JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; }\ + } \ + LEAF_EDGE_SET(IndexLSB, cDigits); \ + } + +#else // JUDYNEXT + +#define JSLE_EVEN(Addr,Pop0,cDigits,LeafType) \ + { \ + LeafType * PjllLSB = ((LeafType *) (Addr)) + (Pop0); \ + LeafType IndexLSB = Index; /* auto-masking */ \ + \ + /* Index at or after end of leaf: */ \ + \ + if (*PjllLSB <= IndexLSB) /* no need to search */ \ + { \ + if (*PjllLSB < IndexLSB) RET_SUCCESS; /* Index empty */\ + LEAF_EDGE(*PjllLSB, cDigits); \ + } \ + \ + /* Index before or in leaf: */ \ + \ + offset = *PjllLSB - IndexLSB; /* tentative offset */ \ + if (offset <= (Pop0)) /* can check density */ \ + { \ + PjllLSB -= offset; /* move to slot */ \ + \ + if (*PjllLSB >= IndexLSB) /* dense or corrupt */ \ + { \ + if (*PjllLSB == IndexLSB) /* dense, check edge */ \ + LEAF_EDGE_SET(PjllLSB[offset], cDigits); \ + RET_CORRUPT; \ + } \ + ++PjllLSB; /* not dense, start at next */ \ + } \ + else PjllLSB = (LeafType *) (Addr); /* start at minimum */ \ + \ + LEAF_HOLE_EVEN(cDigits, PjllLSB, IndexLSB); \ + } + +#define JSLE_ODD(cDigits,Pjll,Pop0,Search,Copy) \ + { \ + Word_t IndexLSB; /* least bytes only */ \ + Word_t IndexFound; /* in leaf */ \ + int offsetmax; /* in bytes */ \ + \ + if ((offset = Search(Pjll, (Pop0) + 1, Index)) < 0) \ + RET_SUCCESS; /* Index is empty */ \ + \ + IndexLSB = JU_LEASTBYTES(Index, cDigits); \ + offset *= (cDigits); \ + offsetmax = (Pop0) * (cDigits); /* single multiply */ \ + \ + while ((offset += (cDigits)) <= offsetmax) \ + { /* skip until empty or end */ \ + Copy(IndexFound, ((uint8_t *) (Pjll)) + offset); \ + if (IndexFound != (++IndexLSB)) /* found an empty */ \ + { JU_SETDIGITS(Index, IndexLSB, cDigits); RET_SUCCESS; } \ + } \ + LEAF_EDGE_SET(IndexLSB, cDigits); \ + } + +#endif // JUDYNEXT + +// Note: Immediate indexes never fill a single index group, so for odd index +// sizes, save time by calling JSLE_ODD_IMM instead of JSLE_ODD. + +#define j__udySearchLeafEmpty1(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 1, uint8_t) + +#define j__udySearchLeafEmpty2(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 2, uint16_t) + +#define j__udySearchLeafEmpty3(Addr,Pop0) \ + JSLE_ODD(3, Addr, Pop0, j__udySearchLeaf3, JU_COPY3_PINDEX_TO_LONG) + +#ifndef JU_64BIT + +#define j__udySearchLeafEmptyL(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 4, Word_t) + +#else + +#define j__udySearchLeafEmpty4(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 4, uint32_t) + +#define j__udySearchLeafEmpty5(Addr,Pop0) \ + JSLE_ODD(5, Addr, Pop0, j__udySearchLeaf5, JU_COPY5_PINDEX_TO_LONG) + +#define j__udySearchLeafEmpty6(Addr,Pop0) \ + JSLE_ODD(6, Addr, Pop0, j__udySearchLeaf6, JU_COPY6_PINDEX_TO_LONG) + +#define j__udySearchLeafEmpty7(Addr,Pop0) \ + JSLE_ODD(7, Addr, Pop0, j__udySearchLeaf7, JU_COPY7_PINDEX_TO_LONG) + +#define j__udySearchLeafEmptyL(Addr,Pop0) \ + JSLE_EVEN(Addr, Pop0, 8, Word_t) + +#endif // JU_64BIT + + +// ---------------------------------------------------------------------------- +// START OF CODE: +// +// CHECK FOR SHORTCUTS: +// +// Error out if PIndex is null. + + if (PIndex == (PWord_t) NULL) + { + JU_SET_ERRNO(PJError, JU_ERRNO_NULLPINDEX); + return(JERRI); + } + + Index = *PIndex; // fast local copy. + +// Set and pre-decrement/increment Index, watching for underflow/overflow: +// +// An out-of-bounds Index means failure: No previous/next empty index. + +SMGetRestart: // return here with revised Index. + +#ifdef JUDYPREV + if (Index-- == 0) return(0); +#else + if (++Index == 0) return(0); +#endif + +// An empty array with an in-bounds (not underflowed/overflowed) Index means +// success: +// +// Note: This check is redundant after restarting at SMGetRestart, but should +// take insignificant time. + + if (PArray == (Pvoid_t) NULL) RET_SUCCESS; + +// ---------------------------------------------------------------------------- +// ROOT-LEVEL LEAF that starts with a Pop0 word; just look within the leaf: +// +// If Index is not in the leaf, return success; otherwise return the first +// empty Index, if any, below/above where it would belong. + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + pop0 = Pjlw[0]; + +#ifdef JUDY1 + if (pop0 == 0) // special case. + { +#ifdef JUDYPREV + if ((Index != Pjlw[1]) || (Index-- != 0)) RET_SUCCESS; +#else + if ((Index != Pjlw[1]) || (++Index != 0)) RET_SUCCESS; +#endif + return(0); // no previous/next empty index. + } +#endif // JUDY1 + + j__udySearchLeafEmptyL(Pjlw + 1, pop0); + +// No return -- thanks ALAN + + } + else + +// ---------------------------------------------------------------------------- +// HANDLE JRP Branch: +// +// For JRP branches, traverse the JPM; handle LEAFW +// directly; but look for the most common cases first. + + { + Pjpm_t Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); + +// goto SMGetContinue; + } + + +// ============================================================================ +// STATE MACHINE -- GET INDEX: +// +// Search for Index (already decremented/incremented so as to be an inclusive +// search). If not found (empty index), return success. Otherwise do a +// previous/next search, and if successful modify Index to the empty index +// found. See function header comments. +// +// ENTRY: Pjp points to next JP to interpret, whose Decode bytes have not yet +// been checked. +// +// Note: Check Decode bytes at the start of each loop, not after looking up a +// new JP, so its easy to do constant shifts/masks. +// +// EXIT: Return, or branch to SMGetRestart with modified Index, or branch to +// SMGetContinue with a modified Pjp, as described elsewhere. +// +// WARNING: For run-time efficiency the following cases replicate code with +// varying constants, rather than using common code with variable values! + +SMGetContinue: // return here for next branch/leaf. + +#ifdef TRACEJPSE + JudyPrintJP(Pjp, "sf", __LINE__); +#endif + + switch (JU_JPTYPE(Pjp)) + { + + +// ---------------------------------------------------------------------------- +// LINEAR BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_L2: CHECKDCD(2); SMPREPB2(SMBranchL); + case cJU_JPBRANCH_L3: CHECKDCD(3); SMPREPB3(SMBranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: CHECKDCD(4); SMPREPB4(SMBranchL); + case cJU_JPBRANCH_L5: CHECKDCD(5); SMPREPB5(SMBranchL); + case cJU_JPBRANCH_L6: CHECKDCD(6); SMPREPB6(SMBranchL); + case cJU_JPBRANCH_L7: CHECKDCD(7); SMPREPB7(SMBranchL); +#endif + case cJU_JPBRANCH_L: SMPREPBL(SMBranchL); + +// Common code (state-independent) for all cases of linear branches: + +SMBranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + +// First, check if Indexs expanse (digit) is below/above the first/last +// populated expanse in the BranchL, in which case Index is empty; otherwise +// find the offset of the lowest/highest populated expanse at or above/below +// digit, if any: +// +// Note: The for-loop is guaranteed to exit eventually because the first/last +// expanse is known to be a terminator. +// +// Note: Cannot use j__udySearchLeaf*Empty1() here because it only applies to +// leaves and does not know about partial versus full JPs, unlike the use of +// j__udySearchLeaf1() for BranchLs in SearchValid code. Also, since linear +// leaf expanse lists are small, dont waste time calling j__udySearchLeaf1(), +// just scan the expanse list. + +#ifdef JUDYPREV + if ((Pjbl->jbl_Expanse[0]) > digit) RET_SUCCESS; + + for (offset = (Pjbl->jbl_NumJPs) - 1; /* null */; --offset) +#else + if ((Pjbl->jbl_Expanse[(Pjbl->jbl_NumJPs) - 1]) < digit) + RET_SUCCESS; + + for (offset = 0; /* null */; ++offset) +#endif + { + +// Too low/high, keep going; or too high/low, meaning the loop passed a hole +// and the initial Index is empty: + +#ifdef JUDYPREV + if ((Pjbl->jbl_Expanse[offset]) > digit) continue; + if ((Pjbl->jbl_Expanse[offset]) < digit) RET_SUCCESS; +#else + if ((Pjbl->jbl_Expanse[offset]) < digit) continue; + if ((Pjbl->jbl_Expanse[offset]) > digit) RET_SUCCESS; +#endif + +// Found expanse matching digit; if its not full, traverse through it: + + if (! JPFULL((Pjbl->jbl_jp) + offset)) + { + Pjp = (Pjbl->jbl_jp) + offset; + goto SMGetContinue; + } + +// Common code: While searching for a lower/higher hole or a non-full JP, upon +// finding a lower/higher hole, adjust Index using the revised digit and +// return; or upon finding a consecutive lower/higher expanse, if the expanses +// JP is non-full, modify Index and traverse through the JP: + +#define BRANCHL_CHECK(OpIncDec,OpLeastDigits,Digit,Digits) \ + { \ + if ((Pjbl->jbl_Expanse[offset]) != OpIncDec digit) \ + SET_AND_RETURN(OpLeastDigits, Digit, Digits); \ + \ + if (! JPFULL((Pjbl->jbl_jp) + offset)) \ + { \ + Pjp = (Pjbl->jbl_jp) + offset; \ + SET_AND_CONTINUE(OpLeastDigits, Digit, Digits); \ + } \ + } + +// BranchL primary dead end: Expanse matching Index/digit is full (rare except +// for dense/sequential indexes): +// +// Search for a lower/higher hole, a non-full JP, or the end of the expanse +// list, while decrementing/incrementing digit. + +#ifdef JUDYPREV + while (--offset >= 0) + BRANCHL_CHECK(--, SETLEASTDIGITS_D, digit, digits) +#else + while (++offset < Pjbl->jbl_NumJPs) + BRANCHL_CHECK(++, CLEARLEASTDIGITS_D, digit, digits) +#endif + +// Passed end of BranchL expanse list after finding a matching but full +// expanse: +// +// Digit now matches the lowest/highest expanse, which is a full expanse; if +// digit is at the end of BranchLs expanse (no hole before/after), break out +// of the loop; otherwise modify Index to the next lower/higher digit and +// return success: + +#ifdef JUDYPREV + if (digit == 0) break; + --digit; SET_AND_RETURN(SETLEASTDIGITS_D, digit, digits); +#else + if (digit == JU_LEASTBYTES(cJU_ALLONES, 1)) break; + ++digit; SET_AND_RETURN(CLEARLEASTDIGITS_D, digit, digits); +#endif + } // for-loop + +// BranchL secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// BITMAP BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_B2: CHECKDCD(2); SMPREPB2(SMBranchB); + case cJU_JPBRANCH_B3: CHECKDCD(3); SMPREPB3(SMBranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: CHECKDCD(4); SMPREPB4(SMBranchB); + case cJU_JPBRANCH_B5: CHECKDCD(5); SMPREPB5(SMBranchB); + case cJU_JPBRANCH_B6: CHECKDCD(6); SMPREPB6(SMBranchB); + case cJU_JPBRANCH_B7: CHECKDCD(7); SMPREPB7(SMBranchB); +#endif + case cJU_JPBRANCH_B: SMPREPBL(SMBranchB); + +// Common code (state-independent) for all cases of bitmap branches: + +SMBranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + +// Locate the digits JP in the subexpanse list, if present: + + subexp = digit / cJU_BITSPERSUBEXPB; + assert(subexp < cJU_NUMSUBEXPB); // falls in expected range. + bitposmaskB = JU_BITPOSMASKB(digit); + +// Absent JP = no JP matches current digit in Index: + +// if (! JU_BITMAPTESTB(Pjbb, digit)) // slower. + if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB)) // faster. + RET_SUCCESS; + +// Non-full JP matches current digit in Index: +// +// Iterate to the subsidiary non-full JP. + + offset = SEARCHBITMAPB(JU_JBB_BITMAP(Pjbb, subexp), digit, + bitposmaskB); + // not negative since at least one bit is set: + assert(offset >= 0); + assert(offset < (int) cJU_BITSPERSUBEXPB); + +// Watch for null JP subarray pointer with non-null bitmap (a corruption): + + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) + == (Pjp_t) NULL) RET_CORRUPT; + + Pjp += offset; + if (! JPFULL(Pjp)) goto SMGetContinue; + +// BranchB primary dead end: +// +// Upon hitting a full JP in a BranchB for the next digit in Index, search +// sideways for a previous/next absent JP (unset bit) or non-full JP (set bit +// with non-full JP); first in the current bitmap subexpanse, then in +// lower/higher subexpanses. Upon entry, Pjp points to a known-unusable JP, +// ready to decrement/increment. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. +// +// TBD: For speed, shift bitposmaskB instead of using JU_BITMAPTESTB or +// JU_BITPOSMASKB, but this shift has knowledge of bit order that really should +// be encapsulated in a header file. + +#define BRANCHB_CHECKBIT(OpLeastDigits) \ + if (! (JU_JBB_BITMAP(Pjbb, subexp) & bitposmaskB)) /* absent JP */ \ + SET_AND_RETURN(OpLeastDigits, digit, digits) + +#define BRANCHB_CHECKJPFULL(OpLeastDigits) \ + if (! JPFULL(Pjp)) \ + SET_AND_CONTINUE(OpLeastDigits, digit, digits) + +#define BRANCHB_STARTSUBEXP(OpLeastDigits) \ + if (! JU_JBB_BITMAP(Pjbb, subexp)) /* empty subexpanse, shortcut */ \ + SET_AND_RETURN(OpLeastDigits, digit, digits) \ + if ((Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp))) == (Pjp_t) NULL) RET_CORRUPT + +#ifdef JUDYPREV + + --digit; // skip initial digit. + bitposmaskB >>= 1; // see TBD above. + +BranchBNextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskB) // more bits to check in subexp. + { + BRANCHB_CHECKBIT(SETLEASTDIGITS_D); + --Pjp; // previous in subarray. + BRANCHB_CHECKJPFULL(SETLEASTDIGITS_D); + assert(digit >= 0); + --digit; + bitposmaskB >>= 1; + } + + if (subexp-- > 0) // more subexpanses. + { + BRANCHB_STARTSUBEXP(SETLEASTDIGITS_D); + Pjp += SEARCHBITMAPMAXB(JU_JBB_BITMAP(Pjbb, subexp)) + 1; + bitposmaskB = (1U << (cJU_BITSPERSUBEXPB - 1)); + goto BranchBNextSubexp; + } + +#else // JUDYNEXT + + ++digit; // skip initial digit. + bitposmaskB <<= 1; // note: BITMAPB_t. + +BranchBNextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskB) // more bits to check in subexp. + { + BRANCHB_CHECKBIT(CLEARLEASTDIGITS_D); + ++Pjp; // previous in subarray. + BRANCHB_CHECKJPFULL(CLEARLEASTDIGITS_D); + assert(digit < cJU_SUBEXPPERSTATE); + ++digit; + bitposmaskB <<= 1; // note: BITMAPB_t. + } + + if (++subexp < cJU_NUMSUBEXPB) // more subexpanses. + { + BRANCHB_STARTSUBEXP(CLEARLEASTDIGITS_D); + --Pjp; // pre-decrement. + bitposmaskB = 1; + goto BranchBNextSubexp; + } + +#endif // JUDYNEXT + +// BranchB secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// UNCOMPRESSED BRANCH: +// +// Check Decode bytes, if any, in the current JP, then search for a JP for the +// next digit in Index. + + case cJU_JPBRANCH_U2: CHECKDCD(2); SMPREPB2(SMBranchU); + case cJU_JPBRANCH_U3: CHECKDCD(3); SMPREPB3(SMBranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: CHECKDCD(4); SMPREPB4(SMBranchU); + case cJU_JPBRANCH_U5: CHECKDCD(5); SMPREPB5(SMBranchU); + case cJU_JPBRANCH_U6: CHECKDCD(6); SMPREPB6(SMBranchU); + case cJU_JPBRANCH_U7: CHECKDCD(7); SMPREPB7(SMBranchU); +#endif + case cJU_JPBRANCH_U: SMPREPBL(SMBranchU); + +// Common code (state-independent) for all cases of uncompressed branches: + +SMBranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + Pjp = (Pjbu->jbu_jp) + digit; + +// Absent JP = null JP for current digit in Index: + + if (JPNULL(JU_JPTYPE(Pjp))) RET_SUCCESS; + +// Non-full JP matches current digit in Index: +// +// Iterate to the subsidiary JP. + + if (! JPFULL(Pjp)) goto SMGetContinue; + +// BranchU primary dead end: +// +// Upon hitting a full JP in a BranchU for the next digit in Index, search +// sideways for a previous/next null or non-full JP. BRANCHU_CHECKJP() is +// shorthand for common code. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. + +#define BRANCHU_CHECKJP(OpIncDec,OpLeastDigits) \ + { \ + OpIncDec Pjp; \ + \ + if (JPNULL(JU_JPTYPE(Pjp))) \ + SET_AND_RETURN(OpLeastDigits, digit, digits) \ + \ + if (! JPFULL(Pjp)) \ + SET_AND_CONTINUE(OpLeastDigits, digit, digits) \ + } + +#ifdef JUDYPREV + while (digit-- > 0) + BRANCHU_CHECKJP(--, SETLEASTDIGITS_D); +#else + while (++digit < cJU_BRANCHUNUMJPS) + BRANCHU_CHECKJP(++, CLEARLEASTDIGITS_D); +#endif + +// BranchU secondary dead end, no non-full previous/next JP: + + SMRESTART(digits); + + +// ---------------------------------------------------------------------------- +// LINEAR LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for the +// previous/next empty index starting at Index. Primary leaf dead end is +// hidden within j__udySearchLeaf*Empty*(). In case of secondary leaf dead +// end, restart at the top of the tree. +// +// Note: Pword is the name known to GET*; think of it as Pjlw. + +#define SMLEAFL(cDigits,Func) \ + Pword = (PWord_t) P_JLW(Pjp->jp_Addr); \ + pop0 = JU_JPLEAF_POP0(Pjp); \ + Func(Pword, pop0) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKDCD(1); SMLEAFL(1, j__udySearchLeafEmpty1); +#endif + case cJU_JPLEAF2: CHECKDCD(2); SMLEAFL(2, j__udySearchLeafEmpty2); + case cJU_JPLEAF3: CHECKDCD(3); SMLEAFL(3, j__udySearchLeafEmpty3); + +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKDCD(4); SMLEAFL(4, j__udySearchLeafEmpty4); + case cJU_JPLEAF5: CHECKDCD(5); SMLEAFL(5, j__udySearchLeafEmpty5); + case cJU_JPLEAF6: CHECKDCD(6); SMLEAFL(6, j__udySearchLeafEmpty6); + case cJU_JPLEAF7: CHECKDCD(7); SMLEAFL(7, j__udySearchLeafEmpty7); +#endif + + +// ---------------------------------------------------------------------------- +// BITMAP LEAF: +// +// Check Decode bytes, if any, in the current JP, then search the leaf for the +// previous/next empty index starting at Index. + + case cJU_JPLEAF_B1: + + CHECKDCD(1); + + Pjlb = P_JLB(Pjp->jp_Addr); + digit = JU_DIGITATSTATE(Index, 1); + subexp = digit / cJU_BITSPERSUBEXPL; + bitposmaskL = JU_BITPOSMASKL(digit); + assert(subexp < cJU_NUMSUBEXPL); // falls in expected range. + +// Absent index = no index matches current digit in Index: + +// if (! JU_BITMAPTESTL(Pjlb, digit)) // slower. + if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL)) // faster. + RET_SUCCESS; + +// LeafB1 primary dead end: +// +// Upon hitting a valid (non-empty) index in a LeafB1 for the last digit in +// Index, search sideways for a previous/next absent index, first in the +// current bitmap subexpanse, then in lower/higher subexpanses. +// LEAFB1_CHECKBIT() is shorthand for common code to handle one bit in one +// bitmap subexpanse. +// +// Note: The preceding code is separate from this loop because Index does not +// need revising (see SET_AND_*()) if the initial index is an empty index. +// +// TBD: For speed, shift bitposmaskL instead of using JU_BITMAPTESTL or +// JU_BITPOSMASKL, but this shift has knowledge of bit order that really should +// be encapsulated in a header file. + +#define LEAFB1_CHECKBIT(OpLeastDigits) \ + if (! (JU_JLB_BITMAP(Pjlb, subexp) & bitposmaskL)) \ + SET_AND_RETURN(OpLeastDigits, digit, 1) + +#define LEAFB1_STARTSUBEXP(OpLeastDigits) \ + if (! JU_JLB_BITMAP(Pjlb, subexp)) /* empty subexp */ \ + SET_AND_RETURN(OpLeastDigits, digit, 1) + +#ifdef JUDYPREV + + --digit; // skip initial digit. + bitposmaskL >>= 1; // see TBD above. + +LeafB1NextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskL) // more bits to check in subexp. + { + LEAFB1_CHECKBIT(SETLEASTDIGITS_D); + assert(digit >= 0); + --digit; + bitposmaskL >>= 1; + } + + if (subexp-- > 0) // more subexpanses. + { + LEAFB1_STARTSUBEXP(SETLEASTDIGITS_D); + bitposmaskL = (1UL << (cJU_BITSPERSUBEXPL - 1)); + goto LeafB1NextSubexp; + } + +#else // JUDYNEXT + + ++digit; // skip initial digit. + bitposmaskL <<= 1; // note: BITMAPL_t. + +LeafB1NextSubexp: // return here to check next bitmap subexpanse. + + while (bitposmaskL) // more bits to check in subexp. + { + LEAFB1_CHECKBIT(CLEARLEASTDIGITS_D); + assert(digit < cJU_SUBEXPPERSTATE); + ++digit; + bitposmaskL <<= 1; // note: BITMAPL_t. + } + + if (++subexp < cJU_NUMSUBEXPL) // more subexpanses. + { + LEAFB1_STARTSUBEXP(CLEARLEASTDIGITS_D); + bitposmaskL = 1; + goto LeafB1NextSubexp; + } + +#endif // JUDYNEXT + +// LeafB1 secondary dead end, no empty index: + + SMRESTART(1); + + +#ifdef JUDY1 +// ---------------------------------------------------------------------------- +// FULL POPULATION: +// +// If the Decode bytes do not match, Index is empty (without modification); +// otherwise restart. + + case cJ1_JPFULLPOPU1: + + CHECKDCD(1); + SMRESTART(1); +#endif + + +// ---------------------------------------------------------------------------- +// IMMEDIATE: +// +// Pop1 = 1 Immediate JPs: +// +// If Index is not in the immediate JP, return success; otherwise check if +// there is an empty index below/above the immediate JPs index, and if so, +// return success with modified Index, else restart. +// +// Note: Doug says its fast enough to calculate the index size (digits) in +// the following; no need to set it separately for each case. + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + if (JU_JPDCDPOP0(Pjp) != JU_TRIMTODCDSIZE(Index)) RET_SUCCESS; + digits = JU_JPTYPE(Pjp) - cJU_JPIMMED_1_01 + 1; + LEAF_EDGE(JU_LEASTBYTES(JU_JPDCDPOP0(Pjp), digits), digits); + +// Immediate JPs with Pop1 > 1: + +#define IMM_MULTI(Func,BaseJPType) \ + JUDY1CODE(Pword = (PWord_t) (Pjp->jp_1Index);) \ + JUDYLCODE(Pword = (PWord_t) (Pjp->jp_LIndex);) \ + Func(Pword, JU_JPTYPE(Pjp) - (BaseJPType) + 1) + + case cJU_JPIMMED_1_02: + case cJU_JPIMMED_1_03: +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: + case cJU_JPIMMED_1_05: + case cJU_JPIMMED_1_06: + case cJU_JPIMMED_1_07: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: + case cJ1_JPIMMED_1_09: + case cJ1_JPIMMED_1_10: + case cJ1_JPIMMED_1_11: + case cJ1_JPIMMED_1_12: + case cJ1_JPIMMED_1_13: + case cJ1_JPIMMED_1_14: + case cJ1_JPIMMED_1_15: +#endif + IMM_MULTI(j__udySearchLeafEmpty1, cJU_JPIMMED_1_02); + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: + case cJU_JPIMMED_2_03: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: + case cJ1_JPIMMED_2_05: + case cJ1_JPIMMED_2_06: + case cJ1_JPIMMED_2_07: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + IMM_MULTI(j__udySearchLeafEmpty2, cJU_JPIMMED_2_02); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: + case cJ1_JPIMMED_3_04: + case cJ1_JPIMMED_3_05: +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + IMM_MULTI(j__udySearchLeafEmpty3, cJU_JPIMMED_3_02); +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_4_02: + case cJ1_JPIMMED_4_03: + IMM_MULTI(j__udySearchLeafEmpty4, cJ1_JPIMMED_4_02); + + case cJ1_JPIMMED_5_02: + case cJ1_JPIMMED_5_03: + IMM_MULTI(j__udySearchLeafEmpty5, cJ1_JPIMMED_5_02); + + case cJ1_JPIMMED_6_02: + IMM_MULTI(j__udySearchLeafEmpty6, cJ1_JPIMMED_6_02); + + case cJ1_JPIMMED_7_02: + IMM_MULTI(j__udySearchLeafEmpty7, cJ1_JPIMMED_7_02); +#endif + + +// ---------------------------------------------------------------------------- +// INVALID JP TYPE: + + default: RET_CORRUPT; + + } // SMGet switch. + +} // Judy1PrevEmpty() / Judy1NextEmpty() / JudyLPrevEmpty() / JudyLNextEmpty() diff --git a/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c b/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c new file mode 100644 index 000000000..cb8b13ff7 --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/JudyLTablesGen.c @@ -0,0 +1,296 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.37 $ $Source: /judy/src/JudyCommon/JudyTables.c $ + +#ifndef JU_WIN +#include <unistd.h> // unavailable on win_*. +#endif + +#include <stdlib.h> +#include <stdio.h> + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#define TERMINATOR 999 // terminator for Alloc tables + +#define BPW sizeof(Word_t) // define bytes per word + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +FILE *fd; + +// Definitions come from header files Judy1.h and JudyL.h: + +int AllocSizes[] = ALLOCSIZES; + +#define ROUNDUP(BYTES,BPW,OFFSETW) \ + ((((BYTES) + (BPW) - 1) / (BPW)) + (OFFSETW)) + + +// **************************************************************************** +// G E N T A B L E +// +// Note: "const" is required for newer compilers. + +FUNCTION void GenTable( + const char * TableName, // name of table string + const char * TableSize, // dimentioned size string + int IndexBytes, // bytes per Index + int LeafSize, // number elements in object + int ValueBytes, // bytes per Value + int OffsetWords) // 1 for LEAFW +{ + int * PAllocSizes = AllocSizes; + int OWord; + int CurWord; + int IWord; + int ii; + int BytesOfIndex; + int BytesOfObject; + int Index; + int LastWords; + int Words [1000] = { 0 }; + int Offset[1000] = { 0 }; + int MaxWords; + + MaxWords = ROUNDUP((IndexBytes + ValueBytes) * LeafSize, BPW, OffsetWords); + Words[0] = 0; + Offset[0] = 0; + CurWord = TERMINATOR; + +// Walk through all number of Indexes in table: + + for (Index = 1; /* null */; ++Index) + { + +// Calculate byte required for next size: + + BytesOfIndex = IndexBytes * Index; + BytesOfObject = (IndexBytes + ValueBytes) * Index; + +// Round up and calculate words required for next size: + + OWord = ROUNDUP(BytesOfObject, BPW, OffsetWords); + IWord = ROUNDUP(BytesOfIndex, BPW, OffsetWords); + +// Root-level leaves of population of 1 and 2 do not have the 1 word offset: + +// Save minimum value of offset: + + Offset[Index] = IWord; + +// Round up to next available size of words: + + while (OWord > *PAllocSizes) PAllocSizes++; + + if (Index == LeafSize) + { + CurWord = Words[Index] = OWord; + break; + } +// end of available sizes ? + + if (*PAllocSizes == TERMINATOR) + { + fprintf(stderr, "BUG, in %sPopToWords, sizes not big enough for object\n", TableName); + exit(1); + } + +// Save words required and last word: + + if (*PAllocSizes < MaxWords) { CurWord = Words[Index] = *PAllocSizes; } + else { CurWord = Words[Index] = MaxWords; } + + } // for each index + + LastWords = TERMINATOR; + +// Round up to largest size in each group of malloc sizes: + + for (ii = LeafSize; ii > 0; ii--) + { + if (LastWords > (Words[ii] - ii)) LastWords = Offset[ii]; + else Offset[ii] = LastWords; + } + +// Print the PopToWords[] table: + + fprintf(fd,"\n//\tobject uses %d words\n", CurWord); + fprintf(fd,"//\t%s = %d\n", TableSize, LeafSize); + + fprintf(fd,"const uint8_t\n"); + fprintf(fd,"%sPopToWords[%s + 1] =\n", TableName, TableSize); + fprintf(fd,"{\n\t 0,"); + + for (ii = 1; ii <= LeafSize; ii++) + { + +// 8 columns per line, starting with 1: + + if ((ii % 8) == 1) fprintf(fd,"\n\t"); + + fprintf(fd,"%2d", Words[ii]); + +// If not last number place comma: + + if (ii != LeafSize) fprintf(fd,", "); + } + fprintf(fd,"\n};\n"); + +// Print the Offset table if needed: + + if (! ValueBytes) return; + + fprintf(fd,"const uint8_t\n"); + fprintf(fd,"%sOffset[%s + 1] =\n", TableName, TableSize); + fprintf(fd,"{\n"); + fprintf(fd,"\t 0,"); + + for (ii = 1; ii <= LeafSize; ii++) + { + if ((ii % 8) == 1) fprintf(fd,"\n\t"); + + fprintf(fd,"%2d", Offset[ii]); + + if (ii != LeafSize) fprintf(fd,", "); + } + fprintf(fd,"\n};\n"); + +} // GenTable() + + +// **************************************************************************** +// M A I N + +FUNCTION int main() +{ + int ii; + +#ifdef JUDY1 + char *fname = "Judy1Tables.c"; +#else + char *fname = "JudyLTables.c"; +#endif + + if ((fd = fopen(fname, "w")) == NULL){ + perror("FATAL ERROR: could not write to Judy[1L]Tables.c file\n"); + return (-1); + } + + + fprintf(fd,"// @(#) From generation tool: $Revision: 4.37 $ $Source: /judy/src/JudyCommon/JudyTables.c $\n"); + fprintf(fd,"//\n\n"); + + +// ================================ Judy1 ================================= +#ifdef JUDY1 + + fprintf(fd,"#include \"Judy1.h\"\n"); + + fprintf(fd,"// Leave the malloc() sizes readable in the binary (via " + "strings(1)):\n"); + fprintf(fd,"const char * Judy1MallocSizes = \"Judy1MallocSizes ="); + + for (ii = 0; AllocSizes[ii] != TERMINATOR; ii++) + fprintf(fd," %d,", AllocSizes[ii]); + +#ifndef JU_64BIT + fprintf(fd," Leaf1 = %d\";\n\n", cJ1_LEAF1_MAXPOP1); +#else + fprintf(fd,"\";\n\n"); // no Leaf1 in this case. +#endif + +// ================================ 32 bit ================================ +#ifndef JU_64BIT + + GenTable("j__1_BranchBJP","cJU_BITSPERSUBEXPB", 8, cJU_BITSPERSUBEXPB,0,0); + + GenTable("j__1_Leaf1", "cJ1_LEAF1_MAXPOP1", 1, cJ1_LEAF1_MAXPOP1, 0, 0); + GenTable("j__1_Leaf2", "cJ1_LEAF2_MAXPOP1", 2, cJ1_LEAF2_MAXPOP1, 0, 0); + GenTable("j__1_Leaf3", "cJ1_LEAF3_MAXPOP1", 3, cJ1_LEAF3_MAXPOP1, 0, 0); + GenTable("j__1_LeafW", "cJ1_LEAFW_MAXPOP1", 4, cJ1_LEAFW_MAXPOP1, 0, 1); + +#endif + +// ================================ 64 bit ================================ +#ifdef JU_64BIT + GenTable("j__1_BranchBJP","cJU_BITSPERSUBEXPB",16, cJU_BITSPERSUBEXPB,0,0); + + GenTable("j__1_Leaf2", "cJ1_LEAF2_MAXPOP1", 2, cJ1_LEAF2_MAXPOP1, 0, 0); + GenTable("j__1_Leaf3", "cJ1_LEAF3_MAXPOP1", 3, cJ1_LEAF3_MAXPOP1, 0, 0); + GenTable("j__1_Leaf4", "cJ1_LEAF4_MAXPOP1", 4, cJ1_LEAF4_MAXPOP1, 0, 0); + GenTable("j__1_Leaf5", "cJ1_LEAF5_MAXPOP1", 5, cJ1_LEAF5_MAXPOP1, 0, 0); + GenTable("j__1_Leaf6", "cJ1_LEAF6_MAXPOP1", 6, cJ1_LEAF6_MAXPOP1, 0, 0); + GenTable("j__1_Leaf7", "cJ1_LEAF7_MAXPOP1", 7, cJ1_LEAF7_MAXPOP1, 0, 0); + GenTable("j__1_LeafW", "cJ1_LEAFW_MAXPOP1", 8, cJ1_LEAFW_MAXPOP1, 0, 1); +#endif +#endif // JUDY1 + + +// ================================ JudyL ================================= +#ifdef JUDYL + + fprintf(fd,"#include \"JudyL.h\"\n"); + + fprintf(fd,"// Leave the malloc() sizes readable in the binary (via " + "strings(1)):\n"); + fprintf(fd,"const char * JudyLMallocSizes = \"JudyLMallocSizes ="); + + for (ii = 0; AllocSizes[ii] != TERMINATOR; ii++) + fprintf(fd," %d,", AllocSizes[ii]); + + fprintf(fd," Leaf1 = %ld\";\n\n", (Word_t)cJL_LEAF1_MAXPOP1); + +#ifndef JU_64BIT +// ================================ 32 bit ================================ + GenTable("j__L_BranchBJP","cJU_BITSPERSUBEXPB", 8, cJU_BITSPERSUBEXPB, 0,0); + + GenTable("j__L_Leaf1", "cJL_LEAF1_MAXPOP1", 1, cJL_LEAF1_MAXPOP1, BPW,0); + GenTable("j__L_Leaf2", "cJL_LEAF2_MAXPOP1", 2, cJL_LEAF2_MAXPOP1, BPW,0); + GenTable("j__L_Leaf3", "cJL_LEAF3_MAXPOP1", 3, cJL_LEAF3_MAXPOP1, BPW,0); + GenTable("j__L_LeafW", "cJL_LEAFW_MAXPOP1", 4, cJL_LEAFW_MAXPOP1, BPW,1); + GenTable("j__L_LeafV", "cJU_BITSPERSUBEXPL", 4, cJU_BITSPERSUBEXPL, 0,0); +#endif // 32 BIT + +#ifdef JU_64BIT +// ================================ 64 bit ================================ + GenTable("j__L_BranchBJP","cJU_BITSPERSUBEXPB",16, cJU_BITSPERSUBEXPB, 0,0); + + GenTable("j__L_Leaf1", "cJL_LEAF1_MAXPOP1", 1, cJL_LEAF1_MAXPOP1, BPW,0); + GenTable("j__L_Leaf2", "cJL_LEAF2_MAXPOP1", 2, cJL_LEAF2_MAXPOP1, BPW,0); + GenTable("j__L_Leaf3", "cJL_LEAF3_MAXPOP1", 3, cJL_LEAF3_MAXPOP1, BPW,0); + GenTable("j__L_Leaf4", "cJL_LEAF4_MAXPOP1", 4, cJL_LEAF4_MAXPOP1, BPW,0); + GenTable("j__L_Leaf5", "cJL_LEAF5_MAXPOP1", 5, cJL_LEAF5_MAXPOP1, BPW,0); + GenTable("j__L_Leaf6", "cJL_LEAF6_MAXPOP1", 6, cJL_LEAF6_MAXPOP1, BPW,0); + GenTable("j__L_Leaf7", "cJL_LEAF7_MAXPOP1", 7, cJL_LEAF7_MAXPOP1, BPW,0); + GenTable("j__L_LeafW", "cJL_LEAFW_MAXPOP1", 8, cJL_LEAFW_MAXPOP1, BPW,1); + GenTable("j__L_LeafV", "cJU_BITSPERSUBEXPL", 8, cJU_BITSPERSUBEXPL, 0,0); +#endif // 64 BIT + +#endif // JUDYL + fclose(fd); + + return(0); + +} // main() diff --git a/libnetdata/libjudy/src/JudyL/j__udyLGet.c b/libnetdata/libjudy/src/JudyL/j__udyLGet.c new file mode 100644 index 000000000..0bb9971cc --- /dev/null +++ b/libnetdata/libjudy/src/JudyL/j__udyLGet.c @@ -0,0 +1,1094 @@ +// Copyright (C) 2000 - 2002 Hewlett-Packard Company +// +// This program is free software; you can redistribute it and/or modify it +// under the term of the GNU Lesser General Public License as published by the +// Free Software Foundation; either version 2 of the License, or (at your +// option) any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License +// for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program; if not, write to the Free Software Foundation, +// Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// _________________ + +// @(#) $Revision: 4.43 $ $Source: /judy/src/JudyCommon/JudyGet.c $ +// +// Judy1Test() and JudyLGet() functions for Judy1 and JudyL. +// Compile with one of -DJUDY1 or -DJUDYL. + +#if (! (defined(JUDY1) || defined(JUDYL))) +#error: One of -DJUDY1 or -DJUDYL must be specified. +#endif + +#ifdef JUDY1 +#include "Judy1.h" +#else +#include "JudyL.h" +#endif + +#include "JudyPrivate1L.h" + +#ifdef TRACEJPR // different macro name, for "retrieval" only. +#include "JudyPrintJP.c" +#endif + + +// **************************************************************************** +// J U D Y 1 T E S T +// J U D Y L G E T +// +// See the manual entry for details. Note support for "shortcut" entries to +// trees known to start with a JPM. + +#ifdef JUDY1 + +#ifdef JUDYGETINLINE +FUNCTION int j__udy1Test +#else +FUNCTION int Judy1Test +#endif + +#else // JUDYL + +#ifdef JUDYGETINLINE +FUNCTION PPvoid_t j__udyLGet +#else +FUNCTION PPvoid_t JudyLGet +#endif + +#endif // JUDYL + ( +#ifdef JUDYGETINLINE + Pvoid_t PArray, // from which to retrieve. + Word_t Index // to retrieve. +#else + Pcvoid_t PArray, // from which to retrieve. + Word_t Index, // to retrieve. + PJError_t PJError // optional, for returning error info. +#endif + ) +{ + Pjp_t Pjp; // current JP while walking the tree. + Pjpm_t Pjpm; // for global accounting. + uint8_t Digit; // byte just decoded from Index. + Word_t Pop1; // leaf population (number of indexes). + Pjll_t Pjll; // pointer to LeafL. + DBGCODE(uint8_t ParentJPType;) + +#ifndef JUDYGETINLINE + + if (PArray == (Pcvoid_t) NULL) // empty array. + { + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + } + +// **************************************************************************** +// PROCESS TOP LEVEL BRANCHES AND LEAF: + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + { + Pjlw_t Pjlw = P_JLW(PArray); // first word of leaf. + int posidx; // signed offset in leaf. + + Pop1 = Pjlw[0] + 1; + posidx = j__udySearchLeafW(Pjlw + 1, Pop1, Index); + + if (posidx >= 0) + { + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAFWVALUEAREA(Pjlw, Pop1) + posidx));) + } + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + } + +#endif // ! JUDYGETINLINE + + Pjpm = P_JPM(PArray); + Pjp = &(Pjpm->jpm_JP); // top branch is below JPM. + +// **************************************************************************** +// WALK THE JUDY TREE USING A STATE MACHINE: + +ContinueWalk: // for going down one level; come here with Pjp set. + +#ifdef TRACEJPR + JudyPrintJP(Pjp, "g", __LINE__); +#endif + switch (JU_JPTYPE(Pjp)) + { + +// Ensure the switch table starts at 0 for speed; otherwise more code is +// executed: + + case 0: goto ReturnCorrupt; // save a little code. + + +// **************************************************************************** +// JPNULL*: +// +// Note: These are legitimate in a BranchU (only) and do not constitute a +// fault. + + case cJU_JPNULL1: + case cJU_JPNULL2: + case cJU_JPNULL3: +#ifdef JU_64BIT + case cJU_JPNULL4: + case cJU_JPNULL5: + case cJU_JPNULL6: + case cJU_JPNULL7: +#endif + assert(ParentJPType >= cJU_JPBRANCH_U2); + assert(ParentJPType <= cJU_JPBRANCH_U); + JUDY1CODE(return(0);) + JUDYLCODE(return((PPvoid_t) NULL);) + + +// **************************************************************************** +// JPBRANCH_L*: +// +// Note: The use of JU_DCDNOTMATCHINDEX() in branches is not strictly +// required,since this can be done at leaf level, but it costs nothing to do it +// sooner, and it aborts an unnecessary traversal sooner. + + case cJU_JPBRANCH_L2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + Digit = JU_DIGITATSTATE(Index, 2); + goto JudyBranchL; + + case cJU_JPBRANCH_L3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + Digit = JU_DIGITATSTATE(Index, 3); + goto JudyBranchL; + +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + Digit = JU_DIGITATSTATE(Index, 4); + goto JudyBranchL; + + case cJU_JPBRANCH_L5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + Digit = JU_DIGITATSTATE(Index, 5); + goto JudyBranchL; + + case cJU_JPBRANCH_L6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + Digit = JU_DIGITATSTATE(Index, 6); + goto JudyBranchL; + + case cJU_JPBRANCH_L7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Digit = JU_DIGITATSTATE(Index, 7); + goto JudyBranchL; + +#endif // JU_64BIT + + case cJU_JPBRANCH_L: + { + Pjbl_t Pjbl; + int posidx; + + Digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + +// Common code for all BranchLs; come here with Digit set: + +JudyBranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + + posidx = 0; + + do { + if (Pjbl->jbl_Expanse[posidx] == Digit) + { // found Digit; continue traversal: + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = Pjbl->jbl_jp + posidx; + goto ContinueWalk; + } + } while (++posidx != Pjbl->jbl_NumJPs); + + break; + } + + +// **************************************************************************** +// JPBRANCH_B*: + + case cJU_JPBRANCH_B2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + Digit = JU_DIGITATSTATE(Index, 2); + goto JudyBranchB; + + case cJU_JPBRANCH_B3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + Digit = JU_DIGITATSTATE(Index, 3); + goto JudyBranchB; + + +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + Digit = JU_DIGITATSTATE(Index, 4); + goto JudyBranchB; + + case cJU_JPBRANCH_B5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + Digit = JU_DIGITATSTATE(Index, 5); + goto JudyBranchB; + + case cJU_JPBRANCH_B6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + Digit = JU_DIGITATSTATE(Index, 6); + goto JudyBranchB; + + case cJU_JPBRANCH_B7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Digit = JU_DIGITATSTATE(Index, 7); + goto JudyBranchB; + +#endif // JU_64BIT + + case cJU_JPBRANCH_B: + { + Pjbb_t Pjbb; + Word_t subexp; // in bitmap, 0..7. + BITMAPB_t BitMap; // for one subexpanse. + BITMAPB_t BitMask; // bit in BitMap for Indexs Digit. + + Digit = JU_DIGITATSTATE(Index, cJU_ROOTSTATE); + +// Common code for all BranchBs; come here with Digit set: + +JudyBranchB: + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjbb = P_JBB(Pjp->jp_Addr); + subexp = Digit / cJU_BITSPERSUBEXPB; + + BitMap = JU_JBB_BITMAP(Pjbb, subexp); + Pjp = P_JP(JU_JBB_PJP(Pjbb, subexp)); + + BitMask = JU_BITPOSMASKB(Digit); + +// No JP in subexpanse for Index => Index not found: + + if (! (BitMap & BitMask)) break; + +// Count JPs in the subexpanse below the one for Index: + + Pjp += j__udyCountBitsB(BitMap & (BitMask - 1)); + + goto ContinueWalk; + + } // case cJU_JPBRANCH_B* + + +// **************************************************************************** +// JPBRANCH_U*: +// +// Notice the reverse order of the cases, and falling through to the next case, +// for performance. + + case cJU_JPBRANCH_U: + + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, cJU_ROOTSTATE); + +// If not a BranchU, traverse; otherwise fall into the next case, which makes +// this very fast code for a large Judy array (mainly BranchUs), especially +// when branches are already in the cache, such as for prev/next: + +#ifndef JU_64BIT + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U3) goto ContinueWalk; +#else + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U7) goto ContinueWalk; +#endif + +#ifdef JU_64BIT + case cJU_JPBRANCH_U7: + + // JU_DCDNOTMATCHINDEX() would be a no-op. + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 7); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U6) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U6: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 6); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U5) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U5: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 5); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U4) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U4: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 4); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U3) goto ContinueWalk; + // and fall through. + +#endif // JU_64BIT + + case cJU_JPBRANCH_U3: + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 3); + + if (JU_JPTYPE(Pjp) != cJU_JPBRANCH_U2) goto ContinueWalk; + // and fall through. + + case cJU_JPBRANCH_U2: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + DBGCODE(ParentJPType = JU_JPTYPE(Pjp);) + Pjp = JU_JBU_PJP(Pjp, Index, 2); + +// Note: BranchU2 is a special case that must continue traversal to a leaf, +// immed, full, or null type: + + goto ContinueWalk; + + +// **************************************************************************** +// JPLEAF*: +// +// Note: Here the calls of JU_DCDNOTMATCHINDEX() are necessary and check +// whether Index is out of the expanse of a narrow pointer. + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + + case cJU_JPLEAF1: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf1(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF1VALUEAREA(Pjll, Pop1) + posidx));) + } + +#endif // (JUDYL || (! JU_64BIT)) + + case cJU_JPLEAF2: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 2)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf2(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF2VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF3: + { + int posidx; // signed offset in leaf. + +#ifdef JU_64BIT // otherwise its a no-op: + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 3)) break; +#endif + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf3(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF3VALUEAREA(Pjll, Pop1) + posidx));) + } +#ifdef JU_64BIT + case cJU_JPLEAF4: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 4)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf4(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF4VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF5: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 5)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf5(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF5VALUEAREA(Pjll, Pop1) + posidx));) + } + + case cJU_JPLEAF6: + { + int posidx; // signed offset in leaf. + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 6)) break; + + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf6(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF6VALUEAREA(Pjll, Pop1) + posidx));) + } + case cJU_JPLEAF7: + { + int posidx; // signed offset in leaf. + + // JU_DCDNOTMATCHINDEX() would be a no-op. + Pop1 = JU_JPLEAF_POP0(Pjp) + 1; + Pjll = P_JLL(Pjp->jp_Addr); + + if ((posidx = j__udySearchLeaf7(Pjll, Pop1, Index)) < 0) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) (JL_LEAF7VALUEAREA(Pjll, Pop1) + posidx));) + } +#endif // JU_64BIT + + +// **************************************************************************** +// JPLEAF_B1: + + case cJU_JPLEAF_B1: + { + Pjlb_t Pjlb; +#ifdef JUDYL + int posidx; + Word_t subexp; // in bitmap, 0..7. + BITMAPL_t BitMap; // for one subexpanse. + BITMAPL_t BitMask; // bit in BitMap for Indexs Digit. + Pjv_t Pjv; +#endif + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + Pjlb = P_JLB(Pjp->jp_Addr); + +#ifdef JUDY1 + +// Simply check if Indexs bit is set in the bitmap: + + if (JU_BITMAPTESTL(Pjlb, Index)) return(1); + break; + +#else // JUDYL + +// JudyL is much more complicated because of value area subarrays: + + Digit = JU_DIGITATSTATE(Index, 1); + subexp = Digit / cJU_BITSPERSUBEXPL; + BitMap = JU_JLB_BITMAP(Pjlb, subexp); + BitMask = JU_BITPOSMASKL(Digit); + +// No value in subexpanse for Index => Index not found: + + if (! (BitMap & BitMask)) break; + +// Count value areas in the subexpanse below the one for Index: + + Pjv = P_JV(JL_JLB_PVALUE(Pjlb, subexp)); + assert(Pjv != (Pjv_t) NULL); + posidx = j__udyCountBitsL(BitMap & (BitMask - 1)); + + return((PPvoid_t) (Pjv + posidx)); + +#endif // JUDYL + + } // case cJU_JPLEAF_B1 + +#ifdef JUDY1 + +// **************************************************************************** +// JPFULLPOPU1: +// +// If the Index is in the expanse, it is necessarily valid (found). + + case cJ1_JPFULLPOPU1: + + if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + return(1); + +#ifdef notdef // for future enhancements +#ifdef JU_64BIT + +// Note: Need ? if (JU_DCDNOTMATCHINDEX(Index, Pjp, 1)) break; + + case cJ1_JPFULLPOPU1m15: + if (Pjp->jp_1Index[14] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m14: + if (Pjp->jp_1Index[13] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m13: + if (Pjp->jp_1Index[12] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m12: + if (Pjp->jp_1Index[11] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m11: + if (Pjp->jp_1Index[10] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m10: + if (Pjp->jp_1Index[9] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m9: + if (Pjp->jp_1Index[8] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m8: + if (Pjp->jp_1Index[7] == (uint8_t)Index) break; +#endif + case cJ1_JPFULLPOPU1m7: + if (Pjp->jp_1Index[6] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m6: + if (Pjp->jp_1Index[5] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m5: + if (Pjp->jp_1Index[4] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m4: + if (Pjp->jp_1Index[3] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m3: + if (Pjp->jp_1Index[2] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m2: + if (Pjp->jp_1Index[1] == (uint8_t)Index) break; + case cJ1_JPFULLPOPU1m1: + if (Pjp->jp_1Index[0] == (uint8_t)Index) break; + + return(1); // found, not in exclusion list + +#endif // JUDY1 +#endif // notdef + +// **************************************************************************** +// JPIMMED*: +// +// Note that the contents of jp_DcdPopO are different for cJU_JPIMMED_*_01: + + case cJU_JPIMMED_1_01: + case cJU_JPIMMED_2_01: + case cJU_JPIMMED_3_01: +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: + case cJU_JPIMMED_5_01: + case cJU_JPIMMED_6_01: + case cJU_JPIMMED_7_01: +#endif + if (JU_JPDCDPOP0(Pjp) != JU_TRIMTODCDSIZE(Index)) break; + + JUDY1CODE(return(1);) + JUDYLCODE(return((PPvoid_t) &(Pjp->jp_Addr));) // immediate value area. + + +// Macros to make code more readable and avoid dup errors + +#ifdef JUDY1 + +#define CHECKINDEXNATIVE(LEAF_T, PJP, IDX, INDEX) \ +if (((LEAF_T *)((PJP)->jp_1Index))[(IDX) - 1] == (LEAF_T)(INDEX)) \ + return(1) + +#define CHECKLEAFNONNAT(LFBTS, PJP, INDEX, IDX, COPY) \ +{ \ + Word_t i_ndex; \ + uint8_t *a_ddr; \ + a_ddr = (PJP)->jp_1Index + (((IDX) - 1) * (LFBTS)); \ + COPY(i_ndex, a_ddr); \ + if (i_ndex == JU_LEASTBYTES((INDEX), (LFBTS))) \ + return(1); \ +} +#endif + +#ifdef JUDYL + +#define CHECKINDEXNATIVE(LEAF_T, PJP, IDX, INDEX) \ +if (((LEAF_T *)((PJP)->jp_LIndex))[(IDX) - 1] == (LEAF_T)(INDEX)) \ + return((PPvoid_t)(P_JV((PJP)->jp_Addr) + (IDX) - 1)) + +#define CHECKLEAFNONNAT(LFBTS, PJP, INDEX, IDX, COPY) \ +{ \ + Word_t i_ndex; \ + uint8_t *a_ddr; \ + a_ddr = (PJP)->jp_LIndex + (((IDX) - 1) * (LFBTS)); \ + COPY(i_ndex, a_ddr); \ + if (i_ndex == JU_LEASTBYTES((INDEX), (LFBTS))) \ + return((PPvoid_t)(P_JV((PJP)->jp_Addr) + (IDX) - 1)); \ +} +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_15: CHECKINDEXNATIVE(uint8_t, Pjp, 15, Index); + case cJ1_JPIMMED_1_14: CHECKINDEXNATIVE(uint8_t, Pjp, 14, Index); + case cJ1_JPIMMED_1_13: CHECKINDEXNATIVE(uint8_t, Pjp, 13, Index); + case cJ1_JPIMMED_1_12: CHECKINDEXNATIVE(uint8_t, Pjp, 12, Index); + case cJ1_JPIMMED_1_11: CHECKINDEXNATIVE(uint8_t, Pjp, 11, Index); + case cJ1_JPIMMED_1_10: CHECKINDEXNATIVE(uint8_t, Pjp, 10, Index); + case cJ1_JPIMMED_1_09: CHECKINDEXNATIVE(uint8_t, Pjp, 9, Index); + case cJ1_JPIMMED_1_08: CHECKINDEXNATIVE(uint8_t, Pjp, 8, Index); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_07: CHECKINDEXNATIVE(uint8_t, Pjp, 7, Index); + case cJU_JPIMMED_1_06: CHECKINDEXNATIVE(uint8_t, Pjp, 6, Index); + case cJU_JPIMMED_1_05: CHECKINDEXNATIVE(uint8_t, Pjp, 5, Index); + case cJU_JPIMMED_1_04: CHECKINDEXNATIVE(uint8_t, Pjp, 4, Index); +#endif + case cJU_JPIMMED_1_03: CHECKINDEXNATIVE(uint8_t, Pjp, 3, Index); + case cJU_JPIMMED_1_02: CHECKINDEXNATIVE(uint8_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint8_t, Pjp, 1, Index); + break; + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_07: CHECKINDEXNATIVE(uint16_t, Pjp, 7, Index); + case cJ1_JPIMMED_2_06: CHECKINDEXNATIVE(uint16_t, Pjp, 6, Index); + case cJ1_JPIMMED_2_05: CHECKINDEXNATIVE(uint16_t, Pjp, 5, Index); + case cJ1_JPIMMED_2_04: CHECKINDEXNATIVE(uint16_t, Pjp, 4, Index); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_03: CHECKINDEXNATIVE(uint16_t, Pjp, 3, Index); + case cJU_JPIMMED_2_02: CHECKINDEXNATIVE(uint16_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint16_t, Pjp, 1, Index); + break; +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_05: + CHECKLEAFNONNAT(3, Pjp, Index, 5, JU_COPY3_PINDEX_TO_LONG); + case cJ1_JPIMMED_3_04: + CHECKLEAFNONNAT(3, Pjp, Index, 4, JU_COPY3_PINDEX_TO_LONG); + case cJ1_JPIMMED_3_03: + CHECKLEAFNONNAT(3, Pjp, Index, 3, JU_COPY3_PINDEX_TO_LONG); +#endif +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: + CHECKLEAFNONNAT(3, Pjp, Index, 2, JU_COPY3_PINDEX_TO_LONG); + CHECKLEAFNONNAT(3, Pjp, Index, 1, JU_COPY3_PINDEX_TO_LONG); + break; +#endif + +#if (defined(JUDY1) && defined(JU_64BIT)) + + case cJ1_JPIMMED_4_03: CHECKINDEXNATIVE(uint32_t, Pjp, 3, Index); + case cJ1_JPIMMED_4_02: CHECKINDEXNATIVE(uint32_t, Pjp, 2, Index); + CHECKINDEXNATIVE(uint32_t, Pjp, 1, Index); + break; + + case cJ1_JPIMMED_5_03: + CHECKLEAFNONNAT(5, Pjp, Index, 3, JU_COPY5_PINDEX_TO_LONG); + case cJ1_JPIMMED_5_02: + CHECKLEAFNONNAT(5, Pjp, Index, 2, JU_COPY5_PINDEX_TO_LONG); + CHECKLEAFNONNAT(5, Pjp, Index, 1, JU_COPY5_PINDEX_TO_LONG); + break; + + case cJ1_JPIMMED_6_02: + CHECKLEAFNONNAT(6, Pjp, Index, 2, JU_COPY6_PINDEX_TO_LONG); + CHECKLEAFNONNAT(6, Pjp, Index, 1, JU_COPY6_PINDEX_TO_LONG); + break; + + case cJ1_JPIMMED_7_02: + CHECKLEAFNONNAT(7, Pjp, Index, 2, JU_COPY7_PINDEX_TO_LONG); + CHECKLEAFNONNAT(7, Pjp, Index, 1, JU_COPY7_PINDEX_TO_LONG); + break; + +#endif // (JUDY1 && JU_64BIT) + + +// **************************************************************************** +// INVALID JP TYPE: + + default: + +ReturnCorrupt: + +#ifdef JUDYGETINLINE // Pjpm is known to be non-null: + JU_SET_ERRNO_NONNULL(Pjpm, JU_ERRNO_CORRUPT); +#else + JU_SET_ERRNO(PJError, JU_ERRNO_CORRUPT); +#endif + JUDY1CODE(return(JERRI );) + JUDYLCODE(return(PPJERR);) + + } // switch on JP type + +JUDY1CODE(return(0);) +JUDYLCODE(return((PPvoid_t) NULL);) + +} // Judy1Test() / JudyLGet() + + +#ifndef JUDYGETINLINE // only compile the following function once: +#ifdef DEBUG + +// **************************************************************************** +// J U D Y C H E C K P O P +// +// Given a pointer to a Judy array, traverse the entire array to ensure +// population counts add up correctly. This can catch various coding errors. +// +// Since walking the entire tree is probably time-consuming, enable this +// function by setting env parameter $CHECKPOP to first call at which to start +// checking. Note: This function is called both from insert and delete code. +// +// Note: Even though this function does nothing useful for LEAFW leaves, its +// good practice to call it anyway, and cheap too. +// +// TBD: This is a debug-only check function similar to JudyCheckSorted(), but +// since it walks the tree it is Judy1/JudyL-specific and must live in a source +// file that is built both ways. +// +// TBD: As feared, enabling this code for every insert/delete makes Judy +// deathly slow, even for a small tree (10K indexes). Its not so bad if +// present but disabled (<1% slowdown measured). Still, should it be ifdefd +// other than DEBUG and/or called less often? +// +// TBD: Should this "population checker" be expanded to a comprehensive tree +// checker? It currently detects invalid LEAFW/JP types as well as inconsistent +// pop1s. Other possible checks, all based on essentially redundant data in +// the Judy tree, include: +// +// - Zero LS bits in jp_Addr field. +// +// - Correct Dcd bits. +// +// - Consistent JP types (always descending down the tree). +// +// - Sorted linear lists in BranchLs and leaves (using JudyCheckSorted(), but +// ideally that function is already called wherever appropriate after any +// linear list is modified). +// +// - Any others possible? + +#include <stdlib.h> // for getenv() and atol(). + +static Word_t JudyCheckPopSM(Pjp_t Pjp, Word_t RootPop1); + +FUNCTION void JudyCheckPop( + Pvoid_t PArray) +{ +static bool_t checked = FALSE; // already checked env parameter. +static bool_t enabled = FALSE; // env parameter set. +static bool_t active = FALSE; // calls >= callsmin. +static Word_t callsmin; // start point from $CHECKPOP. +static Word_t calls = 0; // times called so far. + + +// CHECK FOR EXTERNAL ENABLING: + + if (! checked) // only check once. + { + char * value; // for getenv(). + + checked = TRUE; + + if ((value = getenv("CHECKPOP")) == (char *) NULL) + { +#ifdef notdef +// Take this out because nightly tests want to be flavor-independent; its not +// OK to emit special non-error output from the debug flavor: + + (void) puts("JudyCheckPop() present but not enabled by " + "$CHECKPOP env parameter; set it to the number of " + "calls at which to begin checking"); +#endif + return; + } + + callsmin = atol(value); // note: non-number evaluates to 0. + enabled = TRUE; + + (void) printf("JudyCheckPop() present and enabled; callsmin = " + "%lu\n", callsmin); + } + else if (! enabled) return; + +// Previously or just now enabled; check if non-active or newly active: + + if (! active) + { + if (++calls < callsmin) return; + + (void) printf("JudyCheckPop() activated at call %lu\n", calls); + active = TRUE; + } + +// IGNORE LEAFW AT TOP OF TREE: + + if (JU_LEAFW_POP0(PArray) < cJU_LEAFW_MAXPOP1) // must be a LEAFW + return; + +// Check JPM pop0 against tree, recursively: +// +// Note: The traversal code in JudyCheckPopSM() is simplest when the case +// statement for each JP type compares the pop1 for that JP to its subtree (if +// any) after traversing the subtree (thats the hard part) and adding up +// actual pop1s. A top branchs JP in the JPM does not have room for a +// full-word pop1, so pass it in as a special case. + + { + Pjpm_t Pjpm = P_JPM(PArray); + (void) JudyCheckPopSM(&(Pjpm->jpm_JP), Pjpm->jpm_Pop0 + 1); + return; + } + +} // JudyCheckPop() + + +// **************************************************************************** +// J U D Y C H E C K P O P S M +// +// Recursive state machine (subroutine) for JudyCheckPop(): Given a Pjp (other +// than JPNULL*; caller should shortcut) and the root population for top-level +// branches, check the subtrees actual pop1 against its nominal value, and +// return the total pop1 for the subtree. +// +// Note: Expect RootPop1 to be ignored at lower levels, so pass down 0, which +// should pop an assertion if this expectation is violated. + +FUNCTION static Word_t JudyCheckPopSM( + Pjp_t Pjp, // top of subtree. + Word_t RootPop1) // whole array, for top-level branches only. +{ + Word_t pop1_jp; // nominal population from the JP. + Word_t pop1 = 0; // actual population at this level. + Word_t offset; // in a branch. + +#define PREPBRANCH(cPopBytes,Next) \ + pop1_jp = JU_JPBRANCH_POP0(Pjp, cPopBytes) + 1; goto Next + +assert((((Word_t) (Pjp->jp_Addr)) & 7) == 3); + switch (JU_JPTYPE(Pjp)) + { + + case cJU_JPBRANCH_L2: PREPBRANCH(2, BranchL); + case cJU_JPBRANCH_L3: PREPBRANCH(3, BranchL); +#ifdef JU_64BIT + case cJU_JPBRANCH_L4: PREPBRANCH(4, BranchL); + case cJU_JPBRANCH_L5: PREPBRANCH(5, BranchL); + case cJU_JPBRANCH_L6: PREPBRANCH(6, BranchL); + case cJU_JPBRANCH_L7: PREPBRANCH(7, BranchL); +#endif + case cJU_JPBRANCH_L: pop1_jp = RootPop1; + { + Pjbl_t Pjbl; +BranchL: + Pjbl = P_JBL(Pjp->jp_Addr); + + for (offset = 0; offset < (Pjbl->jbl_NumJPs); ++offset) + pop1 += JudyCheckPopSM((Pjbl->jbl_jp) + offset, 0); + + assert(pop1_jp == pop1); + return(pop1); + } + + case cJU_JPBRANCH_B2: PREPBRANCH(2, BranchB); + case cJU_JPBRANCH_B3: PREPBRANCH(3, BranchB); +#ifdef JU_64BIT + case cJU_JPBRANCH_B4: PREPBRANCH(4, BranchB); + case cJU_JPBRANCH_B5: PREPBRANCH(5, BranchB); + case cJU_JPBRANCH_B6: PREPBRANCH(6, BranchB); + case cJU_JPBRANCH_B7: PREPBRANCH(7, BranchB); +#endif + case cJU_JPBRANCH_B: pop1_jp = RootPop1; + { + Word_t subexp; + Word_t jpcount; + Pjbb_t Pjbb; +BranchB: + Pjbb = P_JBB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPB; ++subexp) + { + jpcount = j__udyCountBitsB(JU_JBB_BITMAP(Pjbb, subexp)); + + for (offset = 0; offset < jpcount; ++offset) + { + pop1 += JudyCheckPopSM(P_JP(JU_JBB_PJP(Pjbb, subexp)) + + offset, 0); + } + } + + assert(pop1_jp == pop1); + return(pop1); + } + + case cJU_JPBRANCH_U2: PREPBRANCH(2, BranchU); + case cJU_JPBRANCH_U3: PREPBRANCH(3, BranchU); +#ifdef JU_64BIT + case cJU_JPBRANCH_U4: PREPBRANCH(4, BranchU); + case cJU_JPBRANCH_U5: PREPBRANCH(5, BranchU); + case cJU_JPBRANCH_U6: PREPBRANCH(6, BranchU); + case cJU_JPBRANCH_U7: PREPBRANCH(7, BranchU); +#endif + case cJU_JPBRANCH_U: pop1_jp = RootPop1; + { + Pjbu_t Pjbu; +BranchU: + Pjbu = P_JBU(Pjp->jp_Addr); + + for (offset = 0; offset < cJU_BRANCHUNUMJPS; ++offset) + { + if (((Pjbu->jbu_jp[offset].jp_Type) >= cJU_JPNULL1) + && ((Pjbu->jbu_jp[offset].jp_Type) <= cJU_JPNULLMAX)) + { + continue; // skip null JP to save time. + } + + pop1 += JudyCheckPopSM((Pjbu->jbu_jp) + offset, 0); + } + + assert(pop1_jp == pop1); + return(pop1); + } + + +// -- Cases below here terminate and do not recurse. -- +// +// For all of these cases except JPLEAF_B1, there is no way to check the JPs +// pop1 against the object itself; just return the pop1; but for linear leaves, +// a bounds check is possible. + +#define CHECKLEAF(MaxPop1) \ + pop1 = JU_JPLEAF_POP0(Pjp) + 1; \ + assert(pop1 >= 1); \ + assert(pop1 <= (MaxPop1)); \ + return(pop1) + +#if (defined(JUDYL) || (! defined(JU_64BIT))) + case cJU_JPLEAF1: CHECKLEAF(cJU_LEAF1_MAXPOP1); +#endif + case cJU_JPLEAF2: CHECKLEAF(cJU_LEAF2_MAXPOP1); + case cJU_JPLEAF3: CHECKLEAF(cJU_LEAF3_MAXPOP1); +#ifdef JU_64BIT + case cJU_JPLEAF4: CHECKLEAF(cJU_LEAF4_MAXPOP1); + case cJU_JPLEAF5: CHECKLEAF(cJU_LEAF5_MAXPOP1); + case cJU_JPLEAF6: CHECKLEAF(cJU_LEAF6_MAXPOP1); + case cJU_JPLEAF7: CHECKLEAF(cJU_LEAF7_MAXPOP1); +#endif + + case cJU_JPLEAF_B1: + { + Word_t subexp; + Pjlb_t Pjlb; + + pop1_jp = JU_JPLEAF_POP0(Pjp) + 1; + + Pjlb = P_JLB(Pjp->jp_Addr); + + for (subexp = 0; subexp < cJU_NUMSUBEXPL; ++subexp) + pop1 += j__udyCountBitsL(JU_JLB_BITMAP(Pjlb, subexp)); + + assert(pop1_jp == pop1); + return(pop1); + } + + JUDY1CODE(case cJ1_JPFULLPOPU1: return(cJU_JPFULLPOPU1_POP0);) + + case cJU_JPIMMED_1_01: return(1); + case cJU_JPIMMED_2_01: return(1); + case cJU_JPIMMED_3_01: return(1); +#ifdef JU_64BIT + case cJU_JPIMMED_4_01: return(1); + case cJU_JPIMMED_5_01: return(1); + case cJU_JPIMMED_6_01: return(1); + case cJU_JPIMMED_7_01: return(1); +#endif + + case cJU_JPIMMED_1_02: return(2); + case cJU_JPIMMED_1_03: return(3); +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_1_04: return(4); + case cJU_JPIMMED_1_05: return(5); + case cJU_JPIMMED_1_06: return(6); + case cJU_JPIMMED_1_07: return(7); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_1_08: return(8); + case cJ1_JPIMMED_1_09: return(9); + case cJ1_JPIMMED_1_10: return(10); + case cJ1_JPIMMED_1_11: return(11); + case cJ1_JPIMMED_1_12: return(12); + case cJ1_JPIMMED_1_13: return(13); + case cJ1_JPIMMED_1_14: return(14); + case cJ1_JPIMMED_1_15: return(15); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_2_02: return(2); + case cJU_JPIMMED_2_03: return(3); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_2_04: return(4); + case cJ1_JPIMMED_2_05: return(5); + case cJ1_JPIMMED_2_06: return(6); + case cJ1_JPIMMED_2_07: return(7); +#endif + +#if (defined(JUDY1) || defined(JU_64BIT)) + case cJU_JPIMMED_3_02: return(2); +#endif +#if (defined(JUDY1) && defined(JU_64BIT)) + case cJ1_JPIMMED_3_03: return(3); + case cJ1_JPIMMED_3_04: return(4); + case cJ1_JPIMMED_3_05: return(5); + + case cJ1_JPIMMED_4_02: return(2); + case cJ1_JPIMMED_4_03: return(3); + case cJ1_JPIMMED_5_02: return(2); + case cJ1_JPIMMED_5_03: return(3); + case cJ1_JPIMMED_6_02: return(2); + case cJ1_JPIMMED_7_02: return(2); +#endif + + } // switch (JU_JPTYPE(Pjp)) + + assert(FALSE); // unrecognized JP type => corruption. + return(0); // to make some compilers happy. + +} // JudyCheckPopSM() + +#endif // DEBUG +#endif // ! JUDYGETINLINE diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c index 2997ce19e..5b6c541ed 100644 --- a/libnetdata/libnetdata.c +++ b/libnetdata/libnetdata.c @@ -1534,3 +1534,19 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c return value; } + + +bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) { + if (unlikely(!ptr)) + return false; + return (ptr->data[idx / 64] & (1ULL << (idx % 64))); +} + +void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) { + if (unlikely(!ptr)) + return; + if (likely(value)) + ptr->data[idx / 64] |= (1ULL << (idx % 64)); + else + ptr->data[idx / 64] &= ~(1ULL << (idx % 64)); +} diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index 34062f2a6..8cc6cce9f 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -311,6 +311,12 @@ extern char *find_and_replace(const char *src, const char *find, const char *rep // Taken from linux kernel #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) +typedef struct bitmap256 { + uint64_t data[4]; +} BITMAP256; + +extern bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx); +extern void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value); extern void netdata_cleanup_and_exit(int ret) NORETURN; extern void send_statistics(const char *action, const char *action_result, const char *action_data); @@ -345,12 +351,25 @@ extern char *netdata_configured_host_prefix; #include "json/json.h" #include "health/health.h" #include "string/utf8.h" +#include "arrayalloc/arrayalloc.h" #include "onewayalloc/onewayalloc.h" #include "worker_utilization/worker_utilization.h" // BEWARE: Outside of the C code this also exists in alarm-notify.sh #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" +#define RRD_STORAGE_TIERS 5 + +static inline size_t struct_natural_alignment(size_t size) __attribute__((const)); + +#define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) +static inline size_t struct_natural_alignment(size_t size) { + if(unlikely(size % STRUCT_NATURAL_ALIGNMENT)) + size = size + STRUCT_NATURAL_ALIGNMENT - (size % STRUCT_NATURAL_ALIGNMENT); + + return size; +} + # ifdef __cplusplus } # endif diff --git a/libnetdata/log/log.c b/libnetdata/log/log.c index 90581269d..d6793b69b 100644 --- a/libnetdata/log/log.c +++ b/libnetdata/log/log.c @@ -6,7 +6,7 @@ int web_server_is_multithreaded = 1; const char *program_name = ""; -uint64_t debug_flags = DEBUG; +uint64_t debug_flags = 0; int access_log_syslog = 1; int error_log_syslog = 1; @@ -20,6 +20,14 @@ const char *stderr_filename = NULL; const char *stdout_filename = NULL; const char *facility_log = NULL; +#ifdef ENABLE_ACLK +const char *aclklog_filename = NULL; +int aclklog_fd = -1; +FILE *aclklog = NULL; +int aclklog_syslog = 1; +int aclklog_enabled = 0; +#endif + // ---------------------------------------------------------------------------- // Log facility(https://tools.ietf.org/html/rfc5424) // @@ -562,6 +570,11 @@ void reopen_all_log_files() { if(stderr_filename) open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL); +#ifdef ENABLE_ACLK + if (aclklog_enabled) + aclklog = open_log_file(aclklog_fd, aclklog, aclklog_filename, NULL, 0, &aclklog_fd); +#endif + if(stdaccess_filename) stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd); } @@ -572,6 +585,12 @@ void open_all_log_files() { open_log_file(STDOUT_FILENO, stdout, stdout_filename, &output_log_syslog, 0, NULL); open_log_file(STDERR_FILENO, stderr, stderr_filename, &error_log_syslog, 0, NULL); + +#ifdef ENABLE_ACLK + if(aclklog_enabled) + aclklog = open_log_file(aclklog_fd, aclklog, aclklog_filename, NULL, 0, &aclklog_fd); +#endif + stdaccess = open_log_file(stdaccess_fd, stdaccess, stdaccess_filename, &access_log_syslog, 1, &stdaccess_fd); } @@ -681,6 +700,26 @@ int error_log_limit(int reset) { return 0; } +void error_log_limit_reset(void) { + log_lock(); + + error_log_errors_per_period = error_log_errors_per_period_backup; + error_log_limit(1); + + log_unlock(); +} + +void error_log_limit_unlimited(void) { + log_lock(); + + error_log_errors_per_period = error_log_errors_per_period_backup; + error_log_limit(1); + + error_log_errors_per_period = ((error_log_errors_per_period_backup * 10) < 10000) ? 10000 : (error_log_errors_per_period_backup * 10); + + log_unlock(); +} + // ---------------------------------------------------------------------------- // debug log @@ -691,7 +730,7 @@ void debug_int( const char *file, const char *function, const unsigned long line log_date(date, LOG_DATE_LENGTH); va_start( args, fmt ); - printf("%s: %s DEBUG : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); + printf("%s: %s DEBUG : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); vprintf(fmt, args); va_end( args ); putchar('\n'); @@ -712,8 +751,13 @@ void info_int( const char *file, const char *function, const unsigned long line, { va_list args; + log_lock(); + // prevent logging too much - if(error_log_limit(0)) return; + if (error_log_limit(0)) { + log_unlock(); + return; + } if(error_log_syslog) { va_start( args, fmt ); @@ -724,11 +768,12 @@ void info_int( const char *file, const char *function, const unsigned long line, char date[LOG_DATE_LENGTH]; log_date(date, LOG_DATE_LENGTH); - log_lock(); - va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: %s INFO : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); - else fprintf(stderr, "%s: %s INFO : %s : ", date, program_name, netdata_thread_tag()); +#ifdef NETDATA_INTERNAL_CHECKS + fprintf(stderr, "%s: %s INFO : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, netdata_thread_tag(), line, file, function); +#else + fprintf(stderr, "%s: %s INFO : %s : ", date, program_name, netdata_thread_tag()); +#endif vfprintf( stderr, fmt, args ); va_end( args ); @@ -768,8 +813,13 @@ void error_int( const char *prefix, const char *file, const char *function, cons va_list args; + log_lock(); + // prevent logging too much - if(error_log_limit(0)) return; + if (error_log_limit(0)) { + log_unlock(); + return; + } if(error_log_syslog) { va_start( args, fmt ); @@ -780,11 +830,12 @@ void error_int( const char *prefix, const char *file, const char *function, cons char date[LOG_DATE_LENGTH]; log_date(date, LOG_DATE_LENGTH); - log_lock(); - va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: %s %-5.5s : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, prefix, netdata_thread_tag(), line, file, function); - else fprintf(stderr, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag()); +#ifdef NETDATA_INTERNAL_CHECKS + fprintf(stderr, "%s: %s %-5.5s : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, prefix, netdata_thread_tag(), line, file, function); +#else + fprintf(stderr, "%s: %s %-5.5s : %s : ", date, program_name, prefix, netdata_thread_tag()); +#endif vfprintf( stderr, fmt, args ); va_end( args ); @@ -826,8 +877,11 @@ void fatal_int( const char *file, const char *function, const unsigned long line log_lock(); va_start( args, fmt ); - if(debug_flags) fprintf(stderr, "%s: %s FATAL : %s : (%04lu@%-10.10s:%-15.15s): ", date, program_name, thread_tag, line, file, function); - else fprintf(stderr, "%s: %s FATAL : %s : ", date, program_name, thread_tag); +#ifdef NETDATA_INTERNAL_CHECKS + fprintf(stderr, "%s: %s FATAL : %s : (%04lu@%-20.20s:%-15.15s): ", date, program_name, thread_tag, line, file, function); +#else + fprintf(stderr, "%s: %s FATAL : %s : ", date, program_name, thread_tag); +#endif vfprintf( stderr, fmt, args ); va_end( args ); @@ -877,3 +931,22 @@ void log_access( const char *fmt, ... ) { netdata_mutex_unlock(&access_mutex); } } + +#ifdef ENABLE_ACLK +void log_aclk_message_bin( const char *data, const size_t data_len, int tx, const char *mqtt_topic, const char *message_name) { + if (aclklog) { + static netdata_mutex_t aclklog_mutex = NETDATA_MUTEX_INITIALIZER; + netdata_mutex_lock(&aclklog_mutex); + + char date[LOG_DATE_LENGTH]; + log_date(date, LOG_DATE_LENGTH); + fprintf(aclklog, "%s: %s Msg:\"%s\", MQTT-topic:\"%s\": ", date, tx ? "OUTGOING" : "INCOMING", message_name, mqtt_topic); + + fwrite(data, data_len, 1, aclklog); + + fputc('\n', aclklog); + + netdata_mutex_unlock(&aclklog_mutex); + } +} +#endif diff --git a/libnetdata/log/log.h b/libnetdata/log/log.h index 4cb62e955..ae86720cb 100644 --- a/libnetdata/log/log.h +++ b/libnetdata/log/log.h @@ -47,10 +47,6 @@ extern "C" { #define D_ACLK_SYNC 0x0000000800000000 #define D_SYSTEM 0x8000000000000000 -//#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS) -//#define DEBUG 0xffffffff -#define DEBUG (0) - extern int web_server_is_multithreaded; extern uint64_t debug_flags; @@ -65,6 +61,13 @@ extern const char *stderr_filename; extern const char *stdout_filename; extern const char *facility_log; +#ifdef ENABLE_ACLK +extern const char *aclklog_filename; +extern int aclklog_fd; +extern FILE *aclklog; +extern int aclklog_enabled; +#endif + extern int access_log_syslog; extern int error_log_syslog; extern int output_log_syslog; @@ -78,16 +81,15 @@ extern void reopen_all_log_files(); static inline void debug_dummy(void) {} -#define error_log_limit_reset() do { error_log_errors_per_period = error_log_errors_per_period_backup; error_log_limit(1); } while(0) -#define error_log_limit_unlimited() do { \ - error_log_limit_reset(); \ - error_log_errors_per_period = ((error_log_errors_per_period_backup * 10) < 10000) ? 10000 : (error_log_errors_per_period_backup * 10); \ - } while(0) +void error_log_limit_reset(void); +void error_log_limit_unlimited(void); #ifdef NETDATA_INTERNAL_CHECKS #define debug(type, args...) do { if(unlikely(debug_flags & type)) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0) +#define internal_error(condition, args...) do { if(unlikely(condition)) error_int("IERR", __FILE__, __FUNCTION__, __LINE__, ##args); } while(0) #else #define debug(type, args...) debug_dummy() +#define internal_error(args...) debug_dummy() #endif #define info(args...) info_int(__FILE__, __FUNCTION__, __LINE__, ##args) @@ -103,6 +105,10 @@ extern void error_int( const char *prefix, const char *file, const char *functio extern void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... ) NORETURN PRINTFLIKE(4, 5); extern void log_access( const char *fmt, ... ) PRINTFLIKE(1, 2); +#ifdef ENABLE_ACLK +extern void log_aclk_message_bin( const char *data, const size_t data_len, int tx, const char *mqtt_topic, const char *message_name); +#endif + # ifdef __cplusplus } # endif diff --git a/libnetdata/onewayalloc/onewayalloc.c b/libnetdata/onewayalloc/onewayalloc.c index a048aebf6..59c3b6859 100644 --- a/libnetdata/onewayalloc/onewayalloc.c +++ b/libnetdata/onewayalloc/onewayalloc.c @@ -1,7 +1,7 @@ #include "onewayalloc.h" -static size_t OWA_NATURAL_PAGE_SIZE = 0; -static size_t OWA_NATURAL_ALIGNMENT = sizeof(int*); +// https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html +#define OWA_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2) typedef struct owa_page { size_t stats_pages; @@ -28,15 +28,22 @@ static inline size_t natural_alignment(size_t size) { // any number of times, for any amount of memory. static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) { - if(unlikely(!OWA_NATURAL_PAGE_SIZE)) - OWA_NATURAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE); + static size_t OWA_NATURAL_PAGE_SIZE = 0; + + if(unlikely(!OWA_NATURAL_PAGE_SIZE)) { + long int page_size = sysconf(_SC_PAGE_SIZE); + if (unlikely(page_size == -1)) + OWA_NATURAL_PAGE_SIZE = 4096; + else + OWA_NATURAL_PAGE_SIZE = page_size; + } // our default page size size_t size = OWA_NATURAL_PAGE_SIZE; // make sure the new page will fit both the requested size // and the OWA_PAGE structure at its beginning - size_hint += sizeof(OWA_PAGE); + size_hint += natural_alignment(sizeof(OWA_PAGE)); // prefer the user size if it is bigger than our size if(size_hint > size) size = size_hint; @@ -75,11 +82,11 @@ static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) { head->stats_pages++; head->stats_pages_size += size; - return (ONEWAYALLOC *)page; + return page; } ONEWAYALLOC *onewayalloc_create(size_t size_hint) { - return onewayalloc_create_internal(NULL, size_hint); + return (ONEWAYALLOC *)onewayalloc_create_internal(NULL, size_hint); } void *onewayalloc_mallocz(ONEWAYALLOC *owa, size_t size) { @@ -138,11 +145,11 @@ void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_ OWA_PAGE *head = (OWA_PAGE *)owa; OWA_PAGE *page; - size_t seeking = (size_t)ptr; + uintptr_t seeking = (uintptr_t)ptr; for(page = head; page ;page = page->next) { - size_t start = (size_t)page; - size_t end = start + page->size; + uintptr_t start = (uintptr_t)page; + uintptr_t end = start + page->size; if(seeking >= start && seeking <= end) { // found it - it is ours @@ -159,6 +166,14 @@ void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_ return; } +void *onewayalloc_doublesize(ONEWAYALLOC *owa, const void *src, size_t oldsize) { + size_t newsize = oldsize * 2; + void *dst = onewayalloc_mallocz(owa, newsize); + memcpy(dst, src, oldsize); + onewayalloc_freez(owa, src); + return dst; +} + void onewayalloc_destroy(ONEWAYALLOC *owa) { if(!owa) return; diff --git a/libnetdata/onewayalloc/onewayalloc.h b/libnetdata/onewayalloc/onewayalloc.h index ed8f12f39..9eb908bfb 100644 --- a/libnetdata/onewayalloc/onewayalloc.h +++ b/libnetdata/onewayalloc/onewayalloc.h @@ -14,4 +14,6 @@ extern char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s); extern void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size); extern void onewayalloc_freez(ONEWAYALLOC *owa, const void *ptr); +extern void *onewayalloc_doublesize(ONEWAYALLOC *owa, const void *src, size_t oldsize); + #endif // ONEWAYALLOC_H diff --git a/libnetdata/required_dummies.h b/libnetdata/required_dummies.h index aa87e3964..6d51bfedd 100644 --- a/libnetdata/required_dummies.h +++ b/libnetdata/required_dummies.h @@ -23,7 +23,7 @@ void signals_unblock(void){}; void signals_reset(void){}; // callback required by eval() -int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result) +int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, NETDATA_DOUBLE *result) { (void)variable; (void)hash; diff --git a/libnetdata/socket/security.h b/libnetdata/socket/security.h index 17ecc6d05..dbf71a6fe 100644 --- a/libnetdata/socket/security.h +++ b/libnetdata/socket/security.h @@ -22,13 +22,21 @@ #define OPENSSL_VERSION_097 0x0907000L #define OPENSSL_VERSION_110 0x10100000L #define OPENSSL_VERSION_111 0x10101000L +#define OPENSSL_VERSION_300 0x30000000L # include <openssl/ssl.h> # include <openssl/err.h> +# include <openssl/evp.h> +# include <openssl/pem.h> # if (SSLEAY_VERSION_NUMBER >= OPENSSL_VERSION_097) && (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) # include <openssl/conf.h> # endif +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300 +#include <openssl/core_names.h> +#include <openssl/decoder.h> +#endif + struct netdata_ssl{ SSL *conn; //SSL connection uint32_t flags; //The flags for SSL connection diff --git a/libnetdata/statistical/statistical.c b/libnetdata/statistical/statistical.c index ba8f0c45a..ef9fe4e56 100644 --- a/libnetdata/statistical/statistical.c +++ b/libnetdata/statistical/statistical.c @@ -2,28 +2,28 @@ #include "../libnetdata.h" -LONG_DOUBLE default_single_exponential_smoothing_alpha = 0.1; +NETDATA_DOUBLE default_single_exponential_smoothing_alpha = 0.1; -void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg) { - const LONG_DOUBLE *value, *end = &series[entries]; +void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg) { + const NETDATA_DOUBLE *value, *end = &series[entries]; fprintf(stderr, "%s of %zu entries [ ", msg, entries); for(value = series; value < end ;value++) { if(value != series) fprintf(stderr, ", "); - fprintf(stderr, "%" LONG_DOUBLE_MODIFIER, *value); + fprintf(stderr, "%" NETDATA_DOUBLE_MODIFIER, *value); } - fprintf(stderr, " ] results in " CALCULATED_NUMBER_FORMAT "\n", result); + fprintf(stderr, " ] results in " NETDATA_DOUBLE_FORMAT "\n", result); } // -------------------------------------------------------------------------------------------------------------------- -inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count) { - const LONG_DOUBLE *value, *end = &series[entries]; - LONG_DOUBLE sum = 0; +inline NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count) { + const NETDATA_DOUBLE *value, *end = &series[entries]; + NETDATA_DOUBLE sum = 0; size_t c = 0; for(value = series; value < end ; value++) { - if(calculated_number_isnumber(*value)) { + if(netdata_double_isnumber(*value)) { sum += *value; c++; } @@ -35,42 +35,42 @@ inline LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size return sum; } -inline LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries) { +inline NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries) { return sum_and_count(series, entries, NULL); } -inline LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries) { +inline NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries) { size_t count = 0; - LONG_DOUBLE sum = sum_and_count(series, entries, &count); + NETDATA_DOUBLE sum = sum_and_count(series, entries, &count); if(unlikely(!count)) return NAN; - return sum / (LONG_DOUBLE)count; + return sum / (NETDATA_DOUBLE)count; } // -------------------------------------------------------------------------------------------------------------------- -LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period) { +NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period) { if(unlikely(period <= 0)) return 0.0; size_t i, count; - LONG_DOUBLE sum = 0, avg = 0; - LONG_DOUBLE p[period]; + NETDATA_DOUBLE sum = 0, avg = 0; + NETDATA_DOUBLE p[period]; for(count = 0; count < period ; count++) p[count] = 0.0; for(i = 0, count = 0; i < entries; i++) { - LONG_DOUBLE value = series[i]; - if(unlikely(!calculated_number_isnumber(value))) continue; + NETDATA_DOUBLE value = series[i]; + if(unlikely(!netdata_double_isnumber(value))) continue; if(unlikely(count < period)) { sum += value; - avg = (count == period - 1) ? sum / (LONG_DOUBLE)period : 0; + avg = (count == period - 1) ? sum / (NETDATA_DOUBLE)period : 0; } else { sum = sum - p[count % period] + value; - avg = sum / (LONG_DOUBLE)period; + avg = sum / (NETDATA_DOUBLE)period; } p[count % period] = value; @@ -83,8 +83,8 @@ LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t per // -------------------------------------------------------------------------------------------------------------------- static int qsort_compare(const void *a, const void *b) { - LONG_DOUBLE *p1 = (LONG_DOUBLE *)a, *p2 = (LONG_DOUBLE *)b; - LONG_DOUBLE n1 = *p1, n2 = *p2; + NETDATA_DOUBLE *p1 = (NETDATA_DOUBLE *)a, *p2 = (NETDATA_DOUBLE *)b; + NETDATA_DOUBLE n1 = *p1, n2 = *p2; if(unlikely(isnan(n1) || isnan(n2))) { if(isnan(n1) && !isnan(n2)) return -1; @@ -102,22 +102,22 @@ static int qsort_compare(const void *a, const void *b) { return 0; } -inline void sort_series(LONG_DOUBLE *series, size_t entries) { - qsort(series, entries, sizeof(LONG_DOUBLE), qsort_compare); +inline void sort_series(NETDATA_DOUBLE *series, size_t entries) { + qsort(series, entries, sizeof(NETDATA_DOUBLE), qsort_compare); } -inline LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries) { - LONG_DOUBLE *copy = mallocz(sizeof(LONG_DOUBLE) * entries); - memcpy(copy, series, sizeof(LONG_DOUBLE) * entries); +inline NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries) { + NETDATA_DOUBLE *copy = mallocz(sizeof(NETDATA_DOUBLE) * entries); + memcpy(copy, series, sizeof(NETDATA_DOUBLE) * entries); return copy; } -LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) { +NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries) { if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 1)) return series[0]; if(unlikely(entries == 2)) return (series[0] + series[1]) / 2; - LONG_DOUBLE average; + NETDATA_DOUBLE average; if(entries % 2 == 0) { size_t m = entries / 2; average = (series[m] + series[m + 1]) / 2; @@ -129,17 +129,17 @@ LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries) { return average; } -LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) { +NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries) { if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 1)) return series[0]; if(unlikely(entries == 2)) return (series[0] + series[1]) / 2; - LONG_DOUBLE *copy = copy_series(series, entries); + NETDATA_DOUBLE *copy = copy_series(series, entries); sort_series(copy, entries); - LONG_DOUBLE avg = median_on_sorted_series(copy, entries); + NETDATA_DOUBLE avg = median_on_sorted_series(copy, entries); freez(copy); return avg; @@ -147,18 +147,18 @@ LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries) { // -------------------------------------------------------------------------------------------------------------------- -LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period) { +NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period) { if(entries <= period) return median(series, entries); - LONG_DOUBLE *data = copy_series(series, entries); + NETDATA_DOUBLE *data = copy_series(series, entries); size_t i; for(i = period; i < entries; i++) { data[i - period] = median(&series[i - period], period); } - LONG_DOUBLE avg = median(data, entries - period); + NETDATA_DOUBLE avg = median(data, entries - period); freez(data); return avg; } @@ -166,17 +166,17 @@ LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t peri // -------------------------------------------------------------------------------------------------------------------- // http://stackoverflow.com/a/15150143/4525767 -LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) { - LONG_DOUBLE median = 0.0f; - LONG_DOUBLE average = 0.0f; +NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries) { + NETDATA_DOUBLE median = 0.0f; + NETDATA_DOUBLE average = 0.0f; size_t i; for(i = 0; i < entries ; i++) { - LONG_DOUBLE value = series[i]; - if(unlikely(!calculated_number_isnumber(value))) continue; + NETDATA_DOUBLE value = series[i]; + if(unlikely(!netdata_double_isnumber(value))) continue; average += ( value - average ) * 0.1f; // rough running average. - median += copysignl( average * 0.01, value - median ); + median += copysignndd( average * 0.01, value - median ); } return median; @@ -184,16 +184,16 @@ LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries) { // -------------------------------------------------------------------------------------------------------------------- -LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) { +NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries) { if(unlikely(entries == 0)) return NAN; if(unlikely(entries == 1)) return series[0]; - const LONG_DOUBLE *value, *end = &series[entries]; + const NETDATA_DOUBLE *value, *end = &series[entries]; size_t count; - LONG_DOUBLE sum; + NETDATA_DOUBLE sum; for(count = 0, sum = 0, value = series ; value < end ;value++) { - if(likely(calculated_number_isnumber(*value))) { + if(likely(netdata_double_isnumber(*value))) { count++; sum += *value; } @@ -202,55 +202,55 @@ LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries) { if(unlikely(count == 0)) return NAN; if(unlikely(count == 1)) return sum; - LONG_DOUBLE average = sum / (LONG_DOUBLE)count; + NETDATA_DOUBLE average = sum / (NETDATA_DOUBLE)count; for(count = 0, sum = 0, value = series ; value < end ;value++) { - if(calculated_number_isnumber(*value)) { + if(netdata_double_isnumber(*value)) { count++; - sum += powl(*value - average, 2); + sum += powndd(*value - average, 2); } } if(unlikely(count == 0)) return NAN; if(unlikely(count == 1)) return average; - LONG_DOUBLE variance = sum / (LONG_DOUBLE)(count); // remove -1 from count to have a population stddev - LONG_DOUBLE stddev = sqrtl(variance); + NETDATA_DOUBLE variance = sum / (NETDATA_DOUBLE)(count); // remove -1 from count to have a population stddev + NETDATA_DOUBLE stddev = sqrtndd(variance); return stddev; } // -------------------------------------------------------------------------------------------------------------------- -LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { +NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) { if(unlikely(entries == 0)) return NAN; if(unlikely(isnan(alpha))) alpha = default_single_exponential_smoothing_alpha; - const LONG_DOUBLE *value = series, *end = &series[entries]; - LONG_DOUBLE level = (1.0 - alpha) * (*value); + const NETDATA_DOUBLE *value = series, *end = &series[entries]; + NETDATA_DOUBLE level = (1.0 - alpha) * (*value); for(value++ ; value < end; value++) { - if(likely(calculated_number_isnumber(*value))) + if(likely(netdata_double_isnumber(*value))) level = alpha * (*value) + (1.0 - alpha) * level; } return level; } -LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha) { +NETDATA_DOUBLE single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha) { if(unlikely(entries == 0)) return NAN; if(unlikely(isnan(alpha))) alpha = default_single_exponential_smoothing_alpha; - const LONG_DOUBLE *value = &series[entries -1]; - LONG_DOUBLE level = (1.0 - alpha) * (*value); + const NETDATA_DOUBLE *value = &series[entries -1]; + NETDATA_DOUBLE level = (1.0 - alpha) * (*value); for(value++ ; value >= series; value--) { - if(likely(calculated_number_isnumber(*value))) + if(likely(netdata_double_isnumber(*value))) level = alpha * (*value) + (1.0 - alpha) * level; } @@ -260,11 +260,14 @@ LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size // -------------------------------------------------------------------------------------------------------------------- // http://grisha.org/blog/2016/02/16/triple-exponential-smoothing-forecasting-part-ii/ -LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast) { +NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, + NETDATA_DOUBLE alpha, + NETDATA_DOUBLE beta, + NETDATA_DOUBLE *forecast) { if(unlikely(entries == 0)) return NAN; - LONG_DOUBLE level, trend; + NETDATA_DOUBLE level, trend; if(unlikely(isnan(alpha))) alpha = 0.3; @@ -279,11 +282,10 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri else trend = 0; - const LONG_DOUBLE *value = series; + const NETDATA_DOUBLE *value = series; for(value++ ; value >= series; value--) { - if(likely(calculated_number_isnumber(*value))) { - - LONG_DOUBLE last_level = level; + if(likely(netdata_double_isnumber(*value))) { + NETDATA_DOUBLE last_level = level; level = alpha * *value + (1.0 - alpha) * (level + trend); trend = beta * (level - last_level) + (1.0 - beta) * trend; @@ -320,24 +322,26 @@ LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entri * s[t] = γ (Y[t] / a[t]) + (1-γ) s[t-p] */ static int __HoltWinters( - const LONG_DOUBLE *series, + const NETDATA_DOUBLE *series, int entries, // start_time + h - LONG_DOUBLE alpha, // alpha parameter of Holt-Winters Filter. - LONG_DOUBLE beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing. - LONG_DOUBLE gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted. + NETDATA_DOUBLE alpha, // alpha parameter of Holt-Winters Filter. + NETDATA_DOUBLE + beta, // beta parameter of Holt-Winters Filter. If set to 0, the function will do exponential smoothing. + NETDATA_DOUBLE + gamma, // gamma parameter used for the seasonal component. If set to 0, an non-seasonal model is fitted. const int *seasonal, const int *period, - const LONG_DOUBLE *a, // Start value for level (a[0]). - const LONG_DOUBLE *b, // Start value for trend (b[0]). - LONG_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0]) + const NETDATA_DOUBLE *a, // Start value for level (a[0]). + const NETDATA_DOUBLE *b, // Start value for trend (b[0]). + NETDATA_DOUBLE *s, // Vector of start values for the seasonal component (s_1[0] ... s_p[0]) /* return values */ - LONG_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing - LONG_DOUBLE *level, // Estimated values for the level component (size entries - t + 2) - LONG_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2) - LONG_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2) + NETDATA_DOUBLE *SSE, // The final sum of squared errors achieved in optimizing + NETDATA_DOUBLE *level, // Estimated values for the level component (size entries - t + 2) + NETDATA_DOUBLE *trend, // Estimated values for the trend component (size entries - t + 2) + NETDATA_DOUBLE *season // Estimated values for the seasonal component (size entries - t + 2) ) { if(unlikely(entries < 4)) @@ -345,13 +349,13 @@ static int __HoltWinters( int start_time = 2; - LONG_DOUBLE res = 0, xhat = 0, stmp = 0; + NETDATA_DOUBLE res = 0, xhat = 0, stmp = 0; int i, i0, s0; /* copy start values to the beginning of the vectors */ level[0] = *a; if(beta > 0) trend[0] = *b; - if(gamma > 0) memcpy(season, s, *period * sizeof(LONG_DOUBLE)); + if(gamma > 0) memcpy(season, s, *period * sizeof(NETDATA_DOUBLE)); for(i = start_time - 1; i < entries; i++) { /* indices for period i */ @@ -397,7 +401,11 @@ static int __HoltWinters( return 1; } -LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast) { +NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries, + NETDATA_DOUBLE alpha, + NETDATA_DOUBLE beta, + NETDATA_DOUBLE gamma, + NETDATA_DOUBLE *forecast) { if(unlikely(isnan(alpha))) alpha = 0.3; @@ -409,15 +417,15 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a int seasonal = 0; int period = 0; - LONG_DOUBLE a0 = series[0]; - LONG_DOUBLE b0 = 0; - LONG_DOUBLE s[] = {}; + NETDATA_DOUBLE a0 = series[0]; + NETDATA_DOUBLE b0 = 0; + NETDATA_DOUBLE s[] = {}; - LONG_DOUBLE errors = 0.0; + NETDATA_DOUBLE errors = 0.0; size_t nb_computations = entries; - LONG_DOUBLE *estimated_level = callocz(nb_computations, sizeof(LONG_DOUBLE)); - LONG_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(LONG_DOUBLE)); - LONG_DOUBLE *estimated_season = callocz(nb_computations, sizeof(LONG_DOUBLE)); + NETDATA_DOUBLE *estimated_level = callocz(nb_computations, sizeof(NETDATA_DOUBLE)); + NETDATA_DOUBLE *estimated_trend = callocz(nb_computations, sizeof(NETDATA_DOUBLE)); + NETDATA_DOUBLE *estimated_season = callocz(nb_computations, sizeof(NETDATA_DOUBLE)); int ret = __HoltWinters( series, @@ -436,7 +444,7 @@ LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE a estimated_season ); - LONG_DOUBLE value = estimated_level[nb_computations - 1]; + NETDATA_DOUBLE value = estimated_level[nb_computations - 1]; if(forecast) *forecast = 0.0; diff --git a/libnetdata/statistical/statistical.h b/libnetdata/statistical/statistical.h index bbf241ff2..9496e0e7f 100644 --- a/libnetdata/statistical/statistical.h +++ b/libnetdata/statistical/statistical.h @@ -5,22 +5,30 @@ #include "../libnetdata.h" -extern void log_series_to_stderr(LONG_DOUBLE *series, size_t entries, calculated_number result, const char *msg); +extern void log_series_to_stderr(NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE result, const char *msg); -extern LONG_DOUBLE average(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE moving_average(const LONG_DOUBLE *series, size_t entries, size_t period); -extern LONG_DOUBLE median(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE moving_median(const LONG_DOUBLE *series, size_t entries, size_t period); -extern LONG_DOUBLE running_median_estimate(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE standard_deviation(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE single_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); -extern LONG_DOUBLE single_exponential_smoothing_reverse(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha); -extern LONG_DOUBLE double_exponential_smoothing(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE *forecast); -extern LONG_DOUBLE holtwinters(const LONG_DOUBLE *series, size_t entries, LONG_DOUBLE alpha, LONG_DOUBLE beta, LONG_DOUBLE gamma, LONG_DOUBLE *forecast); -extern LONG_DOUBLE sum_and_count(const LONG_DOUBLE *series, size_t entries, size_t *count); -extern LONG_DOUBLE sum(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE median_on_sorted_series(const LONG_DOUBLE *series, size_t entries); -extern LONG_DOUBLE *copy_series(const LONG_DOUBLE *series, size_t entries); -extern void sort_series(LONG_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE average(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE moving_average(const NETDATA_DOUBLE *series, size_t entries, size_t period); +extern NETDATA_DOUBLE median(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE moving_median(const NETDATA_DOUBLE *series, size_t entries, size_t period); +extern NETDATA_DOUBLE running_median_estimate(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE standard_deviation(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE single_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha); +extern NETDATA_DOUBLE +single_exponential_smoothing_reverse(const NETDATA_DOUBLE *series, size_t entries, NETDATA_DOUBLE alpha); +extern NETDATA_DOUBLE double_exponential_smoothing(const NETDATA_DOUBLE *series, size_t entries, + NETDATA_DOUBLE alpha, + NETDATA_DOUBLE beta, + NETDATA_DOUBLE *forecast); +extern NETDATA_DOUBLE holtwinters(const NETDATA_DOUBLE *series, size_t entries, + NETDATA_DOUBLE alpha, + NETDATA_DOUBLE beta, + NETDATA_DOUBLE gamma, + NETDATA_DOUBLE *forecast); +extern NETDATA_DOUBLE sum_and_count(const NETDATA_DOUBLE *series, size_t entries, size_t *count); +extern NETDATA_DOUBLE sum(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE median_on_sorted_series(const NETDATA_DOUBLE *series, size_t entries); +extern NETDATA_DOUBLE *copy_series(const NETDATA_DOUBLE *series, size_t entries); +extern void sort_series(NETDATA_DOUBLE *series, size_t entries); #endif //NETDATA_STATISTICAL_H diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c index ba7a66874..6a8b68c11 100644 --- a/libnetdata/storage_number/storage_number.c +++ b/libnetdata/storage_number/storage_number.c @@ -2,12 +2,7 @@ #include "../libnetdata.h" -#define get_storage_number_flags(value) \ - ((((storage_number)(value)) & (1 << 24)) | \ - (((storage_number)(value)) & (1 << 25)) | \ - (((storage_number)(value)) & (1 << 26))) - -storage_number pack_storage_number(calculated_number value, uint32_t flags) { +storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) { // bit 32 = sign 0:positive, 1:negative // bit 31 = 0:divide, 1:multiply // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) @@ -16,44 +11,48 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) { // bit 25 SN_ANOMALY_BIT = 0: anomalous, 1: not anomalous // bit 24 to bit 1 = the value - storage_number r = get_storage_number_flags(flags); - if(!value) - goto RET_SN; + if(unlikely(fpclassify(value) == FP_NAN || fpclassify(value) == FP_INFINITE)) + return SN_EMPTY_SLOT; + + storage_number r = flags & SN_USER_FLAGS; + + if(unlikely(fpclassify(value) == FP_ZERO || fpclassify(value) == FP_SUBNORMAL)) + return r; int m = 0; - calculated_number n = value, factor = 10; + NETDATA_DOUBLE n = value, factor = 10; // if the value is negative // add the sign bit and make it positive if(n < 0) { - r += (1 << 31); // the sign bit 32 + r += SN_FLAG_NEGATIVE; // the sign bit 32 n = -n; } if(n / 10000000.0 > 0x00ffffff) { factor = 100; - r |= SN_EXISTS_100; + r |= SN_FLAG_NOT_EXISTS_MUL100; } // make its integer part fit in 0x00ffffff // by dividing it by 10 up to 7 times // and increasing the multiplier - while(m < 7 && n > (calculated_number)0x00ffffff) { + while(m < 7 && n > (NETDATA_DOUBLE)0x00ffffff) { n /= factor; m++; } if(m) { - // the value was too big and we divided it - // so we add a multiplier to unpack it - r += (1 << 30) + (m << 27); // the multiplier m + // the value was too big, and we divided it + // so, we add a multiplier to unpack it + r += SN_FLAG_MULTIPLY + (m << 27); // the multiplier m - if(n > (calculated_number)0x00ffffff) { + if(n > (NETDATA_DOUBLE)0x00ffffff) { #ifdef NETDATA_INTERNAL_CHECKS - error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); + error("Number " NETDATA_DOUBLE_FORMAT " is too big.", value); #endif r += 0x00ffffff; - goto RET_SN; + return r; } } else { @@ -62,18 +61,18 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) { // while the value is below 0x0019999e we can // multiply it by 10, up to 7 times, increasing // the multiplier - while(m < 7 && n < (calculated_number)0x0019999e) { + while(m < 7 && n < (NETDATA_DOUBLE)0x0019999e) { n *= 10; m++; } - if (unlikely(n > (calculated_number) (0x00ffffff))) { + if (unlikely(n > (NETDATA_DOUBLE)0x00ffffff)) { n /= 10; m--; } - // the value was small enough and we multiplied it - // so we add a divider to unpack it - r += (0 << 30) + (m << 27); // the divider m + // the value was small enough, and we multiplied it + // so, we add a divider to unpack it + r += (m << 27); // the divider m } #ifdef STORAGE_WITH_MATH @@ -84,15 +83,11 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) { r += (storage_number)n; #endif -RET_SN: - if (r == SN_EMPTY_SLOT) - r = SN_ANOMALOUS_ZERO; - return r; } // Lookup table to make storage number unpacking efficient. -calculated_number unpack_storage_number_lut10x[4 * 8]; +NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8]; __attribute__((constructor)) void initialize_lut(void) { // The lookup table is partitioned in 4 subtables based on the @@ -109,7 +104,7 @@ __attribute__((constructor)) void initialize_lut(void) { } /* -int print_calculated_number(char *str, calculated_number value) +int print_netdata_double(char *str, NETDATA_DOUBLE value) { char *wstr = str; @@ -119,9 +114,9 @@ int print_calculated_number(char *str, calculated_number value) #ifdef STORAGE_WITH_MATH // without llrintl() there are rounding problems // for example 0.9 becomes 0.89 - unsigned long long uvalue = (unsigned long long int) llrintl(value * (calculated_number)100000); + unsigned long long uvalue = (unsigned long long int) llrintl(value * (NETDATA_DOUBLE)100000); #else - unsigned long long uvalue = value * (calculated_number)100000; + unsigned long long uvalue = value * (NETDATA_DOUBLE)100000; #endif wstr = print_number_llu_r_smart(str, uvalue); @@ -165,8 +160,8 @@ int print_calculated_number(char *str, calculated_number value) } */ -int print_calculated_number(char *str, calculated_number value) { - // info("printing number " CALCULATED_NUMBER_FORMAT, value); +int print_netdata_double(char *str, NETDATA_DOUBLE value) { + // info("printing number " NETDATA_DOUBLE_FORMAT, value); char integral_str[50], fractional_str[50]; char *wstr = str; @@ -176,22 +171,22 @@ int print_calculated_number(char *str, calculated_number value) { value = -value; } - calculated_number integral, fractional; + NETDATA_DOUBLE integral, fractional; #ifdef STORAGE_WITH_MATH - fractional = calculated_number_modf(value, &integral) * 10000000.0; + fractional = modfndd(value, &integral) * 10000000.0; #else fractional = ((unsigned long long)(value * 10000000ULL) % 10000000ULL); #endif unsigned long long integral_int = (unsigned long long)integral; - unsigned long long fractional_int = (unsigned long long)calculated_number_llrint(fractional); + unsigned long long fractional_int = (unsigned long long)llrintndd(fractional); if(unlikely(fractional_int >= 10000000)) { integral_int += 1; fractional_int -= 10000000; } - // info("integral " CALCULATED_NUMBER_FORMAT " (%llu), fractional " CALCULATED_NUMBER_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int); + // info("integral " NETDATA_DOUBLE_FORMAT " (%llu), fractional " NETDATA_DOUBLE_FORMAT " (%llu)", integral, integral_int, fractional, fractional_int); char *istre; if(unlikely(integral_int == 0)) { diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h index 7e7b511b0..faea47751 100644 --- a/libnetdata/storage_number/storage_number.h +++ b/libnetdata/storage_number/storage_number.h @@ -3,92 +3,114 @@ #ifndef NETDATA_STORAGE_NUMBER_H #define NETDATA_STORAGE_NUMBER_H 1 +#include <math.h> #include "../libnetdata.h" -#ifdef NETDATA_WITHOUT_LONG_DOUBLE +#ifdef NETDATA_WITH_LONG_DOUBLE -#define powl pow -#define modfl modf -#define llrintl llrint -#define roundl round -#define sqrtl sqrt -#define copysignl copysign -#define strtold strtod +typedef long double NETDATA_DOUBLE; +#define NETDATA_DOUBLE_FORMAT "%0.7Lf" +#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0Lf" +#define NETDATA_DOUBLE_FORMAT_AUTO "%Lf" +#define NETDATA_DOUBLE_MODIFIER "Lf" -typedef double calculated_number; -#define CALCULATED_NUMBER_FORMAT "%0.7f" -#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0f" -#define CALCULATED_NUMBER_FORMAT_AUTO "%f" +#define NETDATA_DOUBLE_MAX LDBL_MAX -#define LONG_DOUBLE_MODIFIER "f" -typedef double LONG_DOUBLE; +#define strtondd(s, endptr) strtold(s, endptr) +#define powndd(x, y) powl(x, y) +#define llrintndd(x) llrintl(x) +#define roundndd(x) roundl(x) +#define sqrtndd(x) sqrtl(x) +#define copysignndd(x, y) copysignl(x, y) +#define modfndd(x, y) modfl(x, y) +#define fabsndd(x) fabsl(x) -#else // NETDATA_WITHOUT_LONG_DOUBLE +#else // NETDATA_WITH_LONG_DOUBLE -typedef long double calculated_number; -#define CALCULATED_NUMBER_FORMAT "%0.7Lf" -#define CALCULATED_NUMBER_FORMAT_ZERO "%0.0Lf" -#define CALCULATED_NUMBER_FORMAT_AUTO "%Lf" +typedef double NETDATA_DOUBLE; +#define NETDATA_DOUBLE_FORMAT "%0.7f" +#define NETDATA_DOUBLE_FORMAT_ZERO "%0.0f" +#define NETDATA_DOUBLE_FORMAT_AUTO "%f" +#define NETDATA_DOUBLE_MODIFIER "f" -#define LONG_DOUBLE_MODIFIER "Lf" -typedef long double LONG_DOUBLE; +#define NETDATA_DOUBLE_MAX DBL_MAX -#endif // NETDATA_WITHOUT_LONG_DOUBLE +#define strtondd(s, endptr) strtod(s, endptr) +#define powndd(x, y) pow(x, y) +#define llrintndd(x) llrint(x) +#define roundndd(x) round(x) +#define sqrtndd(x) sqrt(x) +#define copysignndd(x, y) copysign(x, y) +#define modfndd(x, y) modf(x, y) +#define fabsndd(x) fabs(x) -//typedef long long calculated_number; -//#define CALCULATED_NUMBER_FORMAT "%lld" +#endif // NETDATA_WITH_LONG_DOUBLE typedef long long collected_number; #define COLLECTED_NUMBER_FORMAT "%lld" -/* -typedef long double collected_number; -#define COLLECTED_NUMBER_FORMAT "%0.7Lf" -*/ +#define epsilonndd (NETDATA_DOUBLE)0.0000001 +#define considered_equal_ndd(a, b) (fabsndd((a) - (b)) < epsilonndd) -#define calculated_number_modf(x, y) modfl(x, y) -#define calculated_number_llrint(x) llrintl(x) -#define calculated_number_round(x) roundl(x) -#define calculated_number_fabs(x) fabsl(x) -#define calculated_number_pow(x, y) powl(x, y) -#define calculated_number_epsilon (calculated_number)0.0000001 +#if defined(HAVE_ISFINITE) || defined(isfinite) +// The isfinite() macro shall determine whether its argument has a +// finite value (zero, subnormal, or normal, and not infinite or NaN). +#define netdata_double_isnumber(a) (isfinite(a)) +#elif defined(HAVE_FINITE) || defined(finite) +#define netdata_double_isnumber(a) (finite(a)) +#else +#define netdata_double_isnumber(a) (fpclassify(a) != FP_NAN && fpclassify(a) != FP_INFINITE) +#endif -#define calculated_number_equal(a, b) (calculated_number_fabs((a) - (b)) < calculated_number_epsilon) +typedef uint32_t storage_number; -#define calculated_number_isnumber(a) (!(fpclassify(a) & (FP_NAN|FP_INFINITE))) +typedef struct storage_number_tier1 { + float sum_value; + float min_value; + float max_value; + uint16_t count; + uint16_t anomaly_count; +} storage_number_tier1_t; -typedef uint32_t storage_number; #define STORAGE_NUMBER_FORMAT "%u" -#define SN_ANOMALY_BIT (1 << 24) // the anomaly bit of the value -#define SN_EXISTS_RESET (1 << 25) // the value has been overflown -#define SN_EXISTS_100 (1 << 26) // very large value (multiplier is 100 instead of 10) +typedef enum { + SN_FLAG_NONE = 0, + SN_FLAG_NOT_ANOMALOUS = (1 << 24), // the anomaly bit of the value (0:anomalous, 1:not anomalous) + SN_FLAG_RESET = (1 << 25), // the value has been overflown + SN_FLAG_NOT_EXISTS_MUL100 = (1 << 26), // very large value (multiplier is 100 instead of 10) + SN_FLAG_MULTIPLY = (1 << 30), // multiply, else divide + SN_FLAG_NEGATIVE = (1 << 31), // negative, else positive +} SN_FLAGS; -#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT +#define SN_USER_FLAGS (SN_FLAG_NOT_ANOMALOUS | SN_FLAG_RESET) -#define SN_EMPTY_SLOT 0x00000000 +// default flags for all storage numbers +// anomaly bit is reversed, so we set it by default +#define SN_DEFAULT_FLAGS SN_FLAG_NOT_ANOMALOUS // When the calculated number is zero and the value is anomalous (ie. it's bit // is zero) we want to return a storage_number representation that is // different from the empty slot. We achieve this by mapping zero to // SN_EXISTS_100. Unpacking the SN_EXISTS_100 value will return zero because // its fraction field (as well as its exponent factor field) will be zero. -#define SN_ANOMALOUS_ZERO SN_EXISTS_100 +#define SN_EMPTY_SLOT SN_FLAG_NOT_EXISTS_MUL100 // checks -#define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT) -#define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0) +#define does_storage_number_exist(value) (((storage_number)(value)) != SN_EMPTY_SLOT) +#define did_storage_number_reset(value) ((((storage_number)(value)) & SN_FLAG_RESET)) +#define is_storage_number_anomalous(value) (does_storage_number_exist(value) && !(((storage_number)(value)) & SN_FLAG_NOT_ANOMALOUS)) -storage_number pack_storage_number(calculated_number value, uint32_t flags); -static inline calculated_number unpack_storage_number(storage_number value) __attribute__((const)); +storage_number pack_storage_number(NETDATA_DOUBLE value, SN_FLAGS flags) __attribute__((const)); +static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) __attribute__((const)); -int print_calculated_number(char *str, calculated_number value); +int print_netdata_double(char *str, NETDATA_DOUBLE value); -// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE -#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) -#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) -#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) -#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1<<27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) +// sign div/mul <--- multiplier / divider ---> 10/100 RESET EXISTS VALUE +#define STORAGE_NUMBER_POSITIVE_MAX_RAW (storage_number)( (0 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) +#define STORAGE_NUMBER_POSITIVE_MIN_RAW (storage_number)( (0 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) +#define STORAGE_NUMBER_NEGATIVE_MAX_RAW (storage_number)( (1 << 31) | (0 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (0 << 26) | (0 << 25) | (1 << 24) | 0x00000001 ) +#define STORAGE_NUMBER_NEGATIVE_MIN_RAW (storage_number)( (1 << 31) | (1 << 30) | (1 << 29) | (1 << 28) | (1 << 27) | (1 << 26) | (0 << 25) | (1 << 24) | 0x00ffffff ) // accepted accuracy loss #define ACCURACY_LOSS_ACCEPTED_PERCENT 0.0001 @@ -99,40 +121,112 @@ int print_calculated_number(char *str, calculated_number value); #define MAX_INCREMENTAL_PERCENT_RATE 10 -static inline calculated_number unpack_storage_number(storage_number value) { - extern calculated_number unpack_storage_number_lut10x[4 * 8]; +static inline NETDATA_DOUBLE unpack_storage_number(storage_number value) { + extern NETDATA_DOUBLE unpack_storage_number_lut10x[4 * 8]; - if(!value) return 0; + if(unlikely(value == SN_EMPTY_SLOT)) + return NAN; int sign = 1, exp = 0; int factor = 0; // bit 32 = 0:positive, 1:negative - if(unlikely(value & (1 << 31))) + if(unlikely(value & SN_FLAG_NEGATIVE)) sign = -1; // bit 31 = 0:divide, 1:multiply - if(unlikely(value & (1 << 30))) + if(unlikely(value & SN_FLAG_MULTIPLY)) exp = 1; - // bit 27 SN_EXISTS_100 - if(unlikely(value & (1 << 26))) + // bit 27 SN_FLAG_NOT_EXISTS_MUL100 + if(unlikely(value & SN_FLAG_NOT_EXISTS_MUL100)) factor = 1; - // bit 26 SN_EXISTS_RESET - // bit 25 SN_ANOMALY_BIT + // bit 26 SN_FLAG_RESET + // bit 25 SN_FLAG_NOT_ANOMALOUS // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) - int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27; + int mul = (int)((value & ((1<<29)|(1<<28)|(1<<27))) >> 27); // bit 24 to bit 1 = the value, so remove all other bits value ^= value & ((1<<31)|(1<<30)|(1<<29)|(1<<28)|(1<<27)|(1<<26)|(1<<25)|(1<<24)); - calculated_number n = value; + NETDATA_DOUBLE n = value; // fprintf(stderr, "UNPACK: %08X, sign = %d, exp = %d, mul = %d, factor = %d, n = " CALCULATED_NUMBER_FORMAT "\n", value, sign, exp, mul, factor, n); return sign * unpack_storage_number_lut10x[(factor * 16) + (exp * 8) + mul] * n; } +static inline NETDATA_DOUBLE str2ndd(const char *s, char **endptr) { + int negative = 0; + const char *start = s; + unsigned long long integer_part = 0; + unsigned long decimal_part = 0; + size_t decimal_digits = 0; + + switch(*s) { + case '-': + s++; + negative = 1; + break; + + case '+': + s++; + break; + + case 'n': + if(s[1] == 'a' && s[2] == 'n') { + if(endptr) *endptr = (char *)&s[3]; + return NAN; + } + break; + + case 'i': + if(s[1] == 'n' && s[2] == 'f') { + if(endptr) *endptr = (char *)&s[3]; + return INFINITY; + } + break; + + default: + break; + } + + while (*s >= '0' && *s <= '9') { + integer_part = (integer_part * 10) + (*s - '0'); + s++; + } + + if(unlikely(*s == '.')) { + decimal_part = 0; + s++; + + while (*s >= '0' && *s <= '9') { + decimal_part = (decimal_part * 10) + (*s - '0'); + s++; + decimal_digits++; + } + } + + if(unlikely(*s == 'e' || *s == 'E')) + return strtondd(start, endptr); + + if(unlikely(endptr)) + *endptr = (char *)s; + + if(unlikely(negative)) { + if(unlikely(decimal_digits)) + return -((NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits)); + else + return -((NETDATA_DOUBLE)integer_part); + } + else { + if(unlikely(decimal_digits)) + return (NETDATA_DOUBLE)integer_part + (NETDATA_DOUBLE)decimal_part / powndd(10.0, decimal_digits); + else + return (NETDATA_DOUBLE)integer_part; + } +} + #endif /* NETDATA_STORAGE_NUMBER_H */ diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c index f90521cab..19309e5c2 100644 --- a/libnetdata/storage_number/tests/test_storage_number.c +++ b/libnetdata/storage_number/tests/test_storage_number.c @@ -11,34 +11,34 @@ static void test_number_printing(void **state) char value[50]; - print_calculated_number(value, 0); + print_netdata_double(value, 0); assert_string_equal(value, "0"); - print_calculated_number(value, 0.0000001); + print_netdata_double(value, 0.0000001); assert_string_equal(value, "0.0000001"); - print_calculated_number(value, 0.00000009); + print_netdata_double(value, 0.00000009); assert_string_equal(value, "0.0000001"); - print_calculated_number(value, 0.000000001); + print_netdata_double(value, 0.000000001); assert_string_equal(value, "0"); - print_calculated_number(value, 99.99999999999999999); + print_netdata_double(value, 99.99999999999999999); assert_string_equal(value, "100"); - print_calculated_number(value, -99.99999999999999999); + print_netdata_double(value, -99.99999999999999999); assert_string_equal(value, "-100"); - print_calculated_number(value, 123.4567890123456789); + print_netdata_double(value, 123.4567890123456789); assert_string_equal(value, "123.456789"); - print_calculated_number(value, 9999.9999999); + print_netdata_double(value, 9999.9999999); assert_string_equal(value, "9999.9999999"); - print_calculated_number(value, -9999.9999999); + print_netdata_double(value, -9999.9999999); assert_string_equal(value, "-9999.9999999"); - print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS))); + print_netdata_double(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS))); assert_string_equal(value, "16.77722"); } diff --git a/libnetdata/tests/test_str2ld.c b/libnetdata/tests/test_str2ld.c index 01d8677f0..8b97a70f8 100644 --- a/libnetdata/tests/test_str2ld.c +++ b/libnetdata/tests/test_str2ld.c @@ -24,8 +24,8 @@ static void test_str2ld(void **state) for (int i = 0; values[i]; i++) { char *e_mine = "hello", *e_sys = "world"; - LONG_DOUBLE mine = str2ld(values[i], &e_mine); - LONG_DOUBLE sys = strtold(values[i], &e_sys); + NETDATA_DOUBLE mine = str2ndd(values[i], &e_mine); + NETDATA_DOUBLE sys = strtondd(values[i], &e_sys); if (isnan(mine)) assert_true(isnan(sys)); |