diff options
Diffstat (limited to 'tools/testing/radix-tree')
37 files changed, 3231 insertions, 0 deletions
diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore new file mode 100644 index 000000000..d97151640 --- /dev/null +++ b/tools/testing/radix-tree/.gitignore @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +generated/map-shift.h +idr.c +idr-test +main +multiorder +radix-tree.c +xarray diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile new file mode 100644 index 000000000..aa6abfe07 --- /dev/null +++ b/tools/testing/radix-tree/Makefile @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0 + +CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ + -fsanitize=undefined +LDFLAGS += -fsanitize=address -fsanitize=undefined +LDLIBS+= -lpthread -lurcu +TARGETS = main idr-test multiorder xarray +CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o +OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ + regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ + iteration_check_2.o benchmark.o + +ifndef SHIFT + SHIFT=3 +endif + +ifeq ($(BUILD), 32) + CFLAGS += -m32 + LDFLAGS += -m32 +endif + +targets: generated/map-shift.h $(TARGETS) + +main: $(OFILES) + +idr-test.o: ../../../lib/test_ida.c +idr-test: idr-test.o $(CORE_OFILES) + +xarray: $(CORE_OFILES) + +multiorder: multiorder.o $(CORE_OFILES) + +clean: + $(RM) $(TARGETS) *.o radix-tree.c idr.c generated/map-shift.h + +vpath %.c ../../lib + +$(OFILES): Makefile *.h */*.h generated/map-shift.h \ + ../../include/linux/*.h \ + ../../include/asm/*.h \ + ../../../include/linux/xarray.h \ + ../../../include/linux/radix-tree.h \ + ../../../include/linux/idr.h + +radix-tree.c: ../../../lib/radix-tree.c + sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@ + +idr.c: ../../../lib/idr.c + sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@ + +xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c + +generated/map-shift.h: + @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ + echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ + generated/map-shift.h; \ + fi diff --git a/tools/testing/radix-tree/benchmark.c b/tools/testing/radix-tree/benchmark.c new file mode 100644 index 000000000..523c79f22 --- /dev/null +++ b/tools/testing/radix-tree/benchmark.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * benchmark.c: + * Author: Konstantin Khlebnikov <koct9i@gmail.com> + */ +#include <linux/radix-tree.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <time.h> +#include "test.h" + +#define NSEC_PER_SEC 1000000000L + +static long long benchmark_iter(struct radix_tree_root *root, bool tagged) +{ + volatile unsigned long sink = 0; + struct radix_tree_iter iter; + struct timespec start, finish; + long long nsec; + int l, loops = 1; + void **slot; + +#ifdef BENCHMARK +again: +#endif + clock_gettime(CLOCK_MONOTONIC, &start); + for (l = 0; l < loops; l++) { + if (tagged) { + radix_tree_for_each_tagged(slot, root, &iter, 0, 0) + sink ^= (unsigned long)slot; + } else { + radix_tree_for_each_slot(slot, root, &iter, 0) + sink ^= (unsigned long)slot; + } + } + clock_gettime(CLOCK_MONOTONIC, &finish); + + nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + + (finish.tv_nsec - start.tv_nsec); + +#ifdef BENCHMARK + if (loops == 1 && nsec * 5 < NSEC_PER_SEC) { + loops = NSEC_PER_SEC / nsec / 4 + 1; + goto again; + } +#endif + + nsec /= loops; + return nsec; +} + +static void benchmark_insert(struct radix_tree_root *root, + unsigned long size, unsigned long step) +{ + struct timespec start, finish; + unsigned long index; + long long nsec; + + clock_gettime(CLOCK_MONOTONIC, &start); + + for (index = 0 ; index < size ; index += step) + item_insert(root, index); + + clock_gettime(CLOCK_MONOTONIC, &finish); + + nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + + (finish.tv_nsec - start.tv_nsec); + + printv(2, "Size: %8ld, step: %8ld, insertion: %15lld ns\n", + size, step, nsec); +} + +static void benchmark_tagging(struct radix_tree_root *root, + unsigned long size, unsigned long step) +{ + struct timespec start, finish; + unsigned long index; + long long nsec; + + clock_gettime(CLOCK_MONOTONIC, &start); + + for (index = 0 ; index < size ; index += step) + radix_tree_tag_set(root, index, 0); + + clock_gettime(CLOCK_MONOTONIC, &finish); + + nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + + (finish.tv_nsec - start.tv_nsec); + + printv(2, "Size: %8ld, step: %8ld, tagging: %17lld ns\n", + size, step, nsec); +} + +static void benchmark_delete(struct radix_tree_root *root, + unsigned long size, unsigned long step) +{ + struct timespec start, finish; + unsigned long index; + long long nsec; + + clock_gettime(CLOCK_MONOTONIC, &start); + + for (index = 0 ; index < size ; index += step) + item_delete(root, index); + + clock_gettime(CLOCK_MONOTONIC, &finish); + + nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC + + (finish.tv_nsec - start.tv_nsec); + + printv(2, "Size: %8ld, step: %8ld, deletion: %16lld ns\n", + size, step, nsec); +} + +static void benchmark_size(unsigned long size, unsigned long step) +{ + RADIX_TREE(tree, GFP_KERNEL); + long long normal, tagged; + + benchmark_insert(&tree, size, step); + benchmark_tagging(&tree, size, step); + + tagged = benchmark_iter(&tree, true); + normal = benchmark_iter(&tree, false); + + printv(2, "Size: %8ld, step: %8ld, tagged iteration: %8lld ns\n", + size, step, tagged); + printv(2, "Size: %8ld, step: %8ld, normal iteration: %8lld ns\n", + size, step, normal); + + benchmark_delete(&tree, size, step); + + item_kill_tree(&tree); + rcu_barrier(); +} + +void benchmark(void) +{ + unsigned long size[] = {1 << 10, 1 << 20, 0}; + unsigned long step[] = {1, 2, 7, 15, 63, 64, 65, + 128, 256, 512, 12345, 0}; + int c, s; + + printv(1, "starting benchmarks\n"); + printv(1, "RADIX_TREE_MAP_SHIFT = %d\n", RADIX_TREE_MAP_SHIFT); + + for (c = 0; size[c]; c++) + for (s = 0; step[s]; s++) + benchmark_size(size[c], step[s]); +} diff --git a/tools/testing/radix-tree/bitmap.c b/tools/testing/radix-tree/bitmap.c new file mode 100644 index 000000000..66ec4a24a --- /dev/null +++ b/tools/testing/radix-tree/bitmap.c @@ -0,0 +1,23 @@ +/* lib/bitmap.c pulls in at least two other files. */ + +#include <linux/bitmap.h> + +void bitmap_clear(unsigned long *map, unsigned int start, int len) +{ + unsigned long *p = map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + while (len - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + len -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + p++; + } + if (len) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *p &= ~mask_to_clear; + } +} diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h new file mode 100644 index 000000000..2218b3cc1 --- /dev/null +++ b/tools/testing/radix-tree/generated/autoconf.h @@ -0,0 +1 @@ +#define CONFIG_XARRAY_MULTI 1 diff --git a/tools/testing/radix-tree/idr-test.c b/tools/testing/radix-tree/idr-test.c new file mode 100644 index 000000000..6ce7460f3 --- /dev/null +++ b/tools/testing/radix-tree/idr-test.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * idr-test.c: Test the IDR API + * Copyright (c) 2016 Matthew Wilcox <willy@infradead.org> + */ +#include <linux/bitmap.h> +#include <linux/idr.h> +#include <linux/slab.h> +#include <linux/kernel.h> +#include <linux/errno.h> + +#include "test.h" + +#define DUMMY_PTR ((void *)0x10) + +int item_idr_free(int id, void *p, void *data) +{ + struct item *item = p; + assert(item->index == id); + free(p); + + return 0; +} + +void item_idr_remove(struct idr *idr, int id) +{ + struct item *item = idr_find(idr, id); + assert(item->index == id); + idr_remove(idr, id); + free(item); +} + +void idr_alloc_test(void) +{ + unsigned long i; + DEFINE_IDR(idr); + + assert(idr_alloc_cyclic(&idr, DUMMY_PTR, 0, 0x4000, GFP_KERNEL) == 0); + assert(idr_alloc_cyclic(&idr, DUMMY_PTR, 0x3ffd, 0x4000, GFP_KERNEL) == 0x3ffd); + idr_remove(&idr, 0x3ffd); + idr_remove(&idr, 0); + + for (i = 0x3ffe; i < 0x4003; i++) { + int id; + struct item *item; + + if (i < 0x4000) + item = item_create(i, 0); + else + item = item_create(i - 0x3fff, 0); + + id = idr_alloc_cyclic(&idr, item, 1, 0x4000, GFP_KERNEL); + assert(id == item->index); + } + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); +} + +void idr_replace_test(void) +{ + DEFINE_IDR(idr); + + idr_alloc(&idr, (void *)-1, 10, 11, GFP_KERNEL); + idr_replace(&idr, &idr, 10); + + idr_destroy(&idr); +} + +/* + * Unlike the radix tree, you can put a NULL pointer -- with care -- into + * the IDR. Some interfaces, like idr_find() do not distinguish between + * "present, value is NULL" and "not present", but that's exactly what some + * users want. + */ +void idr_null_test(void) +{ + int i; + DEFINE_IDR(idr); + + assert(idr_is_empty(&idr)); + + assert(idr_alloc(&idr, NULL, 0, 0, GFP_KERNEL) == 0); + assert(!idr_is_empty(&idr)); + idr_remove(&idr, 0); + assert(idr_is_empty(&idr)); + + assert(idr_alloc(&idr, NULL, 0, 0, GFP_KERNEL) == 0); + assert(!idr_is_empty(&idr)); + idr_destroy(&idr); + assert(idr_is_empty(&idr)); + + for (i = 0; i < 10; i++) { + assert(idr_alloc(&idr, NULL, 0, 0, GFP_KERNEL) == i); + } + + assert(idr_replace(&idr, DUMMY_PTR, 3) == NULL); + assert(idr_replace(&idr, DUMMY_PTR, 4) == NULL); + assert(idr_replace(&idr, NULL, 4) == DUMMY_PTR); + assert(idr_replace(&idr, DUMMY_PTR, 11) == ERR_PTR(-ENOENT)); + idr_remove(&idr, 5); + assert(idr_alloc(&idr, NULL, 0, 0, GFP_KERNEL) == 5); + idr_remove(&idr, 5); + + for (i = 0; i < 9; i++) { + idr_remove(&idr, i); + assert(!idr_is_empty(&idr)); + } + idr_remove(&idr, 8); + assert(!idr_is_empty(&idr)); + idr_remove(&idr, 9); + assert(idr_is_empty(&idr)); + + assert(idr_alloc(&idr, NULL, 0, 0, GFP_KERNEL) == 0); + assert(idr_replace(&idr, DUMMY_PTR, 3) == ERR_PTR(-ENOENT)); + assert(idr_replace(&idr, DUMMY_PTR, 0) == NULL); + assert(idr_replace(&idr, NULL, 0) == DUMMY_PTR); + + idr_destroy(&idr); + assert(idr_is_empty(&idr)); + + for (i = 1; i < 10; i++) { + assert(idr_alloc(&idr, NULL, 1, 0, GFP_KERNEL) == i); + } + + idr_destroy(&idr); + assert(idr_is_empty(&idr)); +} + +void idr_nowait_test(void) +{ + unsigned int i; + DEFINE_IDR(idr); + + idr_preload(GFP_KERNEL); + + for (i = 0; i < 3; i++) { + struct item *item = item_create(i, 0); + assert(idr_alloc(&idr, item, i, i + 1, GFP_NOWAIT) == i); + } + + idr_preload_end(); + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); +} + +void idr_get_next_test(int base) +{ + unsigned long i; + int nextid; + DEFINE_IDR(idr); + idr_init_base(&idr, base); + + int indices[] = {4, 7, 9, 15, 65, 128, 1000, 99999, 0}; + + for(i = 0; indices[i]; i++) { + struct item *item = item_create(indices[i], 0); + assert(idr_alloc(&idr, item, indices[i], indices[i+1], + GFP_KERNEL) == indices[i]); + } + + for(i = 0, nextid = 0; indices[i]; i++) { + idr_get_next(&idr, &nextid); + assert(nextid == indices[i]); + nextid++; + } + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); +} + +int idr_u32_cb(int id, void *ptr, void *data) +{ + BUG_ON(id < 0); + BUG_ON(ptr != DUMMY_PTR); + return 0; +} + +void idr_u32_test1(struct idr *idr, u32 handle) +{ + static bool warned = false; + u32 id = handle; + int sid = 0; + void *ptr; + + BUG_ON(idr_alloc_u32(idr, DUMMY_PTR, &id, id, GFP_KERNEL)); + BUG_ON(id != handle); + BUG_ON(idr_alloc_u32(idr, DUMMY_PTR, &id, id, GFP_KERNEL) != -ENOSPC); + BUG_ON(id != handle); + if (!warned && id > INT_MAX) + printk("vvv Ignore these warnings\n"); + ptr = idr_get_next(idr, &sid); + if (id > INT_MAX) { + BUG_ON(ptr != NULL); + BUG_ON(sid != 0); + } else { + BUG_ON(ptr != DUMMY_PTR); + BUG_ON(sid != id); + } + idr_for_each(idr, idr_u32_cb, NULL); + if (!warned && id > INT_MAX) { + printk("^^^ Warnings over\n"); + warned = true; + } + BUG_ON(idr_remove(idr, id) != DUMMY_PTR); + BUG_ON(!idr_is_empty(idr)); +} + +void idr_u32_test(int base) +{ + DEFINE_IDR(idr); + idr_init_base(&idr, base); + idr_u32_test1(&idr, 10); + idr_u32_test1(&idr, 0x7fffffff); + idr_u32_test1(&idr, 0x80000000); + idr_u32_test1(&idr, 0x80000001); + idr_u32_test1(&idr, 0xffe00000); + idr_u32_test1(&idr, 0xffffffff); +} + +static void idr_align_test(struct idr *idr) +{ + char name[] = "Motorola 68000"; + int i, id; + void *entry; + + for (i = 0; i < 9; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 1; i < 10; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 1); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 2; i < 11; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 2); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 3; i < 12; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != i - 3); + idr_for_each_entry(idr, entry, id); + } + idr_destroy(idr); + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0); + BUG_ON(idr_alloc(idr, &name[i + 1], 0, 0, GFP_KERNEL) != 1); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 1); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 0); + BUG_ON(!idr_is_empty(idr)); + } + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 0); + idr_for_each_entry(idr, entry, id); + idr_replace(idr, &name[i], 0); + idr_for_each_entry(idr, entry, id); + BUG_ON(idr_find(idr, 0) != &name[i]); + idr_remove(idr, 0); + } + + for (i = 0; i < 8; i++) { + BUG_ON(idr_alloc(idr, &name[i], 0, 0, GFP_KERNEL) != 0); + BUG_ON(idr_alloc(idr, NULL, 0, 0, GFP_KERNEL) != 1); + idr_remove(idr, 1); + idr_for_each_entry(idr, entry, id); + idr_replace(idr, &name[i + 1], 0); + idr_for_each_entry(idr, entry, id); + idr_remove(idr, 0); + } +} + +DEFINE_IDR(find_idr); + +static void *idr_throbber(void *arg) +{ + time_t start = time(NULL); + int id = *(int *)arg; + + rcu_register_thread(); + do { + idr_alloc(&find_idr, xa_mk_value(id), id, id + 1, GFP_KERNEL); + idr_remove(&find_idr, id); + } while (time(NULL) < start + 10); + rcu_unregister_thread(); + + return NULL; +} + +void idr_find_test_1(int anchor_id, int throbber_id) +{ + pthread_t throbber; + time_t start = time(NULL); + + BUG_ON(idr_alloc(&find_idr, xa_mk_value(anchor_id), anchor_id, + anchor_id + 1, GFP_KERNEL) != anchor_id); + + pthread_create(&throbber, NULL, idr_throbber, &throbber_id); + + rcu_read_lock(); + do { + int id = 0; + void *entry = idr_get_next(&find_idr, &id); + rcu_read_unlock(); + BUG_ON(entry != xa_mk_value(id)); + rcu_read_lock(); + } while (time(NULL) < start + 11); + rcu_read_unlock(); + + pthread_join(throbber, NULL); + + idr_remove(&find_idr, anchor_id); + BUG_ON(!idr_is_empty(&find_idr)); +} + +void idr_find_test(void) +{ + idr_find_test_1(100000, 0); + idr_find_test_1(0, 100000); +} + +void idr_checks(void) +{ + unsigned long i; + DEFINE_IDR(idr); + + for (i = 0; i < 10000; i++) { + struct item *item = item_create(i, 0); + assert(idr_alloc(&idr, item, 0, 20000, GFP_KERNEL) == i); + } + + assert(idr_alloc(&idr, DUMMY_PTR, 5, 30, GFP_KERNEL) < 0); + + for (i = 0; i < 5000; i++) + item_idr_remove(&idr, i); + + idr_remove(&idr, 3); + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); + + assert(idr_is_empty(&idr)); + + idr_remove(&idr, 3); + idr_remove(&idr, 0); + + assert(idr_alloc(&idr, DUMMY_PTR, 0, 0, GFP_KERNEL) == 0); + idr_remove(&idr, 1); + for (i = 1; i < RADIX_TREE_MAP_SIZE; i++) + assert(idr_alloc(&idr, DUMMY_PTR, 0, 0, GFP_KERNEL) == i); + idr_remove(&idr, 1 << 30); + idr_destroy(&idr); + + for (i = INT_MAX - 3UL; i < INT_MAX + 1UL; i++) { + struct item *item = item_create(i, 0); + assert(idr_alloc(&idr, item, i, i + 10, GFP_KERNEL) == i); + } + assert(idr_alloc(&idr, DUMMY_PTR, i - 2, i, GFP_KERNEL) == -ENOSPC); + assert(idr_alloc(&idr, DUMMY_PTR, i - 2, i + 10, GFP_KERNEL) == -ENOSPC); + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); + idr_destroy(&idr); + + assert(idr_is_empty(&idr)); + + idr_set_cursor(&idr, INT_MAX - 3UL); + for (i = INT_MAX - 3UL; i < INT_MAX + 3UL; i++) { + struct item *item; + unsigned int id; + if (i <= INT_MAX) + item = item_create(i, 0); + else + item = item_create(i - INT_MAX - 1, 0); + + id = idr_alloc_cyclic(&idr, item, 0, 0, GFP_KERNEL); + assert(id == item->index); + } + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); + assert(idr_is_empty(&idr)); + + for (i = 1; i < 10000; i++) { + struct item *item = item_create(i, 0); + assert(idr_alloc(&idr, item, 1, 20000, GFP_KERNEL) == i); + } + + idr_for_each(&idr, item_idr_free, &idr); + idr_destroy(&idr); + + idr_replace_test(); + idr_alloc_test(); + idr_null_test(); + idr_nowait_test(); + idr_get_next_test(0); + idr_get_next_test(1); + idr_get_next_test(4); + idr_u32_test(4); + idr_u32_test(1); + idr_u32_test(0); + idr_align_test(&idr); + idr_find_test(); +} + +#define module_init(x) +#define module_exit(x) +#define MODULE_AUTHOR(x) +#define MODULE_LICENSE(x) +#define dump_stack() assert(0) +void ida_dump(struct ida *); + +#include "../../../lib/test_ida.c" + +/* + * Check that we get the correct error when we run out of memory doing + * allocations. In userspace, GFP_NOWAIT will always fail an allocation. + * The first test is for not having a bitmap available, and the second test + * is for not being able to allocate a level of the radix tree. + */ +void ida_check_nomem(void) +{ + DEFINE_IDA(ida); + int id; + + id = ida_alloc_min(&ida, 256, GFP_NOWAIT); + IDA_BUG_ON(&ida, id != -ENOMEM); + id = ida_alloc_min(&ida, 1UL << 30, GFP_NOWAIT); + IDA_BUG_ON(&ida, id != -ENOMEM); + IDA_BUG_ON(&ida, !ida_is_empty(&ida)); +} + +/* + * Check handling of conversions between exceptional entries and full bitmaps. + */ +void ida_check_conv_user(void) +{ + DEFINE_IDA(ida); + unsigned long i; + + for (i = 0; i < 1000000; i++) { + int id = ida_alloc(&ida, GFP_NOWAIT); + if (id == -ENOMEM) { + IDA_BUG_ON(&ida, ((i % IDA_BITMAP_BITS) != + BITS_PER_XA_VALUE) && + ((i % IDA_BITMAP_BITS) != 0)); + id = ida_alloc(&ida, GFP_KERNEL); + } else { + IDA_BUG_ON(&ida, (i % IDA_BITMAP_BITS) == + BITS_PER_XA_VALUE); + } + IDA_BUG_ON(&ida, id != i); + } + ida_destroy(&ida); +} + +void ida_check_random(void) +{ + DEFINE_IDA(ida); + DECLARE_BITMAP(bitmap, 2048); + unsigned int i; + time_t s = time(NULL); + + repeat: + memset(bitmap, 0, sizeof(bitmap)); + for (i = 0; i < 100000; i++) { + int i = rand(); + int bit = i & 2047; + if (test_bit(bit, bitmap)) { + __clear_bit(bit, bitmap); + ida_free(&ida, bit); + } else { + __set_bit(bit, bitmap); + IDA_BUG_ON(&ida, ida_alloc_min(&ida, bit, GFP_KERNEL) + != bit); + } + } + ida_destroy(&ida); + if (time(NULL) < s + 10) + goto repeat; +} + +void ida_simple_get_remove_test(void) +{ + DEFINE_IDA(ida); + unsigned long i; + + for (i = 0; i < 10000; i++) { + assert(ida_simple_get(&ida, 0, 20000, GFP_KERNEL) == i); + } + assert(ida_simple_get(&ida, 5, 30, GFP_KERNEL) < 0); + + for (i = 0; i < 10000; i++) { + ida_simple_remove(&ida, i); + } + assert(ida_is_empty(&ida)); + + ida_destroy(&ida); +} + +void user_ida_checks(void) +{ + radix_tree_cpu_dead(1); + + ida_check_nomem(); + ida_check_conv_user(); + ida_check_random(); + ida_simple_get_remove_test(); + + radix_tree_cpu_dead(1); +} + +static void *ida_random_fn(void *arg) +{ + rcu_register_thread(); + ida_check_random(); + rcu_unregister_thread(); + return NULL; +} + +static void *ida_leak_fn(void *arg) +{ + struct ida *ida = arg; + time_t s = time(NULL); + int i, ret; + + rcu_register_thread(); + + do for (i = 0; i < 1000; i++) { + ret = ida_alloc_range(ida, 128, 128, GFP_KERNEL); + if (ret >= 0) + ida_free(ida, 128); + } while (time(NULL) < s + 2); + + rcu_unregister_thread(); + return NULL; +} + +void ida_thread_tests(void) +{ + DEFINE_IDA(ida); + pthread_t threads[20]; + int i; + + for (i = 0; i < ARRAY_SIZE(threads); i++) + if (pthread_create(&threads[i], NULL, ida_random_fn, NULL)) { + perror("creating ida thread"); + exit(1); + } + + while (i--) + pthread_join(threads[i], NULL); + + for (i = 0; i < ARRAY_SIZE(threads); i++) + if (pthread_create(&threads[i], NULL, ida_leak_fn, &ida)) { + perror("creating ida thread"); + exit(1); + } + + while (i--) + pthread_join(threads[i], NULL); + assert(ida_is_empty(&ida)); +} + +void ida_tests(void) +{ + user_ida_checks(); + ida_checks(); + ida_exit(); + ida_thread_tests(); +} + +int __weak main(void) +{ + rcu_register_thread(); + radix_tree_init(); + idr_checks(); + ida_tests(); + radix_tree_cpu_dead(1); + rcu_barrier(); + if (nr_allocated) + printf("nr_allocated = %d\n", nr_allocated); + rcu_unregister_thread(); + return 0; +} diff --git a/tools/testing/radix-tree/iteration_check.c b/tools/testing/radix-tree/iteration_check.c new file mode 100644 index 000000000..e9908bcb0 --- /dev/null +++ b/tools/testing/radix-tree/iteration_check.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * iteration_check.c: test races having to do with xarray iteration + * Copyright (c) 2016 Intel Corporation + * Author: Ross Zwisler <ross.zwisler@linux.intel.com> + */ +#include <pthread.h> +#include "test.h" + +#define NUM_THREADS 5 +#define MAX_IDX 100 +#define TAG XA_MARK_0 +#define NEW_TAG XA_MARK_1 + +static pthread_t threads[NUM_THREADS]; +static unsigned int seeds[3]; +static DEFINE_XARRAY(array); +static bool test_complete; +static int max_order; + +void my_item_insert(struct xarray *xa, unsigned long index) +{ + XA_STATE(xas, xa, index); + struct item *item = item_create(index, 0); + int order; + +retry: + xas_lock(&xas); + for (order = max_order; order >= 0; order--) { + xas_set_order(&xas, index, order); + item->order = order; + if (xas_find_conflict(&xas)) + continue; + xas_store(&xas, item); + xas_set_mark(&xas, TAG); + break; + } + xas_unlock(&xas); + if (xas_nomem(&xas, GFP_KERNEL)) + goto retry; + if (order < 0) + free(item); +} + +/* relentlessly fill the array with tagged entries */ +static void *add_entries_fn(void *arg) +{ + rcu_register_thread(); + + while (!test_complete) { + unsigned long pgoff; + + for (pgoff = 0; pgoff < MAX_IDX; pgoff++) { + my_item_insert(&array, pgoff); + } + } + + rcu_unregister_thread(); + + return NULL; +} + +/* + * Iterate over tagged entries, retrying when we find ourselves in a deleted + * node and randomly pausing the iteration. + */ +static void *tagged_iteration_fn(void *arg) +{ + XA_STATE(xas, &array, 0); + void *entry; + + rcu_register_thread(); + + while (!test_complete) { + xas_set(&xas, 0); + rcu_read_lock(); + xas_for_each_marked(&xas, entry, ULONG_MAX, TAG) { + if (xas_retry(&xas, entry)) + continue; + + if (rand_r(&seeds[0]) % 50 == 0) { + xas_pause(&xas); + rcu_read_unlock(); + rcu_barrier(); + rcu_read_lock(); + } + } + rcu_read_unlock(); + } + + rcu_unregister_thread(); + + return NULL; +} + +/* + * Iterate over the entries, retrying when we find ourselves in a deleted + * node and randomly pausing the iteration. + */ +static void *untagged_iteration_fn(void *arg) +{ + XA_STATE(xas, &array, 0); + void *entry; + + rcu_register_thread(); + + while (!test_complete) { + xas_set(&xas, 0); + rcu_read_lock(); + xas_for_each(&xas, entry, ULONG_MAX) { + if (xas_retry(&xas, entry)) + continue; + + if (rand_r(&seeds[1]) % 50 == 0) { + xas_pause(&xas); + rcu_read_unlock(); + rcu_barrier(); + rcu_read_lock(); + } + } + rcu_read_unlock(); + } + + rcu_unregister_thread(); + + return NULL; +} + +/* + * Randomly remove entries to help induce retries in the + * two iteration functions. + */ +static void *remove_entries_fn(void *arg) +{ + rcu_register_thread(); + + while (!test_complete) { + int pgoff; + struct item *item; + + pgoff = rand_r(&seeds[2]) % MAX_IDX; + + item = xa_erase(&array, pgoff); + if (item) + item_free(item, pgoff); + } + + rcu_unregister_thread(); + + return NULL; +} + +static void *tag_entries_fn(void *arg) +{ + rcu_register_thread(); + + while (!test_complete) { + tag_tagged_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG); + } + rcu_unregister_thread(); + return NULL; +} + +/* This is a unit test for a bug found by the syzkaller tester */ +void iteration_test(unsigned order, unsigned test_duration) +{ + int i; + + printv(1, "Running %siteration tests for %d seconds\n", + order > 0 ? "multiorder " : "", test_duration); + + max_order = order; + test_complete = false; + + for (i = 0; i < 3; i++) + seeds[i] = rand(); + + if (pthread_create(&threads[0], NULL, tagged_iteration_fn, NULL)) { + perror("create tagged iteration thread"); + exit(1); + } + if (pthread_create(&threads[1], NULL, untagged_iteration_fn, NULL)) { + perror("create untagged iteration thread"); + exit(1); + } + if (pthread_create(&threads[2], NULL, add_entries_fn, NULL)) { + perror("create add entry thread"); + exit(1); + } + if (pthread_create(&threads[3], NULL, remove_entries_fn, NULL)) { + perror("create remove entry thread"); + exit(1); + } + if (pthread_create(&threads[4], NULL, tag_entries_fn, NULL)) { + perror("create tag entry thread"); + exit(1); + } + + sleep(test_duration); + test_complete = true; + + for (i = 0; i < NUM_THREADS; i++) { + if (pthread_join(threads[i], NULL)) { + perror("pthread_join"); + exit(1); + } + } + + item_kill_tree(&array); +} diff --git a/tools/testing/radix-tree/iteration_check_2.c b/tools/testing/radix-tree/iteration_check_2.c new file mode 100644 index 000000000..aac5c50a3 --- /dev/null +++ b/tools/testing/radix-tree/iteration_check_2.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * iteration_check_2.c: Check that deleting a tagged entry doesn't cause + * an RCU walker to finish early. + * Copyright (c) 2020 Oracle + * Author: Matthew Wilcox <willy@infradead.org> + */ +#include <pthread.h> +#include "test.h" + +static volatile bool test_complete; + +static void *iterator(void *arg) +{ + XA_STATE(xas, arg, 0); + void *entry; + + rcu_register_thread(); + + while (!test_complete) { + xas_set(&xas, 0); + rcu_read_lock(); + xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0) + ; + rcu_read_unlock(); + assert(xas.xa_index >= 100); + } + + rcu_unregister_thread(); + return NULL; +} + +static void *throbber(void *arg) +{ + struct xarray *xa = arg; + + rcu_register_thread(); + + while (!test_complete) { + int i; + + for (i = 0; i < 100; i++) { + xa_store(xa, i, xa_mk_value(i), GFP_KERNEL); + xa_set_mark(xa, i, XA_MARK_0); + } + for (i = 0; i < 100; i++) + xa_erase(xa, i); + } + + rcu_unregister_thread(); + return NULL; +} + +void iteration_test2(unsigned test_duration) +{ + pthread_t threads[2]; + DEFINE_XARRAY(array); + int i; + + printv(1, "Running iteration test 2 for %d seconds\n", test_duration); + + test_complete = false; + + xa_store(&array, 100, xa_mk_value(100), GFP_KERNEL); + xa_set_mark(&array, 100, XA_MARK_0); + + if (pthread_create(&threads[0], NULL, iterator, &array)) { + perror("create iterator thread"); + exit(1); + } + if (pthread_create(&threads[1], NULL, throbber, &array)) { + perror("create throbber thread"); + exit(1); + } + + sleep(test_duration); + test_complete = true; + + for (i = 0; i < 2; i++) { + if (pthread_join(threads[i], NULL)) { + perror("pthread_join"); + exit(1); + } + } + + xa_destroy(&array); +} diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c new file mode 100644 index 000000000..2d9c59df6 --- /dev/null +++ b/tools/testing/radix-tree/linux.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdlib.h> +#include <string.h> +#include <malloc.h> +#include <pthread.h> +#include <unistd.h> +#include <assert.h> + +#include <linux/gfp.h> +#include <linux/poison.h> +#include <linux/slab.h> +#include <linux/radix-tree.h> +#include <urcu/uatomic.h> + +int nr_allocated; +int preempt_count; +int kmalloc_verbose; +int test_verbose; + +struct kmem_cache { + pthread_mutex_t lock; + unsigned int size; + unsigned int align; + int nr_objs; + void *objs; + void (*ctor)(void *); +}; + +void *kmem_cache_alloc(struct kmem_cache *cachep, int gfp) +{ + void *p; + + if (!(gfp & __GFP_DIRECT_RECLAIM)) + return NULL; + + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs) { + struct radix_tree_node *node = cachep->objs; + cachep->nr_objs--; + cachep->objs = node->parent; + pthread_mutex_unlock(&cachep->lock); + node->parent = NULL; + p = node; + } else { + pthread_mutex_unlock(&cachep->lock); + if (cachep->align) + posix_memalign(&p, cachep->align, cachep->size); + else + p = malloc(cachep->size); + if (cachep->ctor) + cachep->ctor(p); + else if (gfp & __GFP_ZERO) + memset(p, 0, cachep->size); + } + + uatomic_inc(&nr_allocated); + if (kmalloc_verbose) + printf("Allocating %p from slab\n", p); + return p; +} + +void kmem_cache_free(struct kmem_cache *cachep, void *objp) +{ + assert(objp); + uatomic_dec(&nr_allocated); + if (kmalloc_verbose) + printf("Freeing %p to slab\n", objp); + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs > 10 || cachep->align) { + memset(objp, POISON_FREE, cachep->size); + free(objp); + } else { + struct radix_tree_node *node = objp; + cachep->nr_objs++; + node->parent = cachep->objs; + cachep->objs = node; + } + pthread_mutex_unlock(&cachep->lock); +} + +void *kmalloc(size_t size, gfp_t gfp) +{ + void *ret; + + if (!(gfp & __GFP_DIRECT_RECLAIM)) + return NULL; + + ret = malloc(size); + uatomic_inc(&nr_allocated); + if (kmalloc_verbose) + printf("Allocating %p from malloc\n", ret); + if (gfp & __GFP_ZERO) + memset(ret, 0, size); + return ret; +} + +void kfree(void *p) +{ + if (!p) + return; + uatomic_dec(&nr_allocated); + if (kmalloc_verbose) + printf("Freeing %p to malloc\n", p); + free(p); +} + +struct kmem_cache * +kmem_cache_create(const char *name, unsigned int size, unsigned int align, + unsigned int flags, void (*ctor)(void *)) +{ + struct kmem_cache *ret = malloc(sizeof(*ret)); + + pthread_mutex_init(&ret->lock, NULL); + ret->size = size; + ret->align = align; + ret->nr_objs = 0; + ret->objs = NULL; + ret->ctor = ctor; + return ret; +} diff --git a/tools/testing/radix-tree/linux/bug.h b/tools/testing/radix-tree/linux/bug.h new file mode 100644 index 000000000..03dc8a57e --- /dev/null +++ b/tools/testing/radix-tree/linux/bug.h @@ -0,0 +1,2 @@ +#include <stdio.h> +#include "asm/bug.h" diff --git a/tools/testing/radix-tree/linux/compiler_types.h b/tools/testing/radix-tree/linux/compiler_types.h new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tools/testing/radix-tree/linux/compiler_types.h diff --git a/tools/testing/radix-tree/linux/cpu.h b/tools/testing/radix-tree/linux/cpu.h new file mode 100644 index 000000000..a45530d78 --- /dev/null +++ b/tools/testing/radix-tree/linux/cpu.h @@ -0,0 +1 @@ +#define cpuhp_setup_state_nocalls(a, b, c, d) (0) diff --git a/tools/testing/radix-tree/linux/gfp.h b/tools/testing/radix-tree/linux/gfp.h new file mode 100644 index 000000000..32159c08a --- /dev/null +++ b/tools/testing/radix-tree/linux/gfp.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _GFP_H +#define _GFP_H + +#include <linux/types.h> + +#define __GFP_BITS_SHIFT 26 +#define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) + +#define __GFP_HIGH 0x20u +#define __GFP_IO 0x40u +#define __GFP_FS 0x80u +#define __GFP_NOWARN 0x200u +#define __GFP_ZERO 0x8000u +#define __GFP_ATOMIC 0x80000u +#define __GFP_ACCOUNT 0x100000u +#define __GFP_DIRECT_RECLAIM 0x400000u +#define __GFP_KSWAPD_RECLAIM 0x2000000u + +#define __GFP_RECLAIM (__GFP_DIRECT_RECLAIM|__GFP_KSWAPD_RECLAIM) + +#define GFP_ZONEMASK 0x0fu +#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) +#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) +#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) + + +static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) +{ + return !!(gfp_flags & __GFP_DIRECT_RECLAIM); +} + +#endif diff --git a/tools/testing/radix-tree/linux/idr.h b/tools/testing/radix-tree/linux/idr.h new file mode 100644 index 000000000..4e342f2e3 --- /dev/null +++ b/tools/testing/radix-tree/linux/idr.h @@ -0,0 +1 @@ +#include "../../../../include/linux/idr.h" diff --git a/tools/testing/radix-tree/linux/init.h b/tools/testing/radix-tree/linux/init.h new file mode 100644 index 000000000..1bb0afc21 --- /dev/null +++ b/tools/testing/radix-tree/linux/init.h @@ -0,0 +1 @@ +#define __init diff --git a/tools/testing/radix-tree/linux/kconfig.h b/tools/testing/radix-tree/linux/kconfig.h new file mode 100644 index 000000000..6c8675859 --- /dev/null +++ b/tools/testing/radix-tree/linux/kconfig.h @@ -0,0 +1 @@ +#include "../../../../include/linux/kconfig.h" diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h new file mode 100644 index 000000000..39867fd80 --- /dev/null +++ b/tools/testing/radix-tree/linux/kernel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_H +#define _KERNEL_H + +#include "../../include/linux/kernel.h" +#include <string.h> +#include <stdio.h> +#include <limits.h> + +#include <linux/compiler.h> +#include <linux/err.h> +#include <linux/bitops.h> +#include <linux/log2.h> +#include "../../../include/linux/kconfig.h" + +#define printk printf +#define pr_info printk +#define pr_debug printk +#define pr_cont printk + +#define __acquires(x) +#define __releases(x) +#define __must_hold(x) + +#define EXPORT_PER_CPU_SYMBOL_GPL(x) +#endif /* _KERNEL_H */ diff --git a/tools/testing/radix-tree/linux/kmemleak.h b/tools/testing/radix-tree/linux/kmemleak.h new file mode 100644 index 000000000..155f11278 --- /dev/null +++ b/tools/testing/radix-tree/linux/kmemleak.h @@ -0,0 +1 @@ +static inline void kmemleak_update_trace(const void *ptr) { } diff --git a/tools/testing/radix-tree/linux/local_lock.h b/tools/testing/radix-tree/linux/local_lock.h new file mode 100644 index 000000000..b3cf8b233 --- /dev/null +++ b/tools/testing/radix-tree/linux/local_lock.h @@ -0,0 +1,8 @@ +#ifndef _LINUX_LOCAL_LOCK +#define _LINUX_LOCAL_LOCK +typedef struct { } local_lock_t; + +static inline void local_lock(local_lock_t *lock) { } +static inline void local_unlock(local_lock_t *lock) { } +#define INIT_LOCAL_LOCK(x) { } +#endif diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h new file mode 100644 index 000000000..565fccdfe --- /dev/null +++ b/tools/testing/radix-tree/linux/lockdep.h @@ -0,0 +1,11 @@ +#ifndef _LINUX_LOCKDEP_H +#define _LINUX_LOCKDEP_H +struct lock_class_key { + unsigned int a; +}; + +static inline void lockdep_set_class(spinlock_t *lock, + struct lock_class_key *key) +{ +} +#endif /* _LINUX_LOCKDEP_H */ diff --git a/tools/testing/radix-tree/linux/percpu.h b/tools/testing/radix-tree/linux/percpu.h new file mode 100644 index 000000000..b2403aa74 --- /dev/null +++ b/tools/testing/radix-tree/linux/percpu.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#define DECLARE_PER_CPU(type, val) extern type val +#define DEFINE_PER_CPU(type, val) type val + +#define __get_cpu_var(var) var +#define this_cpu_ptr(var) var +#define this_cpu_read(var) var +#define this_cpu_xchg(var, val) uatomic_xchg(&var, val) +#define this_cpu_cmpxchg(var, old, new) uatomic_cmpxchg(&var, old, new) +#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) +#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu)) diff --git a/tools/testing/radix-tree/linux/preempt.h b/tools/testing/radix-tree/linux/preempt.h new file mode 100644 index 000000000..edb10302b --- /dev/null +++ b/tools/testing/radix-tree/linux/preempt.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_PREEMPT_H +#define __LINUX_PREEMPT_H + +extern int preempt_count; + +#define preempt_disable() uatomic_inc(&preempt_count) +#define preempt_enable() uatomic_dec(&preempt_count) + +static inline int in_interrupt(void) +{ + return 0; +} + +#endif /* __LINUX_PREEMPT_H */ diff --git a/tools/testing/radix-tree/linux/radix-tree.h b/tools/testing/radix-tree/linux/radix-tree.h new file mode 100644 index 000000000..d1635a5be --- /dev/null +++ b/tools/testing/radix-tree/linux/radix-tree.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TEST_RADIX_TREE_H +#define _TEST_RADIX_TREE_H + +#include "../../../../include/linux/radix-tree.h" + +extern int kmalloc_verbose; +extern int test_verbose; + +static inline void trace_call_rcu(struct rcu_head *head, + void (*func)(struct rcu_head *head)) +{ + if (kmalloc_verbose) + printf("Delaying free of %p to slab\n", (char *)head - + offsetof(struct radix_tree_node, rcu_head)); + call_rcu(head, func); +} + +#define printv(verbosity_level, fmt, ...) \ + if(test_verbose >= verbosity_level) \ + printf(fmt, ##__VA_ARGS__) + +#undef call_rcu +#define call_rcu(x, y) trace_call_rcu(x, y) + +#endif /* _TEST_RADIX_TREE_H */ diff --git a/tools/testing/radix-tree/linux/rcupdate.h b/tools/testing/radix-tree/linux/rcupdate.h new file mode 100644 index 000000000..fed468fb0 --- /dev/null +++ b/tools/testing/radix-tree/linux/rcupdate.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _RCUPDATE_H +#define _RCUPDATE_H + +#include <urcu.h> + +#define rcu_dereference_raw(p) rcu_dereference(p) +#define rcu_dereference_protected(p, cond) rcu_dereference(p) +#define rcu_dereference_check(p, cond) rcu_dereference(p) +#define RCU_INIT_POINTER(p, v) do { (p) = (v); } while (0) + +#endif diff --git a/tools/testing/radix-tree/linux/slab.h b/tools/testing/radix-tree/linux/slab.h new file mode 100644 index 000000000..2958830ce --- /dev/null +++ b/tools/testing/radix-tree/linux/slab.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef SLAB_H +#define SLAB_H + +#include <linux/types.h> +#include <linux/gfp.h> + +#define SLAB_HWCACHE_ALIGN 1 +#define SLAB_PANIC 2 +#define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ + +void *kmalloc(size_t size, gfp_t); +void kfree(void *); + +static inline void *kzalloc(size_t size, gfp_t gfp) +{ + return kmalloc(size, gfp | __GFP_ZERO); +} + +void *kmem_cache_alloc(struct kmem_cache *cachep, int flags); +void kmem_cache_free(struct kmem_cache *cachep, void *objp); + +struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, + unsigned int align, unsigned int flags, + void (*ctor)(void *)); + +#endif /* SLAB_H */ diff --git a/tools/testing/radix-tree/linux/xarray.h b/tools/testing/radix-tree/linux/xarray.h new file mode 100644 index 000000000..df3812cda --- /dev/null +++ b/tools/testing/radix-tree/linux/xarray.h @@ -0,0 +1,2 @@ +#include "generated/map-shift.h" +#include "../../../../include/linux/xarray.h" diff --git a/tools/testing/radix-tree/main.c b/tools/testing/radix-tree/main.c new file mode 100644 index 000000000..f2cbc8e5b --- /dev/null +++ b/tools/testing/radix-tree/main.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> +#include <assert.h> +#include <limits.h> + +#include <linux/slab.h> +#include <linux/radix-tree.h> + +#include "test.h" +#include "regression.h" + +void __gang_check(unsigned long middle, long down, long up, int chunk, int hop) +{ + long idx; + RADIX_TREE(tree, GFP_KERNEL); + + middle = 1 << 30; + + for (idx = -down; idx < up; idx++) + item_insert(&tree, middle + idx); + + item_check_absent(&tree, middle - down - 1); + for (idx = -down; idx < up; idx++) + item_check_present(&tree, middle + idx); + item_check_absent(&tree, middle + up); + + if (chunk > 0) { + item_gang_check_present(&tree, middle - down, up + down, + chunk, hop); + item_full_scan(&tree, middle - down, down + up, chunk); + } + item_kill_tree(&tree); +} + +void gang_check(void) +{ + __gang_check(1UL << 30, 128, 128, 35, 2); + __gang_check(1UL << 31, 128, 128, 32, 32); + __gang_check(1UL << 31, 128, 128, 32, 100); + __gang_check(1UL << 31, 128, 128, 17, 7); + __gang_check(0xffff0000UL, 0, 65536, 17, 7); + __gang_check(0xfffffffeUL, 1, 1, 17, 7); +} + +void __big_gang_check(void) +{ + unsigned long start; + int wrapped = 0; + + start = 0; + do { + unsigned long old_start; + +// printf("0x%08lx\n", start); + __gang_check(start, rand() % 113 + 1, rand() % 71, + rand() % 157, rand() % 91 + 1); + old_start = start; + start += rand() % 1000000; + start %= 1ULL << 33; + if (start < old_start) + wrapped = 1; + } while (!wrapped); +} + +void big_gang_check(bool long_run) +{ + int i; + + for (i = 0; i < (long_run ? 1000 : 3); i++) { + __big_gang_check(); + printv(2, "%d ", i); + fflush(stdout); + } +} + +void add_and_check(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + + item_insert(&tree, 44); + item_check_present(&tree, 44); + item_check_absent(&tree, 43); + item_kill_tree(&tree); +} + +void dynamic_height_check(void) +{ + int i; + RADIX_TREE(tree, GFP_KERNEL); + tree_verify_min_height(&tree, 0); + + item_insert(&tree, 42); + tree_verify_min_height(&tree, 42); + + item_insert(&tree, 1000000); + tree_verify_min_height(&tree, 1000000); + + assert(item_delete(&tree, 1000000)); + tree_verify_min_height(&tree, 42); + + assert(item_delete(&tree, 42)); + tree_verify_min_height(&tree, 0); + + for (i = 0; i < 1000; i++) { + item_insert(&tree, i); + tree_verify_min_height(&tree, i); + } + + i--; + for (;;) { + assert(item_delete(&tree, i)); + if (i == 0) { + tree_verify_min_height(&tree, 0); + break; + } + i--; + tree_verify_min_height(&tree, i); + } + + item_kill_tree(&tree); +} + +void check_copied_tags(struct radix_tree_root *tree, unsigned long start, unsigned long end, unsigned long *idx, int count, int fromtag, int totag) +{ + int i; + + for (i = 0; i < count; i++) { +/* if (i % 1000 == 0) + putchar('.'); */ + if (idx[i] < start || idx[i] > end) { + if (item_tag_get(tree, idx[i], totag)) { + printv(2, "%lu-%lu: %lu, tags %d-%d\n", start, + end, idx[i], item_tag_get(tree, idx[i], + fromtag), + item_tag_get(tree, idx[i], totag)); + } + assert(!item_tag_get(tree, idx[i], totag)); + continue; + } + if (item_tag_get(tree, idx[i], fromtag) ^ + item_tag_get(tree, idx[i], totag)) { + printv(2, "%lu-%lu: %lu, tags %d-%d\n", start, end, + idx[i], item_tag_get(tree, idx[i], fromtag), + item_tag_get(tree, idx[i], totag)); + } + assert(!(item_tag_get(tree, idx[i], fromtag) ^ + item_tag_get(tree, idx[i], totag))); + } +} + +#define ITEMS 50000 + +void copy_tag_check(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + unsigned long idx[ITEMS]; + unsigned long start, end, count = 0, tagged, cur, tmp; + int i; + +// printf("generating radix tree indices...\n"); + start = rand(); + end = rand(); + if (start > end && (rand() % 10)) { + cur = start; + start = end; + end = cur; + } + /* Specifically create items around the start and the end of the range + * with high probability to check for off by one errors */ + cur = rand(); + if (cur & 1) { + item_insert(&tree, start); + if (cur & 2) { + if (start <= end) + count++; + item_tag_set(&tree, start, 0); + } + } + if (cur & 4) { + item_insert(&tree, start-1); + if (cur & 8) + item_tag_set(&tree, start-1, 0); + } + if (cur & 16) { + item_insert(&tree, end); + if (cur & 32) { + if (start <= end) + count++; + item_tag_set(&tree, end, 0); + } + } + if (cur & 64) { + item_insert(&tree, end+1); + if (cur & 128) + item_tag_set(&tree, end+1, 0); + } + + for (i = 0; i < ITEMS; i++) { + do { + idx[i] = rand(); + } while (item_lookup(&tree, idx[i])); + + item_insert(&tree, idx[i]); + if (rand() & 1) { + item_tag_set(&tree, idx[i], 0); + if (idx[i] >= start && idx[i] <= end) + count++; + } +/* if (i % 1000 == 0) + putchar('.'); */ + } + +// printf("\ncopying tags...\n"); + tagged = tag_tagged_items(&tree, start, end, ITEMS, XA_MARK_0, XA_MARK_1); + +// printf("checking copied tags\n"); + assert(tagged == count); + check_copied_tags(&tree, start, end, idx, ITEMS, 0, 1); + + /* Copy tags in several rounds */ +// printf("\ncopying tags...\n"); + tmp = rand() % (count / 10 + 2); + tagged = tag_tagged_items(&tree, start, end, tmp, XA_MARK_0, XA_MARK_2); + assert(tagged == count); + +// printf("%lu %lu %lu\n", tagged, tmp, count); +// printf("checking copied tags\n"); + check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2); + verify_tag_consistency(&tree, 0); + verify_tag_consistency(&tree, 1); + verify_tag_consistency(&tree, 2); +// printf("\n"); + item_kill_tree(&tree); +} + +static void single_thread_tests(bool long_run) +{ + int i; + + printv(1, "starting single_thread_tests: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + multiorder_checks(); + rcu_barrier(); + printv(2, "after multiorder_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + tag_check(); + rcu_barrier(); + printv(2, "after tag_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + gang_check(); + rcu_barrier(); + printv(2, "after gang_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + add_and_check(); + rcu_barrier(); + printv(2, "after add_and_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + dynamic_height_check(); + rcu_barrier(); + printv(2, "after dynamic_height_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + idr_checks(); + ida_tests(); + rcu_barrier(); + printv(2, "after idr_checks: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + big_gang_check(long_run); + rcu_barrier(); + printv(2, "after big_gang_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + for (i = 0; i < (long_run ? 2000 : 3); i++) { + copy_tag_check(); + printv(2, "%d ", i); + fflush(stdout); + } + rcu_barrier(); + printv(2, "after copy_tag_check: %d allocated, preempt %d\n", + nr_allocated, preempt_count); +} + +int main(int argc, char **argv) +{ + bool long_run = false; + int opt; + unsigned int seed = time(NULL); + + while ((opt = getopt(argc, argv, "ls:v")) != -1) { + if (opt == 'l') + long_run = true; + else if (opt == 's') + seed = strtoul(optarg, NULL, 0); + else if (opt == 'v') + test_verbose++; + } + + printf("random seed %u\n", seed); + srand(seed); + + printf("running tests\n"); + + rcu_register_thread(); + radix_tree_init(); + + xarray_tests(); + regression1_test(); + regression2_test(); + regression3_test(); + regression4_test(); + iteration_test(0, 10 + 90 * long_run); + iteration_test(7, 10 + 90 * long_run); + iteration_test2(10 + 90 * long_run); + single_thread_tests(long_run); + + /* Free any remaining preallocated nodes */ + radix_tree_cpu_dead(0); + + benchmark(); + + rcu_barrier(); + printv(2, "after rcu_barrier: %d allocated, preempt %d\n", + nr_allocated, preempt_count); + rcu_unregister_thread(); + + printf("tests completed\n"); + + exit(0); +} diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c new file mode 100644 index 000000000..e00520cc6 --- /dev/null +++ b/tools/testing/radix-tree/multiorder.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * multiorder.c: Multi-order radix tree entry testing + * Copyright (c) 2016 Intel Corporation + * Author: Ross Zwisler <ross.zwisler@linux.intel.com> + * Author: Matthew Wilcox <matthew.r.wilcox@intel.com> + */ +#include <linux/radix-tree.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <pthread.h> + +#include "test.h" + +static int item_insert_order(struct xarray *xa, unsigned long index, + unsigned order) +{ + XA_STATE_ORDER(xas, xa, index, order); + struct item *item = item_create(index, order); + + do { + xas_lock(&xas); + xas_store(&xas, item); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + + if (!xas_error(&xas)) + return 0; + + free(item); + return xas_error(&xas); +} + +void multiorder_iteration(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + struct item *item; + int i, j, err; + +#define NUM_ENTRIES 11 + int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128}; + int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7}; + + printv(1, "Multiorder iteration test\n"); + + for (i = 0; i < NUM_ENTRIES; i++) { + err = item_insert_order(xa, index[i], order[i]); + assert(!err); + } + + for (j = 0; j < 256; j++) { + for (i = 0; i < NUM_ENTRIES; i++) + if (j <= (index[i] | ((1 << order[i]) - 1))) + break; + + xas_set(&xas, j); + xas_for_each(&xas, item, ULONG_MAX) { + int height = order[i] / XA_CHUNK_SHIFT; + int shift = height * XA_CHUNK_SHIFT; + unsigned long mask = (1UL << order[i]) - 1; + + assert((xas.xa_index | mask) == (index[i] | mask)); + assert(xas.xa_node->shift == shift); + assert(!radix_tree_is_internal_node(item)); + assert((item->index | mask) == (index[i] | mask)); + assert(item->order == order[i]); + i++; + } + } + + item_kill_tree(xa); +} + +void multiorder_tagged_iteration(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + struct item *item; + int i, j; + +#define MT_NUM_ENTRIES 9 + int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128}; + int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7}; + +#define TAG_ENTRIES 7 + int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128}; + + printv(1, "Multiorder tagged iteration test\n"); + + for (i = 0; i < MT_NUM_ENTRIES; i++) + assert(!item_insert_order(xa, index[i], order[i])); + + assert(!xa_marked(xa, XA_MARK_1)); + + for (i = 0; i < TAG_ENTRIES; i++) + xa_set_mark(xa, tag_index[i], XA_MARK_1); + + for (j = 0; j < 256; j++) { + int k; + + for (i = 0; i < TAG_ENTRIES; i++) { + for (k = i; index[k] < tag_index[i]; k++) + ; + if (j <= (index[k] | ((1 << order[k]) - 1))) + break; + } + + xas_set(&xas, j); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_1) { + unsigned long mask; + for (k = i; index[k] < tag_index[i]; k++) + ; + mask = (1UL << order[k]) - 1; + + assert((xas.xa_index | mask) == (tag_index[i] | mask)); + assert(!xa_is_internal(item)); + assert((item->index | mask) == (tag_index[i] | mask)); + assert(item->order == order[k]); + i++; + } + } + + assert(tag_tagged_items(xa, 0, ULONG_MAX, TAG_ENTRIES, XA_MARK_1, + XA_MARK_2) == TAG_ENTRIES); + + for (j = 0; j < 256; j++) { + int mask, k; + + for (i = 0; i < TAG_ENTRIES; i++) { + for (k = i; index[k] < tag_index[i]; k++) + ; + if (j <= (index[k] | ((1 << order[k]) - 1))) + break; + } + + xas_set(&xas, j); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_2) { + for (k = i; index[k] < tag_index[i]; k++) + ; + mask = (1 << order[k]) - 1; + + assert((xas.xa_index | mask) == (tag_index[i] | mask)); + assert(!xa_is_internal(item)); + assert((item->index | mask) == (tag_index[i] | mask)); + assert(item->order == order[k]); + i++; + } + } + + assert(tag_tagged_items(xa, 1, ULONG_MAX, MT_NUM_ENTRIES * 2, XA_MARK_1, + XA_MARK_0) == TAG_ENTRIES); + i = 0; + xas_set(&xas, 0); + xas_for_each_marked(&xas, item, ULONG_MAX, XA_MARK_0) { + assert(xas.xa_index == tag_index[i]); + i++; + } + assert(i == TAG_ENTRIES); + + item_kill_tree(xa); +} + +bool stop_iteration = false; + +static void *creator_func(void *ptr) +{ + /* 'order' is set up to ensure we have sibling entries */ + unsigned int order = RADIX_TREE_MAP_SHIFT - 1; + struct radix_tree_root *tree = ptr; + int i; + + for (i = 0; i < 10000; i++) { + item_insert_order(tree, 0, order); + item_delete_rcu(tree, 0); + } + + stop_iteration = true; + return NULL; +} + +static void *iterator_func(void *ptr) +{ + XA_STATE(xas, ptr, 0); + struct item *item; + + while (!stop_iteration) { + rcu_read_lock(); + xas_for_each(&xas, item, ULONG_MAX) { + if (xas_retry(&xas, item)) + continue; + + item_sanity(item, xas.xa_index); + } + rcu_read_unlock(); + } + return NULL; +} + +static void multiorder_iteration_race(struct xarray *xa) +{ + const int num_threads = sysconf(_SC_NPROCESSORS_ONLN); + pthread_t worker_thread[num_threads]; + int i; + + pthread_create(&worker_thread[0], NULL, &creator_func, xa); + for (i = 1; i < num_threads; i++) + pthread_create(&worker_thread[i], NULL, &iterator_func, xa); + + for (i = 0; i < num_threads; i++) + pthread_join(worker_thread[i], NULL); + + item_kill_tree(xa); +} + +static DEFINE_XARRAY(array); + +void multiorder_checks(void) +{ + multiorder_iteration(&array); + multiorder_tagged_iteration(&array); + multiorder_iteration_race(&array); + + radix_tree_cpu_dead(0); +} + +int __weak main(void) +{ + rcu_register_thread(); + radix_tree_init(); + multiorder_checks(); + rcu_unregister_thread(); + return 0; +} diff --git a/tools/testing/radix-tree/regression.h b/tools/testing/radix-tree/regression.h new file mode 100644 index 000000000..135145af1 --- /dev/null +++ b/tools/testing/radix-tree/regression.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __REGRESSION_H__ +#define __REGRESSION_H__ + +void regression1_test(void); +void regression2_test(void); +void regression3_test(void); +void regression4_test(void); + +#endif diff --git a/tools/testing/radix-tree/regression1.c b/tools/testing/radix-tree/regression1.c new file mode 100644 index 000000000..63f468bf8 --- /dev/null +++ b/tools/testing/radix-tree/regression1.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression1 + * Description: + * Salman Qazi describes the following radix-tree bug: + * + * In the following case, we get can get a deadlock: + * + * 0. The radix tree contains two items, one has the index 0. + * 1. The reader (in this case find_get_pages) takes the rcu_read_lock. + * 2. The reader acquires slot(s) for item(s) including the index 0 item. + * 3. The non-zero index item is deleted, and as a consequence the other item + * is moved to the root of the tree. The place where it used to be is queued + * for deletion after the readers finish. + * 3b. The zero item is deleted, removing it from the direct slot, it remains in + * the rcu-delayed indirect node. + * 4. The reader looks at the index 0 slot, and finds that the page has 0 ref + * count + * 5. The reader looks at it again, hoping that the item will either be freed + * or the ref count will increase. This never happens, as the slot it is + * looking at will never be updated. Also, this slot can never be reclaimed + * because the reader is holding rcu_read_lock and is in an infinite loop. + * + * The fix is to re-use the same "indirect" pointer case that requires a slot + * lookup retry into a general "retry the lookup" bit. + * + * Running: + * This test should run to completion in a few seconds. The above bug would + * cause it to hang indefinitely. + * + * Upstream commit: + * Not yet + */ +#include <linux/kernel.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/radix-tree.h> +#include <linux/rcupdate.h> +#include <stdlib.h> +#include <pthread.h> +#include <stdio.h> +#include <assert.h> + +#include "regression.h" + +static RADIX_TREE(mt_tree, GFP_KERNEL); + +struct page { + pthread_mutex_t lock; + struct rcu_head rcu; + int count; + unsigned long index; +}; + +static struct page *page_alloc(int index) +{ + struct page *p; + p = malloc(sizeof(struct page)); + p->count = 1; + p->index = index; + pthread_mutex_init(&p->lock, NULL); + + return p; +} + +static void page_rcu_free(struct rcu_head *rcu) +{ + struct page *p = container_of(rcu, struct page, rcu); + assert(!p->count); + pthread_mutex_destroy(&p->lock); + free(p); +} + +static void page_free(struct page *p) +{ + call_rcu(&p->rcu, page_rcu_free); +} + +static unsigned find_get_pages(unsigned long start, + unsigned int nr_pages, struct page **pages) +{ + XA_STATE(xas, &mt_tree, start); + struct page *page; + unsigned int ret = 0; + + rcu_read_lock(); + xas_for_each(&xas, page, ULONG_MAX) { + if (xas_retry(&xas, page)) + continue; + + pthread_mutex_lock(&page->lock); + if (!page->count) + goto unlock; + + /* don't actually update page refcount */ + pthread_mutex_unlock(&page->lock); + + /* Has the page moved? */ + if (unlikely(page != xas_reload(&xas))) + goto put_page; + + pages[ret] = page; + ret++; + continue; +unlock: + pthread_mutex_unlock(&page->lock); +put_page: + xas_reset(&xas); + } + rcu_read_unlock(); + return ret; +} + +static pthread_barrier_t worker_barrier; + +static void *regression1_fn(void *arg) +{ + rcu_register_thread(); + + if (pthread_barrier_wait(&worker_barrier) == + PTHREAD_BARRIER_SERIAL_THREAD) { + int j; + + for (j = 0; j < 1000000; j++) { + struct page *p; + + p = page_alloc(0); + xa_lock(&mt_tree); + radix_tree_insert(&mt_tree, 0, p); + xa_unlock(&mt_tree); + + p = page_alloc(1); + xa_lock(&mt_tree); + radix_tree_insert(&mt_tree, 1, p); + xa_unlock(&mt_tree); + + xa_lock(&mt_tree); + p = radix_tree_delete(&mt_tree, 1); + pthread_mutex_lock(&p->lock); + p->count--; + pthread_mutex_unlock(&p->lock); + xa_unlock(&mt_tree); + page_free(p); + + xa_lock(&mt_tree); + p = radix_tree_delete(&mt_tree, 0); + pthread_mutex_lock(&p->lock); + p->count--; + pthread_mutex_unlock(&p->lock); + xa_unlock(&mt_tree); + page_free(p); + } + } else { + int j; + + for (j = 0; j < 100000000; j++) { + struct page *pages[10]; + + find_get_pages(0, 10, pages); + } + } + + rcu_unregister_thread(); + + return NULL; +} + +static pthread_t *threads; +void regression1_test(void) +{ + int nr_threads; + int i; + long arg; + + /* Regression #1 */ + printv(1, "running regression test 1, should finish in under a minute\n"); + nr_threads = 2; + pthread_barrier_init(&worker_barrier, NULL, nr_threads); + + threads = malloc(nr_threads * sizeof(*threads)); + + for (i = 0; i < nr_threads; i++) { + arg = i; + if (pthread_create(&threads[i], NULL, regression1_fn, (void *)arg)) { + perror("pthread_create"); + exit(1); + } + } + + for (i = 0; i < nr_threads; i++) { + if (pthread_join(threads[i], NULL)) { + perror("pthread_join"); + exit(1); + } + } + + free(threads); + + printv(1, "regression test 1, done\n"); +} diff --git a/tools/testing/radix-tree/regression2.c b/tools/testing/radix-tree/regression2.c new file mode 100644 index 000000000..f2c7e640a --- /dev/null +++ b/tools/testing/radix-tree/regression2.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression2 + * Description: + * Toshiyuki Okajima describes the following radix-tree bug: + * + * In the following case, we can get a hangup on + * radix_radix_tree_gang_lookup_tag_slot. + * + * 0. The radix tree contains RADIX_TREE_MAP_SIZE items. And the tag of + * a certain item has PAGECACHE_TAG_DIRTY. + * 1. radix_tree_range_tag_if_tagged(, start, end, , PAGECACHE_TAG_DIRTY, + * PAGECACHE_TAG_TOWRITE) is called to add PAGECACHE_TAG_TOWRITE tag + * for the tag which has PAGECACHE_TAG_DIRTY. However, there is no tag with + * PAGECACHE_TAG_DIRTY within the range from start to end. As the result, + * There is no tag with PAGECACHE_TAG_TOWRITE but the root tag has + * PAGECACHE_TAG_TOWRITE. + * 2. An item is added into the radix tree and then the level of it is + * extended into 2 from 1. At that time, the new radix tree node succeeds + * the tag status of the root tag. Therefore the tag of the new radix tree + * node has PAGECACHE_TAG_TOWRITE but there is not slot with + * PAGECACHE_TAG_TOWRITE tag in the child node of the new radix tree node. + * 3. The tag of a certain item is cleared with PAGECACHE_TAG_DIRTY. + * 4. All items within the index range from 0 to RADIX_TREE_MAP_SIZE - 1 are + * released. (Only the item which index is RADIX_TREE_MAP_SIZE exist in the + * radix tree.) As the result, the slot of the radix tree node is NULL but + * the tag which corresponds to the slot has PAGECACHE_TAG_TOWRITE. + * 5. radix_tree_gang_lookup_tag_slot(PAGECACHE_TAG_TOWRITE) calls + * __lookup_tag. __lookup_tag returns with 0. And __lookup_tag doesn't + * change the index that is the input and output parameter. Because the 1st + * slot of the radix tree node is NULL, but the tag which corresponds to + * the slot has PAGECACHE_TAG_TOWRITE. + * Therefore radix_tree_gang_lookup_tag_slot tries to get some items by + * calling __lookup_tag, but it cannot get any items forever. + * + * The fix is to change that radix_tree_tag_if_tagged doesn't tag the root tag + * if it doesn't set any tags within the specified range. + * + * Running: + * This test should run to completion immediately. The above bug would cause it + * to hang indefinitely. + * + * Upstream commit: + * Not yet + */ +#include <linux/kernel.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/radix-tree.h> +#include <stdlib.h> +#include <stdio.h> + +#include "regression.h" +#include "test.h" + +#define PAGECACHE_TAG_DIRTY XA_MARK_0 +#define PAGECACHE_TAG_WRITEBACK XA_MARK_1 +#define PAGECACHE_TAG_TOWRITE XA_MARK_2 + +static RADIX_TREE(mt_tree, GFP_KERNEL); +unsigned long page_count = 0; + +struct page { + unsigned long index; +}; + +static struct page *page_alloc(void) +{ + struct page *p; + p = malloc(sizeof(struct page)); + p->index = page_count++; + + return p; +} + +void regression2_test(void) +{ + int i; + struct page *p; + int max_slots = RADIX_TREE_MAP_SIZE; + unsigned long int start, end; + struct page *pages[1]; + + printv(1, "running regression test 2 (should take milliseconds)\n"); + /* 0. */ + for (i = 0; i <= max_slots - 1; i++) { + p = page_alloc(); + radix_tree_insert(&mt_tree, i, p); + } + radix_tree_tag_set(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY); + + /* 1. */ + start = 0; + end = max_slots - 2; + tag_tagged_items(&mt_tree, start, end, 1, + PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE); + + /* 2. */ + p = page_alloc(); + radix_tree_insert(&mt_tree, max_slots, p); + + /* 3. */ + radix_tree_tag_clear(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY); + + /* 4. */ + for (i = max_slots - 1; i >= 0; i--) + free(radix_tree_delete(&mt_tree, i)); + + /* 5. */ + // NOTE: start should not be 0 because radix_tree_gang_lookup_tag_slot + // can return. + start = 1; + end = max_slots - 2; + radix_tree_gang_lookup_tag_slot(&mt_tree, (void ***)pages, start, end, + PAGECACHE_TAG_TOWRITE); + + /* We remove all the remained nodes */ + free(radix_tree_delete(&mt_tree, max_slots)); + + BUG_ON(!radix_tree_empty(&mt_tree)); + + printv(1, "regression test 2, done\n"); +} diff --git a/tools/testing/radix-tree/regression3.c b/tools/testing/radix-tree/regression3.c new file mode 100644 index 000000000..9f9a3b280 --- /dev/null +++ b/tools/testing/radix-tree/regression3.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression3 + * Description: + * Helper radix_tree_iter_retry resets next_index to the current index. + * In following radix_tree_next_slot current chunk size becomes zero. + * This isn't checked and it tries to dereference null pointer in slot. + * + * Helper radix_tree_iter_resume reset slot to NULL and next_index to index + 1, + * for tagger iteraction it also must reset cached tags in iterator to abort + * next radix_tree_next_slot and go to slow-path into radix_tree_next_chunk. + * + * Running: + * This test should run to completion immediately. The above bug would + * cause it to segfault. + * + * Upstream commit: + * Not yet + */ +#include <linux/kernel.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/radix-tree.h> +#include <stdlib.h> +#include <stdio.h> + +#include "regression.h" + +void regression3_test(void) +{ + RADIX_TREE(root, GFP_KERNEL); + void *ptr0 = (void *)4ul; + void *ptr = (void *)8ul; + struct radix_tree_iter iter; + void **slot; + bool first; + + printv(1, "running regression test 3 (should take milliseconds)\n"); + + radix_tree_insert(&root, 0, ptr0); + radix_tree_tag_set(&root, 0, 0); + + first = true; + radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) { + printv(2, "tagged %ld %p\n", iter.index, *slot); + if (first) { + radix_tree_insert(&root, 1, ptr); + radix_tree_tag_set(&root, 1, 0); + first = false; + } + if (radix_tree_deref_retry(*slot)) { + printv(2, "retry at %ld\n", iter.index); + slot = radix_tree_iter_retry(&iter); + continue; + } + } + radix_tree_delete(&root, 1); + + first = true; + radix_tree_for_each_slot(slot, &root, &iter, 0) { + printv(2, "slot %ld %p\n", iter.index, *slot); + if (first) { + radix_tree_insert(&root, 1, ptr); + first = false; + } + if (radix_tree_deref_retry(*slot)) { + printv(2, "retry at %ld\n", iter.index); + slot = radix_tree_iter_retry(&iter); + continue; + } + } + + radix_tree_for_each_slot(slot, &root, &iter, 0) { + printv(2, "slot %ld %p\n", iter.index, *slot); + if (!iter.index) { + printv(2, "next at %ld\n", iter.index); + slot = radix_tree_iter_resume(slot, &iter); + } + } + + radix_tree_tag_set(&root, 0, 0); + radix_tree_tag_set(&root, 1, 0); + radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) { + printv(2, "tagged %ld %p\n", iter.index, *slot); + if (!iter.index) { + printv(2, "next at %ld\n", iter.index); + slot = radix_tree_iter_resume(slot, &iter); + } + } + + radix_tree_delete(&root, 0); + radix_tree_delete(&root, 1); + + printv(1, "regression test 3 passed\n"); +} diff --git a/tools/testing/radix-tree/regression4.c b/tools/testing/radix-tree/regression4.c new file mode 100644 index 000000000..cf4e5aba6 --- /dev/null +++ b/tools/testing/radix-tree/regression4.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/kernel.h> +#include <linux/gfp.h> +#include <linux/slab.h> +#include <linux/radix-tree.h> +#include <linux/rcupdate.h> +#include <stdlib.h> +#include <pthread.h> +#include <stdio.h> +#include <assert.h> + +#include "regression.h" + +static pthread_barrier_t worker_barrier; +static int obj0, obj1; +static RADIX_TREE(mt_tree, GFP_KERNEL); + +static void *reader_fn(void *arg) +{ + int i; + void *entry; + + rcu_register_thread(); + pthread_barrier_wait(&worker_barrier); + + for (i = 0; i < 1000000; i++) { + rcu_read_lock(); + entry = radix_tree_lookup(&mt_tree, 0); + rcu_read_unlock(); + if (entry != &obj0) { + printf("iteration %d bad entry = %p\n", i, entry); + abort(); + } + } + + rcu_unregister_thread(); + + return NULL; +} + +static void *writer_fn(void *arg) +{ + int i; + + rcu_register_thread(); + pthread_barrier_wait(&worker_barrier); + + for (i = 0; i < 1000000; i++) { + radix_tree_insert(&mt_tree, 1, &obj1); + radix_tree_delete(&mt_tree, 1); + } + + rcu_unregister_thread(); + + return NULL; +} + +void regression4_test(void) +{ + pthread_t reader, writer; + + printv(1, "regression test 4 starting\n"); + + radix_tree_insert(&mt_tree, 0, &obj0); + pthread_barrier_init(&worker_barrier, NULL, 2); + + if (pthread_create(&reader, NULL, reader_fn, NULL) || + pthread_create(&writer, NULL, writer_fn, NULL)) { + perror("pthread_create"); + exit(1); + } + + if (pthread_join(reader, NULL) || pthread_join(writer, NULL)) { + perror("pthread_join"); + exit(1); + } + + printv(1, "regression test 4 passed\n"); +} diff --git a/tools/testing/radix-tree/tag_check.c b/tools/testing/radix-tree/tag_check.c new file mode 100644 index 000000000..f898957b1 --- /dev/null +++ b/tools/testing/radix-tree/tag_check.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdlib.h> +#include <assert.h> +#include <stdio.h> +#include <string.h> + +#include <linux/slab.h> +#include <linux/radix-tree.h> + +#include "test.h" + + +static void +__simple_checks(struct radix_tree_root *tree, unsigned long index, int tag) +{ + unsigned long first = 0; + int ret; + + item_check_absent(tree, index); + assert(item_tag_get(tree, index, tag) == 0); + + item_insert(tree, index); + assert(item_tag_get(tree, index, tag) == 0); + item_tag_set(tree, index, tag); + ret = item_tag_get(tree, index, tag); + assert(ret != 0); + ret = tag_tagged_items(tree, first, ~0UL, 10, tag, !tag); + assert(ret == 1); + ret = item_tag_get(tree, index, !tag); + assert(ret != 0); + ret = item_delete(tree, index); + assert(ret != 0); + item_insert(tree, index); + ret = item_tag_get(tree, index, tag); + assert(ret == 0); + ret = item_delete(tree, index); + assert(ret != 0); + ret = item_delete(tree, index); + assert(ret == 0); +} + +void simple_checks(void) +{ + unsigned long index; + RADIX_TREE(tree, GFP_KERNEL); + + for (index = 0; index < 10000; index++) { + __simple_checks(&tree, index, 0); + __simple_checks(&tree, index, 1); + } + verify_tag_consistency(&tree, 0); + verify_tag_consistency(&tree, 1); + printv(2, "before item_kill_tree: %d allocated\n", nr_allocated); + item_kill_tree(&tree); + rcu_barrier(); + printv(2, "after item_kill_tree: %d allocated\n", nr_allocated); +} + +/* + * Check that tags propagate correctly when extending a tree. + */ +static void extend_checks(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + + item_insert(&tree, 43); + assert(item_tag_get(&tree, 43, 0) == 0); + item_tag_set(&tree, 43, 0); + assert(item_tag_get(&tree, 43, 0) == 1); + item_insert(&tree, 1000000); + assert(item_tag_get(&tree, 43, 0) == 1); + + item_insert(&tree, 0); + item_tag_set(&tree, 0, 0); + item_delete(&tree, 1000000); + assert(item_tag_get(&tree, 43, 0) != 0); + item_delete(&tree, 43); + assert(item_tag_get(&tree, 43, 0) == 0); /* crash */ + assert(item_tag_get(&tree, 0, 0) == 1); + + verify_tag_consistency(&tree, 0); + + item_kill_tree(&tree); +} + +/* + * Check that tags propagate correctly when contracting a tree. + */ +static void contract_checks(void) +{ + struct item *item; + int tmp; + RADIX_TREE(tree, GFP_KERNEL); + + tmp = 1<<RADIX_TREE_MAP_SHIFT; + item_insert(&tree, tmp); + item_insert(&tree, tmp+1); + item_tag_set(&tree, tmp, 0); + item_tag_set(&tree, tmp, 1); + item_tag_set(&tree, tmp+1, 0); + item_delete(&tree, tmp+1); + item_tag_clear(&tree, tmp, 1); + + assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 0) == 1); + assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 1) == 0); + + assert(item_tag_get(&tree, tmp, 0) == 1); + assert(item_tag_get(&tree, tmp, 1) == 0); + + verify_tag_consistency(&tree, 0); + item_kill_tree(&tree); +} + +/* + * Stupid tag thrasher + * + * Create a large linear array corresponding to the tree. Each element in + * the array is coherent with each node in the tree + */ + +enum { + NODE_ABSENT = 0, + NODE_PRESENT = 1, + NODE_TAGGED = 2, +}; + +#define THRASH_SIZE (1000 * 1000) +#define N 127 +#define BATCH 33 + +static void gang_check(struct radix_tree_root *tree, + char *thrash_state, int tag) +{ + struct item *items[BATCH]; + int nr_found; + unsigned long index = 0; + unsigned long last_index = 0; + + while ((nr_found = radix_tree_gang_lookup_tag(tree, (void **)items, + index, BATCH, tag))) { + int i; + + for (i = 0; i < nr_found; i++) { + struct item *item = items[i]; + + while (last_index < item->index) { + assert(thrash_state[last_index] != NODE_TAGGED); + last_index++; + } + assert(thrash_state[last_index] == NODE_TAGGED); + last_index++; + } + index = items[nr_found - 1]->index + 1; + } +} + +static void do_thrash(struct radix_tree_root *tree, char *thrash_state, int tag) +{ + int insert_chunk; + int delete_chunk; + int tag_chunk; + int untag_chunk; + int total_tagged = 0; + int total_present = 0; + + for (insert_chunk = 1; insert_chunk < THRASH_SIZE; insert_chunk *= N) + for (delete_chunk = 1; delete_chunk < THRASH_SIZE; delete_chunk *= N) + for (tag_chunk = 1; tag_chunk < THRASH_SIZE; tag_chunk *= N) + for (untag_chunk = 1; untag_chunk < THRASH_SIZE; untag_chunk *= N) { + int i; + unsigned long index; + int nr_inserted = 0; + int nr_deleted = 0; + int nr_tagged = 0; + int nr_untagged = 0; + int actual_total_tagged; + int actual_total_present; + + for (i = 0; i < insert_chunk; i++) { + index = rand() % THRASH_SIZE; + if (thrash_state[index] != NODE_ABSENT) + continue; + item_check_absent(tree, index); + item_insert(tree, index); + assert(thrash_state[index] != NODE_PRESENT); + thrash_state[index] = NODE_PRESENT; + nr_inserted++; + total_present++; + } + + for (i = 0; i < delete_chunk; i++) { + index = rand() % THRASH_SIZE; + if (thrash_state[index] == NODE_ABSENT) + continue; + item_check_present(tree, index); + if (item_tag_get(tree, index, tag)) { + assert(thrash_state[index] == NODE_TAGGED); + total_tagged--; + } else { + assert(thrash_state[index] == NODE_PRESENT); + } + item_delete(tree, index); + assert(thrash_state[index] != NODE_ABSENT); + thrash_state[index] = NODE_ABSENT; + nr_deleted++; + total_present--; + } + + for (i = 0; i < tag_chunk; i++) { + index = rand() % THRASH_SIZE; + if (thrash_state[index] != NODE_PRESENT) { + if (item_lookup(tree, index)) + assert(item_tag_get(tree, index, tag)); + continue; + } + item_tag_set(tree, index, tag); + item_tag_set(tree, index, tag); + assert(thrash_state[index] != NODE_TAGGED); + thrash_state[index] = NODE_TAGGED; + nr_tagged++; + total_tagged++; + } + + for (i = 0; i < untag_chunk; i++) { + index = rand() % THRASH_SIZE; + if (thrash_state[index] != NODE_TAGGED) + continue; + item_check_present(tree, index); + assert(item_tag_get(tree, index, tag)); + item_tag_clear(tree, index, tag); + item_tag_clear(tree, index, tag); + assert(thrash_state[index] != NODE_PRESENT); + thrash_state[index] = NODE_PRESENT; + nr_untagged++; + total_tagged--; + } + + actual_total_tagged = 0; + actual_total_present = 0; + for (index = 0; index < THRASH_SIZE; index++) { + switch (thrash_state[index]) { + case NODE_ABSENT: + item_check_absent(tree, index); + break; + case NODE_PRESENT: + item_check_present(tree, index); + assert(!item_tag_get(tree, index, tag)); + actual_total_present++; + break; + case NODE_TAGGED: + item_check_present(tree, index); + assert(item_tag_get(tree, index, tag)); + actual_total_present++; + actual_total_tagged++; + break; + } + } + + gang_check(tree, thrash_state, tag); + + printv(2, "%d(%d) %d(%d) %d(%d) %d(%d) / " + "%d(%d) present, %d(%d) tagged\n", + insert_chunk, nr_inserted, + delete_chunk, nr_deleted, + tag_chunk, nr_tagged, + untag_chunk, nr_untagged, + total_present, actual_total_present, + total_tagged, actual_total_tagged); + } +} + +static void thrash_tags(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + char *thrash_state; + + thrash_state = malloc(THRASH_SIZE); + memset(thrash_state, 0, THRASH_SIZE); + + do_thrash(&tree, thrash_state, 0); + + verify_tag_consistency(&tree, 0); + item_kill_tree(&tree); + free(thrash_state); +} + +static void leak_check(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + + item_insert(&tree, 1000000); + item_delete(&tree, 1000000); + item_kill_tree(&tree); +} + +static void __leak_check(void) +{ + RADIX_TREE(tree, GFP_KERNEL); + + printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated); + item_insert(&tree, 1000000); + printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated); + item_delete(&tree, 1000000); + printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated); + item_kill_tree(&tree); + printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated); +} + +static void single_check(void) +{ + struct item *items[BATCH]; + RADIX_TREE(tree, GFP_KERNEL); + int ret; + unsigned long first = 0; + + item_insert(&tree, 0); + item_tag_set(&tree, 0, 0); + ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0); + assert(ret == 1); + ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 1, BATCH, 0); + assert(ret == 0); + verify_tag_consistency(&tree, 0); + verify_tag_consistency(&tree, 1); + ret = tag_tagged_items(&tree, first, 10, 10, XA_MARK_0, XA_MARK_1); + assert(ret == 1); + ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 1); + assert(ret == 1); + item_tag_clear(&tree, 0, 0); + ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0); + assert(ret == 0); + item_kill_tree(&tree); +} + +void tag_check(void) +{ + single_check(); + extend_checks(); + contract_checks(); + rcu_barrier(); + printv(2, "after extend_checks: %d allocated\n", nr_allocated); + __leak_check(); + leak_check(); + rcu_barrier(); + printv(2, "after leak_check: %d allocated\n", nr_allocated); + simple_checks(); + rcu_barrier(); + printv(2, "after simple_checks: %d allocated\n", nr_allocated); + thrash_tags(); + rcu_barrier(); + printv(2, "after thrash_tags: %d allocated\n", nr_allocated); +} diff --git a/tools/testing/radix-tree/test.c b/tools/testing/radix-tree/test.c new file mode 100644 index 000000000..a15d0512e --- /dev/null +++ b/tools/testing/radix-tree/test.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdlib.h> +#include <assert.h> +#include <stdio.h> +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/bitops.h> + +#include "test.h" + +struct item * +item_tag_set(struct radix_tree_root *root, unsigned long index, int tag) +{ + return radix_tree_tag_set(root, index, tag); +} + +struct item * +item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag) +{ + return radix_tree_tag_clear(root, index, tag); +} + +int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag) +{ + return radix_tree_tag_get(root, index, tag); +} + +struct item *item_create(unsigned long index, unsigned int order) +{ + struct item *ret = malloc(sizeof(*ret)); + + ret->index = index; + ret->order = order; + return ret; +} + +int item_insert(struct radix_tree_root *root, unsigned long index) +{ + struct item *item = item_create(index, 0); + int err = radix_tree_insert(root, item->index, item); + if (err) + free(item); + return err; +} + +void item_sanity(struct item *item, unsigned long index) +{ + unsigned long mask; + assert(!radix_tree_is_internal_node(item)); + assert(item->order < BITS_PER_LONG); + mask = (1UL << item->order) - 1; + assert((item->index | mask) == (index | mask)); +} + +void item_free(struct item *item, unsigned long index) +{ + item_sanity(item, index); + free(item); +} + +int item_delete(struct radix_tree_root *root, unsigned long index) +{ + struct item *item = radix_tree_delete(root, index); + + if (!item) + return 0; + + item_free(item, index); + return 1; +} + +static void item_free_rcu(struct rcu_head *head) +{ + struct item *item = container_of(head, struct item, rcu_head); + + free(item); +} + +int item_delete_rcu(struct xarray *xa, unsigned long index) +{ + struct item *item = xa_erase(xa, index); + + if (item) { + item_sanity(item, index); + call_rcu(&item->rcu_head, item_free_rcu); + return 1; + } + return 0; +} + +void item_check_present(struct radix_tree_root *root, unsigned long index) +{ + struct item *item; + + item = radix_tree_lookup(root, index); + assert(item != NULL); + item_sanity(item, index); +} + +struct item *item_lookup(struct radix_tree_root *root, unsigned long index) +{ + return radix_tree_lookup(root, index); +} + +void item_check_absent(struct radix_tree_root *root, unsigned long index) +{ + struct item *item; + + item = radix_tree_lookup(root, index); + assert(item == NULL); +} + +/* + * Scan only the passed (start, start+nr] for present items + */ +void item_gang_check_present(struct radix_tree_root *root, + unsigned long start, unsigned long nr, + int chunk, int hop) +{ + struct item *items[chunk]; + unsigned long into; + + for (into = 0; into < nr; ) { + int nfound; + int nr_to_find = chunk; + int i; + + if (nr_to_find > (nr - into)) + nr_to_find = nr - into; + + nfound = radix_tree_gang_lookup(root, (void **)items, + start + into, nr_to_find); + assert(nfound == nr_to_find); + for (i = 0; i < nfound; i++) + assert(items[i]->index == start + into + i); + into += hop; + } +} + +/* + * Scan the entire tree, only expecting present items (start, start+nr] + */ +void item_full_scan(struct radix_tree_root *root, unsigned long start, + unsigned long nr, int chunk) +{ + struct item *items[chunk]; + unsigned long into = 0; + unsigned long this_index = start; + int nfound; + int i; + +// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk); + + while ((nfound = radix_tree_gang_lookup(root, (void **)items, into, + chunk))) { +// printf("At 0x%08lx, nfound=%d\n", into, nfound); + for (i = 0; i < nfound; i++) { + assert(items[i]->index == this_index); + this_index++; + } +// printf("Found 0x%08lx->0x%08lx\n", +// items[0]->index, items[nfound-1]->index); + into = this_index; + } + if (chunk) + assert(this_index == start + nr); + nfound = radix_tree_gang_lookup(root, (void **)items, + this_index, chunk); + assert(nfound == 0); +} + +/* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */ +int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end, + unsigned batch, xa_mark_t iftag, xa_mark_t thentag) +{ + XA_STATE(xas, xa, start); + unsigned int tagged = 0; + struct item *item; + + if (batch == 0) + batch = 1; + + xas_lock_irq(&xas); + xas_for_each_marked(&xas, item, end, iftag) { + xas_set_mark(&xas, thentag); + if (++tagged % batch) + continue; + + xas_pause(&xas); + xas_unlock_irq(&xas); + rcu_barrier(); + xas_lock_irq(&xas); + } + xas_unlock_irq(&xas); + + return tagged; +} + +static int verify_node(struct radix_tree_node *slot, unsigned int tag, + int tagged) +{ + int anyset = 0; + int i; + int j; + + slot = entry_to_node(slot); + + /* Verify consistency at this level */ + for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) { + if (slot->tags[tag][i]) { + anyset = 1; + break; + } + } + if (tagged != anyset) { + printf("tag: %u, shift %u, tagged: %d, anyset: %d\n", + tag, slot->shift, tagged, anyset); + for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) { + printf("tag %d: ", j); + for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) + printf("%016lx ", slot->tags[j][i]); + printf("\n"); + } + return 1; + } + assert(tagged == anyset); + + /* Go for next level */ + if (slot->shift > 0) { + for (i = 0; i < RADIX_TREE_MAP_SIZE; i++) + if (slot->slots[i]) + if (verify_node(slot->slots[i], tag, + !!test_bit(i, slot->tags[tag]))) { + printf("Failure at off %d\n", i); + for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) { + printf("tag %d: ", j); + for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) + printf("%016lx ", slot->tags[j][i]); + printf("\n"); + } + return 1; + } + } + return 0; +} + +void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag) +{ + struct radix_tree_node *node = root->xa_head; + if (!radix_tree_is_internal_node(node)) + return; + verify_node(node, tag, !!root_tag_get(root, tag)); +} + +void item_kill_tree(struct xarray *xa) +{ + XA_STATE(xas, xa, 0); + void *entry; + + xas_for_each(&xas, entry, ULONG_MAX) { + if (!xa_is_value(entry)) { + item_free(entry, xas.xa_index); + } + xas_store(&xas, NULL); + } + + assert(xa_empty(xa)); +} + +void tree_verify_min_height(struct radix_tree_root *root, int maxindex) +{ + unsigned shift; + struct radix_tree_node *node = root->xa_head; + if (!radix_tree_is_internal_node(node)) { + assert(maxindex == 0); + return; + } + + node = entry_to_node(node); + assert(maxindex <= node_maxindex(node)); + + shift = node->shift; + if (shift > 0) + assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT)); + else + assert(maxindex > 0); +} diff --git a/tools/testing/radix-tree/test.h b/tools/testing/radix-tree/test.h new file mode 100644 index 000000000..7ef7067e9 --- /dev/null +++ b/tools/testing/radix-tree/test.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/gfp.h> +#include <linux/types.h> +#include <linux/radix-tree.h> +#include <linux/rcupdate.h> + +struct item { + struct rcu_head rcu_head; + unsigned long index; + unsigned int order; +}; + +struct item *item_create(unsigned long index, unsigned int order); +int item_insert(struct radix_tree_root *root, unsigned long index); +void item_sanity(struct item *item, unsigned long index); +void item_free(struct item *item, unsigned long index); +int item_delete(struct radix_tree_root *root, unsigned long index); +int item_delete_rcu(struct xarray *xa, unsigned long index); +struct item *item_lookup(struct radix_tree_root *root, unsigned long index); + +void item_check_present(struct radix_tree_root *root, unsigned long index); +void item_check_absent(struct radix_tree_root *root, unsigned long index); +void item_gang_check_present(struct radix_tree_root *root, + unsigned long start, unsigned long nr, + int chunk, int hop); +void item_full_scan(struct radix_tree_root *root, unsigned long start, + unsigned long nr, int chunk); +void item_kill_tree(struct radix_tree_root *root); + +int tag_tagged_items(struct xarray *, unsigned long start, unsigned long end, + unsigned batch, xa_mark_t iftag, xa_mark_t thentag); + +void xarray_tests(void); +void tag_check(void); +void multiorder_checks(void); +void iteration_test(unsigned order, unsigned duration); +void iteration_test2(unsigned duration); +void benchmark(void); +void idr_checks(void); +void ida_tests(void); + +struct item * +item_tag_set(struct radix_tree_root *root, unsigned long index, int tag); +struct item * +item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag); +int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag); +void tree_verify_min_height(struct radix_tree_root *root, int maxindex); +void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag); + +extern int nr_allocated; + +/* Normally private parts of lib/radix-tree.c */ +struct radix_tree_node *entry_to_node(void *ptr); +void radix_tree_dump(struct radix_tree_root *root); +int root_tag_get(struct radix_tree_root *root, unsigned int tag); +unsigned long node_maxindex(struct radix_tree_node *); +unsigned long shift_maxindex(unsigned int shift); +int radix_tree_cpu_dead(unsigned int cpu); +extern struct radix_tree_preload radix_tree_preloads; diff --git a/tools/testing/radix-tree/xarray.c b/tools/testing/radix-tree/xarray.c new file mode 100644 index 000000000..f20e12cbb --- /dev/null +++ b/tools/testing/radix-tree/xarray.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * xarray.c: Userspace shim for XArray test-suite + * Copyright (c) 2018 Matthew Wilcox <willy@infradead.org> + */ + +#define XA_DEBUG +#include "test.h" + +#define module_init(x) +#define module_exit(x) +#define MODULE_AUTHOR(x) +#define MODULE_LICENSE(x) +#define dump_stack() assert(0) + +#include "../../../lib/xarray.c" +#undef XA_DEBUG +#include "../../../lib/test_xarray.c" + +void xarray_tests(void) +{ + xarray_checks(); + xarray_exit(); +} + +int __weak main(void) +{ + rcu_register_thread(); + radix_tree_init(); + xarray_tests(); + radix_tree_cpu_dead(1); + rcu_barrier(); + if (nr_allocated) + printf("nr_allocated = %d\n", nr_allocated); + rcu_unregister_thread(); + return 0; +} |