diff options
Diffstat (limited to '')
| -rw-r--r-- | include/linux/slab.h | 734 |
1 files changed, 734 insertions, 0 deletions
diff --git a/include/linux/slab.h b/include/linux/slab.h new file mode 100644 index 000000000..d6393413e --- /dev/null +++ b/include/linux/slab.h @@ -0,0 +1,734 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). + * + * (C) SGI 2006, Christoph Lameter + * Cleaned up and restructured to ease the addition of alternative + * implementations of SLAB allocators. + * (C) Linux Foundation 2008-2013 + * Unified interface for all slab allocators + */ + +#ifndef _LINUX_SLAB_H +#define _LINUX_SLAB_H + +#include <linux/gfp.h> +#include <linux/overflow.h> +#include <linux/types.h> +#include <linux/workqueue.h> + + +/* + * Flags to pass to kmem_cache_create(). + * The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set. + */ +/* DEBUG: Perform (expensive) checks on alloc/free */ +#define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) +/* DEBUG: Red zone objs in a cache */ +#define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) +/* DEBUG: Poison objects */ +#define SLAB_POISON ((slab_flags_t __force)0x00000800U) +/* Align objs on cache lines */ +#define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) +/* Use GFP_DMA memory */ +#define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) +/* Use GFP_DMA32 memory */ +#define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) +/* DEBUG: Store the last owner for bug hunting */ +#define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) +/* Panic if kmem_cache_create() fails */ +#define SLAB_PANIC ((slab_flags_t __force)0x00040000U) +/* + * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! + * + * This delays freeing the SLAB page by a grace period, it does _NOT_ + * delay object freeing. This means that if you do kmem_cache_free() + * that memory location is free to be reused at any time. Thus it may + * be possible to see another object there in the same RCU grace period. + * + * This feature only ensures the memory location backing the object + * stays valid, the trick to using this is relying on an independent + * object validation pass. Something like: + * + * rcu_read_lock() + * again: + * obj = lockless_lookup(key); + * if (obj) { + * if (!try_get_ref(obj)) // might fail for free objects + * goto again; + * + * if (obj->key != key) { // not the object we expected + * put_ref(obj); + * goto again; + * } + * } + * rcu_read_unlock(); + * + * This is useful if we need to approach a kernel structure obliquely, + * from its address obtained without the usual locking. We can lock + * the structure to stabilize it and check it's still at the given address, + * only if we can be sure that the memory has not been meanwhile reused + * for some other kind of object (which our subsystem's lock might corrupt). + * + * rcu_read_lock before reading the address, then rcu_read_unlock after + * taking the spinlock within the structure expected at that address. + * + * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. + */ +/* Defer freeing slabs to RCU */ +#define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) +/* Spread some memory over cpuset */ +#define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) +/* Trace allocations and frees */ +#define SLAB_TRACE ((slab_flags_t __force)0x00200000U) + +/* Flag to prevent checks on free */ +#ifdef CONFIG_DEBUG_OBJECTS +# define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) +#else +# define SLAB_DEBUG_OBJECTS 0 +#endif + +/* Avoid kmemleak tracing */ +#define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) + +/* Fault injection mark */ +#ifdef CONFIG_FAILSLAB +# define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) +#else +# define SLAB_FAILSLAB 0 +#endif +/* Account to memcg */ +#ifdef CONFIG_MEMCG_KMEM +# define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) +#else +# define SLAB_ACCOUNT 0 +#endif + +#ifdef CONFIG_KASAN +#define SLAB_KASAN ((slab_flags_t __force)0x08000000U) +#else +#define SLAB_KASAN 0 +#endif + +/* The following flags affect the page allocator grouping pages by mobility */ +/* Objects are reclaimable */ +#define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) +#define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ +/* + * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. + * + * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. + * + * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. + * Both make kfree a no-op. + */ +#define ZERO_SIZE_PTR ((void *)16) + +#define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ + (unsigned long)ZERO_SIZE_PTR) + +#include <linux/kasan.h> + +struct mem_cgroup; +/* + * struct kmem_cache related prototypes + */ +void __init kmem_cache_init(void); +bool slab_is_available(void); + +extern bool usercopy_fallback; + +struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, + unsigned int align, slab_flags_t flags, + void (*ctor)(void *)); +struct kmem_cache *kmem_cache_create_usercopy(const char *name, + unsigned int size, unsigned int align, + slab_flags_t flags, + unsigned int useroffset, unsigned int usersize, + void (*ctor)(void *)); +void kmem_cache_destroy(struct kmem_cache *); +int kmem_cache_shrink(struct kmem_cache *); + +void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *); +void memcg_deactivate_kmem_caches(struct mem_cgroup *); +void memcg_destroy_kmem_caches(struct mem_cgroup *); + +/* + * Please use this macro to create slab caches. Simply specify the + * name of the structure and maybe some flags that are listed above. + * + * The alignment of the struct determines object alignment. If you + * f.e. add ____cacheline_aligned_in_smp to the struct declaration + * then the objects will be properly aligned in SMP configurations. + */ +#define KMEM_CACHE(__struct, __flags) \ + kmem_cache_create(#__struct, sizeof(struct __struct), \ + __alignof__(struct __struct), (__flags), NULL) + +/* + * To whitelist a single field for copying to/from usercopy, use this + * macro instead for KMEM_CACHE() above. + */ +#define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ + kmem_cache_create_usercopy(#__struct, \ + sizeof(struct __struct), \ + __alignof__(struct __struct), (__flags), \ + offsetof(struct __struct, __field), \ + sizeof_field(struct __struct, __field), NULL) + +/* + * Common kmalloc functions provided by all allocators + */ +void * __must_check __krealloc(const void *, size_t, gfp_t); +void * __must_check krealloc(const void *, size_t, gfp_t); +void kfree(const void *); +void kzfree(const void *); +size_t ksize(const void *); + +#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR +void __check_heap_object(const void *ptr, unsigned long n, struct page *page, + bool to_user); +#else +static inline void __check_heap_object(const void *ptr, unsigned long n, + struct page *page, bool to_user) { } +#endif + +/* + * Some archs want to perform DMA into kmalloc caches and need a guaranteed + * alignment larger than the alignment of a 64-bit integer. + * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. + */ +#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 +#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN +#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN +#define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) +#else +#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) +#endif + +/* + * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. + * Intended for arches that get misalignment faults even for 64 bit integer + * aligned buffers. + */ +#ifndef ARCH_SLAB_MINALIGN +#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) +#endif + +/* + * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned + * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN + * aligned pointers. + */ +#define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) +#define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) +#define __assume_page_alignment __assume_aligned(PAGE_SIZE) + +/* + * Kmalloc array related definitions + */ + +#ifdef CONFIG_SLAB +/* + * The largest kmalloc size supported by the SLAB allocators is + * 32 megabyte (2^25) or the maximum allocatable page order if that is + * less than 32 MB. + * + * WARNING: Its not easy to increase this value since the allocators have + * to do various tricks to work around compiler limitations in order to + * ensure proper constant folding. + */ +#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ + (MAX_ORDER + PAGE_SHIFT - 1) : 25) +#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifndef KMALLOC_SHIFT_LOW +#define KMALLOC_SHIFT_LOW 5 +#endif +#endif + +#ifdef CONFIG_SLUB +/* + * SLUB directly allocates requests fitting in to an order-1 page + * (PAGE_SIZE*2). Larger requests are passed to the page allocator. + */ +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) +#ifndef KMALLOC_SHIFT_LOW +#define KMALLOC_SHIFT_LOW 3 +#endif +#endif + +#ifdef CONFIG_SLOB +/* + * SLOB passes all requests larger than one page to the page allocator. + * No kmalloc array is necessary since objects of different sizes can + * be allocated from the same page. + */ +#define KMALLOC_SHIFT_HIGH PAGE_SHIFT +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) +#ifndef KMALLOC_SHIFT_LOW +#define KMALLOC_SHIFT_LOW 3 +#endif +#endif + +/* Maximum allocatable size */ +#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) +/* Maximum size for which we actually use a slab cache */ +#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) +/* Maximum order allocatable via the slab allocagtor */ +#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) + +/* + * Kmalloc subsystem. + */ +#ifndef KMALLOC_MIN_SIZE +#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) +#endif + +/* + * This restriction comes from byte sized index implementation. + * Page size is normally 2^12 bytes and, in this case, if we want to use + * byte sized index which can represent 2^8 entries, the size of the object + * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. + * If minimum size of kmalloc is less than 16, we use it as minimum object + * size and give up to use byte sized index. + */ +#define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ + (KMALLOC_MIN_SIZE) : 16) + +#ifndef CONFIG_SLOB +extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +#ifdef CONFIG_ZONE_DMA +extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; +#endif + +/* + * Figure out which kmalloc slab an allocation of a certain size + * belongs to. + * 0 = zero alloc + * 1 = 65 .. 96 bytes + * 2 = 129 .. 192 bytes + * n = 2^(n-1)+1 .. 2^n + */ +static __always_inline unsigned int kmalloc_index(size_t size) +{ + if (!size) + return 0; + + if (size <= KMALLOC_MIN_SIZE) + return KMALLOC_SHIFT_LOW; + + if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) + return 1; + if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) + return 2; + if (size <= 8) return 3; + if (size <= 16) return 4; + if (size <= 32) return 5; + if (size <= 64) return 6; + if (size <= 128) return 7; + if (size <= 256) return 8; + if (size <= 512) return 9; + if (size <= 1024) return 10; + if (size <= 2 * 1024) return 11; + if (size <= 4 * 1024) return 12; + if (size <= 8 * 1024) return 13; + if (size <= 16 * 1024) return 14; + if (size <= 32 * 1024) return 15; + if (size <= 64 * 1024) return 16; + if (size <= 128 * 1024) return 17; + if (size <= 256 * 1024) return 18; + if (size <= 512 * 1024) return 19; + if (size <= 1024 * 1024) return 20; + if (size <= 2 * 1024 * 1024) return 21; + if (size <= 4 * 1024 * 1024) return 22; + if (size <= 8 * 1024 * 1024) return 23; + if (size <= 16 * 1024 * 1024) return 24; + if (size <= 32 * 1024 * 1024) return 25; + if (size <= 64 * 1024 * 1024) return 26; + BUG(); + + /* Will never be reached. Needed because the compiler may complain */ + return -1; +} +#endif /* !CONFIG_SLOB */ + +void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc; +void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags) __assume_slab_alignment __malloc; +void kmem_cache_free(struct kmem_cache *, void *); + +/* + * Bulk allocation and freeing operations. These are accelerated in an + * allocator specific way to avoid taking locks repeatedly or building + * metadata structures unnecessarily. + * + * Note that interrupts must be enabled when calling these functions. + */ +void kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); +int kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); + +/* + * Caller must not use kfree_bulk() on memory not originally allocated + * by kmalloc(), because the SLOB allocator cannot handle this. + */ +static __always_inline void kfree_bulk(size_t size, void **p) +{ + kmem_cache_free_bulk(NULL, size, p); +} + +#ifdef CONFIG_NUMA +void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc; +void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc; +#else +static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ + return __kmalloc(size, flags); +} + +static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) +{ + return kmem_cache_alloc(s, flags); +} +#endif + +#ifdef CONFIG_TRACING +extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t) __assume_slab_alignment __malloc; + +#ifdef CONFIG_NUMA +extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size) __assume_slab_alignment __malloc; +#else +static __always_inline void * +kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size) +{ + return kmem_cache_alloc_trace(s, gfpflags, size); +} +#endif /* CONFIG_NUMA */ + +#else /* CONFIG_TRACING */ +static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, + gfp_t flags, size_t size) +{ + void *ret = kmem_cache_alloc(s, flags); + + kasan_kmalloc(s, ret, size, flags); + return ret; +} + +static __always_inline void * +kmem_cache_alloc_node_trace(struct kmem_cache *s, + gfp_t gfpflags, + int node, size_t size) +{ + void *ret = kmem_cache_alloc_node(s, gfpflags, node); + + kasan_kmalloc(s, ret, size, gfpflags); + return ret; +} +#endif /* CONFIG_TRACING */ + +extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; + +#ifdef CONFIG_TRACING +extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; +#else +static __always_inline void * +kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) +{ + return kmalloc_order(size, flags, order); +} +#endif + +static __always_inline void *kmalloc_large(size_t size, gfp_t flags) +{ + unsigned int order = get_order(size); + return kmalloc_order_trace(size, flags, order); +} + +/** + * kmalloc - allocate memory + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate. + * + * kmalloc is the normal method of allocating memory + * for objects smaller than page size in the kernel. + * + * The @flags argument may be one of: + * + * %GFP_USER - Allocate memory on behalf of user. May sleep. + * + * %GFP_KERNEL - Allocate normal kernel ram. May sleep. + * + * %GFP_ATOMIC - Allocation will not sleep. May use emergency pools. + * For example, use this inside interrupt handlers. + * + * %GFP_HIGHUSER - Allocate pages from high memory. + * + * %GFP_NOIO - Do not do any I/O at all while trying to get memory. + * + * %GFP_NOFS - Do not make any fs calls while trying to get memory. + * + * %GFP_NOWAIT - Allocation will not sleep. + * + * %__GFP_THISNODE - Allocate node-local memory only. + * + * %GFP_DMA - Allocation suitable for DMA. + * Should only be used for kmalloc() caches. Otherwise, use a + * slab created with SLAB_DMA. + * + * Also it is possible to set different flags by OR'ing + * in one or more of the following additional @flags: + * + * %__GFP_HIGH - This allocation has high priority and may use emergency pools. + * + * %__GFP_NOFAIL - Indicate that this allocation is in no way allowed to fail + * (think twice before using). + * + * %__GFP_NORETRY - If memory is not immediately available, + * then give up at once. + * + * %__GFP_NOWARN - If allocation fails, don't issue any warnings. + * + * %__GFP_RETRY_MAYFAIL - Try really hard to succeed the allocation but fail + * eventually. + * + * There are other flags available as well, but these are not intended + * for general use, and so are not documented here. For a full list of + * potential flags, always refer to linux/gfp.h. + */ +static __always_inline void *kmalloc(size_t size, gfp_t flags) +{ + if (__builtin_constant_p(size)) { + if (size > KMALLOC_MAX_CACHE_SIZE) + return kmalloc_large(size, flags); +#ifndef CONFIG_SLOB + if (!(flags & GFP_DMA)) { + unsigned int index = kmalloc_index(size); + + if (!index) + return ZERO_SIZE_PTR; + + return kmem_cache_alloc_trace(kmalloc_caches[index], + flags, size); + } +#endif + } + return __kmalloc(size, flags); +} + +/* + * Determine size used for the nth kmalloc cache. + * return size or 0 if a kmalloc cache for that + * size does not exist + */ +static __always_inline unsigned int kmalloc_size(unsigned int n) +{ +#ifndef CONFIG_SLOB + if (n > 2) + return 1U << n; + + if (n == 1 && KMALLOC_MIN_SIZE <= 32) + return 96; + + if (n == 2 && KMALLOC_MIN_SIZE <= 64) + return 192; +#endif + return 0; +} + +static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) +{ +#ifndef CONFIG_SLOB + if (__builtin_constant_p(size) && + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { + unsigned int i = kmalloc_index(size); + + if (!i) + return ZERO_SIZE_PTR; + + return kmem_cache_alloc_node_trace(kmalloc_caches[i], + flags, node, size); + } +#endif + return __kmalloc_node(size, flags, node); +} + +struct memcg_cache_array { + struct rcu_head rcu; + struct kmem_cache *entries[0]; +}; + +/* + * This is the main placeholder for memcg-related information in kmem caches. + * Both the root cache and the child caches will have it. For the root cache, + * this will hold a dynamically allocated array large enough to hold + * information about the currently limited memcgs in the system. To allow the + * array to be accessed without taking any locks, on relocation we free the old + * version only after a grace period. + * + * Root and child caches hold different metadata. + * + * @root_cache: Common to root and child caches. NULL for root, pointer to + * the root cache for children. + * + * The following fields are specific to root caches. + * + * @memcg_caches: kmemcg ID indexed table of child caches. This table is + * used to index child cachces during allocation and cleared + * early during shutdown. + * + * @root_caches_node: List node for slab_root_caches list. + * + * @children: List of all child caches. While the child caches are also + * reachable through @memcg_caches, a child cache remains on + * this list until it is actually destroyed. + * + * The following fields are specific to child caches. + * + * @memcg: Pointer to the memcg this cache belongs to. + * + * @children_node: List node for @root_cache->children list. + * + * @kmem_caches_node: List node for @memcg->kmem_caches list. + */ +struct memcg_cache_params { + struct kmem_cache *root_cache; + union { + struct { + struct memcg_cache_array __rcu *memcg_caches; + struct list_head __root_caches_node; + struct list_head children; + bool dying; + }; + struct { + struct mem_cgroup *memcg; + struct list_head children_node; + struct list_head kmem_caches_node; + + void (*deact_fn)(struct kmem_cache *); + union { + struct rcu_head deact_rcu_head; + struct work_struct deact_work; + }; + }; + }; +}; + +int memcg_update_all_caches(int num_memcgs); + +/** + * kmalloc_array - allocate memory for an array. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate (see kmalloc). + */ +static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags) +{ + size_t bytes; + + if (unlikely(check_mul_overflow(n, size, &bytes))) + return NULL; + if (__builtin_constant_p(n) && __builtin_constant_p(size)) + return kmalloc(bytes, flags); + return __kmalloc(bytes, flags); +} + +/** + * kcalloc - allocate memory for an array. The memory is set to zero. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate (see kmalloc). + */ +static inline void *kcalloc(size_t n, size_t size, gfp_t flags) +{ + return kmalloc_array(n, size, flags | __GFP_ZERO); +} + +/* + * kmalloc_track_caller is a special version of kmalloc that records the + * calling function of the routine calling it for slab leak tracking instead + * of just the calling function (confusing, eh?). + * It's useful when the call to kmalloc comes from a widely-used standard + * allocator where we care about the real place the memory allocation + * request comes from. + */ +extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); +#define kmalloc_track_caller(size, flags) \ + __kmalloc_track_caller(size, flags, _RET_IP_) + +static inline void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, + int node) +{ + size_t bytes; + + if (unlikely(check_mul_overflow(n, size, &bytes))) + return NULL; + if (__builtin_constant_p(n) && __builtin_constant_p(size)) + return kmalloc_node(bytes, flags, node); + return __kmalloc_node(bytes, flags, node); +} + +static inline void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) +{ + return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); +} + + +#ifdef CONFIG_NUMA +extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long); +#define kmalloc_node_track_caller(size, flags, node) \ + __kmalloc_node_track_caller(size, flags, node, \ + _RET_IP_) + +#else /* CONFIG_NUMA */ + +#define kmalloc_node_track_caller(size, flags, node) \ + kmalloc_track_caller(size, flags) + +#endif /* CONFIG_NUMA */ + +/* + * Shortcuts + */ +static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) +{ + return kmem_cache_alloc(k, flags | __GFP_ZERO); +} + +/** + * kzalloc - allocate memory. The memory is set to zero. + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate (see kmalloc). + */ +static inline void *kzalloc(size_t size, gfp_t flags) +{ + return kmalloc(size, flags | __GFP_ZERO); +} + +/** + * kzalloc_node - allocate zeroed memory from a particular memory node. + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate (see kmalloc). + * @node: memory node from which to allocate + */ +static inline void *kzalloc_node(size_t size, gfp_t flags, int node) +{ + return kmalloc_node(size, flags | __GFP_ZERO, node); +} + +unsigned int kmem_cache_size(struct kmem_cache *s); +void __init kmem_cache_init_late(void); + +#if defined(CONFIG_SMP) && defined(CONFIG_SLAB) +int slab_prepare_cpu(unsigned int cpu); +int slab_dead_cpu(unsigned int cpu); +#else +#define slab_prepare_cpu NULL +#define slab_dead_cpu NULL +#endif + +#endif /* _LINUX_SLAB_H */ |