summaryrefslogtreecommitdiffstats
path: root/src/pool.c
blob: 54ae25b077865fa30822df08f16d98f369393772 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
/*
 * Memory management functions.
 *
 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 */

#include <sys/mman.h>
#include <errno.h>

#include <haproxy/activity.h>
#include <haproxy/api.h>
#include <haproxy/applet-t.h>
#include <haproxy/cfgparse.h>
#include <haproxy/channel.h>
#include <haproxy/cli.h>
#include <haproxy/errors.h>
#include <haproxy/global.h>
#include <haproxy/list.h>
#include <haproxy/pool.h>
#include <haproxy/sc_strm.h>
#include <haproxy/stats-t.h>
#include <haproxy/stconn.h>
#include <haproxy/thread.h>
#include <haproxy/tools.h>


/* These ones are initialized per-thread on startup by init_pools() */
THREAD_LOCAL size_t pool_cache_bytes = 0;                /* total cache size */
THREAD_LOCAL size_t pool_cache_count = 0;                /* #cache objects   */

static struct list pools __read_mostly = LIST_HEAD_INIT(pools);
int mem_poison_byte __read_mostly = 'P';
uint pool_debugging __read_mostly =               /* set of POOL_DBG_* flags */
#ifdef DEBUG_FAIL_ALLOC
	POOL_DBG_FAIL_ALLOC |
#endif
#ifdef DEBUG_DONT_SHARE_POOLS
	POOL_DBG_DONT_MERGE |
#endif
#ifdef DEBUG_POOL_INTEGRITY
	POOL_DBG_COLD_FIRST |
#endif
#ifdef DEBUG_POOL_INTEGRITY
	POOL_DBG_INTEGRITY  |
#endif
#ifdef CONFIG_HAP_NO_GLOBAL_POOLS
	POOL_DBG_NO_GLOBAL  |
#endif
#ifndef CONFIG_HAP_POOLS
	POOL_DBG_NO_CACHE   |
#endif
#if defined(DEBUG_POOL_TRACING)
	POOL_DBG_CALLER     |
#endif
#if defined(DEBUG_MEMORY_POOLS)
	POOL_DBG_TAG        |
#endif
	0;

static const struct {
	uint flg;
	const char *set;
	const char *clr;
	const char *hlp;
} dbg_options[] = {
	/* flg,                 set,          clr,            hlp */
	{ POOL_DBG_FAIL_ALLOC, "fail",       "no-fail",      "randomly fail allocations" },
	{ POOL_DBG_DONT_MERGE, "no-merge",   "merge",        "disable merging of similar pools" },
	{ POOL_DBG_COLD_FIRST, "cold-first", "hot-first",    "pick cold objects first" },
	{ POOL_DBG_INTEGRITY,  "integrity",  "no-integrity", "enable cache integrity checks" },
	{ POOL_DBG_NO_GLOBAL,  "no-global",  "global",       "disable global shared cache" },
	{ POOL_DBG_NO_CACHE,   "no-cache",   "cache",        "disable thread-local cache" },
	{ POOL_DBG_CALLER,     "caller",     "no-caller",    "save caller information in cache" },
	{ POOL_DBG_TAG,        "tag",        "no-tag",       "add tag at end of allocated objects" },
	{ POOL_DBG_POISON,     "poison",     "no-poison",    "poison newly allocated objects" },
	{ 0 /* end */ }
};

static int mem_fail_rate __read_mostly = 0;
static int using_default_allocator __read_mostly = 1;
static int disable_trim __read_mostly = 0;
static int(*my_mallctl)(const char *, void *, size_t *, void *, size_t) = NULL;

/* ask the allocator to trim memory pools.
 * This must run under thread isolation so that competing threads trying to
 * allocate or release memory do not prevent the allocator from completing
 * its job. We just have to be careful as callers might already be isolated
 * themselves.
 */
static void trim_all_pools(void)
{
	int isolated = thread_isolated();

	if (disable_trim)
		return;

	if (!isolated)
		thread_isolate();

	if (my_mallctl) {
		unsigned int i, narenas = 0;
		size_t len = sizeof(narenas);

		if (my_mallctl("arenas.narenas", &narenas, &len, NULL, 0) == 0) {
			for (i = 0; i < narenas; i ++) {
				char mib[32] = {0};
				snprintf(mib, sizeof(mib), "arena.%u.purge", i);
				(void)my_mallctl(mib, NULL, NULL, NULL, 0);
			}
		}
	} else {
#if defined(HA_HAVE_MALLOC_TRIM)
		if (using_default_allocator)
			malloc_trim(0);
#elif defined(HA_HAVE_MALLOC_ZONE)
		if (using_default_allocator) {
			vm_address_t *zones;
			unsigned int i, nzones;

			if (malloc_get_all_zones(0, NULL, &zones, &nzones) == KERN_SUCCESS) {
				for (i = 0; i < nzones; i ++) {
					malloc_zone_t *zone = (malloc_zone_t *)zones[i];

					/* we cannot purge anonymous zones */
					if (zone->zone_name)
						malloc_zone_pressure_relief(zone, 0);
				}
			}
		}
#endif
	}

	if (!isolated)
		thread_release();
}

/* check if we're using the same allocator as the one that provides
 * malloc_trim() and mallinfo(). The principle is that on glibc, both
 * malloc_trim() and mallinfo() are provided, and using mallinfo() we
 * can check if malloc() is performed through glibc or any other one
 * the executable was linked against (e.g. jemalloc). Prior to this we
 * have to check whether we're running on jemalloc by verifying if the
 * mallctl() function is provided. Its pointer will be used later.
 */
static void detect_allocator(void)
{
#if defined(__ELF__)
	extern int mallctl(const char *, void *, size_t *, void *, size_t) __attribute__((weak));

	my_mallctl = mallctl;
#endif

	if (!my_mallctl) {
		my_mallctl = get_sym_curr_addr("mallctl");
		using_default_allocator = (my_mallctl == NULL);
	}

	if (!my_mallctl) {
#if defined(HA_HAVE_MALLOC_TRIM)
#ifdef HA_HAVE_MALLINFO2
		struct mallinfo2 mi1, mi2;
#else
		struct mallinfo mi1, mi2;
#endif
		void *ptr;

#ifdef HA_HAVE_MALLINFO2
		mi1 = mallinfo2();
#else
		mi1 = mallinfo();
#endif
		ptr = DISGUISE(malloc(1));
#ifdef HA_HAVE_MALLINFO2
		mi2 = mallinfo2();
#else
		mi2 = mallinfo();
#endif
		free(DISGUISE(ptr));

		using_default_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1));
#elif defined(HA_HAVE_MALLOC_ZONE)
		using_default_allocator = (malloc_default_zone() != NULL);
#endif
	}
}

static int is_trim_enabled(void)
{
	return using_default_allocator;
}

static int mem_should_fail(const struct pool_head *pool)
{
	int ret = 0;

	if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
		if (mem_fail_rate > statistical_prng_range(100))
			ret = 1;
		else
			ret = 0;
	}
	return ret;
}

/* Try to find an existing shared pool with the same characteristics and
 * returns it, otherwise creates this one. NULL is returned if no memory
 * is available for a new creation. Two flags are supported :
 *   - MEM_F_SHARED to indicate that the pool may be shared with other users
 *   - MEM_F_EXACT to indicate that the size must not be rounded up
 */
struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags)
{
	unsigned int extra_mark, extra_caller, extra;
	struct pool_head *pool;
	struct pool_head *entry;
	struct list *start;
	unsigned int align;
	int thr __maybe_unused;

	/* We need to store a (void *) at the end of the chunks. Since we know
	 * that the malloc() function will never return such a small size,
	 * let's round the size up to something slightly bigger, in order to
	 * ease merging of entries. Note that the rounding is a power of two.
	 * This extra (void *) is not accounted for in the size computation
	 * so that the visible parts outside are not affected.
	 *
	 * Note: for the LRU cache, we need to store 2 doubly-linked lists.
	 */

	extra_mark = (pool_debugging & POOL_DBG_TAG) ? POOL_EXTRA_MARK : 0;
	extra_caller = (pool_debugging & POOL_DBG_CALLER) ? POOL_EXTRA_CALLER : 0;
	extra = extra_mark + extra_caller;

	if (!(flags & MEM_F_EXACT)) {
		align = 4 * sizeof(void *); // 2 lists = 4 pointers min
		size  = ((size + extra + align - 1) & -align) - extra;
	}

	if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
		/* we'll store two lists there, we need the room for this. This is
		 * guaranteed by the test above, except if MEM_F_EXACT is set, or if
		 * the only EXTRA part is in fact the one that's stored in the cache
		 * in addition to the pci struct.
		 */
		if (size + extra - extra_caller < sizeof(struct pool_cache_item))
			size = sizeof(struct pool_cache_item) + extra_caller - extra;
	}

	/* TODO: thread: we do not lock pool list for now because all pools are
	 * created during HAProxy startup (so before threads creation) */
	start = &pools;
	pool = NULL;

	list_for_each_entry(entry, &pools, list) {
		if (entry->size == size) {
			/* either we can share this place and we take it, or
			 * we look for a shareable one or for the next position
			 * before which we will insert a new one.
			 */
			if ((flags & entry->flags & MEM_F_SHARED) &&
			    (!(pool_debugging & POOL_DBG_DONT_MERGE) ||
			     strcmp(name, entry->name) == 0)) {
				/* we can share this one */
				pool = entry;
				DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
				break;
			}
		}
		else if (entry->size > size) {
			/* insert before this one */
			start = &entry->list;
			break;
		}
	}

	if (!pool) {
		void *pool_addr;

		pool_addr = calloc(1, sizeof(*pool) + __alignof__(*pool));
		if (!pool_addr)
			return NULL;

		/* always provide an aligned pool */
		pool = (struct pool_head*)((((size_t)pool_addr) + __alignof__(*pool)) & -(size_t)__alignof__(*pool));
		pool->base_addr = pool_addr; // keep it, it's the address to free later

		if (name)
			strlcpy2(pool->name, name, sizeof(pool->name));
		pool->alloc_sz = size + extra;
		pool->size = size;
		pool->flags = flags;
		LIST_APPEND(start, &pool->list);

		if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
			/* update per-thread pool cache if necessary */
			for (thr = 0; thr < MAX_THREADS; thr++) {
				LIST_INIT(&pool->cache[thr].list);
				pool->cache[thr].tid = thr;
				pool->cache[thr].pool = pool;
			}
		}
	}
	pool->users++;
	return pool;
}

/* Tries to allocate an object for the pool <pool> using the system's allocator
 * and directly returns it. The pool's allocated counter is checked and updated,
 * but no other checks are performed.
 */
void *pool_get_from_os(struct pool_head *pool)
{
	if (!pool->limit || pool->allocated < pool->limit) {
		void *ptr = pool_alloc_area(pool->alloc_sz);
		if (ptr) {
			_HA_ATOMIC_INC(&pool->allocated);
			return ptr;
		}
		_HA_ATOMIC_INC(&pool->failed);
	}
	activity[tid].pool_fail++;
	return NULL;

}

/* Releases a pool item back to the operating system and atomically updates
 * the allocation counter.
 */
void pool_put_to_os(struct pool_head *pool, void *ptr)
{
#ifdef DEBUG_UAF
	/* This object will be released for real in order to detect a use after
	 * free. We also force a write to the area to ensure we crash on double
	 * free or free of a const area.
	 */
	*(uint32_t *)ptr = 0xDEADADD4;
#endif /* DEBUG_UAF */

	pool_free_area(ptr, pool->alloc_sz);
	_HA_ATOMIC_DEC(&pool->allocated);
}

/* Tries to allocate an object for the pool <pool> using the system's allocator
 * and directly returns it. The pool's counters are updated but the object is
 * never cached, so this is usable with and without local or shared caches.
 */
void *pool_alloc_nocache(struct pool_head *pool)
{
	void *ptr = NULL;

	ptr = pool_get_from_os(pool);
	if (!ptr)
		return NULL;

	swrate_add_scaled(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used, POOL_AVG_SAMPLES/4);
	_HA_ATOMIC_INC(&pool->used);

	/* keep track of where the element was allocated from */
	POOL_DEBUG_SET_MARK(pool, ptr);
	POOL_DEBUG_TRACE_CALLER(pool, (struct pool_cache_item *)ptr, NULL);
	return ptr;
}

/* Release a pool item back to the OS and keeps the pool's counters up to date.
 * This is always defined even when pools are not enabled (their usage stats
 * are maintained).
 */
void pool_free_nocache(struct pool_head *pool, void *ptr)
{
	_HA_ATOMIC_DEC(&pool->used);
	swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
	pool_put_to_os(pool, ptr);
}


/* Updates <pch>'s fill_pattern and fills the free area after <item> with it,
 * up to <size> bytes. The item part is left untouched.
 */
void pool_fill_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
{
	ulong *ptr = (ulong *)item;
	uint ofs;
	ulong u;

	if (size <= sizeof(*item))
		return;

	/* Upgrade the fill_pattern to change about half of the bits
	 * (to be sure to catch static flag corruption), and apply it.
	 */
	u = pch->fill_pattern += ~0UL / 3; // 0x55...55
	ofs = sizeof(*item) / sizeof(*ptr);
	while (ofs < size / sizeof(*ptr))
		ptr[ofs++] = u;
}

/* check for a pool_cache_item integrity after extracting it from the cache. It
 * must have been previously initialized using pool_fill_pattern(). If any
 * corruption is detected, the function provokes an immediate crash.
 */
void pool_check_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
{
	const ulong *ptr = (const ulong *)item;
	uint ofs;
	ulong u;

	if (size <= sizeof(*item))
		return;

	/* let's check that all words past *item are equal */
	ofs = sizeof(*item) / sizeof(*ptr);
	u = ptr[ofs++];
	while (ofs < size / sizeof(*ptr)) {
		if (unlikely(ptr[ofs] != u))
			ABORT_NOW();
		ofs++;
	}
}

/* removes up to <count> items from the end of the local pool cache <ph> for
 * pool <pool>. The shared pool is refilled with these objects in the limit
 * of the number of acceptable objects, and the rest will be released to the
 * OS. It is not a problem is <count> is larger than the number of objects in
 * the local cache. The counters are automatically updated. Must not be used
 * with pools disabled.
 */
static void pool_evict_last_items(struct pool_head *pool, struct pool_cache_head *ph, uint count)
{
	struct pool_cache_item *item;
	struct pool_item *pi, *head = NULL;
	uint released = 0;
	uint cluster = 0;
	uint to_free_max;

	BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

	/* Note: this will be zero when global pools are disabled */
	to_free_max = pool_releasable(pool);

	while (released < count && !LIST_ISEMPTY(&ph->list)) {
		item = LIST_PREV(&ph->list, typeof(item), by_pool);
		BUG_ON(&item->by_pool == &ph->list);
		if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
			pool_check_pattern(ph, item, pool->size);
		LIST_DELETE(&item->by_pool);
		LIST_DELETE(&item->by_lru);

		if (to_free_max > released || cluster) {
			/* will never match when global pools are disabled */
			pi = (struct pool_item *)item;
			pi->next = NULL;
			pi->down = head;
			head = pi;
			cluster++;
			if (cluster >= CONFIG_HAP_POOL_CLUSTER_SIZE) {
				/* enough to make a cluster */
				pool_put_to_shared_cache(pool, head, cluster);
				cluster = 0;
				head = NULL;
			}
		} else
			pool_free_nocache(pool, item);

		released++;
	}

	/* incomplete cluster left */
	if (cluster)
		pool_put_to_shared_cache(pool, head, cluster);

	ph->count -= released;
	pool_cache_count -= released;
	pool_cache_bytes -= released * pool->size;
}

/* Evicts some of the oldest objects from one local cache, until its number of
 * objects is no more than 16+1/8 of the total number of locally cached objects
 * or the total size of the local cache is no more than 75% of its maximum (i.e.
 * we don't want a single cache to use all the cache for itself). For this, the
 * list is scanned in reverse. If <full> is non-null, all objects are evicted.
 * Must not be used when pools are disabled.
 */
void pool_evict_from_local_cache(struct pool_head *pool, int full)
{
	struct pool_cache_head *ph = &pool->cache[tid];

	BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

	while ((ph->count && full) ||
	       (ph->count >= CONFIG_HAP_POOL_CLUSTER_SIZE &&
	        ph->count >= 16 + pool_cache_count / 8 &&
	        pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
		pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
	}
}

/* Evicts some of the oldest objects from the local cache, pushing them to the
 * global pool. Must not be used when pools are disabled.
 */
void pool_evict_from_local_caches()
{
	struct pool_cache_item *item;
	struct pool_cache_head *ph;
	struct pool_head *pool;

	BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

	do {
		item = LIST_PREV(&th_ctx->pool_lru_head, struct pool_cache_item *, by_lru);
		BUG_ON(&item->by_lru == &th_ctx->pool_lru_head);
		/* note: by definition we remove oldest objects so they also are the
		 * oldest in their own pools, thus their next is the pool's head.
		 */
		ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
		BUG_ON(ph->tid != tid);

		pool = container_of(ph - tid, struct pool_head, cache);
		BUG_ON(pool != ph->pool);

		pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
	} while (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 7 / 8);
}

/* Frees an object to the local cache, possibly pushing oldest objects to the
 * shared cache, which itself may decide to release some of them to the OS.
 * While it is unspecified what the object becomes past this point, it is
 * guaranteed to be released from the users' perpective. A caller address may
 * be passed and stored into the area when DEBUG_POOL_TRACING is set. Must not
 * be used with pools disabled.
 */
void pool_put_to_cache(struct pool_head *pool, void *ptr, const void *caller)
{
	struct pool_cache_item *item = (struct pool_cache_item *)ptr;
	struct pool_cache_head *ph = &pool->cache[tid];

	BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

	LIST_INSERT(&ph->list, &item->by_pool);
	LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
	POOL_DEBUG_TRACE_CALLER(pool, item, caller);
	ph->count++;
	if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
		pool_fill_pattern(ph, item, pool->size);
	pool_cache_count++;
	pool_cache_bytes += pool->size;

	if (unlikely(pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4)) {
		if (ph->count >= 16 + pool_cache_count / 8 + CONFIG_HAP_POOL_CLUSTER_SIZE)
			pool_evict_from_local_cache(pool, 0);
		if (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE)
			pool_evict_from_local_caches();
	}
}

/* Tries to refill the local cache <pch> from the shared one for pool <pool>.
 * This is only used when pools are in use and shared pools are enabled. No
 * malloc() is attempted, and poisonning is never performed. The purpose is to
 * get the fastest possible refilling so that the caller can easily check if
 * the cache has enough objects for its use. Must not be used when pools are
 * disabled.
 */
void pool_refill_local_from_shared(struct pool_head *pool, struct pool_cache_head *pch)
{
	struct pool_cache_item *item;
	struct pool_item *ret, *down;
	uint count;

	BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

	/* we'll need to reference the first element to figure the next one. We
	 * must temporarily lock it so that nobody allocates then releases it,
	 * or the dereference could fail.
	 */
	ret = _HA_ATOMIC_LOAD(&pool->free_list);
	do {
		while (unlikely(ret == POOL_BUSY)) {
			__ha_cpu_relax();
			ret = _HA_ATOMIC_LOAD(&pool->free_list);
		}
		if (ret == NULL)
			return;
	} while (unlikely((ret = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));

	if (unlikely(ret == NULL)) {
		HA_ATOMIC_STORE(&pool->free_list, NULL);
		return;
	}

	/* this releases the lock */
	HA_ATOMIC_STORE(&pool->free_list, ret->next);

	/* now store the retrieved object(s) into the local cache */
	count = 0;
	for (; ret; ret = down) {
		down = ret->down;
		item = (struct pool_cache_item *)ret;
		POOL_DEBUG_TRACE_CALLER(pool, item, NULL);
		LIST_INSERT(&pch->list, &item->by_pool);
		LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
		count++;
		if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
			pool_fill_pattern(pch, item, pool->size);
	}
	HA_ATOMIC_ADD(&pool->used, count);
	pch->count += count;
	pool_cache_count += count;
	pool_cache_bytes += count * pool->size;
}

/* Adds pool item cluster <item> to the shared cache, which contains <count>
 * elements. The caller is advised to first check using pool_releasable() if
 * it's wise to add this series of objects there. Both the pool and the item's
 * head must be valid.
 */
void pool_put_to_shared_cache(struct pool_head *pool, struct pool_item *item, uint count)
{
	struct pool_item *free_list;

	_HA_ATOMIC_SUB(&pool->used, count);
	free_list = _HA_ATOMIC_LOAD(&pool->free_list);
	do {
		while (unlikely(free_list == POOL_BUSY)) {
			__ha_cpu_relax();
			free_list = _HA_ATOMIC_LOAD(&pool->free_list);
		}
		_HA_ATOMIC_STORE(&item->next, free_list);
		__ha_barrier_atomic_store();
	} while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, item));
	__ha_barrier_atomic_store();
	swrate_add(&pool->needed_avg, POOL_AVG_SAMPLES, pool->used);
}

/*
 * This function frees whatever can be freed in pool <pool>.
 */
void pool_flush(struct pool_head *pool)
{
	struct pool_item *next, *temp, *down;

	if (!pool || (pool_debugging & (POOL_DBG_NO_CACHE|POOL_DBG_NO_GLOBAL)))
		return;

	/* The loop below atomically detaches the head of the free list and
	 * replaces it with a NULL. Then the list can be released.
	 */
	next = pool->free_list;
	do {
		while (unlikely(next == POOL_BUSY)) {
			__ha_cpu_relax();
			next = _HA_ATOMIC_LOAD(&pool->free_list);
		}
		if (next == NULL)
			return;
	} while (unlikely((next = _HA_ATOMIC_XCHG(&pool->free_list, POOL_BUSY)) == POOL_BUSY));
	_HA_ATOMIC_STORE(&pool->free_list, NULL);
	__ha_barrier_atomic_store();

	while (next) {
		temp = next;
		next = temp->next;
		for (; temp; temp = down) {
			down = temp->down;
			pool_put_to_os(pool, temp);
		}
	}
	/* here, we should have pool->allocated == pool->used */
}

/*
 * This function frees whatever can be freed in all pools, but respecting
 * the minimum thresholds imposed by owners. It makes sure to be alone to
 * run by using thread_isolate(). <pool_ctx> is unused.
 */
void pool_gc(struct pool_head *pool_ctx)
{
	struct pool_head *entry;
	int isolated = thread_isolated();

	if (!isolated)
		thread_isolate();

	list_for_each_entry(entry, &pools, list) {
		struct pool_item *temp, *down;

		while (entry->free_list &&
		       (int)(entry->allocated - entry->used) > (int)entry->minavail) {
			temp = entry->free_list;
			entry->free_list = temp->next;
			for (; temp; temp = down) {
				down = temp->down;
				pool_put_to_os(entry, temp);
			}
		}
	}

	trim_all_pools();

	if (!isolated)
		thread_release();
}

/*
 * Returns a pointer to type <type> taken from the pool <pool_type> or
 * dynamically allocated. In the first case, <pool_type> is updated to point to
 * the next element in the list. <flags> is a binary-OR of POOL_F_* flags.
 * Prefer using pool_alloc() which does the right thing without flags.
 */
void *__pool_alloc(struct pool_head *pool, unsigned int flags)
{
	void *p = NULL;
	void *caller = __builtin_return_address(0);

	if (unlikely(pool_debugging & POOL_DBG_FAIL_ALLOC))
		if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool))
			return NULL;

	if (likely(!(pool_debugging & POOL_DBG_NO_CACHE)) && !p)
		p = pool_get_from_cache(pool, caller);

	if (unlikely(!p))
		p = pool_alloc_nocache(pool);

	if (likely(p)) {
		if (unlikely(flags & POOL_F_MUST_ZERO))
			memset(p, 0, pool->size);
		else if (unlikely(!(flags & POOL_F_NO_POISON) && (pool_debugging & POOL_DBG_POISON)))
			memset(p, mem_poison_byte, pool->size);
	}
	return p;
}

/*
 * Puts a memory area back to the corresponding pool. <ptr> be valid. Using
 * pool_free() is preferred.
 */
void __pool_free(struct pool_head *pool, void *ptr)
{
	const void *caller = __builtin_return_address(0);

	/* we'll get late corruption if we refill to the wrong pool or double-free */
	POOL_DEBUG_CHECK_MARK(pool, ptr);
	POOL_DEBUG_RESET_MARK(pool, ptr);

	if (unlikely(pool_debugging & POOL_DBG_NO_CACHE)) {
		pool_free_nocache(pool, ptr);
		return;
	}

	pool_put_to_cache(pool, ptr, caller);
}


#ifdef DEBUG_UAF

/************* use-after-free allocator *************/

/* allocates an area of size <size> and returns it. The semantics are similar
 * to those of malloc(). However the allocation is rounded up to 4kB so that a
 * full page is allocated. This ensures the object can be freed alone so that
 * future dereferences are easily detected. The returned object is always
 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
 * some padding is added, the area's start address is copied at the end of the
 * padding to help detect underflows.
 */
void *pool_alloc_area_uaf(size_t size)
{
	size_t pad = (4096 - size) & 0xFF0;
	void *ret;

	ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
	if (ret != MAP_FAILED) {
		/* let's dereference the page before returning so that the real
		 * allocation in the system is performed without holding the lock.
		 */
		*(int *)ret = 0;
		if (pad >= sizeof(void *))
			*(void **)(ret + pad - sizeof(void *)) = ret + pad;
		ret += pad;
	} else {
		ret = NULL;
	}
	return ret;
}

/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
 * semantics are identical to free() except that the size must absolutely match
 * the one passed to pool_alloc_area(). In case some padding is added, the
 * area's start address is compared to the one at the end of the padding, and
 * a segfault is triggered if they don't match, indicating an underflow.
 */
void pool_free_area_uaf(void *area, size_t size)
{
	size_t pad = (4096 - size) & 0xFF0;

	if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
		ABORT_NOW();

	munmap(area - pad, (size + 4095) & -4096);
}

#endif /* DEBUG_UAF */

/*
 * This function destroys a pool by freeing it completely, unless it's still
 * in use. This should be called only under extreme circumstances. It always
 * returns NULL if the resulting pool is empty, easing the clearing of the old
 * pointer, otherwise it returns the pool.
 * .
 */
void *pool_destroy(struct pool_head *pool)
{
	if (pool) {
		if (!(pool_debugging & POOL_DBG_NO_CACHE))
			pool_evict_from_local_cache(pool, 1);

		pool_flush(pool);
		if (pool->used)
			return pool;
		pool->users--;
		if (!pool->users) {
			LIST_DELETE(&pool->list);
			/* note that if used == 0, the cache is empty */
			free(pool->base_addr);
		}
	}
	return NULL;
}

/* This destroys all pools on exit. It is *not* thread safe. */
void pool_destroy_all()
{
	struct pool_head *entry, *back;

	list_for_each_entry_safe(entry, back, &pools, list) {
		/* there's only one occurrence of each pool in the list,
		 * and we're existing instead of looping on the whole
		 * list just to decrement users, force it to 1 here.
		 */
		entry->users = 1;
		pool_destroy(entry);
	}
}

/* This function dumps memory usage information into the trash buffer. */
void dump_pools_to_trash()
{
	struct pool_head *entry;
	unsigned long long allocated, used;
	int nbpools;
	unsigned long long cached_bytes = 0;
	uint cached = 0;

	allocated = used = nbpools = 0;
	chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n");
	list_for_each_entry(entry, &pools, list) {
		if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
			int i;
			for (cached = i = 0; i < global.nbthread; i++)
				cached += entry->cache[i].count;
			cached_bytes += cached * (ullong)entry->size;
		}
		chunk_appendf(&trash, "  - Pool %s (%u bytes) : %u allocated (%llu bytes), %u used"
			      " (~%u by thread caches)"
			      ", needed_avg %u, %u failures, %u users, @%p%s\n",
		              entry->name, entry->size, entry->allocated,
		              (ullong)entry->size * entry->allocated, entry->used,
		              cached,
		              swrate_avg(entry->needed_avg, POOL_AVG_SAMPLES), entry->failed,
		              entry->users, entry,
		              (entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");

		allocated += entry->allocated * (ullong)entry->size;
		used += entry->used * (ullong)entry->size;
		nbpools++;
	}
	chunk_appendf(&trash, "Total: %d pools, %llu bytes allocated, %llu used"
		      " (~%llu by thread caches)"
		      ".\n",
	              nbpools, allocated, used, cached_bytes
		      );
}

/* Dump statistics on pools usage. */
void dump_pools(void)
{
	dump_pools_to_trash();
	qfprintf(stderr, "%s", trash.area);
}

/* This function returns the total number of failed pool allocations */
int pool_total_failures()
{
	struct pool_head *entry;
	int failed = 0;

	list_for_each_entry(entry, &pools, list)
		failed += entry->failed;
	return failed;
}

/* This function returns the total amount of memory allocated in pools (in bytes) */
unsigned long long pool_total_allocated()
{
	struct pool_head *entry;
	unsigned long long allocated = 0;

	list_for_each_entry(entry, &pools, list)
		allocated += entry->allocated * (ullong)entry->size;
	return allocated;
}

/* This function returns the total amount of memory used in pools (in bytes) */
unsigned long long pool_total_used()
{
	struct pool_head *entry;
	unsigned long long used = 0;

	list_for_each_entry(entry, &pools, list)
		used += entry->used * (ullong)entry->size;
	return used;
}

/* This function parses a string made of a set of debugging features as
 * specified after -dM on the command line, and will set pool_debugging
 * accordingly. On success it returns a strictly positive value. It may zero
 * with the first warning in <err>, -1 with a help message in <err>, or -2 with
 * the first error in <err> return the first error in <err>. <err> is undefined
 * on success, and will be non-null and locally allocated on help/error/warning.
 * The caller must free it. Warnings are used to report features that were not
 * enabled at build time, and errors are used to report unknown features.
 */
int pool_parse_debugging(const char *str, char **err)
{
	struct ist args;
	char *end;
	uint new_dbg;
	int v;


	/* if it's empty or starts with a number, it's the mem poisonning byte */
	v = strtol(str, &end, 0);
	if (!*end || *end == ',') {
		mem_poison_byte = *str ? v : 'P';
		if (mem_poison_byte >= 0)
			pool_debugging |=  POOL_DBG_POISON;
		else
			pool_debugging &= ~POOL_DBG_POISON;
		str = end;
	}

	new_dbg = pool_debugging;

	for (args = ist(str); istlen(args); args = istadv(istfind(args, ','), 1)) {
		struct ist feat = iststop(args, ',');

		if (!istlen(feat))
			continue;

		if (isteq(feat, ist("help"))) {
			ha_free(err);
			memprintf(err,
				  "-dM alone enables memory poisonning with byte 0x50 on allocation. A numeric\n"
				  "value may be appended immediately after -dM to use another value (0 supported).\n"
				  "Then an optional list of comma-delimited keywords may be appended to set or\n"
				  "clear some debugging options ('*' marks the current setting):\n\n"
				  "    set               clear            description\n"
				  "  -----------------+-----------------+-----------------------------------------\n");

			for (v = 0; dbg_options[v].flg; v++) {
				memprintf(err, "%s  %c %-15s|%c %-15s| %s\n",
					  *err,
					  (pool_debugging & dbg_options[v].flg) ? '*' : ' ',
					  dbg_options[v].set,
					  (pool_debugging & dbg_options[v].flg) ? ' ' : '*',
					  dbg_options[v].clr,
					  dbg_options[v].hlp);
			}
			return -1;
		}

		for (v = 0; dbg_options[v].flg; v++) {
			if (isteq(feat, ist(dbg_options[v].set))) {
				new_dbg |= dbg_options[v].flg;
				break;
			}
			else if (isteq(feat, ist(dbg_options[v].clr))) {
				new_dbg &= ~dbg_options[v].flg;
				break;
			}
		}

		if (!dbg_options[v].flg) {
			memprintf(err, "unknown pool debugging feature <%.*s>", (int)istlen(feat), istptr(feat));
			return -2;
		}
	}

	pool_debugging = new_dbg;
	return 1;
}

/* This function dumps memory usage information onto the stream connector's
 * read buffer. It returns 0 as long as it does not complete, non-zero upon
 * completion. No state is used.
 */
static int cli_io_handler_dump_pools(struct appctx *appctx)
{
	dump_pools_to_trash();
	if (applet_putchk(appctx, &trash) == -1)
		return 0;
	return 1;
}

/* callback used to create early pool <name> of size <size> and store the
 * resulting pointer into <ptr>. If the allocation fails, it quits with after
 * emitting an error message.
 */
void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size)
{
	*ptr = create_pool(name, size, MEM_F_SHARED);
	if (!*ptr) {
		ha_alert("Failed to allocate pool '%s' of size %u : %s. Aborting.\n",
			 name, size, strerror(errno));
		exit(1);
	}
}

/* Initializes all per-thread arrays on startup */
static void init_pools()
{
	int thr;

	for (thr = 0; thr < MAX_THREADS; thr++) {
		LIST_INIT(&ha_thread_ctx[thr].pool_lru_head);
	}

	detect_allocator();
}

INITCALL0(STG_PREPARE, init_pools);

/* Report in build options if trim is supported */
static void pools_register_build_options(void)
{
	if (is_trim_enabled()) {
		char *ptr = NULL;
		memprintf(&ptr, "Support for malloc_trim() is enabled.");
		hap_register_build_opts(ptr, 1);
	}
}
INITCALL0(STG_REGISTER, pools_register_build_options);

/* register cli keywords */
static struct cli_kw_list cli_kws = {{ },{
	{ { "show", "pools",  NULL }, "show pools                              : report information about the memory pools usage", NULL, cli_io_handler_dump_pools },
	{{},}
}};

INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);


/* config parser for global "tune.fail-alloc" */
static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
                                       const struct proxy *defpx, const char *file, int line,
                                       char **err)
{
	if (too_many_args(1, args, err, NULL))
		return -1;
	mem_fail_rate = atoi(args[1]);
	if (mem_fail_rate < 0 || mem_fail_rate > 100) {
	    memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
	    return -1;
	}
	return 0;
}

/* config parser for global "no-memory-trimming" */
static int mem_parse_global_no_mem_trim(char **args, int section_type, struct proxy *curpx,
                                       const struct proxy *defpx, const char *file, int line,
                                       char **err)
{
	if (too_many_args(0, args, err, NULL))
		return -1;
	disable_trim = 1;
	return 0;
}

/* register global config keywords */
static struct cfg_kw_list mem_cfg_kws = {ILH, {
	{ CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
	{ CFG_GLOBAL, "no-memory-trimming", mem_parse_global_no_mem_trim },
	{ 0, NULL, NULL }
}};

INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);

/*
 * Local variables:
 *  c-indent-level: 8
 *  c-basic-offset: 8
 * End:
 */