umem_impl.h

Revision 2, 13.6 kB (checked in by wez, 7 years ago)
Initial revision
Property svn:eol-style set to `native`

Line
1	/*
2	* CDDL HEADER START
3	*
4	* The contents of this file are subject to the terms of the
5	* Common Development and Distribution License, Version 1.0 only
6	* (the "License"). You may not use this file except in compliance
7	* with the License.
8	*
9	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10	* or http://www.opensolaris.org/os/licensing.
11	* See the License for the specific language governing permissions
12	* and limitations under the License.
13	*
14	* When distributing Covered Code, include this CDDL HEADER in each
15	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16	* If applicable, add the following below this CDDL HEADER, with the
17	* fields enclosed by brackets "[]" replaced with your own identifying
18	* information: Portions Copyright [yyyy] [name of copyright owner]
19	*
20	* CDDL HEADER END
21	*/
22	/*
23	* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24	* Use is subject to license terms.
25	*/
26	/*
27	* Portions Copyright 2006 OmniTI, Inc.
28	*/
29
30	#ifndef _UMEM_IMPL_H
31	#define _UMEM_IMPL_H
32
33	/* #pragma ident "@(#)umem_impl.h 1.6 05/06/08 SMI" */
34
35	#include <umem.h>
36
37	#ifdef HAVE_SYS_SYSMACROS_H
38	#include <sys/sysmacros.h>
39	#endif
40
41	#if HAVE_SYS_TIME_H
42	#include <sys/time.h>
43	#endif
44
45	#include <sys/vmem.h>
46	#ifdef HAVE_THREAD_H
47	# include <thread.h>
48	#else
49	# include "sol_compat.h"
50	#endif
51
52	#ifdef __cplusplus
53	extern "C" {
54	#endif
55
56	/*
57	* umem memory allocator: implementation-private data structures
58	*/
59
60	/*
61	* Internal flags for umem_cache_create
62	*/
63	#define UMC_QCACHE 0x00100000
64	#define UMC_INTERNAL 0x80000000
65
66	/*
67	* Cache flags
68	*/
69	#define UMF_AUDIT 0x00000001 /* transaction auditing */
70	#define UMF_DEADBEEF 0x00000002 /* deadbeef checking */
71	#define UMF_REDZONE 0x00000004 /* redzone checking */
72	#define UMF_CONTENTS 0x00000008 /* freed-buffer content logging */
73	#define UMF_CHECKSIGNAL 0x00000010 /* abort when in signal context */
74	#define UMF_NOMAGAZINE 0x00000020 /* disable per-cpu magazines */
75	#define UMF_FIREWALL 0x00000040 /* put all bufs before unmapped pages */
76	#define UMF_LITE 0x00000100 /* lightweight debugging */
77
78	#define UMF_HASH 0x00000200 /* cache has hash table */
79	#define UMF_RANDOMIZE 0x00000400 /* randomize other umem_flags */
80
81	#define UMF_BUFTAG (UMF_DEADBEEF \| UMF_REDZONE)
82	#define UMF_TOUCH (UMF_BUFTAG \| UMF_LITE \| UMF_CONTENTS)
83	#define UMF_RANDOM (UMF_TOUCH \| UMF_AUDIT \| UMF_NOMAGAZINE)
84	#define UMF_DEBUG (UMF_RANDOM \| UMF_FIREWALL)
85
86	#define UMEM_STACK_DEPTH umem_stack_depth
87
88	#define UMEM_FREE_PATTERN 0xdeadbeefdeadbeefULL
89	#define UMEM_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL
90	#define UMEM_REDZONE_PATTERN 0xfeedfacefeedfaceULL
91	#define UMEM_REDZONE_BYTE 0xbb
92
93	#define UMEM_FATAL_FLAGS (UMEM_NOFAIL)
94	#define UMEM_SLEEP_FLAGS (0)
95
96	/*
97	* Redzone size encodings for umem_alloc() / umem_free(). We encode the
98	* allocation size, rather than storing it directly, so that umem_free()
99	* can distinguish frees of the wrong size from redzone violations.
100	*/
101	#define UMEM_SIZE_ENCODE(x) (251 * (x) + 1)
102	#define UMEM_SIZE_DECODE(x) ((x) / 251)
103	#define UMEM_SIZE_VALID(x) ((x) % 251 == 1)
104
105	/*
106	* The bufctl (buffer control) structure keeps some minimal information
107	* about each buffer: its address, its slab, and its current linkage,
108	* which is either on the slab's freelist (if the buffer is free), or
109	* on the cache's buf-to-bufctl hash table (if the buffer is allocated).
110	* In the case of non-hashed, or "raw", caches (the common case), only
111	* the freelist linkage is necessary: the buffer address is at a fixed
112	* offset from the bufctl address, and the slab is at the end of the page.
113	*
114	* NOTE: bc_next must be the first field; raw buffers have linkage only.
115	*/
116	typedef struct umem_bufctl {
117	struct umem_bufctl bc_next; / next bufctl struct */
118	void bc_addr; / address of buffer */
119	struct umem_slab bc_slab; / controlling slab */
120	} umem_bufctl_t;
121
122	/*
123	* The UMF_AUDIT version of the bufctl structure. The beginning of this
124	* structure must be identical to the normal bufctl structure so that
125	* pointers are interchangeable.
126	*/
127
128	#define UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
129	((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))
130
131	/*
132	* umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
133	* require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
134	* a single page.
135	*
136	* For ILP32, this is about 1000 frames.
137	* For LP64, this is about 490 frames.
138	*/
139
140	#define UMEM_BUFCTL_AUDIT_ALIGN 32
141
142	#define UMEM_BUFCTL_AUDIT_MAX_SIZE \
143	(P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 - \
144	sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))
145
146	#define UMEM_MAX_STACK_DEPTH \
147	((UMEM_BUFCTL_AUDIT_MAX_SIZE - \
148	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))
149
150	typedef struct umem_bufctl_audit {
151	struct umem_bufctl bc_next; / next bufctl struct */
152	void bc_addr; / address of buffer */
153	struct umem_slab bc_slab; / controlling slab */
154	umem_cache_t bc_cache; / controlling cache */
155	hrtime_t bc_timestamp; /* transaction time */
156	thread_t bc_thread; /* thread doing transaction */
157	struct umem_bufctl bc_lastlog; / last log entry */
158	void bc_contents; / contents at last free */
159	int bc_depth; /* stack depth */
160	uintptr_t bc_stack[1]; /* pc stack */
161	} umem_bufctl_audit_t;
162
163	#define UMEM_LOCAL_BUFCTL_AUDIT(bcpp) \
164	(bcpp) = (umem_bufctl_audit_t ) \
165	alloca(UMEM_BUFCTL_AUDIT_SIZE)
166
167	#define UMEM_BUFCTL_AUDIT_SIZE \
168	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)
169
170	/*
171	* A umem_buftag structure is appended to each buffer whenever any of the
172	* UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
173	*/
174	typedef struct umem_buftag {
175	uint64_t bt_redzone; /* 64-bit redzone pattern */
176	umem_bufctl_t bt_bufctl; / bufctl */
177	intptr_t bt_bxstat; /* bufctl ^ (alloc/free) */
178	} umem_buftag_t;
179
180	#define UMEM_BUFTAG(cp, buf) \
181	((umem_buftag_t )((char )(buf) + (cp)->cache_buftag))
182
183	#define UMEM_BUFCTL(cp, buf) \
184	((umem_bufctl_t )((char )(buf) + (cp)->cache_bufctl))
185
186	#define UMEM_BUF(cp, bcp) \
187	((void )((char )(bcp) - (cp)->cache_bufctl))
188
189	#define UMEM_SLAB(cp, buf) \
190	((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)
191
192	#define UMEM_CPU_CACHE(cp, cpu) \
193	(umem_cpu_cache_t )((char )cp + cpu->cpu_cache_offset)
194
195	#define UMEM_MAGAZINE_VALID(cp, mp) \
196	(((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
197	(cp)->cache_magtype->mt_cache)
198
199	#define UMEM_SLAB_MEMBER(sp, buf) \
200	((size_t)(buf) - (size_t)(sp)->slab_base < \
201	(sp)->slab_cache->cache_slabsize)
202
203	#define UMEM_BUFTAG_ALLOC 0xa110c8edUL
204	#define UMEM_BUFTAG_FREE 0xf4eef4eeUL
205
206	typedef struct umem_slab {
207	struct umem_cache slab_cache; / controlling cache */
208	void slab_base; / base of allocated memory */
209	struct umem_slab slab_next; / next slab on freelist */
210	struct umem_slab slab_prev; / prev slab on freelist */
211	struct umem_bufctl slab_head; / first free buffer */
212	long slab_refcnt; /* outstanding allocations */
213	long slab_chunks; /* chunks (bufs) in this slab */
214	} umem_slab_t;
215
216	#define UMEM_HASH_INITIAL 64
217
218	#define UMEM_HASH(cp, buf) \
219	((cp)->cache_hash_table + \
220	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) & (cp)->cache_hash_mask))
221
222	typedef struct umem_magazine {
223	void *mag_next;
224	void mag_round[1]; / one or more rounds */
225	} umem_magazine_t;
226
227	/*
228	* The magazine types for fast per-cpu allocation
229	*/
230	typedef struct umem_magtype {
231	int mt_magsize; /* magazine size (number of rounds) */
232	int mt_align; /* magazine alignment */
233	size_t mt_minbuf; /* all smaller buffers qualify */
234	size_t mt_maxbuf; /* no larger buffers qualify */
235	umem_cache_t mt_cache; / magazine cache */
236	} umem_magtype_t;
237
238	#if (defined(__PTHREAD_MUTEX_SIZE__) && __PTHREAD_MUTEX_SIZE__ >= 24) \|\| defined(UMEM_PTHREAD_MUTEX_TOO_BIG)
239	#define UMEM_CPU_CACHE_SIZE 128 /* must be power of 2 */
240	#else
241	#define UMEM_CPU_CACHE_SIZE 64 /* must be power of 2 */
242	#endif
243	#define UMEM_CPU_PAD (UMEM_CPU_CACHE_SIZE - sizeof (mutex_t) - \
244	2 * sizeof (uint_t) - 2 * sizeof (void ) - 4 sizeof (int))
245	#define UMEM_CACHE_SIZE(ncpus) \
246	((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))
247
248	typedef struct umem_cpu_cache {
249	mutex_t cc_lock; /* protects this cpu's local cache */
250	uint_t cc_alloc; /* allocations from this cpu */
251	uint_t cc_free; /* frees to this cpu */
252	umem_magazine_t cc_loaded; / the currently loaded magazine */
253	umem_magazine_t cc_ploaded; / the previously loaded magazine */
254	int cc_rounds; /* number of objects in loaded mag */
255	int cc_prounds; /* number of objects in previous mag */
256	int cc_magsize; /* number of rounds in a full mag */
257	int cc_flags; /* CPU-local copy of cache_flags */
258	#if (!defined(_LP64) \|\| defined(UMEM_PTHREAD_MUTEX_TOO_BIG)) && !defined(_WIN32)
259	/* on win32, UMEM_CPU_PAD evaluates to zero, and the MS compiler
260	* won't allow static initialization of arrays containing structures
261	* that contain zero size arrays */
262	char cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
263	#endif
264	} umem_cpu_cache_t;
265
266	/*
267	* The magazine lists used in the depot.
268	*/
269	typedef struct umem_maglist {
270	umem_magazine_t ml_list; / magazine list */
271	long ml_total; /* number of magazines */
272	long ml_min; /* min since last update */
273	long ml_reaplimit; /* max reapable magazines */
274	uint64_t ml_alloc; /* allocations from this list */
275	} umem_maglist_t;
276
277	#define UMEM_CACHE_NAMELEN 31
278
279	struct umem_cache {
280	/*
281	* Statistics
282	*/
283	uint64_t cache_slab_create; /* slab creates */
284	uint64_t cache_slab_destroy; /* slab destroys */
285	uint64_t cache_slab_alloc; /* slab layer allocations */
286	uint64_t cache_slab_free; /* slab layer frees */
287	uint64_t cache_alloc_fail; /* total failed allocations */
288	uint64_t cache_buftotal; /* total buffers */
289	uint64_t cache_bufmax; /* max buffers ever */
290	uint64_t cache_rescale; /* # of hash table rescales */
291	uint64_t cache_lookup_depth; /* hash lookup depth */
292	uint64_t cache_depot_contention; /* mutex contention count */
293	uint64_t cache_depot_contention_prev; /* previous snapshot */
294
295	/*
296	* Cache properties
297	*/
298	char cache_name[UMEM_CACHE_NAMELEN + 1];
299	size_t cache_bufsize; /* object size */
300	size_t cache_align; /* object alignment */
301	umem_constructor_t *cache_constructor;
302	umem_destructor_t *cache_destructor;
303	umem_reclaim_t *cache_reclaim;
304	void cache_private; / opaque arg to callbacks */
305	vmem_t cache_arena; / vmem source for slabs */
306	int cache_cflags; /* cache creation flags */
307	int cache_flags; /* various cache state info */
308	int cache_uflags; /* UMU_* flags */
309	uint32_t cache_mtbf; /* induced alloc failure rate */
310	umem_cache_t cache_next; / forward cache linkage */
311	umem_cache_t cache_prev; / backward cache linkage */
312	umem_cache_t cache_unext; / next in update list */
313	umem_cache_t cache_uprev; / prev in update list */
314	uint32_t cache_cpu_mask; /* mask for cpu offset */
315
316	/*
317	* Slab layer
318	*/
319	mutex_t cache_lock; /* protects slab layer */
320	size_t cache_chunksize; /* buf + alignment [+ debug] */
321	size_t cache_slabsize; /* size of a slab */
322	size_t cache_bufctl; /* buf-to-bufctl distance */
323	size_t cache_buftag; /* buf-to-buftag distance */
324	size_t cache_verify; /* bytes to verify */
325	size_t cache_contents; /* bytes of saved content */
326	size_t cache_color; /* next slab color */
327	size_t cache_mincolor; /* maximum slab color */
328	size_t cache_maxcolor; /* maximum slab color */
329	size_t cache_hash_shift; /* get to interesting bits */
330	size_t cache_hash_mask; /* hash table mask */
331	umem_slab_t cache_freelist; / slab free list */
332	umem_slab_t cache_nullslab; /* end of freelist marker */
333	umem_cache_t cache_bufctl_cache; / source of bufctls */
334	umem_bufctl_t *cache_hash_table; / hash table base */
335	/*
336	* Depot layer
337	*/
338	mutex_t cache_depot_lock; /* protects depot */
339	umem_magtype_t cache_magtype; / magazine type */
340	umem_maglist_t cache_full; /* full magazines */
341	umem_maglist_t cache_empty; /* empty magazines */
342
343	/*
344	* Per-CPU layer
345	*/
346	umem_cpu_cache_t cache_cpu[1]; /* cache_cpu_mask + 1 entries */
347	};
348
349	typedef struct umem_cpu_log_header {
350	mutex_t clh_lock;
351	char *clh_current;
352	size_t clh_avail;
353	int clh_chunk;
354	int clh_hits;
355	char clh_pad[UMEM_CPU_CACHE_SIZE -
356	sizeof (mutex_t) - sizeof (char *) -
357	sizeof (size_t) - 2 * sizeof (int)];
358	} umem_cpu_log_header_t;
359
360	typedef struct umem_log_header {
361	mutex_t lh_lock;
362	char *lh_base;
363	int *lh_free;
364	size_t lh_chunksize;
365	int lh_nchunks;
366	int lh_head;
367	int lh_tail;
368	int lh_hits;
369	umem_cpu_log_header_t lh_cpu[1]; /* actually umem_max_ncpus */
370	} umem_log_header_t;
371
372	typedef struct umem_cpu {
373	uint32_t cpu_cache_offset;
374	uint32_t cpu_number;
375	} umem_cpu_t;
376
377	#define UMEM_MAXBUF 16384
378
379	#define UMEM_ALIGN 8 /* min guaranteed alignment */
380	#define UMEM_ALIGN_SHIFT 3 /* log2(UMEM_ALIGN) */
381	#define UMEM_VOID_FRACTION 8 /* never waste more than 1/8 of slab */
382
383	/*
384	* For 64 bits, buffers >= 16 bytes must be 16-byte aligned
385	*/
386	#ifdef _LP64
387	#define UMEM_SECOND_ALIGN 16
388	#else
389	#define UMEM_SECOND_ALIGN UMEM_ALIGN
390	#endif
391
392	#define MALLOC_MAGIC 0x3a10c000 /* 8-byte tag */
393	#define MEMALIGN_MAGIC 0x3e3a1000
394
395	#ifdef _LP64
396	#define MALLOC_SECOND_MAGIC 0x16ba7000 /* 8-byte tag, 16-aligned */
397	#define MALLOC_OVERSIZE_MAGIC 0x06e47000 /* 16-byte tag, _LP64 */
398	#endif
399
400	#define UMEM_MALLOC_ENCODE(type, sz) (uint32_t)((type) - (sz))
401	#define UMEM_MALLOC_DECODE(stat, sz) (uint32_t)((stat) + (sz))
402	#define UMEM_FREE_PATTERN_32 (uint32_t)(UMEM_FREE_PATTERN)
403
404	#define UMU_MAGAZINE_RESIZE 0x00000001
405	#define UMU_HASH_RESCALE 0x00000002
406	#define UMU_REAP 0x00000004
407	#define UMU_NOTIFY 0x08000000
408	#define UMU_ACTIVE 0x80000000
409
410	#define UMEM_READY_INIT_FAILED -1
411	#define UMEM_READY_STARTUP 1
412	#define UMEM_READY_INITING 2
413	#define UMEM_READY 3
414
415	#ifdef UMEM_STANDALONE
416	extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
417	extern int umem_add(caddr_t, size_t);
418	#endif
419
420	#ifdef __cplusplus
421	}
422	#endif
423
424	#endif /* _UMEM_IMPL_H */

Note: See TracBrowser for help on using the browser.

root/trunk/umem_impl.h

Download in other formats: