1 /* Alloc.c -- Memory allocation functions
2 2024-02-18 : Igor Pavlov : Public domain */
13 #if defined(Z7_LARGE_PAGES) && defined(_WIN32) && \
14 (!defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0502) // < Win2003 (xp-64)
15 #define Z7_USE_DYN_GetLargePageMinimum
20 #if defined(__CHERI__) && defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ == 16)
21 // #pragma message("=== Z7_ALLOC_NO_OFFSET_ALLOCATOR === ")
22 #define Z7_ALLOC_NO_OFFSET_ALLOCATOR
26 // #define SZ_ALLOC_DEBUG
27 /* #define SZ_ALLOC_DEBUG */
29 /* use SZ_ALLOC_DEBUG to debug alloc/free operations */
34 static int g_allocCount = 0;
36 static int g_allocCountMid = 0;
37 static int g_allocCountBig = 0;
41 #define CONVERT_INT_TO_STR(charType, tempSize) \
42 char temp[tempSize]; unsigned i = 0; \
43 while (val >= 10) { temp[i++] = (char)('0' + (unsigned)(val % 10)); val /= 10; } \
44 *s++ = (charType)('0' + (unsigned)val); \
45 while (i != 0) { i--; *s++ = temp[i]; } \
48 static void ConvertUInt64ToString(UInt64 val, char *s)
50 CONVERT_INT_TO_STR(char, 24)
53 #define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))))
55 static void ConvertUInt64ToHex(UInt64 val, char *s)
68 unsigned t = (unsigned)(val & 0xF);
70 s[--i] = GET_HEX_CHAR(t);
75 #define DEBUG_OUT_STREAM stderr
77 static void Print(const char *s)
79 fputs(s, DEBUG_OUT_STREAM);
82 static void PrintAligned(const char *s, size_t align)
84 size_t len = strlen(s);
87 fputc(' ', DEBUG_OUT_STREAM);
95 static void PrintLn(void)
100 static void PrintHex(UInt64 v, size_t align)
103 ConvertUInt64ToHex(v, s);
104 PrintAligned(s, align);
107 static void PrintDec(int v, size_t align)
110 ConvertUInt64ToString((unsigned)v, s);
111 PrintAligned(s, align);
114 static void PrintAddr(void *p)
116 PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12);
120 #define PRINT_REALLOC(name, cnt, size, ptr) { \
122 if (!ptr) PrintDec(cnt++, 10); \
123 PrintHex(size, 10); \
127 #define PRINT_ALLOC(name, cnt, size, ptr) { \
129 PrintDec(cnt++, 10); \
130 PrintHex(size, 10); \
134 #define PRINT_FREE(name, cnt, ptr) if (ptr) { \
136 PrintDec(--cnt, 10); \
143 #define PRINT_ALLOC(name, cnt, size, ptr)
145 #define PRINT_FREE(name, cnt, ptr)
148 #ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR
149 #define PrintHex(v, align)
158 malloc(non_NULL, 0) : returns NULL or a unique pointer value that can later be successfully passed to free()
159 realloc(NULL, size) : the call is equivalent to malloc(size)
160 realloc(non_NULL, 0) : the call is equivalent to free(ptr)
163 malloc(0) : returns non_NULL
164 realloc(NULL, 0) : returns non_NULL
165 realloc(non_NULL, 0) : returns NULL
169 void *MyAlloc(size_t size)
173 // PRINT_ALLOC("Alloc ", g_allocCount, size, NULL)
174 #ifdef SZ_ALLOC_DEBUG
176 void *p = malloc(size);
179 PRINT_ALLOC("Alloc ", g_allocCount, size, p)
188 void MyFree(void *address)
190 PRINT_FREE("Free ", g_allocCount, address)
195 void *MyRealloc(void *address, size_t size)
202 // PRINT_REALLOC("Realloc ", g_allocCount, size, address)
203 #ifdef SZ_ALLOC_DEBUG
205 void *p = realloc(address, size);
208 PRINT_REALLOC("Realloc ", g_allocCount, size, address)
213 return realloc(address, size);
220 void *MidAlloc(size_t size)
224 #ifdef SZ_ALLOC_DEBUG
226 void *p = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
229 PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, p)
234 return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
238 void MidFree(void *address)
240 PRINT_FREE("Free-Mid", g_allocCountMid, address)
244 VirtualFree(address, 0, MEM_RELEASE);
247 #ifdef Z7_LARGE_PAGES
249 #ifdef MEM_LARGE_PAGES
250 #define MY_MEM_LARGE_PAGES MEM_LARGE_PAGES
252 #define MY_MEM_LARGE_PAGES 0x20000000
256 SIZE_T g_LargePageSize;
257 SIZE_T g_LargePageSize = 0;
258 typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID);
260 void SetLargePageSize(void)
263 #ifdef Z7_USE_DYN_GetLargePageMinimum
264 Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION
267 Func_GetLargePageMinimum fn =
268 (Func_GetLargePageMinimum) Z7_CAST_FUNC_C GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
269 "GetLargePageMinimum");
274 size = GetLargePageMinimum();
276 if (size == 0 || (size & (size - 1)) != 0)
278 g_LargePageSize = size;
281 #endif // Z7_LARGE_PAGES
283 void *BigAlloc(size_t size)
288 PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL)
290 #ifdef Z7_LARGE_PAGES
292 SIZE_T ps = g_LargePageSize;
293 if (ps != 0 && ps <= (1 << 30) && size > (ps / 2))
297 size2 = (size + ps) & ~ps;
300 void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY_MEM_LARGE_PAGES, PAGE_READWRITE);
303 PRINT_ALLOC("Alloc-BM ", g_allocCountMid, size2, p)
311 return MidAlloc(size);
314 void BigFree(void *address)
316 PRINT_FREE("Free-Big", g_allocCountBig, address)
323 static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MyAlloc(size); }
324 static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MyFree(address); }
325 const ISzAlloc g_Alloc = { SzAlloc, SzFree };
328 static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MidAlloc(size); }
329 static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MidFree(address); }
330 static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return BigAlloc(size); }
331 static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) BigFree(address); }
332 const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree };
333 const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
336 #ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR
338 #define ADJUST_ALLOC_SIZE 0
340 #define ADJUST_ALLOC_SIZE (sizeof(void *) - 1)
343 Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if
344 MyAlloc() can return address that is NOT multiple of sizeof(void *).
348 uintptr_t : <stdint.h> C99 (optional)
362 || (defined(__CHERI__) \
363 || defined(__SIZEOF_POINTER__) && (__SIZEOF_POINTER__ > 8))
364 // for 128-bit pointers (cheri):
365 #define MY_ALIGN_PTR_DOWN(p, align) \
366 ((void *)((char *)(p) - ((size_t)(MY_uintptr_t)(p) & ((align) - 1))))
368 #define MY_ALIGN_PTR_DOWN(p, align) \
369 ((void *)((((MY_uintptr_t)(p)) & ~((MY_uintptr_t)(align) - 1))))
374 #if !defined(_WIN32) \
375 && (defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR) \
376 || defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L))
377 #define USE_posix_memalign
380 #ifndef USE_posix_memalign
381 #define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align)
385 This posix_memalign() is for test purposes only.
386 We also need special Free() function instead of free(),
387 if this posix_memalign() is used.
391 static int posix_memalign(void **ptr, size_t align, size_t size)
393 size_t newSize = size + align;
399 p = MyAlloc(newSize);
402 pAligned = MY_ALIGN_PTR_UP_PLUS(p, align);
403 ((void **)pAligned)[-1] = p;
410 ALLOC_ALIGN_SIZE >= sizeof(void *)
411 ALLOC_ALIGN_SIZE >= cache_line_size
414 #define ALLOC_ALIGN_SIZE ((size_t)1 << 7)
416 void *z7_AlignedAlloc(size_t size)
418 #ifndef USE_posix_memalign
424 /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned
425 block to prevent cache line sharing with another allocated blocks */
427 newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE;
431 p = MyAlloc(newSize);
435 pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE);
437 Print(" size="); PrintHex(size, 8);
438 Print(" a_size="); PrintHex(newSize, 8);
439 Print(" ptr="); PrintAddr(p);
440 Print(" a_ptr="); PrintAddr(pAligned);
443 ((void **)pAligned)[-1] = p;
450 if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size))
453 Print(" posix_memalign="); PrintAddr(p);
462 void z7_AlignedFree(void *address)
464 #ifndef USE_posix_memalign
466 MyFree(((void **)address)[-1]);
473 static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size)
476 return z7_AlignedAlloc(size);
480 static void SzAlignedFree(ISzAllocPtr pp, void *address)
483 #ifndef USE_posix_memalign
485 MyFree(((void **)address)[-1]);
492 const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree };
496 /* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */
497 #ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR
499 #define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *))
500 #define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1]
502 // we can use this simplified code,
503 // if (CAlignOffsetAlloc::offset == (k * sizeof(void *))
504 #define REAL_BLOCK_PTR_VAR(p) (((void **)(p))[-1])
510 #ifndef Z7_ALLOC_NO_OFFSET_ALLOCATOR
512 static void PrintPtr(const char *s, const void *p)
514 const Byte *p2 = (const Byte *)&p;
516 printf("%s %p ", s, p);
517 for (i = sizeof(p); i != 0;)
520 printf("%02x", p2[i]);
528 static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size)
530 #if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR)
532 return z7_AlignedAlloc(size);
534 const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt);
539 size_t alignSize = (size_t)1 << p->numAlignBits;
541 if (alignSize < sizeof(void *))
542 alignSize = sizeof(void *);
544 if (p->offset >= alignSize)
547 /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned
548 block to prevent cache line sharing with another allocated blocks */
549 extra = p->offset & (sizeof(void *) - 1);
550 newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE;
554 adr = ISzAlloc_Alloc(p->baseAlloc, newSize);
559 pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr +
560 alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset;
563 printf("\nalignSize = %6x, offset=%6x, size=%8x \n", (unsigned)alignSize, (unsigned)p->offset, (unsigned)size);
564 PrintPtr("base", adr);
565 PrintPtr("alig", pAligned);
569 Print("- Aligned: ");
570 Print(" size="); PrintHex(size, 8);
571 Print(" a_size="); PrintHex(newSize, 8);
572 Print(" ptr="); PrintAddr(adr);
573 Print(" a_ptr="); PrintAddr(pAligned);
576 REAL_BLOCK_PTR_VAR(pAligned) = adr;
583 static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address)
585 #if defined(Z7_ALLOC_NO_OFFSET_ALLOCATOR)
587 z7_AlignedFree(address);
591 const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt);
593 Print("- Aligned Free: ");
595 ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address));
601 void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p)
603 p->vt.Alloc = AlignOffsetAlloc_Alloc;
604 p->vt.Free = AlignOffsetAlloc_Free;