NotAlexNoyle/UXP-Jihad
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Issue #2927 - Update mimalloc to 3.3.1

Browse Source
This commit is contained in:
Basilisk-Dev committed 2026-04-21 18:09:24 -04:00
1 parent 336c03ff42
commit b822fa9d33
36 files changed
+637 -366
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

memory/mimalloc/README.MIMALLOC
+2 -2
View File
@@ -1,8 +1,8 @@
mimalloc vendor snapshot
Source: https://github.com/microsoft/mimalloc
Commit: 0ddf397796fbefa35b3278bd4431c2913a9892eb
Release: 3.3.0
Commit: e2e6e70f8333e58ed0116591ca33a56227db9139
Release: 3.3.1
Notes:
- Copied include/, src/, LICENSE, readme.md, and Windows x86/x64
memory/mimalloc/include/mimalloc-override.h
+10 -10
View File
@@ -19,7 +19,7 @@ not accidentally mix pointers from different allocators).
// Standard C allocation
#define malloc(n) mi_malloc(n)
#define calloc(n,c) mi_calloc(n,c)
#define calloc(c,n) mi_calloc(c,n)
#define realloc(p,n) mi_realloc(p,n)
#define free(p) mi_free(p)
@@ -30,26 +30,26 @@ not accidentally mix pointers from different allocators).
// Microsoft extensions
#define _expand(p,n) mi_expand(p,n)
#define _msize(p) mi_usable_size(p)
#define _recalloc(p,n,c) mi_recalloc(p,n,c)
#define _recalloc(p,c,n) mi_recalloc(p,c,n)
#define _strdup(s) mi_strdup(s)
#define _strndup(s,n) mi_strndup(s,n)
#define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s))
#define _wcsdup(s) mi_wcsdup(s)
#define _mbsdup(s) mi_mbsdup(s)
#define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v)
#define _wdupenv_s(b,n,v) mi_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v))
#define _dupenv_s(buf,n,nm) mi_dupenv_s(buf,n,nm)
#define _wdupenv_s(buf,n,nm) mi_wdupenv_s(buf,n,nm)
// Various Posix and Unix variants
#define reallocf(p,n) mi_reallocf(p,n)
#define malloc_size(p) mi_usable_size(p)
#define malloc_usable_size(p) mi_usable_size(p)
#define malloc_good_size(sz) mi_malloc_good_size(sz)
#define malloc_good_size(n) mi_malloc_good_size(n)
#define cfree(p) mi_free(p)
#define valloc(n) mi_valloc(n)
#define pvalloc(n) mi_pvalloc(n)
#define reallocarray(p,s,n) mi_reallocarray(p,s,n)
#define reallocarr(p,s,n) mi_reallocarr(p,s,n)
#define reallocarray(p,c,n) mi_reallocarray(p,c,n)
#define reallocarr(ptrp,c,n) mi_reallocarr(ptrp,c,n)
#define memalign(a,n) mi_memalign(a,n)
#define aligned_alloc(a,n) mi_aligned_alloc(a,n)
#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n)
@@ -58,11 +58,11 @@ not accidentally mix pointers from different allocators).
// Microsoft aligned variants
#define _aligned_malloc(n,a) mi_malloc_aligned(n,a)
#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a)
#define _aligned_recalloc(p,s,n,a) mi_aligned_recalloc(p,s,n,a)
#define _aligned_recalloc(p,c,n,a) mi_aligned_recalloc(p,c,n,a)
#define _aligned_msize(p,a,o) mi_usable_size(p)
#define _aligned_free(p) mi_free(p)
#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o)
#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o)
#define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o)
#define _aligned_offset_recalloc(p,c,n,a,o) mi_recalloc_aligned_at(p,c,n,a,o)
#endif // MIMALLOC_OVERRIDE_H
memory/mimalloc/include/mimalloc-stats.h
+1 -1
View File
@@ -44,7 +44,7 @@ typedef struct mi_stat_counter_s {
MI_STAT_COUNTER(reset) /* reset bytes */ \
MI_STAT_COUNTER(purged) /* purged bytes */ \
MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \
MI_STAT_COUNT(pages_abandoned) /* abandonded pages count */ \
MI_STAT_COUNT(pages_abandoned) /* abandoned pages count */ \
MI_STAT_COUNT(threads) /* number of threads */ \
MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \
MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \
memory/mimalloc/include/mimalloc.h
+11 -9
View File
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MIMALLOC_H
#define MIMALLOC_H
#define MI_MALLOC_VERSION 30300 // major + 2 digits minor + 2 digits patch
#define MI_MALLOC_VERSION 30301 // major + 2 digits minor + 2 digits patch
// ------------------------------------------------------
// Compiler specific attributes
@@ -114,7 +114,7 @@ mi_decl_export void* mi_expand(void* p, size_t newsize)
mi_decl_export void mi_free(void* p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
// ------------------------------------------------------
// Extended allocation functions
@@ -254,7 +254,7 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* theap, void*
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
@@ -525,18 +525,20 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size)
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3);
mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, size_t size) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept;
mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept;
mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept;
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept;
// wide characters
#include <wchar.h> // wchar_t
mi_decl_export int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc;
// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
memory/mimalloc/include/mimalloc/atomic.h
+55 -131
View File
@@ -97,7 +97,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1)
#define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1)
static inline void mi_atomic_yield(void);
static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
@@ -159,7 +158,9 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
#elif defined(_MSC_VER)
// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
// It is recommended to always compile as C++ when using MSVC
#include <intrin.h>
#ifdef _WIN64
typedef LONG64 msc_intptr_t;
@@ -252,7 +253,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_o
}
static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
(void)(mo);
#if defined(x_M_IX86) || defined(_M_X64)
#if defined(_M_X64)
*p = x;
#else
InterlockedExchange64(p, x);
@@ -337,21 +338,12 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
// ----------------------------------------------------------------------
// Once and Guard
// Guard
// ----------------------------------------------------------------------
typedef _Atomic(uintptr_t) mi_atomic_once_t;
// Returns true only on the first invocation
static inline bool mi_atomic_once( mi_atomic_once_t* once ) {
if (mi_atomic_load_relaxed(once) != 0) return false; // quick test
uintptr_t expected = 0;
return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1
}
typedef _Atomic(uintptr_t) mi_atomic_guard_t;
// Allows only one thread to execute at a time
// Allows only one thread to execute at a time (without blocking anyone)
#define mi_atomic_guard(guard) \
uintptr_t _mi_guard_expected = 0; \
for(bool _mi_guard_once = true; \
@@ -359,77 +351,6 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t;
(mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
// ----------------------------------------------------------------------
// Yield
// ----------------------------------------------------------------------
#if defined(_WIN32)
static inline void mi_atomic_yield(void) {
YieldProcessor(); // see issue #1215 and #1225 why this is preferred over __yield or SwitchToThread
}
#elif defined(__SSE2__)
#include <emmintrin.h>
static inline void mi_atomic_yield(void) {
_mm_pause();
}
#elif (defined(__GNUC__) || defined(__clang__)) && \
(defined(__x86_64__) || defined(__i386__) || \
defined(__aarch64__) || defined(__arm__) || \
defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__))
#if defined(__x86_64__) || defined(__i386__)
static inline void mi_atomic_yield(void) {
__asm__ volatile ("pause" ::: "memory");
}
#elif defined(__aarch64__)
static inline void mi_atomic_yield(void) {
__asm__ volatile("isb");
}
#elif defined(__arm__)
#if __ARM_ARCH >= 7
static inline void mi_atomic_yield(void) {
__asm__ volatile("yield" ::: "memory");
}
#else
static inline void mi_atomic_yield(void) {
__asm__ volatile ("nop" ::: "memory");
}
#endif
#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)
#ifdef __APPLE__
static inline void mi_atomic_yield(void) {
__asm__ volatile ("or r27,r27,r27" ::: "memory");
}
#else
static inline void mi_atomic_yield(void) {
__asm__ __volatile__ ("or 27,27,27" ::: "memory");
}
#endif
#endif
#elif defined(__sun)
#include <synch.h>
static inline void mi_atomic_yield(void) {
smt_pause();
}
#elif defined(__wasi__)
#include <sched.h>
static inline void mi_atomic_yield(void) {
sched_yield();
}
// Fallback for other archs
#elif defined(__cplusplus)
#include <thread>
static inline void mi_atomic_yield(void) {
std::this_thread::yield();
}
#else
#include <unistd.h>
static inline void mi_atomic_yield(void) {
sleep(0);
}
#endif
// ----------------------------------------------------------------------
// Locks
// These should be light-weight in-process only locks.
@@ -439,18 +360,18 @@ static inline void mi_atomic_yield(void) {
#pragma warning(disable:26110) // unlock with holding lock
#endif
#define mi_lock(lock) for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) )
#define mi_lock_maybe(lock,acquire) for(bool _go = (acquire ? (mi_lock_acquire(lock),true) : true); _go; _go = (acquire ? (mi_lock_release(lock),false) : false) )
#define mi_lock(lock) for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
#define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
#if defined(_WIN32)
#if 1
typedef union mi_lock_u {
size_t _init; // for static initialization
SRWLOCK mutex; // slim reader-writer lock
typedef struct mi_lock_s {
SRWLOCK mutex; // slim reader-writer lock
} mi_lock_t;
#define MI_LOCK_INITIALIZER { SRWLOCK_INIT }
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryAcquireSRWLockExclusive(&lock->mutex);
}
@@ -467,41 +388,17 @@ static inline void mi_lock_done(mi_lock_t* lock) {
(void)(lock);
}
#else
typedef union mi_lock_u {
size_t _init; // for static initialization
CRITICAL_SECTION mutex;
} mi_lock_t;
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryEnterCriticalSection(&lock->mutex);
}
static inline void mi_lock_acquire(mi_lock_t* lock) {
EnterCriticalSection(&lock->mutex);
}
static inline void mi_lock_release(mi_lock_t* lock) {
LeaveCriticalSection(&lock->mutex);
}
static inline void mi_lock_init(mi_lock_t* lock) {
InitializeCriticalSection(&lock->mutex);
}
static inline void mi_lock_done(mi_lock_t* lock) {
DeleteCriticalSection(&lock->mutex);
}
#endif
#elif defined(MI_USE_PTHREADS)
#include <string.h> // memcpy
void _mi_error_message(int err, const char* fmt, ...);
typedef union mi_lock_u {
size_t _init; // for static initialization
typedef struct mi_lock_s {
pthread_mutex_t mutex;
} mi_lock_t;
#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return (pthread_mutex_trylock(&lock->mutex) == 0);
}
@@ -516,8 +413,8 @@ static inline void mi_lock_release(mi_lock_t* lock) {
}
static inline void mi_lock_init(mi_lock_t* lock) {
if(lock==NULL) return;
// use instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
// use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
memcpy(&lock->mutex,&mutex,sizeof(mutex));
}
static inline void mi_lock_done(mi_lock_t* lock) {
@@ -526,12 +423,15 @@ static inline void mi_lock_done(mi_lock_t* lock) {
#elif defined(__cplusplus)
#include <thread>
#include <mutex>
typedef union mi_lock_u {
size_t _init; // for static initialization
#include <new>
typedef struct mi_lock_s {
std::mutex mutex;
} mi_lock_t;
#define MI_LOCK_INITIALIZER { }
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return lock->mutex.try_lock();
@@ -543,7 +443,7 @@ static inline void mi_lock_release(mi_lock_t* lock) {
lock->mutex.unlock();
}
static inline void mi_lock_init(mi_lock_t* lock) {
(void)(lock);
new(&lock->mutex) std::mutex();
}
static inline void mi_lock_done(mi_lock_t* lock) {
(void)(lock);
@@ -553,27 +453,35 @@ static inline void mi_lock_done(mi_lock_t* lock) {
// fall back to poor man's locks.
// this should only be the case in a single-threaded environment (like __wasi__)
#include <errno.h>
#ifndef EFAULT
#define EFAULT (14)
#endif
void _mi_error_message(int err, const char* fmt, ...);
void _mi_prim_thread_yield(void);
typedef union mi_lock_u {
size_t _init; // for static initialization
typedef struct mi_lock_s {
_Atomic(uintptr_t) mutex;
} mi_lock_t;
#define MI_LOCK_INITIALIZER { MI_ATOMIC_VAR_INIT(0) }
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
uintptr_t expected = 0;
return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
}
static inline void mi_lock_acquire(mi_lock_t* lock) {
for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
if (mi_lock_try_acquire(&lock->mutex)) return;
mi_atomic_yield();
for (int i = 0; i < 10000; i++) { // for at most 10000 tries?
if (mi_lock_try_acquire(lock)) return;
_mi_prim_thread_yield();
}
_mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
}
static inline void mi_lock_release(mi_lock_t* lock) {
mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
}
static inline void mi_lock_init(mi_lock_t* lock) {
mi_lock_release(&lock->mutex);
mi_lock_release(lock);
}
static inline void mi_lock_done(mi_lock_t* lock) {
(void)(lock);
@@ -582,4 +490,20 @@ static inline void mi_lock_done(mi_lock_t* lock) {
#endif
#endif // MI_ATOMIC_H
typedef struct mi_atomic_once_s {
_Atomic(uintptr_t) tid;
mi_lock_t lock;
} mi_atomic_once_t;
// Returns `true` only on the first invocation, signifying we can execute an action once.
// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
bool _mi_atomic_once_enter(mi_atomic_once_t* once); // defined in `libc.c`
void _mi_atomic_once_release(mi_atomic_once_t* once); // defined in `libc.c`
#define mi_atomic_do_once \
static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
#endif // __MIMALLOC_ATOMIC_H
memory/mimalloc/include/mimalloc/bits.h
+6 -8
View File
@@ -108,6 +108,9 @@ typedef int32_t mi_ssize_t;
#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
#define __BMI1__ 1
#endif
#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
#define __LZCNT__ 1
#endif
// Define big endian if needed
// #define MI_BIG_ENDIAN 1
@@ -222,11 +225,11 @@ static inline size_t mi_ctz(size_t x) {
}
static inline size_t mi_clz(size_t x) {
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 lzcnt is defined for 0
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
size_t r;
__asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
return r;
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__)
#elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__)
return _lzcnt_u64(x);
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
unsigned long idx;
@@ -273,12 +276,7 @@ static inline bool mi_bsf(size_t x, size_t* idx) {
// return false if `x==0` (with `*idx` undefined) and true otherwise,
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
static inline bool mi_bsr(size_t x, size_t* idx) {
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9)
// on x64 the carry flag is set on zero which gives better codegen
bool is_zero;
__asm ("lzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc");
return !is_zero;
#elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
#if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
unsigned long i;
return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
#else
memory/mimalloc/include/mimalloc/internal.h
+1
View File
@@ -131,6 +131,7 @@ void _mi_strlcat(char* dest, const char* src, size_t dest_size);
size_t _mi_strlen(const char* s);
size_t _mi_strnlen(const char* s, size_t max_len);
char* _mi_strnstr(char* s, size_t max_len, const char* pat);
bool _mi_streq(const char* s, const char* t);
bool _mi_getenv(const char* name, char* result, size_t result_size);
// "options.c"
memory/mimalloc/include/mimalloc/prim.h
+15 -10
View File
@@ -126,6 +126,9 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
// Is this thread part of a thread pool?
bool _mi_prim_thread_is_in_threadpool(void);
// Yield to other threads. Should be similar to `sleep(0)`.
// Is called only in rare situations and does not have to be lightning fast.
void _mi_prim_thread_yield(void);
//-------------------------------------------------------------------
// Access to TLS (thread local storage) slots.
@@ -253,14 +256,16 @@ extern mi_decl_hidden bool _mi_process_is_initialized; // has mi_
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
// For now, we only enable this for specific platforms.
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
&& !defined(__CYGWIN__) \
&& !defined(MI_LIBC_MUSL) \
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
#define MI_USE_BUILTIN_THREAD_POINTER 1
#if !defined(MI_USE_BUILTIN_THREAD_POINTER) /* allow user override */
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
&& !defined(__CYGWIN__) \
&& !defined(MI_LIBC_MUSL) \
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
#define MI_USE_BUILTIN_THREAD_POINTER 1
#endif
#endif
#endif
@@ -363,14 +368,14 @@ static inline mi_theap_t* _mi_theap_cached(void);
#if defined(_WIN32)
#define MI_TLS_MODEL_DYNAMIC_WIN32 1
#elif defined(__APPLE__) // macOS
#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__) // macOS on arm64 or x64
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS 1 // also works but a bit slower
#define MI_TLS_MODEL_FIXED_SLOT 1
#define MI_TLS_MODEL_FIXED_SLOT_DEFAULT 108 // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
#define MI_TLS_MODEL_FIXED_SLOT_CACHED 109
// we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
#elif defined(__OpenBSD__) || defined(__ANDROID__)
#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
#define MI_TLS_MODEL_DYNAMIC_PTHREADS 1
// #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL 1
#else
memory/mimalloc/include/mimalloc/track.h
+7 -2
View File
@@ -84,7 +84,8 @@ defined, undefined, or not accessible at all:
#include "../src/prim/windows/etw.h"
#define mi_track_init() EventRegistermicrosoft_windows_mimalloc();
#define mi_track_init() EventRegistermicrosoft_windows_mimalloc()
#define mi_track_done() EventUnregistermicrosoft_windows_mimalloc()
#define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
#define mi_track_free_size(p,size) EventWriteETW_MI_FREE((UINT64)(p), size)
@@ -104,7 +105,7 @@ defined, undefined, or not accessible at all:
// Utility definitions
#ifndef mi_track_resize
#define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false)
#define mi_track_resize(p,oldsize,newsize) do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
#endif
#ifndef mi_track_align
@@ -115,6 +116,10 @@ defined, undefined, or not accessible at all:
#define mi_track_init()
#endif
#ifndef mi_track_done
#define mi_track_done()
#endif
#ifndef mi_track_mem_defined
#define mi_track_mem_defined(p,size)
#endif
memory/mimalloc/include/mimalloc/types.h
+9 -15
View File
@@ -54,18 +54,12 @@ terms of the MIT license. A copy of the license can be found in the file
// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
// #define MI_STAT 1
// Define MI_SECURE to enable security mitigations. Level 1 has minimal performance impact,
// but protects most metadata with guard pages:
// #define MI_SECURE 1 // guard page around metadata; check pointer validity on free
//
// Level 2 is only used if `MI_PAGE_META_IS_SEPARATED==0` (which it is not by default)
// #define MI_SECURE 2 // guard page around each mimalloc page (can fragment VMA's with large theaps..)
//
// Level 3 has slightly more performance overhead
// #define MI_SECURE 3 // randomize allocations, encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
//
// Level 4 has (much) more overhead. It also adds guard pages around each mimalloc page (even if `MI_PAGE_META_IS_SEPARATED` is defined).
// #define MI_SECURE 4 // checks also for double free.
// Define MI_SECURE to enable security mitigations
// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages)
// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
#if !defined(MI_SECURE)
#define MI_SECURE 0
@@ -165,7 +159,7 @@ terms of the MIT license. A copy of the license can be found in the file
#ifndef MI_ARENA_SLICE_SHIFT
#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility
#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT
#elif MI_SECURE && __APPLE__ && MI_ARCH_ARM64
#elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
#define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages
#else
#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit)
@@ -346,7 +340,7 @@ typedef size_t mi_page_flags_t;
// There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
// and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
// in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
// Abondoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
#define MI_THREADID_ABANDONED MI_ZU(0)
#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1)
@@ -450,7 +444,7 @@ typedef enum mi_page_kind_e {
MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages
MI_PAGE_LARGE, // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
MI_PAGE_SINGLETON // page containing a single block.
// used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an aligment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
// used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
} mi_page_kind_t;
memory/mimalloc/readme.md
+10 -5
View File
@@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac
Initially developed by Daan Leijen for the runtime systems of the
[Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages.
Latest release : `v3.3.0` (2026-04-15) recommended.
Latest v2 release: `v2.3.0` (2026-04-15) stable.
Latest v1 release: `v1.9.8` (2026-04-15) legacy.
Latest release : `v3.3.1` (2026-04-20) recommended.
Latest v2 release: `v2.3.1` (2026-04-20) stable.
Latest v1 release: `v1.9.9` (2026-04-20) legacy.
mimalloc is a drop-in replacement for `malloc` and can be used in other programs
without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as:
@@ -88,6 +88,9 @@ New development is mostly on v3, while v1 and v2 are maintained with security an
- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible.
### Releases
* 2026-04-20, `v1.9.9`, `v2.3.1`, `v3.3.1`: various bug and security fixes. Special thanks to
@jinpzhanAMD, @res2k, and @GoldJohnKing for their help in improving Windows finalization, and
@Zoxc for his help in finding various issues.
* 2026-04-15, `v1.9.8`, `v2.3.0`, `v3.3.0`: initial support for github (binary) releases,
fix visiting of full pages during collection (performance),
fix THP alignment (performance), fix arm64 cross-compilation on Windows, enable guard pages in debug mode,
@@ -399,8 +402,7 @@ OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the
_mimalloc_ can be build in secure mode by using the `-DMI_SECURE=ON` flags in `cmake`. This build enables various mitigations
to make mimalloc more robust against exploits. In particular:
- All internal mimalloc pages are surrounded by guard pages and the heap metadata is behind a guard page as well (so a buffer overflow
exploit cannot reach into the metadata).
- All internal mimalloc page meta-data is surrounded by guard pages (so a buffer overflow exploit cannot reach into the metadata).
- All free list pointers are
[encoded](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396)
with per-page keys which is used both to prevent overwrites with a known pointer, as well as to detect heap corruption.
@@ -408,6 +410,9 @@ to make mimalloc more robust against exploits. In particular:
- The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to
mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc
from the OS are also address randomized.
- If enabling `-DMI_SECURE_FULL=ON` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving
valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to
reaching the maximum VMA limit on Linux systems if the heap gets too large.
As always, evaluate with care as part of an overall security strategy as all of the above are mitigations but not guarantees.
memory/mimalloc/src/alloc-aligned.c
+8 -6
View File
@@ -34,6 +34,10 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap,
if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); }
#endif
mi_assert_internal(alignment > 0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN);
if mi_unlikely(alignment >= MI_PAGE_MAX_OVERALLOC_ALIGN || size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE - alignment)) {
_mi_error_message(EOVERFLOW, "(guarded) aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
return NULL;
}
const size_t oversize = size + alignment - 1;
void* base = _mi_theap_malloc_guarded(theap, oversize, zero);
void* p = _mi_align_up_ptr(base, alignment);
@@ -74,9 +78,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
// in the first (and single) page such that the page info is `MI_PAGE_ALIGN` bytes before it (and can be found in the _mi_page_map).
if mi_unlikely(offset != 0) {
// todo: cannot support offset alignment for very large alignments yet
#if MI_DEBUG > 0
_mi_error_message(EOVERFLOW, "aligned allocation with a large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
#endif
return NULL;
}
oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
@@ -86,7 +88,9 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap
}
else {
// otherwise over-allocate
oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and aligment 64k, we would allocate a 64k block and pointing just beyond that.
mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN);
mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow
oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that.
p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable);
if (p == NULL) return NULL;
}
@@ -150,9 +154,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t
mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
// we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
#if MI_DEBUG > 0
_mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
#endif
return NULL;
}
@@ -328,7 +330,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap,
static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
mi_assert(alignment > 0);
if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL);
if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL);
size_t size = mi_usable_size(p);
if (newsize <= size && newsize >= (size - (size / 2))
memory/mimalloc/src/alloc-override.c
+7 -8
View File
@@ -48,7 +48,6 @@ typedef void* mi_nothrow_t;
#define MI_FORWARD02(fun,x,y) { fun(x,y); }
#endif
#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE)
// define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for
// functions that are interposed (or the interposing does not work)
@@ -92,12 +91,12 @@ typedef void* mi_nothrow_t;
MI_INTERPOSE_FUN(vfree,mi_cfree),
#endif
};
MI_INTERPOSE_DECLS(_mi_interposes_10_7) __OSX_AVAILABLE(10.7) = {
MI_INTERPOSE_MI(strndup),
};
MI_INTERPOSE_DECLS(_mi_interposes_10_15) __OSX_AVAILABLE(10.15) = {
MI_INTERPOSE_MI(aligned_alloc),
};
#if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
MI_INTERPOSE_DECLS(_mi_interposes_10_7) = { MI_INTERPOSE_MI(strndup) };
#endif
#if defined(MAC_OS_X_VERSION_10_15) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15)
MI_INTERPOSE_DECLS(_mi_interposes_10_15) = { MI_INTERPOSE_MI(aligned_alloc) };
#endif
#ifdef __cplusplus
extern "C" {
@@ -358,7 +357,7 @@ void cfree(void* p) { mi_free(p); }
void* pvalloc(size_t size) { return mi_pvalloc(size); }
void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
#if !defined(_WIN32)
void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
void* _aligned_malloc(size_t size, size_t alignment) { return mi_malloc_aligned(size,alignment); }
#endif
void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }
// some systems define reallocarr so mark it as a weak symbol (#751)
memory/mimalloc/src/alloc-posix.c
+28 -19
View File
@@ -99,7 +99,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size
return p;
}
mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD
mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD <https://man.freebsd.org/cgi/man.cgi?query=reallocarray>
size_t total;
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
errno = EOVERFLOW;
@@ -110,20 +110,28 @@ mi_decl_nodiscard void* mi_reallocarray( void* p, size_t count, size_t size ) mi
return newp;
}
mi_decl_nodiscard int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
mi_assert(p != NULL);
if (p == NULL) {
mi_decl_nodiscard int mi_reallocarr( void* ptrp, size_t count, size_t size ) mi_attr_noexcept { // NetBSD <https://man.netbsd.org/reallocarr.3>
mi_assert(size != 0);
mi_assert(ptrp != NULL);
if (ptrp == NULL || size == 0) {
return (errno = EINVAL);
}
size_t total;
if mi_unlikely(mi_count_size_overflow(count, size, &total)) {
return (errno = EOVERFLOW);
}
void** op = (void**)p;
void* newp = mi_realloc(*op,total);
if (newp == NULL) { return ENOMEM; }
*op = newp;
return 0;
void** op = (void**)ptrp;
if (total == 0) {
free(*op);
*op = NULL;
return 0;
}
else {
void* newp = mi_realloc(*op,total);
if (newp == NULL) { return (errno = ENOMEM); }
*op = newp;
return 0;
}
}
void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
@@ -132,12 +140,13 @@ void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
return res;
}
mi_decl_nodiscard mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
mi_decl_nodiscard mi_decl_restrict wchar_t* mi_wcsdup(const wchar_t* s) mi_attr_noexcept {
if (s==NULL) return NULL;
size_t len;
for(len = 0; s[len] != 0; len++) { }
size_t size = (len+1)*sizeof(unsigned short);
unsigned short* p = (unsigned short*)mi_malloc(size);
size_t wlen;
for(wlen = 0; s[wlen] != 0 && wlen < PTRDIFF_MAX; wlen++) { } // prevent overflow on wlen+1
size_t size;
if (mi_mul_overflow(wlen+1, sizeof(wchar_t), &size) || size > PTRDIFF_MAX) return NULL;
wchar_t* p = (wchar_t*)mi_malloc(size);
if (p != NULL) {
_mi_memcpy(p,s,size);
}
@@ -151,19 +160,19 @@ mi_decl_nodiscard mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char*
int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {
if (size != NULL) *size = 0;
if (buf==NULL || name==NULL) return EINVAL;
char* p = getenv(name); // mscver warning 4996
char* p = getenv(name);
if (p==NULL) {
*buf = NULL;
}
else {
*buf = mi_strdup(p);
if (*buf==NULL) return ENOMEM;
if (size != NULL) { *size = _mi_strlen(p) + 1; }
if (size != NULL) { *size = _mi_strlen(p) + 1; } // cannot overflow as mi_strdup is limited to PTRDIFF_MAX
}
return 0;
}
int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept {
int mi_wdupenv_s(wchar_t** buf, size_t* size, const wchar_t* name) mi_attr_noexcept {
if (size != NULL) *size = 0;
if (buf==NULL || name==NULL) return EINVAL;
#if !defined(_WIN32) || (defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP))
@@ -171,14 +180,14 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name)
*buf = NULL;
return EINVAL;
#else
unsigned short* p = (unsigned short*)_wgetenv((const wchar_t*)name); // msvc warning 4996
wchar_t* p = (wchar_t*)_wgetenv(name);
if (p==NULL) {
*buf = NULL;
}
else {
*buf = mi_wcsdup(p);
if (*buf==NULL) return ENOMEM;
if (size != NULL) { *size = wcslen((const wchar_t*)p) + 1; }
if (size != NULL) { *size = wcslen(p) + 1; } // cannot overflow as wcsdup is limited to PTRDIFF_MAX
}
return 0;
#endif
memory/mimalloc/src/alloc.c
+42 -2
View File
@@ -490,6 +490,7 @@ mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count,
mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strdup(mi_theap_t* theap, const char* s) mi_attr_noexcept {
if (s == NULL) return NULL;
size_t len = _mi_strlen(s);
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
char* t = (char*)mi_theap_malloc(theap,len+1);
if (t == NULL) return NULL;
_mi_memcpy(t, s, len);
@@ -509,6 +510,7 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const c
mi_decl_nodiscard static mi_decl_restrict char* mi_theap_strndup(mi_theap_t* theap, const char* s, size_t n) mi_attr_noexcept {
if (s == NULL) return NULL;
const size_t len = _mi_strnlen(s,n); // len <= n
if (len > MI_MAX_ALLOC_SIZE - 1) return NULL; // prevent overflow on len+1
char* t = (char*)mi_theap_malloc(theap, len+1);
if (t == NULL) return NULL;
_mi_memcpy(t, s, len);
@@ -548,17 +550,51 @@ mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* th
return mi_theap_strndup(theap, buf, PATH_MAX);
}
}
#else
#include <unistd.h> // pathconf
static size_t mi_path_max(void) {
static _Atomic(size_t) path_max = 0;
size_t pmax = mi_atomic_load_acquire(&path_max);
if (pmax == 0) {
long m = 0;
#ifdef _PC_PATH_MAX
m = pathconf("/",_PC_PATH_MAX);
#endif
if (m <= 0) pmax = 4096; // guess
else if (m < 256) pmax = 256; // at least 256
else if (m > 64*1024) pmax = 64*1024; // at most 64 KiB
else pmax = m;
size_t expected = 0;
mi_atomic_cas_strong_acq_rel(&path_max, &expected, pmax);
}
return pmax;
}
char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept {
if (resolved_name != NULL) {
return realpath(fname,resolved_name);
}
else {
/*
char* rname = realpath(fname, NULL);
if (rname == NULL) return NULL;
char* result = mi_theap_strdup(theap, rname);
mi_cfree(rname); // use checked free (which may be redirected to our free but that's ok)
char* result = mi_heap_strdup(heap, rname);
mi_cfree(rname); // note: may leak the original pointer if allocated internally with the system allocator
// note: with ASAN realpath is intercepted and mi_cfree may leak the returned pointer :-(
return result;
*/
const size_t n = mi_path_max();
char* const buf = (char*)mi_zalloc(n+1);
if (buf == NULL) {
errno = ENOMEM;
return NULL;
}
char* rname = realpath(fname,buf);
char* result = mi_theap_strndup(theap,rname,n); // ok if `rname==NULL`
mi_free(buf);
return result;
}
}
@@ -790,6 +826,10 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size,
{
// allocate multiple of page size ending in a guard page
// ensure minimal alignment requirement?
if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow
_mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size);
return NULL;
}
const size_t os_page_size = _mi_os_page_size();
const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
memory/mimalloc/src/arena-meta.c
+2 -2
View File
@@ -118,7 +118,7 @@ mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
mi_meta_page_t* mpage = mpage0;
while (mpage != NULL) {
size_t block_idx;
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) {
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
// found and claimed `block_count` blocks
*pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
return mi_meta_block_start(mpage,block_idx);
@@ -136,7 +136,7 @@ mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid )
mpage = mi_meta_page_zalloc();
if (mpage != NULL) {
size_t block_idx;
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) {
if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) {
// found and claimed `block_count` blocks
*pmemid = _mi_memid_create_meta(mpage, block_idx, block_count);
return mi_meta_block_start(mpage,block_idx);
memory/mimalloc/src/arena.c
+7 -7
View File
@@ -216,7 +216,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
{
size_t slice_index;
if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, slice_count, tseq, &slice_index)) return NULL;
if (!mi_bbitmap_try_find_and_clearN(arena->slices_free, tseq, slice_count, &slice_index)) return NULL;
// claimed it!
void* p = mi_arena_slice_start(arena, slice_index);
@@ -239,7 +239,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
if (already_committed < slice_count) {
// not all committed, try to commit now
bool commit_zero = false;
if (!_mi_os_commit_ex(p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
// if the commit fails, release ownership, and return NULL;
// note: this does not roll back dirty bits but that is ok.
mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
@@ -745,7 +745,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid) {
MI_UNUSED_RELEASE(block_size);
const bool allow_large = (MI_SECURE < 2); // 2 = guard page at end of each arena page
const bool allow_large = (MI_SECURE < 5); // 5 = guard page at end of each arena page
const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
mi_heap_t* const heap = _mi_theap_heap(theap);
@@ -832,7 +832,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou
if (!slice_start) return NULL;
// guard page at the end of mimalloc page?
#if (MI_SECURE >= 2 && (!MI_PAGE_META_IS_SEPARATED || MI_PAGE_META_ALIGNED_FREE_SMALL)) || MI_SECURE >= 4
#if MI_SECURE>=5
mi_assert(alloc_size > _mi_os_secure_guard_page_size());
const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();
if (memid.initially_committed) {
@@ -1065,7 +1065,7 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) {
// recommit guard page at the end?
// we must do this since we may later allocate large spans over this page and cannot have a guard page in between
#if (MI_SECURE >= 2 && (!MI_PAGE_META_IS_SEPARATED || MI_PAGE_META_ALIGNED_FREE_SMALL)) || MI_SECURE >= 4
#if MI_SECURE >= 5
if (!page->memid.is_pinned) {
_mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid);
}
@@ -1266,7 +1266,7 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
}
mi_assert_internal(slice_index < arena->slice_count);
mi_assert_internal(slice_index >= mi_arena_info_slices(arena));
if (slice_index < mi_arena_info_slices(arena) || slice_index > arena->slice_count) {
if (slice_index < mi_arena_info_slices(arena) || slice_index >= arena->slice_count) {
_mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
return;
}
@@ -1653,7 +1653,7 @@ bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is
return mi_manage_os_memory_ex2(_mi_subproc(), start, size, numa_node, exclusive, memid, NULL, NULL, arena_id);
}
bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_zero, bool is_pinned, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinned, bool is_zero, int numa_node, bool exclusive, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) mi_attr_noexcept
{
mi_memid_t memid = _mi_memid_create(MI_MEM_EXTERNAL);
memid.mem.os.base = start;
memory/mimalloc/src/bitmap.c
+31 -12
View File
@@ -12,6 +12,7 @@ Concurrent bitmap that can set/reset sequences of bits atomically
#include "mimalloc.h"
#include "mimalloc/internal.h"
#include "mimalloc/bits.h"
#include "mimalloc/prim.h" // _mi_prim_thread_yield
#include "bitmap.h"
#ifndef MI_OPT_SIMD
@@ -119,7 +120,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_
mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
}
while ((old&mask)==0) { // busy wait
mi_atomic_yield();
_mi_prim_thread_yield();
old = mi_atomic_load_acquire(b);
}
}
@@ -1072,7 +1073,6 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si
// last chunk
if (n > 0) {
mi_assert_internal(n < MI_BCHUNK_BITS);
mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL);
}
}
@@ -1442,21 +1442,27 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis
for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);
#if MI_DEBUG > 1
#if MI_DEBUG > 1
const size_t bpopcount = mi_popcount(b);
size_t rngcount = 0;
#endif
#endif
size_t bidx;
while (mi_bfield_find_least_bit(b, &bidx)) {
size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx
#if MI_DEBUG > 1
#if MI_DEBUG > 1
rngcount += rng;
#endif
#endif
const size_t idx = base_idx + bidx;
mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);
mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);
mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));
if (!visit(idx, rng, arena, arg)) return false;
if (!visit(idx, rng, arena, arg)) {
// break early: reset the non-visited bits
if (b!=0) {
mi_atomic_or_relaxed(&chunk->bfields[j], b);
}
return false;
}
// clear rng bits in b
b = b & ~mi_bfield_mask(rng, bidx);
}
@@ -1501,11 +1507,16 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo
if ((b & rngmask) == rngmask) {
const size_t idx = base_idx + shift;
if (!visit(idx, rngslices, arena, arg)) {
// break early
if (skipped != 0) {
mi_atomic_or_relaxed(&chunk->bfields[j], skipped);
return false;
// break early: restore non-visited entries
mi_bfield_t notyet_visited = 0;
if (shift + rngslices < MI_BFIELD_BITS) {
notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));
}
mi_assert_internal((notyet_visited & skipped) == 0);
if ((notyet_visited | skipped) != 0) {
mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped);
}
return false;
}
}
else {
@@ -1561,11 +1572,19 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
}
bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {
const size_t chunkmap_max = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap), MI_BFIELD_BITS);
const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);
const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);
size_t skip_at_top = chunk_count % MI_BFIELD_BITS;
for (size_t i = chunkmap_max; i > 0; ) {
i--;
mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);
size_t cmap_idx;
// don't consider top 0 bits; set those to 1 here
if (skip_at_top > 0) {
const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);
skip_at_top = 0; // only for the first iteration
cmap |= mask_top;
}
if (mi_bsr(~cmap, &cmap_idx)) {
// highest chunk
const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
memory/mimalloc/src/bitmap.h
+4 -4
View File
@@ -323,13 +323,13 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 1-bit
bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits
// bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS
bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // < MI_BFIELD_BITS
bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // > MI_BCHUNK_BITS
bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS
bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS
// Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx) {
mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx); // small pages
if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx); // medium pages
// if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages
memory/mimalloc/src/free.c
+1 -1
View File
@@ -267,7 +267,7 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
return _mi_arenas_page_try_reabandon_to_mapped(page);
}
// Release ownership of a page. This may free or reabandond the page if other blocks are concurrently
// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
// freed in the meantime. Returns `true` if the page was freed.
// By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
memory/mimalloc/src/heap.c
+17 -8
View File
@@ -57,15 +57,18 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he
// create a fresh theap?
if (theap==NULL) {
// set first an invalid value to ensure the thread local storage is allocated
if (!_mi_thread_local_set(heap->theap, (mi_theap_t*)1)) {
_mi_error_message(EFAULT, "unable to allocate memory for thread local storage\n");
return NULL;
}
// then allocate the theap
theap = _mi_theap_create(heap, _mi_theap_default_safe()->tld);
if (theap==NULL) {
_mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
return NULL;
}
if (!_mi_thread_local_set(heap->theap, theap)) {
_mi_error_message(EFAULT, "unable to allocate memory for a thread local storage\n");
return NULL;
}
_mi_thread_local_set(heap->theap, theap); // this cannot fail now as it was set before to a non-zero value
}
return theap;
}
@@ -157,8 +160,13 @@ static void mi_heap_free_theaps(mi_heap_t* heap) {
theap = next;
}
}
if (!all_freed) { mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); mi_atomic_yield(); }
else { mi_assert_internal(heap->theaps==NULL); }
if (!all_freed) {
mi_heap_stat_counter_increase(heap,heaps_delete_wait,1);
_mi_prim_thread_yield();
}
else {
mi_assert_internal(heap->theaps==NULL);
}
}
while(!all_freed);
}
@@ -191,6 +199,7 @@ static void mi_heap_free(mi_heap_t* heap) {
_mi_thread_local_free(heap->theap);
mi_lock_done(&heap->theaps_lock);
mi_lock_done(&heap->os_abandoned_pages_lock);
mi_lock_done(&heap->arena_pages_lock);
mi_free(heap);
}
@@ -209,7 +218,7 @@ void _mi_heap_force_destroy(mi_heap_t* heap) {
if (heap==NULL) return;
mi_heap_free_theaps(heap);
_mi_heap_destroy_pages(heap);
if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); }
if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap?
}
void mi_heap_destroy(mi_heap_t* heap) {
@@ -260,5 +269,5 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc
// check utilization
if (page->capacity==0) return false;
if (perc_threshold>=100) return true;
return (perc_threshold >= ((100*page->used) / page->capacity));
return (perc_threshold >= ((100UL*page->used) / page->capacity));
}
memory/mimalloc/src/init.c
+52 -23
View File
@@ -111,7 +111,7 @@ static mi_decl_cache_align mi_tld_t tld_empty = {
0, // default numa node
&subproc_main, // subproc
NULL, // theaps list
{0}, // theaps lock
MI_LOCK_INITIALIZER, // theaps lock
false, // recurse
false, // is_in_threadpool
MI_MEMID_STATIC // memid
@@ -178,7 +178,7 @@ static mi_decl_cache_align mi_tld_t tld_main = {
0, // numa node
&subproc_main, // subproc
&theap_main, // theaps list
{0}, // theaps lock
MI_LOCK_INITIALIZER, // theaps lock
false, // recurse
false, // is_in_threadpool
MI_MEMID_STATIC // memid
@@ -495,10 +495,10 @@ mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
}
// destroy all subproc resources including arena's, heap's etc.
static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc)
static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock)
{
// remove from the subproc list
mi_lock(&subprocs_lock) {
mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) {
if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; }
if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; }
else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; }
@@ -539,7 +539,7 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc)
void mi_subproc_destroy(mi_subproc_id_t subproc_id) {
if (subproc_id == NULL) return;
mi_subproc_unsafe_destroy(_mi_subproc_from_id(subproc_id));
mi_subproc_unsafe_destroy(_mi_subproc_from_id(subproc_id), true /* take lock */);
}
static void mi_subprocs_unsafe_destroy_all(void) {
@@ -548,12 +548,12 @@ static void mi_subprocs_unsafe_destroy_all(void) {
while (subproc!=NULL) {
mi_subproc_t* next = subproc->next;
if (subproc!=&subproc_main) {
mi_subproc_unsafe_destroy(subproc);
mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */);
}
subproc = next;
}
}
mi_subproc_unsafe_destroy(&subproc_main);
}
mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */);
}
@@ -651,8 +651,13 @@ static void mi_thread_theaps_done(mi_tld_t* tld)
theap = next;
}
}
if (!all_freed) { mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); mi_atomic_yield(); }
else { mi_assert_internal(tld->theaps==NULL); }
if (!all_freed) {
mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1);
_mi_prim_thread_yield();
}
else {
mi_assert_internal(tld->theaps==NULL);
}
} while (!all_freed);
mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete
@@ -822,8 +827,7 @@ static void mi_win_tls_slot_free(DWORD* raw_index) {
}
static void mi_tls_slots_init(void) {
static mi_atomic_once_t tls_slots_init;
if (mi_atomic_once(&tls_slots_init)) {
mi_atomic_do_once {
bool ok = mi_win_tls_slot_alloc(&_mi_theap_default_slot, &_mi_theap_default_expansion_slot, &mi_tls_raw_index_default);
if (ok) {
ok = mi_win_tls_slot_alloc(&_mi_theap_cached_slot, &_mi_theap_cached_expansion_slot, &mi_tls_raw_index_cached);
@@ -856,12 +860,29 @@ static void mi_win_tls_slot_set(size_t slot, size_t extended_slot, void* value)
mi_decl_hidden pthread_key_t _mi_theap_default_key = 0;
mi_decl_hidden pthread_key_t _mi_theap_cached_key = 0;
// create a non-zero pthread key
static int mi_pthread_key_create( pthread_key_t* pkey ) {
pthread_key_t key;
int err = pthread_key_create(&key, NULL);
if (err!=0) return err;
if (key==0) {
// if we get a zero key, create another one as we use 0 for an invalid key
pthread_key_t key2;
err = pthread_key_create(&key2, NULL);
pthread_key_delete(key); // delete the old key
if (err!=0) return err;
key = key2;
}
mi_assert_internal(key!=0);
*pkey = key;
return 0;
}
static void mi_tls_slots_init(void) {
static mi_atomic_once_t tls_keys_init;
if (mi_atomic_once(&tls_keys_init)) {
int err = pthread_key_create(&_mi_theap_default_key, NULL);
mi_atomic_do_once {
int err = mi_pthread_key_create(&_mi_theap_default_key);
if (err==0) {
err = pthread_key_create(&_mi_theap_cached_key, NULL);
err = mi_pthread_key_create(&_mi_theap_cached_key);
}
if (err!=0) {
_mi_error_message(EFAULT, "unable to allocate pthread keys (error %d)\n", err);
@@ -1062,13 +1083,8 @@ static void mi_detect_cpu_features(void) {
// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
// ensure we are called once
static mi_atomic_once_t process_init;
// #if _MSC_VER < 1920
// mi_heap_main_init(); // vs2017 can dynamically re-initialize theap_main
// #endif
if (!mi_atomic_once(&process_init)) return;
static void mi_process_init_once(void) mi_attr_noexcept {
_mi_process_is_initialized = true;
_mi_verbose_message("process init: 0x%zx\n", _mi_thread_id());
mi_detect_cpu_features();
@@ -1108,6 +1124,16 @@ void mi_process_init(void) mi_attr_noexcept {
}
}
// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
// #if _MSC_VER < 1920
// mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main
// #endif
mi_atomic_do_once {
mi_process_init_once();
}
}
// Called when the process is done (cdecl as it is used with `at_exit` on some platforms)
void mi_cdecl mi_process_done(void) mi_attr_noexcept {
// only shutdown if we were initialized
@@ -1132,6 +1158,9 @@ void mi_cdecl mi_process_done(void) mi_attr_noexcept {
#endif
#endif
// done with tracking tools
mi_track_done()
// Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free
// since after process_done there might still be other code running that calls `free` (like at_exit routines,
// or C-runtime termination code.
memory/mimalloc/src/libc.c
+53 -8
View File
@@ -22,6 +22,7 @@ char _mi_toupper(char c) {
}
int _mi_strnicmp(const char* s, const char* t, size_t n) {
mi_assert_internal(s!=NULL && t!=NULL);
if (n == 0) return 0;
for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
if (_mi_toupper(*s) != _mi_toupper(*t)) break;
@@ -29,6 +30,15 @@ int _mi_strnicmp(const char* s, const char* t, size_t n) {
return (n == 0 ? 0 : *s - *t);
}
bool _mi_streq(const char* s, const char* t) {
if (s==NULL && t==NULL) return true;
if (s==NULL || t==NULL) return false;
for (; *s != 0 && *t != 0; s++, t++) {
if (*s != *t) break;
}
return (*s == *t);
}
void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
if (dest==NULL || src==NULL || dest_size == 0) return;
// copy until end of src, or when dest is (almost) full
@@ -51,13 +61,6 @@ void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
_mi_strlcpy(dest, src, dest_size);
}
size_t _mi_strlen(const char* s) {
if (s==NULL) return 0;
size_t len = 0;
while(s[len] != 0) { len++; }
return len;
}
size_t _mi_strnlen(const char* s, size_t max_len) {
if (s==NULL) return 0;
size_t len = 0;
@@ -65,6 +68,10 @@ size_t _mi_strnlen(const char* s, size_t max_len) {
return len;
}
size_t _mi_strlen(const char* s) {
return _mi_strnlen(s,PTRDIFF_MAX);
}
char* _mi_strnstr(char* s, size_t max_len, const char* pat) {
if (s==NULL) return NULL;
if (pat==NULL) return s;
@@ -94,6 +101,43 @@ bool _mi_getenv(const char* name, char* result, size_t result_size) {
}
#endif
// --------------------------------------------------------
// Define our own primitives for doing an action once
// --------------------------------------------------------
// Returns `true` only on the first invocation, signifying we can execute an action once.
// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
bool _mi_atomic_once_enter(mi_atomic_once_t* once) {
const uintptr_t once_tid = mi_atomic_load_acquire(&once->tid);
if mi_likely(once_tid == 1) {
return false; // already executed
}
const mi_threadid_t current_tid = _mi_thread_id();
if (once_tid == current_tid) {
return false; // recursive invocation; we need this for process_init for example
}
mi_lock_acquire(&once->lock);
uintptr_t expected = 0;
if (mi_atomic_cas_strong_acq_rel(&once->tid, &expected, current_tid)) { // could use atomic_load/store as well
return true; // should execute and release
}
else {
mi_lock_release(&once->lock);
return false; // already another thread entered and released
}
}
void _mi_atomic_once_release(mi_atomic_once_t* once) {
if (mi_atomic_load_acquire(&once->tid)>1) { // paranoia
mi_atomic_store_release(&once->tid,1); // done executing
mi_lock_release(&once->lock);
}
}
// --------------------------------------------------------
// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
// This is mostly to avoid calling these when libc is not yet
@@ -244,7 +288,8 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
else x = va_arg(args, unsigned int);
}
else if (c == 'p') {
x = va_arg(args, uintptr_t);
void* const p = va_arg(args, void*);
x = (uintptr_t)p;
mi_outs("0x", &out, end);
start = out;
width = (width >= 2 ? width - 2 : 0);
memory/mimalloc/src/options.c
+2 -2
View File
@@ -647,11 +647,11 @@ static void mi_option_init(mi_option_desc_t* desc) {
buf[i] = _mi_toupper(s[i]);
}
buf[len] = 0;
if (buf[0] == 0 || strstr("1;TRUE;YES;ON", buf) != NULL) {
if (buf[0] == 0 || _mi_streq(buf,"1") || _mi_streq(buf,"TRUE") || _mi_streq(buf,"YES") || _mi_streq(buf,"ON")) {
desc->value = 1;
desc->init = MI_OPTION_INITIALIZED;
}
else if (strstr("0;FALSE;NO;OFF", buf) != NULL) {
else if (_mi_streq(buf,"0") || _mi_streq(buf,"FALSE") || _mi_streq(buf,"NO") || _mi_streq(buf,"OFF")) {
desc->value = 0;
desc->init = MI_OPTION_INITIALIZED;
}
memory/mimalloc/src/os.c
+18 -9
View File
@@ -138,7 +138,9 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
uintptr_t init = MI_HINT_BASE;
#if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode
const uintptr_t r = _mi_theap_random_next(mi_theap_get_default());
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point
const uintptr_t r = _mi_theap_random_next(theap);
init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
#endif
uintptr_t expected = hint + size;
@@ -708,10 +710,16 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
if (start == 0) {
// Initialize the start address after the 32TiB area
start = ((uintptr_t)8 << 40); // 8TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_theap_random_next(_mi_theap_default());
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
#endif
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267)
if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness?
const uintptr_t r = _mi_theap_random_next(theap);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
}
else {
_mi_warning_message("failed to randomize the start address of huge pages allocation (%zu bytes at %p)", size, start);
}
#endif
}
end = start + size;
} while (!mi_atomic_cas_weak_acq_rel(&mi_huge_start, &huge_start, end));
@@ -784,16 +792,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
}
}
}
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
const size_t allocated = page * MI_HUGE_OS_PAGE_SIZE;
mi_assert_internal(allocated <= size);
if (pages_reserved != NULL) { *pages_reserved = page; }
if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; }
if (psize != NULL) { *psize = allocated; }
if (page != 0) {
mi_assert(start != NULL);
*memid = _mi_memid_create_os(start, size, true /* is committed */, all_zero, true /* is_large */);
*memid = _mi_memid_create_os(start, allocated, true /* is committed */, all_zero, true /* is_large */);
memid->memkind = MI_MEM_OS_HUGE;
mi_assert(memid->is_pinned);
#ifdef MI_TRACK_ASAN
if (all_zero) { mi_track_mem_defined(start,size); }
if (all_zero) { mi_track_mem_defined(start,allocated); }
#endif
}
return (page == 0 ? NULL : start);
memory/mimalloc/src/page-map.c
+6
View File
@@ -42,6 +42,9 @@ bool _mi_page_map_init(void) {
if (vbits >= 48) { vbits = 47; }
#endif
}
if (vbits < MI_ARENA_SLICE_SHIFT) {
vbits = MI_ARENA_SLICE_SHIFT;
}
// Allocate the page map and commit bits
mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
@@ -236,6 +239,9 @@ bool _mi_page_map_init(void) {
if (vbits >= 48) { vbits = 47; }
#endif
}
if (vbits < MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT) {
vbits = MI_PAGE_MAP_SUB_SHIFT + MI_ARENA_SLICE_SHIFT;
}
// Allocate the page map and commit bits
mi_assert(MI_MAX_VABITS >= vbits);
memory/mimalloc/src/page-queue.c
+9 -6
View File
@@ -115,9 +115,12 @@ mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexce
if (size <= MI_LARGE_MAX_OBJ_SIZE) {
return _mi_bin_size(mi_bin(size + MI_PADDING_SIZE));
}
else {
else if (size <= MI_MAX_ALLOC_SIZE) {
return _mi_align_up(size + MI_PADDING_SIZE,_mi_os_page_size());
}
else {
return size;
}
}
#if (MI_DEBUG>1)
@@ -205,25 +208,25 @@ static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) {
// range of entries in `_mi_page_small_free`.
static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_queue_t* pq) {
mi_assert_internal(mi_theap_contains_queue(theap,pq));
size_t size = pq->block_size;
const size_t size = pq->block_size;
if (size > MI_SMALL_SIZE_MAX) return;
mi_page_t* page = pq->first;
if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty;
// find index in the right direct page array
size_t start;
size_t idx = _mi_wsize_from_size(size);
mi_page_t** pages_free = theap->pages_free_direct;
const size_t idx = _mi_wsize_from_size(size);
mi_page_t** const pages_free = theap->pages_free_direct;
if (pages_free[idx] == page) return; // already set
// find start slot
size_t start;
if (idx<=1) {
start = 0;
}
else {
// find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
mi_assert_internal(pq > &theap->pages[0]); // since idx > 1
size_t bin = mi_bin(size);
const mi_page_queue_t* prev = pq - 1;
while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) {
memory/mimalloc/src/page.c
+9 -4
View File
@@ -154,10 +154,15 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head)
}
// if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
if (count > max_count) {
if mi_unlikely(count > max_count) {
_mi_error_message(EFAULT, "corrupted thread-free list\n");
return; // the thread-free items cannot be freed
}
// if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list)
else if mi_unlikely(count > page->used) {
_mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n");
return; // the thread-free items cannot be freed
}
// and append the current local free list
mi_block_set_next(page, last, page->local_free);
@@ -610,7 +615,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
----------------------------------------------------------- */
#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well.
#if (MI_SECURE>=3)
#if (MI_SECURE>=2)
#define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many
#else
#define MI_MIN_EXTEND (1)
@@ -669,7 +674,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) {
}
// and append the extend the free list
if (extend < MI_MIN_SLICES || MI_SECURE<3) { //!mi_option_is_enabled(mi_option_secure)) {
if (extend < MI_MIN_SLICES || MI_SECURE<2) { //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, bsize, extend );
}
else {
@@ -858,7 +863,7 @@ static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) {
// check the first page: we even do this with candidate search or otherwise we re-search every time
mi_page_t* page = pq->first;
if mi_likely(page != NULL && mi_page_free_quick_collect(page)) {
#if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness
#if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness
if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) {
(void)mi_page_extend_free(theap, page); // ok if this fails
mi_assert_internal(mi_page_immediate_available(page));
memory/mimalloc/src/prim/emscripten/prim.c
+6
View File
@@ -250,3 +250,9 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
bool _mi_prim_thread_is_in_threadpool(void) {
return false;
}
#include <emscripten.h>
void _mi_prim_thread_yield(void) {
emscripten_sleep(0);
}
memory/mimalloc/src/prim/osx/alloc-override-zone.c
+3 -6
View File
@@ -254,11 +254,8 @@ static malloc_zone_t mi_malloc_zone = {
// `malloc_zone_calloc` etc. see <https://github.com/aosm/libmalloc/blob/master/man/malloc_zone_malloc.3>
// ------------------------------------------------------
static inline malloc_zone_t* mi_get_default_zone(void)
{
static bool init;
if mi_unlikely(!init) {
init = true;
static inline malloc_zone_t* mi_get_default_zone(void) {
mi_atomic_do_once {
malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see <http://eatmyrandom.blogspot.com/2010/03/mallocfree-interception-on-mac-os-x.html>)
}
return &mi_malloc_zone;
@@ -329,7 +326,7 @@ static bool zone_check(malloc_zone_t* zone) {
static malloc_zone_t* zone_from_ptr(const void* p) {
MI_UNUSED(p);
return mi_get_default_zone();
return (mi_any_heap_contains(p) ? mi_get_default_zone() : NULL);
}
static void zone_log(malloc_zone_t* zone, void* p) {
memory/mimalloc/src/prim/unix/prim.c
+18 -8
View File
@@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/prim.h"
#include <sys/mman.h> // mmap
#include <unistd.h> // sysconf
#include <unistd.h> // sysconf, sleep
#include <fcntl.h> // open, close, read, access
#include <stdlib.h> // getenv, arc4random_buf
@@ -187,8 +187,16 @@ static void unix_detect_physical_memory( size_t page_size, size_t* physical_memo
MI_UNUSED(page_size);
struct sysinfo info; _mi_memzero_var(info);
const int err = sysinfo(&info);
if (err==0 && info.totalram > 0 && info.totalram <= SIZE_MAX) {
*physical_memory_in_kib = (size_t)info.totalram / MI_KiB;
if (err==0 && info.mem_unit > 0 && info.totalram <= SIZE_MAX) {
if (info.mem_unit==MI_KiB) {
*physical_memory_in_kib = (size_t)info.totalram;
}
else {
size_t total = 0;
if (!mi_mul_overflow((size_t)info.totalram, (size_t)info.mem_unit, &total)) {
*physical_memory_in_kib = (total / MI_KiB);
}
}
}
#elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100)
const long pphys = sysconf(_SC_PHYS_PAGES);
@@ -218,12 +226,10 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option
{
config->has_transparent_huge_pages = false;
int val = 0;
if (prctl(PR_GET_THP_DISABLE, &val, 0, 0, 0) != 0) {
if (prctl(PR_GET_THP_DISABLE, 0, 0, 0, 0) == 0) { // -1 on error, 1 if already disabled
// Most likely since distros often come with always/madvise settings.
val = 1;
// Disabling only for mimalloc process rather than touching system wide settings
(void)prctl(PR_SET_THP_DISABLE, &val, 0, 0, 0);
(void)prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0);
}
}
#endif
@@ -450,7 +456,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm
//---------------------------------------------
static void unix_mprotect_hint(int err) {
#if defined(__linux__) && ((MI_SECURE >= 2 && (!MI_PAGE_META_IS_SEPARATED || MI_PAGE_META_ALIGNED_FREE_SMALL)) || MI_SECURE >= 4) // guard page around every mimalloc page
#if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page
if (err == ENOMEM) {
_mi_warning_message("The next warning may be caused by a low memory map limit.\n"
" On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n"
@@ -987,3 +993,7 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
bool _mi_prim_thread_is_in_threadpool(void) {
return false;
}
void _mi_prim_thread_yield(void) {
sleep(0);
}
memory/mimalloc/src/prim/wasi/prim.c
+5 -2
View File
@@ -13,6 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stdio.h> // fputs
#include <stdlib.h> // getenv
#include <unistd.h> // sbrk, sleep
//---------------------------------------------
// Initialize
@@ -42,8 +43,6 @@ int _mi_prim_free(void* addr, size_t size ) {
//---------------------------------------------
#if defined(MI_USE_SBRK)
#include <unistd.h> // for sbrk
static void* mi_memory_grow( size_t size ) {
void* p = sbrk(size);
if (p == (void*)(-1)) return NULL;
@@ -286,3 +285,7 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
bool _mi_prim_thread_is_in_threadpool(void) {
return false;
}
void _mi_prim_thread_yield(void) {
sleep(0);
}
memory/mimalloc/src/prim/windows/prim.c
+160 -27
View File
@@ -11,6 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/internal.h"
#include "mimalloc/prim.h"
#include <stdio.h> // fputs, stderr
#include <stdlib.h> // atexit
// xbox has no console IO
#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM)
@@ -86,15 +87,13 @@ typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalM
typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation);
//---------------------------------------------
// Enable large page support dynamically (if possible)
//---------------------------------------------
static bool win_enable_large_os_pages(size_t* large_page_size)
static bool win_enable_large_os_pages_once(size_t* large_page_size)
{
static bool large_initialized = false;
if (large_initialized) return (_mi_os_large_page_size() > 0);
large_initialized = true;
if (pGetLargePageMinimum==NULL) return false; // no large page support (xbox etc.)
// Try to see if large OS pages are supported
@@ -128,6 +127,13 @@ static bool win_enable_large_os_pages(size_t* large_page_size)
return (ok!=0);
}
static bool win_enable_large_os_pages(size_t* large_page_size) {
mi_atomic_do_once {
win_enable_large_os_pages_once(large_page_size);
}
return (_mi_os_large_page_size() > 0);
}
//---------------------------------------------
// Initialize
@@ -417,8 +423,8 @@ static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int num
MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} };
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
static bool mi_huge_pages_available = true;
if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
static _Atomic(size_t) mi_huge_pages_available = MI_ATOMIC_VAR_INIT(1);
if (pNtAllocateVirtualMemoryEx != NULL && mi_atomic_load_acquire(&mi_huge_pages_available) != 0) {
params[0].Type.Type = MiMemExtendedParameterAttributeFlags;
params[0].Arg.ULong64 = MI_MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
ULONG param_count = 1;
@@ -435,7 +441,7 @@ static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int num
}
else {
// fall back to regular large pages
mi_huge_pages_available = false; // don't try further huge pages
mi_atomic_store_release(&mi_huge_pages_available,0); // don't try further huge pages
_mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err);
}
}
@@ -561,10 +567,11 @@ void _mi_prim_process_info(mi_process_info_t* pinfo)
pinfo->stime = filetime_msecs(&st);
// load psapi on demand
if (pGetProcessMemoryInfo == NULL) {
mi_atomic_do_once {
HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll"));
if (hDll != NULL) {
pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo");
// FreeLibrary(hDll); // don't free
}
}
@@ -663,13 +670,16 @@ typedef LONG (NTAPI *PBCryptGenRandom)(HANDLE, PUCHAR, ULONG, ULONG);
static PBCryptGenRandom pBCryptGenRandom = NULL;
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
if (pBCryptGenRandom == NULL) {
mi_assert(buf_len <= ULONG_MAX);
if (buf_len > ULONG_MAX) return false;
mi_atomic_do_once {
HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll"));
if (hDll != NULL) {
pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom");
// FreeLibrary(hDll); // don't free
}
if (pBCryptGenRandom == NULL) return false;
}
if (pBCryptGenRandom == NULL) return false;
return (pBCryptGenRandom(NULL, (PUCHAR)buf, (ULONG)buf_len, BCRYPT_USE_SYSTEM_PREFERRED_RNG) >= 0);
}
@@ -693,6 +703,9 @@ bool _mi_prim_thread_is_in_threadpool(void) {
return false;
}
void _mi_prim_thread_yield(void) {
SwitchToThread();
}
//----------------------------------------------------------------
// Process & Thread Init/Done
@@ -718,12 +731,136 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
}
/* ----------------------------------------------------------------------
/* -----------------------------------------------------------------------
Auto initialize and finalize mimalloc on process and thread start/end.
By default we use a combination of _pRawDllMain and TLS sections for
both static and dynamic linkage
By default we use a combination of CRT init and TLS sections for
both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`).
------------------------------------------------------------------------- */
#ifndef MI_WIN_NO_RAW_DLLMAIN
#if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS)
#if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER)
#define MI_WIN_INIT_USE_CRT_TLS 1
#else
#define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */
#endif
#endif
#if defined(MI_WIN_INIT_USE_CRT_TLS)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
void _mi_prim_thread_init_auto_done(void) {}
void _mi_prim_thread_done_auto_done(void) {}
void _mi_prim_thread_associate_default_theap(mi_theap_t* theap) {
MI_UNUSED(theap);
}
static bool mi_module_is_dll(PVOID mod) {
if (mod==NULL) return false;
PIMAGE_DOS_HEADER imageDosHeader = (PIMAGE_DOS_HEADER)mod;
PIMAGE_NT_HEADERS imageNtHeaders = (PIMAGE_NT_HEADERS)((unsigned char*)imageDosHeader + imageDosHeader->e_lfanew);
return ((imageNtHeaders->FileHeader.Characteristics & IMAGE_FILE_DLL) == IMAGE_FILE_DLL);
}
static bool mi_current_module_is_dll(void) {
HMODULE mod = NULL;
const BOOL ok = GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCSTR)mi_current_module_is_dll, &mod);
return (ok && mi_module_is_dll(mod));
}
// Hook into CRT initialization and finalization.
static void mi_cdecl mi_crt_done(void) {
// mi_debug_out(mi_current_module_is_dll() ? "crt dll done\n" : "crt exe done\n");
mi_win_main(NULL, DLL_PROCESS_DETACH, 0);
}
static int mi_cdecl mi_crt_init(void) {
// mi_debug_out(mi_current_module_is_dll() ? "crt dll init\n" : "crt exe init\n");
if (mi_current_module_is_dll()) {
// in a dll, atexit (crt_done) is called after tls process detach
atexit(&mi_crt_done);
}
return 0;
}
// We also hook into the Windows loader TLS initialization and finalization.
static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_ATTACH) {
//mi_debug_out("tls thread attach\n");
mi_win_main(module, reason, reserved);
}
else if (reason == DLL_PROCESS_ATTACH) {
// mi_debug_out(mi_module_is_dll(module) ? "tls dll process attach\n" : "tls exe process attach\n");
// tls process attach is always called before crt init
mi_win_main(module, reason, reserved);
}
}
static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_THREAD_DETACH) {
//mi_debug_out("tls thread detach\n");
mi_win_main(module, reason, reserved);
}
else if (reason == DLL_PROCESS_DETACH) {
// mi_debug_out(mi_module_is_dll(module) ? "tls dll process detach\n" : "tls exe process detach\n");
if (!mi_module_is_dll(module)) {
// in an exe, tls process detach is called after atexit (mi_crt_done)
mi_win_main(module, reason, reserved);
}
}
}
// Set up TLS callbacks in a statically linked library by using special data sections.
// See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
// We use 2 entries to ensure we call attach events before constructors
// are called, and detach events after destructors are called.
// We also set up a CRT initialization callback.
#if defined(__cplusplus)
extern "C" {
#endif
typedef int (mi_cdecl* mi_crt_callback_t)(void);
#if defined(_WIN64)
#pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma const_seg()
#pragma const_seg(".CRT$XLY")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma const_seg()
#pragma const_seg(".CRT$XIB")
extern const mi_crt_callback_t _mi_crt_callback_init[];
const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma const_seg()
#else
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach };
#pragma data_seg()
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach };
#pragma data_seg()
#pragma data_seg(".CRT$XIB")
mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init };
#pragma data_seg()
#endif
#if defined(__cplusplus)
}
#endif
/* ----------------------------------------------------------------
Here we use a combination of the raw DLL main and TLS sections for
both static and dynamic linkage. This is perhaps the most robust as
the raw DLL main ensure we can close mimalloc latest possible.
However, it can lead to link errors with on `_pRawDllMain` if other
libraries also try to hook into this (like Boost).
*/
#elif defined(MI_WIN_INIT_USE_RAW_DLLMAIN)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
void _mi_prim_thread_init_auto_done(void) {}
@@ -735,10 +872,6 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
// If linked into a DLL module, this raw entry is called before the CRT attach and
// after the CRT detach through the CRT _pRawDllMain pointer.
static BOOL NTAPI mi_dll_main_raw(PVOID module, DWORD reason, LPVOID reserved) {
//if (reason == DLL_PROCESS_ATTACH) { mi_debug_out("dll process attach\n"); }
//else if (reason == DLL_PROCESS_DETACH) { mi_debug_out("dll process detach\n"); }
//else if (reason == DLL_THREAD_ATTACH) { mi_debug_out("dll thread attach\n"); }
//else if (reason == DLL_THREAD_DETACH) { mi_debug_out("dll thread detach\n"); }
mi_win_main(module, reason, reserved);
return TRUE;
}
@@ -747,7 +880,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
#if defined(__cplusplus)
extern "C"
#endif
PVOID _pRawDllMain = &mi_dll_main_raw;
PVOID _pRawDllMain = (PVOID)&mi_dll_main_raw;
// We also hook into the Windows loader TLS initialization and finalization.
// If we are linked into an EXE module we rely on these as `mi_dll_main_raw`
@@ -762,7 +895,6 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
static void NTAPI mi_tls_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
if (!mi_module_is_dll(module)) {
// mi_debug_out(reason==DLL_PROCESS_ATTACH ? "exe process attach\n" : "exe thread attach\n");
mi_win_main(module, reason, reserved);
}
}
@@ -771,7 +903,6 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
static void NTAPI mi_tls_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
if (!mi_module_is_dll(module)) {
// mi_debug_out(reason==DLL_PROCESS_DETACH ? "exe process detach\n" : "exe thread detach\n");
mi_win_main(module, reason, reserved);
}
}
@@ -813,10 +944,10 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
}
#endif
/* ----------------------------------------
legacy options: DllMain, TLS, and FLS
*/
#elif defined(MI_SHARED_LIB)
/* --------------------------------------------------------------------
Legacy options: DllMain, TLS, and FLS
--------------------------------------------------------------------*/
#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && defined(MI_SHARED_LIB)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
// Windows DLL: easy to hook into process_init and thread_done
@@ -832,7 +963,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
MI_UNUSED(theap);
}
#elif !defined(MI_WIN_USE_FLS)
#elif defined(MI_WIN_INIT_USE_TLS_DLLMAIN)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {
@@ -889,7 +1020,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
MI_UNUSED(theap);
}
#else // deprecated: statically linked, use fiber api
#elif defined(MI_WIN_INIT_USE_FLS) // deprecated: statically linked, use fiber api
#if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)
// MSVC: use data section magic for static libraries
@@ -946,6 +1077,8 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
mi_assert_internal(mi_fls_key != (DWORD)(-1));
FlsSetValue(mi_fls_key, theap);
}
#else
#error "define windows process and thread auto initialization"
#endif
// ----------------------------------------------------
memory/mimalloc/src/random.c
+5 -2
View File
@@ -178,9 +178,12 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
#endif
uintptr_t x = _mi_os_random_weak(0);
for (size_t i = 0; i < 8; i++, x++) { // key is eight 32-bit words.
for (size_t i = 0; i < 32; i+=4, x++) {
x = _mi_random_shuffle(x);
((uint32_t*)key)[i] = (uint32_t)x;
key[i] = (uint8_t)(x);
key[i+1] = (uint8_t)(x>>8);
key[i+2] = (uint8_t)(x>>16);
key[i+3] = (uint8_t)(x>>24);
}
ctx->weak = true;
}
memory/mimalloc/src/stats.c
+14 -7
View File
@@ -543,12 +543,15 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) {
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept
{
mi_subproc_t* subproc = _mi_subproc_main();
mi_process_info_t pinfo;
_mi_memzero_var(pinfo);
pinfo.elapsed = _mi_clock_end(mi_process_start);
pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
{ const mi_subproc_t* subproc = _mi_subproc_main();
if (subproc!=NULL) {
pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
}
}
pinfo.current_rss = pinfo.current_commit;
pinfo.peak_rss = pinfo.peak_commit;
pinfo.utime = 0;
@@ -581,7 +584,7 @@ size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept {
return _mi_bin_size(bin);
}
static bool _mi_stats_get(mi_stats_t* stats_in, mi_stats_t* stats_out) mi_attr_noexcept {
static bool _mi_stats_get(const mi_stats_t* stats_in, mi_stats_t* stats_out) mi_attr_noexcept {
if (stats_out == NULL || stats_out->size != sizeof(mi_stats_t) || stats_out->version != MI_STAT_VERSION) return false;
if (stats_in == NULL || stats_in->size != stats_out->size) return false;
_mi_memcpy(stats_out, stats_in, stats_out->size);
@@ -589,7 +592,9 @@ static bool _mi_stats_get(mi_stats_t* stats_in, mi_stats_t* stats_out) mi_attr_n
}
bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
return _mi_stats_get(&_mi_subproc_from_id(subproc_id)->stats, stats);
const mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
if (subproc==NULL) return false;
return _mi_stats_get(&subproc->stats, stats);
}
bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept {
@@ -605,8 +610,10 @@ static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) {
bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept {
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
if (stats == NULL || stats->size != sizeof(mi_stats_t) || stats->version != MI_STAT_VERSION) return false;
if (subproc == NULL || stats == NULL || stats->size != sizeof(mi_stats_t) || stats->version != MI_STAT_VERSION) return false;
_mi_memzero(stats,stats->size);
stats->size = sizeof(mi_stats_t);
stats->version = MI_STAT_VERSION;
mi_subproc_visit_heaps(subproc, &mi_heap_aggregate_visitor, stats);
mi_stats_add_into(stats, &subproc->stats);
return true;
@@ -792,7 +799,7 @@ char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, cha
if (subproc==NULL) return NULL;
mi_stats_t_decl(stats);
if (!mi_subproc_stats_get(subproc_id,&stats)) return NULL;
return mi_stats_get_json_from(&subproc->stats, buf_size, buf);
return mi_stats_get_json_from(&stats, buf_size, buf);
}
char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept {
memory/mimalloc/src/threadlocal.c
+3
View File
@@ -71,6 +71,9 @@ static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) {
tls_old = NULL; // so we allocate fresh from mi_thread_locals_empty
count = 16; // start with 16 slots
}
else if (count_old >= MI_TLS_IDX_MASK - 1024) {
return NULL; // too large
}
else if (count_old >= 1024) {
count = count_old + 1024; // at some point increase linearly
}