diff options
Diffstat (limited to 'memory/build')
| -rw-r--r-- | memory/build/jemalloc_config.cpp | 164 | ||||
| -rw-r--r-- | memory/build/malloc_decls.h | 70 | ||||
| -rw-r--r-- | memory/build/moz.build | 55 | ||||
| -rw-r--r-- | memory/build/mozjemalloc_compat.c | 182 | ||||
| -rw-r--r-- | memory/build/mozmemory.h | 91 | ||||
| -rw-r--r-- | memory/build/mozmemory_wrap.c | 178 | ||||
| -rw-r--r-- | memory/build/mozmemory_wrap.h | 219 | ||||
| -rw-r--r-- | memory/build/replace_malloc.c | 546 | ||||
| -rw-r--r-- | memory/build/replace_malloc.h | 133 | ||||
| -rw-r--r-- | memory/build/replace_malloc_bridge.h | 202 |
10 files changed, 1840 insertions, 0 deletions
diff --git a/memory/build/jemalloc_config.cpp b/memory/build/jemalloc_config.cpp new file mode 100644 index 000000000..441fd8a2b --- /dev/null +++ b/memory/build/jemalloc_config.cpp @@ -0,0 +1,164 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_JEMALLOC4 + +#define MOZ_JEMALLOC_IMPL + +/* mozmemory_wrap.h needs to be included before MFBT headers */ +#include "mozmemory_wrap.h" +#include <mozilla/Assertions.h> +#include "mozilla/Types.h" + +#if defined(MOZ_SYSTEM_JEMALLOC) +#include MALLOC_H +#else +#define DLLEXPORT +#include "jemalloc/jemalloc.h" +#endif + +#ifdef XP_WIN +#include <windows.h> +#endif +#ifdef XP_DARWIN +#include <sys/mman.h> +#endif + +/* Override some jemalloc defaults */ +#ifdef DEBUG +#define MOZ_MALLOC_BUILD_OPTIONS ",junk:true" +#else +#define MOZ_MALLOC_BUILD_OPTIONS ",junk:free" +#endif + +#define MOZ_MALLOC_OPTIONS "narenas:1,tcache:false" +MFBT_DATA const char* je_(malloc_conf) = + MOZ_MALLOC_OPTIONS MOZ_MALLOC_BUILD_OPTIONS; + +#ifdef ANDROID +#include <android/log.h> + +static void +_je_malloc_message(void* cbopaque, const char* s) +{ + __android_log_print(ANDROID_LOG_INFO, "GeckoJemalloc", "%s", s); +} + +void (*je_(malloc_message))(void*, const char* s) = _je_malloc_message; +#endif + +/* Jemalloc supports hooks that are called on chunk + * allocate/deallocate/commit/decommit/purge/etc. + * + * We currently only hook commit, decommit and purge. We do this to tweak + * the way chunks are handled so that RSS stays lower than it normally + * would with the default jemalloc uses. + * This somewhat matches the behavior of mozjemalloc, except it doesn't + * rely on a double purge on mac, instead purging directly. (Yes, this + * means we can get rid of jemalloc_purge_freed_pages at some point) + * + * The default for jemalloc is to do the following: + * - commit, decommit: nothing + * - purge: MEM_RESET on Windows, MADV_FREE on Mac/BSD, MADV_DONTNEED on Linux + * + * The hooks we setup do the following: + * on Windows: + * - commit: MEM_COMMIT + * - decommit: MEM_DECOMMIT + * on Mac: + * - purge: mmap new anonymous memory on top of the chunk + * + * We only set the above hooks, others are left with the default. + */ +#if defined(XP_WIN) || defined(XP_DARWIN) +class JemallocInit { +public: + JemallocInit() + { + chunk_hooks_t hooks; + size_t hooks_len; + unsigned narenas; + size_t mib[3]; + size_t size; + + size = sizeof(narenas); + je_(mallctl)("arenas.narenas", &narenas, &size, nullptr, 0); + + size = sizeof(mib) / sizeof(mib[0]); + je_(mallctlnametomib)("arena.0.chunk_hooks", mib, &size); + + /* Set the hooks on all the existing arenas. */ + for (unsigned arena = 0; arena < narenas; arena++) { + mib[1] = arena; + hooks_len = sizeof(hooks); + je_(mallctlbymib)(mib, size, &hooks, &hooks_len, nullptr, 0); + +#ifdef XP_WIN + hooks.commit = CommitHook; + hooks.decommit = DecommitHook; +#endif +#ifdef XP_DARWIN + hooks.purge = PurgeHook; +#endif + + je_(mallctlbymib)(mib, size, nullptr, nullptr, &hooks, hooks_len); + } + } + +private: +#ifdef XP_WIN + static bool + CommitHook(void* chunk, size_t size, size_t offset, size_t length, + unsigned arena_ind) + { + void* addr = reinterpret_cast<void*>( + reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset)); + + if (!VirtualAlloc(addr, length, MEM_COMMIT, PAGE_READWRITE)) + return true; + + return false; + } + + static bool + DecommitHook(void* chunk, size_t size, size_t offset, size_t length, + unsigned arena_ind) + { + void* addr = reinterpret_cast<void*>( + reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset)); + + if (!VirtualFree(addr, length, MEM_DECOMMIT)) + MOZ_CRASH(); + + return false; + } +#endif + +#ifdef XP_DARWIN + static bool + PurgeHook(void* chunk, size_t size, size_t offset, size_t length, + unsigned arena_ind) + { + void* addr = reinterpret_cast<void*>( + reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset)); + + void* new_addr = mmap(addr, length, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0); + return (new_addr != addr); + } +#endif +}; + +/* For the static constructor from the class above */ +JemallocInit gJemallocInit; +#endif + +#else +#include <mozilla/Assertions.h> +#endif /* MOZ_JEMALLOC4 */ + +/* Provide an abort function for use in jemalloc code */ +extern "C" void moz_abort() { + MOZ_CRASH(); +} diff --git a/memory/build/malloc_decls.h b/memory/build/malloc_decls.h new file mode 100644 index 000000000..f3f9570ac --- /dev/null +++ b/memory/build/malloc_decls.h @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Helper header to declare all the supported malloc functions. + * MALLOC_DECL arguments are: + * - function name + * - return type + * - argument types + */ + +#ifndef malloc_decls_h +# define malloc_decls_h + +# include "jemalloc_types.h" + +#ifndef MALLOC_USABLE_SIZE_CONST_PTR +#define MALLOC_USABLE_SIZE_CONST_PTR const +#endif + +typedef MALLOC_USABLE_SIZE_CONST_PTR void * usable_ptr_t; + +# define MALLOC_FUNCS_MALLOC 1 +# define MALLOC_FUNCS_JEMALLOC 2 +# define MALLOC_FUNCS_INIT 4 +# define MALLOC_FUNCS_BRIDGE 8 +# define MALLOC_FUNCS_ALL (MALLOC_FUNCS_INIT | MALLOC_FUNCS_BRIDGE | \ + MALLOC_FUNCS_MALLOC | MALLOC_FUNCS_JEMALLOC) + +#endif /* malloc_decls_h */ + +#ifndef MALLOC_FUNCS +# define MALLOC_FUNCS (MALLOC_FUNCS_MALLOC | MALLOC_FUNCS_JEMALLOC) +#endif + +#ifdef MALLOC_DECL +# ifndef MALLOC_DECL_VOID +# define MALLOC_DECL_VOID(func, ...) MALLOC_DECL(func, void, __VA_ARGS__) +# endif + +# if MALLOC_FUNCS & MALLOC_FUNCS_INIT +MALLOC_DECL(init, void, const malloc_table_t *) +# endif +# if MALLOC_FUNCS & MALLOC_FUNCS_BRIDGE +MALLOC_DECL(get_bridge, struct ReplaceMallocBridge*, void) +# endif +# if MALLOC_FUNCS & MALLOC_FUNCS_MALLOC +MALLOC_DECL(malloc, void *, size_t) +MALLOC_DECL(posix_memalign, int, void **, size_t, size_t) +MALLOC_DECL(aligned_alloc, void *, size_t, size_t) +MALLOC_DECL(calloc, void *, size_t, size_t) +MALLOC_DECL(realloc, void *, void *, size_t) +MALLOC_DECL_VOID(free, void *) +MALLOC_DECL(memalign, void *, size_t, size_t) +MALLOC_DECL(valloc, void *, size_t) +MALLOC_DECL(malloc_usable_size, size_t, usable_ptr_t) +MALLOC_DECL(malloc_good_size, size_t, size_t) +# endif +# if MALLOC_FUNCS & MALLOC_FUNCS_JEMALLOC +MALLOC_DECL_VOID(jemalloc_stats, jemalloc_stats_t *) +MALLOC_DECL_VOID(jemalloc_purge_freed_pages, void) +MALLOC_DECL_VOID(jemalloc_free_dirty_pages, void) +# endif + +# undef MALLOC_DECL_VOID +#endif /* MALLOC_DECL */ + +#undef MALLOC_DECL +#undef MALLOC_FUNCS diff --git a/memory/build/moz.build b/memory/build/moz.build new file mode 100644 index 000000000..8d80561b1 --- /dev/null +++ b/memory/build/moz.build @@ -0,0 +1,55 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + 'mozmemory.h', + 'mozmemory_wrap.h', +] + +LIBRARY_DEFINES['MOZ_HAS_MOZGLUE'] = True +DEFINES['MOZ_MEMORY_IMPL'] = True + +if CONFIG['MOZ_REPLACE_MALLOC']: + EXPORTS += [ + 'malloc_decls.h', + 'replace_malloc.h', + 'replace_malloc_bridge.h', + ] + +SOURCES += [ + 'jemalloc_config.cpp', + 'mozmemory_wrap.c', +] + +if CONFIG['MOZ_JEMALLOC4']: + SOURCES += [ + 'mozjemalloc_compat.c', + ] + LOCAL_INCLUDES += ['!../jemalloc/src/include'] + if CONFIG['_MSC_VER']: + LOCAL_INCLUDES += ['/memory/jemalloc/src/include/msvc_compat'] + if not CONFIG['HAVE_INTTYPES_H']: + LOCAL_INCLUDES += ['/memory/jemalloc/src/include/msvc_compat/C99'] + +if CONFIG['MOZ_REPLACE_MALLOC']: + SOURCES += [ + 'replace_malloc.c', + ] + +Library('memory') + +if CONFIG['MOZ_GLUE_IN_PROGRAM']: + SDK_LIBRARY = True + DIST_INSTALL = True + +# Keep jemalloc separated when mozglue is statically linked +if CONFIG['MOZ_MEMORY'] and (CONFIG['OS_TARGET'] in ('WINNT', 'Darwin', 'Android') or + CONFIG['MOZ_SYSTEM_JEMALLOC']): + FINAL_LIBRARY = 'mozglue' + +if CONFIG['MOZ_REPLACE_MALLOC'] and CONFIG['OS_TARGET'] == 'Darwin': + # The zone allocator for OSX needs some jemalloc internal functions + LOCAL_INCLUDES += ['/memory/jemalloc/src/include'] diff --git a/memory/build/mozjemalloc_compat.c b/memory/build/mozjemalloc_compat.c new file mode 100644 index 000000000..6591892c1 --- /dev/null +++ b/memory/build/mozjemalloc_compat.c @@ -0,0 +1,182 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZ_JEMALLOC4 +# error Should only compile this file when building with jemalloc 3 +#endif + +#define MOZ_JEMALLOC_IMPL + +#include "mozmemory_wrap.h" +#include "jemalloc_types.h" +#include "mozilla/Types.h" + +#include <stdbool.h> + +#if defined(MOZ_SYSTEM_JEMALLOC) +# include MALLOC_H +#else +# include "jemalloc/jemalloc.h" +#endif + +/* + * CTL_* macros are from memory/jemalloc/src/src/stats.c with changes: + * - drop `t' argument to avoid redundancy in calculating type size + * - require `i' argument for arena number explicitly + */ + +#define CTL_GET(n, v) do { \ + size_t sz = sizeof(v); \ + je_(mallctl)(n, &v, &sz, NULL, 0); \ +} while (0) + +#define CTL_I_GET(n, v, i) do { \ + size_t mib[6]; \ + size_t miblen = sizeof(mib) / sizeof(mib[0]); \ + size_t sz = sizeof(v); \ + je_(mallctlnametomib)(n, mib, &miblen); \ + mib[2] = i; \ + je_(mallctlbymib)(mib, miblen, &v, &sz, NULL, 0); \ +} while (0) + +#define CTL_IJ_GET(n, v, i, j) do { \ + size_t mib[6]; \ + size_t miblen = sizeof(mib) / sizeof(mib[0]); \ + size_t sz = sizeof(v); \ + je_(mallctlnametomib)(n, mib, &miblen); \ + mib[2] = i; \ + mib[4] = j; \ + je_(mallctlbymib)(mib, miblen, &v, &sz, NULL, 0); \ +} while (0) + +/* + * VARIABLE_ARRAY is copied from + * memory/jemalloc/src/include/jemalloc/internal/jemalloc_internal.h.in + */ +#if __STDC_VERSION__ < 199901L +# ifdef _MSC_VER +# include <malloc.h> +# define alloca _alloca +# else +# ifdef HAVE_ALLOCA_H +# include <alloca.h> +# else +# include <stdlib.h> +# endif +# endif +# define VARIABLE_ARRAY(type, name, count) \ + type *name = alloca(sizeof(type) * (count)) +#else +# define VARIABLE_ARRAY(type, name, count) type name[(count)] +#endif + +MOZ_MEMORY_API size_t +malloc_good_size_impl(size_t size) +{ + /* je_nallocx crashes when given a size of 0. As + * malloc_usable_size(malloc(0)) and malloc_usable_size(malloc(1)) + * return the same value, use a size of 1. */ + if (size == 0) + size = 1; + return je_(nallocx)(size, 0); +} + +static void +compute_bin_unused_and_bookkeeping(jemalloc_stats_t *stats, unsigned int narenas) +{ + size_t bin_unused = 0; + + uint32_t nregs; // number of regions per run in the j-th bin + size_t reg_size; // size of regions served by the j-th bin + size_t curruns; // number of runs belonging to a bin + size_t curregs; // number of allocated regions in a bin + + unsigned int nbins; // number of bins per arena + unsigned int i, j; + + size_t stats_metadata; + size_t stats_ametadata = 0; // total internal allocations in all arenas + + // narenas also counts uninitialized arenas, and initialized arenas + // are not guaranteed to be adjacent + VARIABLE_ARRAY(bool, initialized, narenas); + size_t isz = sizeof(initialized) / sizeof(initialized[0]); + + je_(mallctl)("arenas.initialized", initialized, &isz, NULL, 0); + CTL_GET("arenas.nbins", nbins); + + for (j = 0; j < nbins; j++) { + CTL_I_GET("arenas.bin.0.nregs", nregs, j); + CTL_I_GET("arenas.bin.0.size", reg_size, j); + + for (i = 0; i < narenas; i++) { + if (!initialized[i]) { + continue; + } + + CTL_IJ_GET("stats.arenas.0.bins.0.curruns", curruns, i, j); + CTL_IJ_GET("stats.arenas.0.bins.0.curregs", curregs, i, j); + + bin_unused += (nregs * curruns - curregs) * reg_size; + } + } + + CTL_GET("stats.metadata", stats_metadata); + + /* get the summation for all arenas, i == narenas */ + CTL_I_GET("stats.arenas.0.metadata.allocated", stats_ametadata, narenas); + + stats->bookkeeping = stats_metadata - stats_ametadata; + stats->bin_unused = bin_unused; +} + +MOZ_JEMALLOC_API void +jemalloc_stats_impl(jemalloc_stats_t *stats) +{ + unsigned narenas; + size_t active, allocated, mapped, page, pdirty; + size_t lg_chunk; + + // Refresh jemalloc's stats by updating its epoch, see ctl_refresh in + // src/ctl.c + uint64_t epoch = 0; + size_t esz = sizeof(epoch); + je_(mallctl)("epoch", &epoch, &esz, &epoch, esz); + + CTL_GET("arenas.narenas", narenas); + CTL_GET("arenas.page", page); + CTL_GET("stats.active", active); + CTL_GET("stats.allocated", allocated); + CTL_GET("stats.mapped", mapped); + CTL_GET("opt.lg_chunk", lg_chunk); + + /* get the summation for all arenas, i == narenas */ + CTL_I_GET("stats.arenas.0.pdirty", pdirty, narenas); + + stats->chunksize = (size_t) 1 << lg_chunk; + stats->mapped = mapped; + stats->allocated = allocated; + stats->waste = active - allocated; + stats->page_cache = pdirty * page; + compute_bin_unused_and_bookkeeping(stats, narenas); + stats->waste -= stats->bin_unused; +} + +MOZ_JEMALLOC_API void +jemalloc_purge_freed_pages_impl() +{ +} + +MOZ_JEMALLOC_API void +jemalloc_free_dirty_pages_impl() +{ + unsigned narenas; + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(mib[0]); + + CTL_GET("arenas.narenas", narenas); + je_(mallctlnametomib)("arena.0.purge", mib, &miblen); + mib[1] = narenas; + je_(mallctlbymib)(mib, miblen, NULL, NULL, NULL, 0); +} diff --git a/memory/build/mozmemory.h b/memory/build/mozmemory.h new file mode 100644 index 000000000..84007fffb --- /dev/null +++ b/memory/build/mozmemory.h @@ -0,0 +1,91 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozmemory_h +#define mozmemory_h + +/* + * This header is meant to be used when the following functions are + * necessary: + * - malloc_good_size (used to be called je_malloc_usable_in_advance) + * - jemalloc_stats + * - jemalloc_purge_freed_pages + * - jemalloc_free_dirty_pages + */ + +#ifndef MOZ_MEMORY +# error Should not include mozmemory.h when MOZ_MEMORY is not set +#endif + +#include "mozmemory_wrap.h" +#include "mozilla/Attributes.h" +#include "mozilla/Types.h" +#include "jemalloc_types.h" + +MOZ_BEGIN_EXTERN_C + +/* + * On OSX, malloc/malloc.h contains the declaration for malloc_good_size, + * which will call back in jemalloc, through the zone allocator so just use it. + */ +#ifdef XP_DARWIN +# include <malloc/malloc.h> +#else +MOZ_MEMORY_API size_t malloc_good_size_impl(size_t size); + +/* Note: the MOZ_GLUE_IN_PROGRAM ifdef below is there to avoid -Werror turning + * the protective if into errors. MOZ_GLUE_IN_PROGRAM is what triggers MFBT_API + * to use weak imports. */ + +static inline size_t _malloc_good_size(size_t size) { +# if defined(MOZ_GLUE_IN_PROGRAM) && !defined(IMPL_MFBT) + if (!malloc_good_size) + return size; +# endif + return malloc_good_size_impl(size); +} + +# define malloc_good_size _malloc_good_size +#endif + +MOZ_JEMALLOC_API void jemalloc_stats(jemalloc_stats_t *stats); + +/* + * On some operating systems (Mac), we use madvise(MADV_FREE) to hand pages + * back to the operating system. On Mac, the operating system doesn't take + * this memory back immediately; instead, the OS takes it back only when the + * machine is running out of physical memory. + * + * This is great from the standpoint of efficiency, but it makes measuring our + * actual RSS difficult, because pages which we've MADV_FREE'd shouldn't count + * against our RSS. + * + * This function explicitly purges any MADV_FREE'd pages from physical memory, + * causing our reported RSS match the amount of memory we're actually using. + * + * Note that this call is expensive in two ways. First, it may be slow to + * execute, because it may make a number of slow syscalls to free memory. This + * function holds the big jemalloc locks, so basically all threads are blocked + * while this function runs. + * + * This function is also expensive in that the next time we go to access a page + * which we've just explicitly decommitted, the operating system has to attach + * to it a physical page! If we hadn't run this function, the OS would have + * less work to do. + * + * If MALLOC_DOUBLE_PURGE is not defined, this function does nothing. + */ +MOZ_JEMALLOC_API void jemalloc_purge_freed_pages(); + +/* + * Free all unused dirty pages in all arenas. Calling this function will slow + * down subsequent allocations so it is recommended to use it only when + * memory needs to be reclaimed at all costs (see bug 805855). This function + * provides functionality similar to mallctl("arenas.purge") in jemalloc 3. + */ +MOZ_JEMALLOC_API void jemalloc_free_dirty_pages(); + +MOZ_END_EXTERN_C + +#endif /* mozmemory_h */ diff --git a/memory/build/mozmemory_wrap.c b/memory/build/mozmemory_wrap.c new file mode 100644 index 000000000..dba3ace56 --- /dev/null +++ b/memory/build/mozmemory_wrap.c @@ -0,0 +1,178 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <string.h> +#include "mozmemory_wrap.h" +#include "mozilla/Types.h" + +/* Declare malloc implementation functions with the right return and + * argument types. */ +#define MALLOC_DECL(name, return_type, ...) \ + MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__); +#include "malloc_decls.h" + +#ifdef MOZ_WRAP_NEW_DELETE +/* operator new(unsigned int) */ +MOZ_MEMORY_API void * +mozmem_malloc_impl(_Znwj)(unsigned int size) +{ + return malloc_impl(size); +} +/* operator new[](unsigned int) */ +MOZ_MEMORY_API void * +mozmem_malloc_impl(_Znaj)(unsigned int size) +{ + return malloc_impl(size); +} +/* operator delete(void*) */ +MOZ_MEMORY_API void +mozmem_malloc_impl(_ZdlPv)(void *ptr) +{ + free_impl(ptr); +} +/* operator delete[](void*) */ +MOZ_MEMORY_API void +mozmem_malloc_impl(_ZdaPv)(void *ptr) +{ + free_impl(ptr); +} +/*operator new(unsigned int, std::nothrow_t const&)*/ +MOZ_MEMORY_API void * +mozmem_malloc_impl(_ZnwjRKSt9nothrow_t)(unsigned int size) +{ + return malloc_impl(size); +} +/*operator new[](unsigned int, std::nothrow_t const&)*/ +MOZ_MEMORY_API void * +mozmem_malloc_impl(_ZnajRKSt9nothrow_t)(unsigned int size) +{ + return malloc_impl(size); +} +/* operator delete(void*, std::nothrow_t const&) */ +MOZ_MEMORY_API void +mozmem_malloc_impl(_ZdlPvRKSt9nothrow_t)(void *ptr) +{ + free_impl(ptr); +} +/* operator delete[](void*, std::nothrow_t const&) */ +MOZ_MEMORY_API void +mozmem_malloc_impl(_ZdaPvRKSt9nothrow_t)(void *ptr) +{ + free_impl(ptr); +} +#endif + +/* strndup and strdup may be defined as macros in string.h, which would + * clash with the definitions below. */ +#undef strndup +#undef strdup + +#ifndef XP_DARWIN +MOZ_MEMORY_API char * +strndup_impl(const char *src, size_t len) +{ + char* dst = (char*) malloc_impl(len + 1); + if (dst) { + strncpy(dst, src, len); + dst[len] = '\0'; + } + return dst; +} + +MOZ_MEMORY_API char * +strdup_impl(const char *src) +{ + size_t len = strlen(src); + return strndup_impl(src, len); +} +#endif /* XP_DARWIN */ + +#ifdef ANDROID +#include <stdarg.h> +#include <stdio.h> + +MOZ_MEMORY_API int +vasprintf_impl(char **str, const char *fmt, va_list ap) +{ + char* ptr, *_ptr; + int ret; + + if (str == NULL || fmt == NULL) { + return -1; + } + + ptr = (char*)malloc_impl(128); + if (ptr == NULL) { + *str = NULL; + return -1; + } + + ret = vsnprintf(ptr, 128, fmt, ap); + if (ret < 0) { + free_impl(ptr); + *str = NULL; + return -1; + } + + _ptr = realloc_impl(ptr, ret + 1); + if (_ptr == NULL) { + free_impl(ptr); + *str = NULL; + return -1; + } + + *str = _ptr; + + return ret; +} + +MOZ_MEMORY_API int +asprintf_impl(char **str, const char *fmt, ...) +{ + int ret; + va_list ap; + va_start(ap, fmt); + + ret = vasprintf_impl(str, fmt, ap); + + va_end(ap); + + return ret; +} +#endif + +#ifdef XP_WIN +/* + * There's a fun allocator mismatch in (at least) the VS 2010 CRT + * (see the giant comment in $(topsrcdir)/mozglue/build/Makefile.in) + * that gets redirected here to avoid a crash on shutdown. + */ +void +dumb_free_thunk(void *ptr) +{ + return; /* shutdown leaks that we don't care about */ +} + +#include <wchar.h> + +/* + * We also need to provide our own impl of wcsdup so that we don't ask + * the CRT for memory from its heap (which will then be unfreeable). + */ +wchar_t * +wcsdup_impl(const wchar_t *src) +{ + size_t len = wcslen(src); + wchar_t *dst = (wchar_t*) malloc_impl((len + 1) * sizeof(wchar_t)); + if (dst) + wcsncpy(dst, src, len + 1); + return dst; +} + +void * +_aligned_malloc(size_t size, size_t alignment) +{ + return memalign_impl(alignment, size); +} +#endif /* XP_WIN */ diff --git a/memory/build/mozmemory_wrap.h b/memory/build/mozmemory_wrap.h new file mode 100644 index 000000000..066d57782 --- /dev/null +++ b/memory/build/mozmemory_wrap.h @@ -0,0 +1,219 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozmemory_wrap_h +#define mozmemory_wrap_h + +/* + * This header contains #defines which tweak the names of various memory + * allocation functions. + * + * There are several types of functions related to memory allocation + * that are meant to be used publicly by the Gecko codebase: + * + * - malloc implementation functions: + * - malloc + * - posix_memalign + * - aligned_alloc + * - calloc + * - realloc + * - free + * - memalign + * - valloc + * - malloc_usable_size + * - malloc_good_size + * Some of these functions are specific to some systems, but for + * convenience, they are treated as being cross-platform, and available + * as such. + * + * - duplication functions: + * - strndup + * - strdup + * - wcsdup (Windows only) + * + * - jemalloc specific functions: + * - jemalloc_stats + * - jemalloc_purge_freed_pages + * - jemalloc_free_dirty_pages + * (these functions are native to mozjemalloc, and have compatibility + * implementations for jemalloc3) + * + * These functions are all exported as part of libmozglue (see + * $(topsrcdir)/mozglue/build/Makefile.in), with a few implementation + * peculiarities: + * + * - On Windows, the malloc implementation functions are all prefixed with + * "je_", the duplication functions are prefixed with "wrap_", and jemalloc + * specific functions are left unprefixed. All these functions are however + * aliased when exporting them, such that the resulting mozglue.dll exports + * them unprefixed (see $(topsrcdir)/mozglue/build/mozglue.def.in). The + * prefixed malloc implementation and duplication functions are not + * exported. + * + * - On MacOSX, the system libc has a zone allocator, which allows us to + * hook custom malloc implementation functions without exporting them. + * The malloc implementation functions are all prefixed with "je_" and used + * this way from the custom zone allocator. They are not exported. + * Duplication functions are not included, since they will call the custom + * zone allocator anyways. Jemalloc-specific functions are also left + * unprefixed. + * + * - On Android and Gonk, all functions are left unprefixed. Additionally, + * C++ allocation functions (operator new/delete) are also exported and + * unprefixed. + * + * - On other systems (mostly Linux), all functions are left unprefixed. + * + * Only Android and Gonk add C++ allocation functions. + * + * Proper exporting of the various functions is done with the MOZ_MEMORY_API + * and MOZ_JEMALLOC_API macros. MOZ_MEMORY_API is meant to be used for malloc + * implementation and duplication functions, while MOZ_JEMALLOC_API is + * dedicated to jemalloc specific functions. + * + * + * All these functions are meant to be called with no prefix from Gecko code. + * In most cases, this is because that's how they are available at runtime. + * However, on Android, this relies on faulty.lib (the custom dynamic linker) + * resolving mozglue symbols before libc symbols, which is guaranteed by the + * way faulty.lib works (it respects the DT_NEEDED order, and libc always + * appears after mozglue ; which we double check when building anyways) + * + * + * Within libmozglue (when MOZ_MEMORY_IMPL is defined), all the functions + * should be suffixed with "_impl" both for declarations and use. + * That is, the implementation declaration for e.g. strdup would look like: + * char* strdup_impl(const char *) + * That implementation would call malloc by using "malloc_impl". + * + * While mozjemalloc uses these "_impl" suffixed helpers, jemalloc3, being + * third-party code, doesn't, but instead has an elaborate way to mangle + * individual functions. See under "Run jemalloc configure script" in + * $(topsrcdir)/configure.in. + * + * + * When building with replace-malloc support, the above still holds, but + * the malloc implementation and jemalloc specific functions are the + * replace-malloc functions from replace_malloc.c. + * + * The actual jemalloc/mozjemalloc implementation is prefixed with "je_". + * + * Thus, when MOZ_REPLACE_MALLOC is defined, the "_impl" suffixed macros + * expand to "je_" prefixed function when building mozjemalloc or + * jemalloc3/mozjemalloc_compat, where MOZ_JEMALLOC_IMPL is defined. + * + * In other cases, the "_impl" suffixed macros follow the original scheme, + * except on Windows and MacOSX, where they would expand to "je_" prefixed + * functions. Instead, they are left unmodified (malloc_impl expands to + * malloc_impl). + */ + +#ifndef MOZ_MEMORY +# error Should only include mozmemory_wrap.h when MOZ_MEMORY is set. +#endif + +#if defined(MOZ_JEMALLOC_IMPL) && !defined(MOZ_MEMORY_IMPL) +# define MOZ_MEMORY_IMPL +#endif +#if defined(MOZ_MEMORY_IMPL) && !defined(IMPL_MFBT) +# ifdef MFBT_API /* mozilla/Types.h was already included */ +# error mozmemory_wrap.h has to be included before mozilla/Types.h when MOZ_MEMORY_IMPL is set and IMPL_MFBT is not. +# endif +# define IMPL_MFBT +#endif + +#include "mozilla/Types.h" + +#if !defined(MOZ_SYSTEM_JEMALLOC) +# ifdef MOZ_MEMORY_IMPL +# if defined(MOZ_JEMALLOC_IMPL) && defined(MOZ_REPLACE_MALLOC) && !defined(MOZ_REPLACE_JEMALLOC) +# define mozmem_malloc_impl(a) je_ ## a +# define mozmem_jemalloc_impl(a) je_ ## a +# else +# define MOZ_JEMALLOC_API MFBT_API +# ifdef MOZ_REPLACE_JEMALLOC +# define MOZ_MEMORY_API MFBT_API +# define mozmem_malloc_impl(a) replace_ ## a +# define mozmem_jemalloc_impl(a) replace_ ## a +# elif (defined(XP_WIN) || defined(XP_DARWIN)) +# if defined(MOZ_REPLACE_MALLOC) +# define mozmem_malloc_impl(a) a ## _impl +# else +# define mozmem_malloc_impl(a) je_ ## a +# endif +# else +# define MOZ_MEMORY_API MFBT_API +# if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK) +# define MOZ_WRAP_NEW_DELETE +# endif +# endif +# endif +# ifdef XP_WIN +# define mozmem_dup_impl(a) wrap_ ## a +# endif +# endif + +/* All other jemalloc3 functions are prefixed with "je_", except when + * building against an unprefixed system jemalloc library */ +# define je_(a) je_ ## a +#else /* defined(MOZ_SYSTEM_JEMALLOC) */ +# define je_(a) a +#endif + +#if !defined(MOZ_MEMORY_IMPL) || defined(MOZ_SYSTEM_JEMALLOC) +# define MOZ_MEMORY_API MFBT_API +# define MOZ_JEMALLOC_API MFBT_API +#endif + +#ifndef MOZ_MEMORY_API +# define MOZ_MEMORY_API +#endif +#ifndef MOZ_JEMALLOC_API +# define MOZ_JEMALLOC_API +#endif + +#ifndef mozmem_malloc_impl +# define mozmem_malloc_impl(a) a +#endif +#ifndef mozmem_dup_impl +# define mozmem_dup_impl(a) a +#endif +#ifndef mozmem_jemalloc_impl +# define mozmem_jemalloc_impl(a) a +#endif + +/* Malloc implementation functions */ +#define malloc_impl mozmem_malloc_impl(malloc) +#define posix_memalign_impl mozmem_malloc_impl(posix_memalign) +#define aligned_alloc_impl mozmem_malloc_impl(aligned_alloc) +#define calloc_impl mozmem_malloc_impl(calloc) +#define realloc_impl mozmem_malloc_impl(realloc) +#define free_impl mozmem_malloc_impl(free) +#define memalign_impl mozmem_malloc_impl(memalign) +#define valloc_impl mozmem_malloc_impl(valloc) +#define malloc_usable_size_impl mozmem_malloc_impl(malloc_usable_size) +#define malloc_good_size_impl mozmem_malloc_impl(malloc_good_size) + +/* Duplication functions */ +#define strndup_impl mozmem_dup_impl(strndup) +#define strdup_impl mozmem_dup_impl(strdup) +#ifdef XP_WIN +# define wcsdup_impl mozmem_dup_impl(wcsdup) +#endif + +/* String functions */ +#ifdef ANDROID +/* Bug 801571 and Bug 879668, libstagefright uses vasprintf, causing malloc()/ + * free() to be mismatched between bionic and mozglue implementation. + */ +#define vasprintf_impl mozmem_dup_impl(vasprintf) +#define asprintf_impl mozmem_dup_impl(asprintf) +#endif + +/* Jemalloc specific function */ +#define jemalloc_stats_impl mozmem_jemalloc_impl(jemalloc_stats) +#define jemalloc_purge_freed_pages_impl mozmem_jemalloc_impl(jemalloc_purge_freed_pages) +#define jemalloc_free_dirty_pages_impl mozmem_jemalloc_impl(jemalloc_free_dirty_pages) + +#endif /* mozmemory_wrap_h */ diff --git a/memory/build/replace_malloc.c b/memory/build/replace_malloc.c new file mode 100644 index 000000000..88dfde33c --- /dev/null +++ b/memory/build/replace_malloc.c @@ -0,0 +1,546 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZ_MEMORY +# error Should not compile this file when MOZ_MEMORY is not set +#endif + +#ifndef MOZ_REPLACE_MALLOC +# error Should not compile this file when replace-malloc is disabled +#endif + +#ifdef MOZ_SYSTEM_JEMALLOC +# error Should not compile this file when we want to use native jemalloc +#endif + +#include "mozmemory_wrap.h" + +/* Declare all je_* functions */ +#define MALLOC_DECL(name, return_type, ...) \ + return_type je_ ## name(__VA_ARGS__); +#include "malloc_decls.h" + +#include "mozilla/Likely.h" + +/* + * Windows doesn't come with weak imports as they are possible with + * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform, + * the replacement functions are defined as variable pointers to the + * function resolved with GetProcAddress() instead of weak definitions + * of functions. On Android, the same needs to happen as well, because + * the Android linker doesn't handle weak linking with non LD_PRELOADed + * libraries, but LD_PRELOADing is not very convenient on Android, with + * the zygote. + */ +#ifdef XP_DARWIN +# define MOZ_REPLACE_WEAK __attribute__((weak_import)) +#elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID) +# define MOZ_NO_REPLACE_FUNC_DECL +#elif defined(__GNUC__) +# define MOZ_REPLACE_WEAK __attribute__((weak)) +#endif + +#include "replace_malloc.h" + +#define MALLOC_DECL(name, return_type, ...) \ + je_ ## name, + +static const malloc_table_t malloc_table = { +#include "malloc_decls.h" +}; + +#ifdef MOZ_NO_REPLACE_FUNC_DECL +# define MALLOC_DECL(name, return_type, ...) \ + typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \ + replace_ ## name ## _impl_t *replace_ ## name = NULL; +# define MALLOC_FUNCS MALLOC_FUNCS_ALL +# include "malloc_decls.h" + +# ifdef XP_WIN +# include <windows.h> +static void +replace_malloc_init_funcs() +{ + char replace_malloc_lib[1024]; + if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib, + sizeof(replace_malloc_lib)) > 0) { + HMODULE handle = LoadLibraryA(replace_malloc_lib); + if (handle) { +#define MALLOC_DECL(name, ...) \ + replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name); + +# define MALLOC_FUNCS MALLOC_FUNCS_ALL +#include "malloc_decls.h" + } + } +} +# elif defined(MOZ_WIDGET_ANDROID) +# include <dlfcn.h> +# include <stdlib.h> +static void +replace_malloc_init_funcs() +{ + const char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB"); + if (replace_malloc_lib && *replace_malloc_lib) { + void *handle = dlopen(replace_malloc_lib, RTLD_LAZY); + if (handle) { +#define MALLOC_DECL(name, ...) \ + replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name); + +# define MALLOC_FUNCS MALLOC_FUNCS_ALL +#include "malloc_decls.h" + } + } +} +# else +# error No implementation for replace_malloc_init_funcs() +# endif + +#endif /* MOZ_NO_REPLACE_FUNC_DECL */ + +/* + * Below is the malloc implementation overriding jemalloc and calling the + * replacement functions if they exist. + */ + +/* + * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc + * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h + */ +#define MALLOC_DECL(name, return_type, ...) \ + MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__); +#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC +#include "malloc_decls.h" + +#define MALLOC_DECL(name, return_type, ...) \ + MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__); +#define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC +#include "malloc_decls.h" + +static int replace_malloc_initialized = 0; +static void +init() +{ +#ifdef MOZ_NO_REPLACE_FUNC_DECL + replace_malloc_init_funcs(); +#endif + // Set this *before* calling replace_init, otherwise if replace_init calls + // malloc() we'll get an infinite loop. + replace_malloc_initialized = 1; + if (replace_init) + replace_init(&malloc_table); +} + +MFBT_API struct ReplaceMallocBridge* +get_bridge(void) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_get_bridge)) + return NULL; + return replace_get_bridge(); +} + +void* +malloc_impl(size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_malloc)) + return je_malloc(size); + return replace_malloc(size); +} + +int +posix_memalign_impl(void **memptr, size_t alignment, size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_posix_memalign)) + return je_posix_memalign(memptr, alignment, size); + return replace_posix_memalign(memptr, alignment, size); +} + +void* +aligned_alloc_impl(size_t alignment, size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_aligned_alloc)) + return je_aligned_alloc(alignment, size); + return replace_aligned_alloc(alignment, size); +} + +void* +calloc_impl(size_t num, size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_calloc)) + return je_calloc(num, size); + return replace_calloc(num, size); +} + +void* +realloc_impl(void *ptr, size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_realloc)) + return je_realloc(ptr, size); + return replace_realloc(ptr, size); +} + +void +free_impl(void *ptr) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_free)) + je_free(ptr); + else + replace_free(ptr); +} + +void* +memalign_impl(size_t alignment, size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_memalign)) + return je_memalign(alignment, size); + return replace_memalign(alignment, size); +} + +void* +valloc_impl(size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_valloc)) + return je_valloc(size); + return replace_valloc(size); +} + +size_t +malloc_usable_size_impl(usable_ptr_t ptr) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_malloc_usable_size)) + return je_malloc_usable_size(ptr); + return replace_malloc_usable_size(ptr); +} + +size_t +malloc_good_size_impl(size_t size) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_malloc_good_size)) + return je_malloc_good_size(size); + return replace_malloc_good_size(size); +} + +void +jemalloc_stats_impl(jemalloc_stats_t *stats) +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_jemalloc_stats)) + je_jemalloc_stats(stats); + else + replace_jemalloc_stats(stats); +} + +void +jemalloc_purge_freed_pages_impl() +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages)) + je_jemalloc_purge_freed_pages(); + else + replace_jemalloc_purge_freed_pages(); +} + +void +jemalloc_free_dirty_pages_impl() +{ + if (MOZ_UNLIKELY(!replace_malloc_initialized)) + init(); + if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages)) + je_jemalloc_free_dirty_pages(); + else + replace_jemalloc_free_dirty_pages(); +} + +/* The following comment and definitions are from jemalloc.c: */ +#if defined(__GLIBC__) && !defined(__UCLIBC__) + +/* + * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible + * to inconsistently reference libc's malloc(3)-compatible functions + * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). + * + * These definitions interpose hooks in glibc. The functions are actually + * passed an extra argument for the caller return address, which will be + * ignored. + */ + +typedef void (* __free_hook_type)(void *ptr); +typedef void *(* __malloc_hook_type)(size_t size); +typedef void *(* __realloc_hook_type)(void *ptr, size_t size); +typedef void *(* __memalign_hook_type)(size_t alignment, size_t size); + +MOZ_MEMORY_API __free_hook_type __free_hook = free_impl; +MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl; +MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl; +MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl; + +#endif + +/* + * The following is a OSX zone allocator implementation. + * /!\ WARNING. It assumes the underlying malloc implementation's + * malloc_usable_size returns 0 when the given pointer is not owned by + * the allocator. Sadly, OSX does call zone_size with pointers not + * owned by the allocator. + */ + +#ifdef XP_DARWIN +#include <stdlib.h> +#include <malloc/malloc.h> +#include "mozilla/Assertions.h" + +static size_t +zone_size(malloc_zone_t *zone, void *ptr) +{ + return malloc_usable_size_impl(ptr); +} + +static void * +zone_malloc(malloc_zone_t *zone, size_t size) +{ + return malloc_impl(size); +} + +static void * +zone_calloc(malloc_zone_t *zone, size_t num, size_t size) +{ + return calloc_impl(num, size); +} + +static void * +zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) +{ + if (malloc_usable_size_impl(ptr)) + return realloc_impl(ptr, size); + return realloc(ptr, size); +} + +static void +zone_free(malloc_zone_t *zone, void *ptr) +{ + if (malloc_usable_size_impl(ptr)) { + free_impl(ptr); + return; + } + free(ptr); +} + +static void +zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) +{ + size_t current_size = malloc_usable_size_impl(ptr); + if (current_size) { + MOZ_ASSERT(current_size == size); + free_impl(ptr); + return; + } + free(ptr); +} + +static void * +zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) +{ + void *ptr; + if (posix_memalign_impl(&ptr, alignment, size) == 0) + return ptr; + return NULL; +} + +static void * +zone_valloc(malloc_zone_t *zone, size_t size) +{ + return valloc_impl(size); +} + +static void * +zone_destroy(malloc_zone_t *zone) +{ + /* This function should never be called. */ + MOZ_CRASH(); +} + +static size_t +zone_good_size(malloc_zone_t *zone, size_t size) +{ + return malloc_good_size_impl(size); +} + +#ifdef MOZ_JEMALLOC + +#include "jemalloc/internal/jemalloc_internal.h" + +static void +zone_force_lock(malloc_zone_t *zone) +{ + /* /!\ This calls into jemalloc. It works because we're linked in the + * same library. Stolen from jemalloc's zone.c. */ + if (isthreaded) + jemalloc_prefork(); +} + +static void +zone_force_unlock(malloc_zone_t *zone) +{ + /* /!\ This calls into jemalloc. It works because we're linked in the + * same library. Stolen from jemalloc's zone.c. */ + if (isthreaded) + jemalloc_postfork_parent(); +} + +#else + +#define JEMALLOC_ZONE_VERSION 6 + +/* Empty implementations are needed, because fork() calls zone->force_(un)lock + * unconditionally. */ +static void +zone_force_lock(malloc_zone_t *zone) +{ +} + +static void +zone_force_unlock(malloc_zone_t *zone) +{ +} + +#endif + +static malloc_zone_t zone; +static struct malloc_introspection_t zone_introspect; + +static malloc_zone_t *get_default_zone() +{ + malloc_zone_t **zones = NULL; + unsigned int num_zones = 0; + + /* + * On OSX 10.12, malloc_default_zone returns a special zone that is not + * present in the list of registered zones. That zone uses a "lite zone" + * if one is present (apparently enabled when malloc stack logging is + * enabled), or the first registered zone otherwise. In practice this + * means unless malloc stack logging is enabled, the first registered + * zone is the default. + * So get the list of zones to get the first one, instead of relying on + * malloc_default_zone. + */ + if (KERN_SUCCESS != malloc_get_all_zones(0, NULL, (vm_address_t**) &zones, + &num_zones)) { + /* Reset the value in case the failure happened after it was set. */ + num_zones = 0; + } + if (num_zones) { + return zones[0]; + } + return malloc_default_zone(); +} + + +__attribute__((constructor)) void +register_zone(void) +{ + malloc_zone_t *default_zone = get_default_zone(); + + zone.size = (void *)zone_size; + zone.malloc = (void *)zone_malloc; + zone.calloc = (void *)zone_calloc; + zone.valloc = (void *)zone_valloc; + zone.free = (void *)zone_free; + zone.realloc = (void *)zone_realloc; + zone.destroy = (void *)zone_destroy; + zone.zone_name = "replace_malloc_zone"; + zone.batch_malloc = NULL; + zone.batch_free = NULL; + zone.introspect = &zone_introspect; + zone.version = JEMALLOC_ZONE_VERSION; + zone.memalign = zone_memalign; + zone.free_definite_size = zone_free_definite_size; +#if (JEMALLOC_ZONE_VERSION >= 8) + zone.pressure_relief = NULL; +#endif + zone_introspect.enumerator = NULL; + zone_introspect.good_size = (void *)zone_good_size; + zone_introspect.check = NULL; + zone_introspect.print = NULL; + zone_introspect.log = NULL; + zone_introspect.force_lock = (void *)zone_force_lock; + zone_introspect.force_unlock = (void *)zone_force_unlock; + zone_introspect.statistics = NULL; + zone_introspect.zone_locked = NULL; +#if (JEMALLOC_ZONE_VERSION >= 7) + zone_introspect.enable_discharge_checking = NULL; + zone_introspect.disable_discharge_checking = NULL; + zone_introspect.discharge = NULL; +#ifdef __BLOCKS__ + zone_introspect.enumerate_discharged_pointers = NULL; +#else + zone_introspect.enumerate_unavailable_without_blocks = NULL; +#endif +#endif + + /* + * The default purgeable zone is created lazily by OSX's libc. It uses + * the default zone when it is created for "small" allocations + * (< 15 KiB), but assumes the default zone is a scalable_zone. This + * obviously fails when the default zone is the jemalloc zone, so + * malloc_default_purgeable_zone is called beforehand so that the + * default purgeable zone is created when the default zone is still + * a scalable_zone. + */ + malloc_zone_t *purgeable_zone = malloc_default_purgeable_zone(); + + /* Register the custom zone. At this point it won't be the default. */ + malloc_zone_register(&zone); + + do { + /* + * Unregister and reregister the default zone. On OSX >= 10.6, + * unregistering takes the last registered zone and places it at the + * location of the specified zone. Unregistering the default zone thus + * makes the last registered one the default. On OSX < 10.6, + * unregistering shifts all registered zones. The first registered zone + * then becomes the default. + */ + malloc_zone_unregister(default_zone); + malloc_zone_register(default_zone); + /* + * On OSX 10.6, having the default purgeable zone appear before the default + * zone makes some things crash because it thinks it owns the default + * zone allocated pointers. We thus unregister/re-register it in order to + * ensure it's always after the default zone. On OSX < 10.6, as + * unregistering shifts registered zones, this simply removes the purgeable + * zone from the list and adds it back at the end, after the default zone. + * On OSX >= 10.6, unregistering replaces the purgeable zone with the last + * registered zone above, i.e the default zone. Registering it again then + * puts it at the end, obviously after the default zone. + */ + malloc_zone_unregister(purgeable_zone); + malloc_zone_register(purgeable_zone); + default_zone = get_default_zone(); + } while (default_zone != &zone); +} +#endif diff --git a/memory/build/replace_malloc.h b/memory/build/replace_malloc.h new file mode 100644 index 000000000..a61744f60 --- /dev/null +++ b/memory/build/replace_malloc.h @@ -0,0 +1,133 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef replace_malloc_h +#define replace_malloc_h + +/* + * The replace_malloc facility allows an external library to replace or + * supplement the jemalloc implementation. + * + * The external library may be hooked by setting one of the following + * environment variables to the library path: + * - LD_PRELOAD on Linux, + * - DYLD_INSERT_LIBRARIES on OSX, + * - MOZ_REPLACE_MALLOC_LIB on Windows and Android. + * + * An initialization function is called before any malloc replacement + * function, and has the following declaration: + * + * void replace_init(const malloc_table_t *) + * + * The const malloc_table_t pointer given to that function is a table + * containing pointers to the original jemalloc implementation, so that + * replacement functions can call them back if they need to. The pointer + * itself can safely be kept around (no need to copy the table itself). + * + * The functions to be implemented in the external library are of the form: + * + * void *replace_malloc(size_t size) + * { + * // Fiddle with the size if necessary. + * // orig->malloc doesn't have to be called if the external library + * // provides its own allocator, but in this case it will have to + * // implement all functions. + * void *ptr = orig->malloc(size); + * // Do whatever you want with the ptr. + * return ptr; + * } + * + * where "orig" is the pointer obtained from replace_init. + * + * See malloc_decls.h for a list of functions that can be replaced this + * way. The implementations are all in the form: + * return_type replace_name(arguments [,...]) + * + * They don't all need to be provided. + * + * Building a replace-malloc library is like rocket science. It can end up + * with things blowing up, especially when trying to use complex types, and + * even more especially when these types come from XPCOM or other parts of the + * Mozilla codebase. + * It is recommended to add the following to a replace-malloc implementation's + * moz.build: + * DISABLE_STL_WRAPPING = True # Avoid STL wrapping + * + * If your replace-malloc implementation lives under memory/replace, these + * are taken care of by memory/replace/defs.mk. + */ + +#ifdef replace_malloc_bridge_h +#error Do not include replace_malloc_bridge.h before replace_malloc.h. \ + In fact, you only need the latter. +#endif + +#define REPLACE_MALLOC_IMPL + +#include "replace_malloc_bridge.h" + +/* Implementing a replace-malloc library is incompatible with using mozalloc. */ +#define MOZ_NO_MOZALLOC 1 + +#include "mozilla/Types.h" + +MOZ_BEGIN_EXTERN_C + +/* MOZ_NO_REPLACE_FUNC_DECL and MOZ_REPLACE_WEAK are only defined in + * replace_malloc.c. Normally including this header will add function + * definitions. */ +#ifndef MOZ_NO_REPLACE_FUNC_DECL + +# ifndef MOZ_REPLACE_WEAK +# define MOZ_REPLACE_WEAK +# endif + +# define MALLOC_DECL(name, return_type, ...) \ + MOZ_EXPORT return_type replace_ ## name(__VA_ARGS__) MOZ_REPLACE_WEAK; + +# define MALLOC_FUNCS MALLOC_FUNCS_ALL +# include "malloc_decls.h" + +#endif /* MOZ_NO_REPLACE_FUNC_DECL */ + +/* + * posix_memalign, aligned_alloc, memalign and valloc all implement some + * kind of aligned memory allocation. For convenience, replace_posix_memalign, + * replace_aligned_alloc and replace_valloc can be automatically derived from + * memalign when MOZ_REPLACE_ONLY_MEMALIGN is defined before including this + * header. PAGE_SIZE also needs to be defined to the appropriate expression. + */ +#ifdef MOZ_REPLACE_ONLY_MEMALIGN +#include <errno.h> + +int replace_posix_memalign(void **ptr, size_t alignment, size_t size) +{ + if (size == 0) { + *ptr = NULL; + return 0; + } + /* alignment must be a power of two and a multiple of sizeof(void *) */ + if (((alignment - 1) & alignment) != 0 || (alignment % sizeof(void *))) + return EINVAL; + *ptr = replace_memalign(alignment, size); + return *ptr ? 0 : ENOMEM; +} + +void *replace_aligned_alloc(size_t alignment, size_t size) +{ + /* size should be a multiple of alignment */ + if (size % alignment) + return NULL; + return replace_memalign(alignment, size); +} + +void *replace_valloc(size_t size) +{ + return replace_memalign(PAGE_SIZE, size); +} +#endif + +MOZ_END_EXTERN_C + +#endif /* replace_malloc_h */ diff --git a/memory/build/replace_malloc_bridge.h b/memory/build/replace_malloc_bridge.h new file mode 100644 index 000000000..301b165eb --- /dev/null +++ b/memory/build/replace_malloc_bridge.h @@ -0,0 +1,202 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef replace_malloc_bridge_h +#define replace_malloc_bridge_h + +/* + * The replace-malloc bridge allows bidirectional method calls between + * a program and the replace-malloc library that has been loaded for it. + * In Firefox, this is used to allow method calls between code in libxul + * and code in the replace-malloc library, without libxul needing to link + * against that library or vice-versa. + * + * Subsystems can add methods for their own need. Replace-malloc libraries + * can decide to implement those methods or not. + * + * Replace-malloc libraries can provide such a bridge by implementing + * a ReplaceMallocBridge-derived class, and a replace_get_bridge function + * returning an instance of that class. The default methods in + * ReplaceMallocBridge are expected to return values that callers would + * understand as "the bridge doesn't implement this method", so that a + * replace-malloc library doesn't have to implement all methods. + * + * The ReplaceMallocBridge class contains definitions for methods for + * all replace-malloc libraries. Each library picks the methods it wants + * to reply to in its ReplaceMallocBridge-derived class instance. + * All methods of ReplaceMallocBridge must be virtual. Similarly, + * anything passed as an argument to those methods must be plain data, or + * an instance of a class with only virtual methods. + * + * Binary compatibility is expected to be maintained, such that a newer + * Firefox can be used with an old replace-malloc library, or an old + * Firefox can be used with a newer replace-malloc library. As such, only + * new virtual methods should be added to ReplaceMallocBridge, and + * each change should have a corresponding bump of the mVersion value. + * At the same time, each virtual method should have a corresponding + * wrapper calling the virtual method on the instance from + * ReplaceMallocBridge::Get(), giving it the version the virtual method + * was added. + * + * Parts that are not relevant to the replace-malloc library end of the + * bridge are hidden when REPLACE_MALLOC_IMPL is not defined, which is + * the case when including replace_malloc.h. + */ + +struct ReplaceMallocBridge; + +#include "mozilla/Types.h" + +MOZ_BEGIN_EXTERN_C + +#ifndef REPLACE_MALLOC_IMPL +/* Returns the replace-malloc bridge if there is one to be returned. */ +MFBT_API ReplaceMallocBridge* get_bridge(); +#endif + +/* Table of malloc functions. + * e.g. void* (*malloc)(size_t), etc. + */ +#define MALLOC_DECL(name, return_type, ...) \ + typedef return_type(name ## _impl_t)(__VA_ARGS__); + +#include "malloc_decls.h" + +#define MALLOC_DECL(name, return_type, ...) \ + name ## _impl_t * name; + +typedef struct { +#include "malloc_decls.h" +} malloc_table_t; + + +/* Table of malloc hook functions. + * Those functions are called with the arguments and results of malloc + * functions after they are called. + * e.g. void* (*malloc_hook)(void*, size_t), etc. + * They can either return the result they're given, or alter it before + * returning it. + * The hooks corresponding to functions, like free(void*), that return no + * value, don't take an extra argument. + * The table must at least contain a pointer for malloc_hook and free_hook + * functions. They will be used as fallback if no pointer is given for + * other allocation functions, like calloc_hook. + */ +#define MALLOC_DECL(name, return_type, ...) \ + return_type (*name ## _hook)(return_type, __VA_ARGS__); +#define MALLOC_DECL_VOID(name, ...) \ + void (*name ## _hook)(__VA_ARGS__); + +typedef struct { +#include "malloc_decls.h" + /* Like free_hook, but called before realloc_hook. free_hook is called + * instead of not given. */ + void (*realloc_hook_before)(void* aPtr); +} malloc_hook_table_t; + +MOZ_END_EXTERN_C + +#ifdef __cplusplus + +namespace mozilla { +namespace dmd { +struct DMDFuncs; +} // namespace dmd + +/* Callbacks to register debug file handles for Poison IO interpose. + * See Mozilla(|Un)RegisterDebugHandle in xpcom/build/PoisonIOInterposer.h */ +struct DebugFdRegistry +{ + virtual void RegisterHandle(intptr_t aFd); + + virtual void UnRegisterHandle(intptr_t aFd); +}; + +} // namespace mozilla + +struct ReplaceMallocBridge +{ + ReplaceMallocBridge() : mVersion(3) {} + + /* This method was added in version 1 of the bridge. */ + virtual mozilla::dmd::DMDFuncs* GetDMDFuncs() { return nullptr; } + + /* Send a DebugFdRegistry instance to the replace-malloc library so that + * it can register/unregister file descriptors whenever needed. The + * instance is valid until the process dies. + * This method was added in version 2 of the bridge. */ + virtual void InitDebugFd(mozilla::DebugFdRegistry&) {} + + /* Register a list of malloc functions and hook functions to the + * replace-malloc library so that it can choose to dispatch to them + * when needed. The details of what is dispatched when is left to the + * replace-malloc library. + * Passing a nullptr for either table will unregister a previously + * registered table under the same name. + * Returns nullptr if registration failed. + * If registration succeeded, a table of "pure" malloc functions is + * returned. Those "pure" malloc functions won't call hooks. + * /!\ Do not rely on registration/unregistration to be instantaneous. + * Functions from a previously registered table may still be called for + * a brief time after RegisterHook returns. + * This method was added in version 3 of the bridge. */ + virtual const malloc_table_t* + RegisterHook(const char* aName, const malloc_table_t* aTable, + const malloc_hook_table_t* aHookTable) { return nullptr; } + +#ifndef REPLACE_MALLOC_IMPL + /* Returns the replace-malloc bridge if its version is at least the + * requested one. */ + static ReplaceMallocBridge* Get(int aMinimumVersion) { + static ReplaceMallocBridge* sSingleton = get_bridge(); + return (sSingleton && sSingleton->mVersion >= aMinimumVersion) + ? sSingleton : nullptr; + } +#endif + +protected: + const int mVersion; +}; + +#ifndef REPLACE_MALLOC_IMPL +/* Class containing wrappers for calls to ReplaceMallocBridge methods. + * Those wrappers need to be static methods in a class because compilers + * complain about unused static global functions, and linkers complain + * about multiple definitions of non-static global functions. + * Using a separate class from ReplaceMallocBridge allows the function + * names to be identical. */ +struct ReplaceMalloc +{ + /* Don't call this method from performance critical code. Use + * mozilla::dmd::DMDFuncs::Get() instead, it has less overhead. */ + static mozilla::dmd::DMDFuncs* GetDMDFuncs() + { + auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 1); + return singleton ? singleton->GetDMDFuncs() : nullptr; + } + + static void InitDebugFd(mozilla::DebugFdRegistry& aRegistry) + { + auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 2); + if (singleton) { + singleton->InitDebugFd(aRegistry); + } + } + + static const malloc_table_t* + RegisterHook(const char* aName, const malloc_table_t* aTable, + const malloc_hook_table_t* aHookTable) + { + auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 3); + return singleton ? singleton->RegisterHook(aName, aTable, aHookTable) + : nullptr; + } +}; +#endif + +#endif /* __cplusplus */ + +#endif /* replace_malloc_bridge_h */ |