summaryrefslogtreecommitdiffstats
path: root/memory/build
diff options
context:
space:
mode:
Diffstat (limited to 'memory/build')
-rw-r--r--memory/build/jemalloc_config.cpp164
-rw-r--r--memory/build/malloc_decls.h70
-rw-r--r--memory/build/moz.build55
-rw-r--r--memory/build/mozjemalloc_compat.c182
-rw-r--r--memory/build/mozmemory.h91
-rw-r--r--memory/build/mozmemory_wrap.c178
-rw-r--r--memory/build/mozmemory_wrap.h219
-rw-r--r--memory/build/replace_malloc.c546
-rw-r--r--memory/build/replace_malloc.h133
-rw-r--r--memory/build/replace_malloc_bridge.h202
10 files changed, 1840 insertions, 0 deletions
diff --git a/memory/build/jemalloc_config.cpp b/memory/build/jemalloc_config.cpp
new file mode 100644
index 000000000..441fd8a2b
--- /dev/null
+++ b/memory/build/jemalloc_config.cpp
@@ -0,0 +1,164 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef MOZ_JEMALLOC4
+
+#define MOZ_JEMALLOC_IMPL
+
+/* mozmemory_wrap.h needs to be included before MFBT headers */
+#include "mozmemory_wrap.h"
+#include <mozilla/Assertions.h>
+#include "mozilla/Types.h"
+
+#if defined(MOZ_SYSTEM_JEMALLOC)
+#include MALLOC_H
+#else
+#define DLLEXPORT
+#include "jemalloc/jemalloc.h"
+#endif
+
+#ifdef XP_WIN
+#include <windows.h>
+#endif
+#ifdef XP_DARWIN
+#include <sys/mman.h>
+#endif
+
+/* Override some jemalloc defaults */
+#ifdef DEBUG
+#define MOZ_MALLOC_BUILD_OPTIONS ",junk:true"
+#else
+#define MOZ_MALLOC_BUILD_OPTIONS ",junk:free"
+#endif
+
+#define MOZ_MALLOC_OPTIONS "narenas:1,tcache:false"
+MFBT_DATA const char* je_(malloc_conf) =
+ MOZ_MALLOC_OPTIONS MOZ_MALLOC_BUILD_OPTIONS;
+
+#ifdef ANDROID
+#include <android/log.h>
+
+static void
+_je_malloc_message(void* cbopaque, const char* s)
+{
+ __android_log_print(ANDROID_LOG_INFO, "GeckoJemalloc", "%s", s);
+}
+
+void (*je_(malloc_message))(void*, const char* s) = _je_malloc_message;
+#endif
+
+/* Jemalloc supports hooks that are called on chunk
+ * allocate/deallocate/commit/decommit/purge/etc.
+ *
+ * We currently only hook commit, decommit and purge. We do this to tweak
+ * the way chunks are handled so that RSS stays lower than it normally
+ * would with the default jemalloc uses.
+ * This somewhat matches the behavior of mozjemalloc, except it doesn't
+ * rely on a double purge on mac, instead purging directly. (Yes, this
+ * means we can get rid of jemalloc_purge_freed_pages at some point)
+ *
+ * The default for jemalloc is to do the following:
+ * - commit, decommit: nothing
+ * - purge: MEM_RESET on Windows, MADV_FREE on Mac/BSD, MADV_DONTNEED on Linux
+ *
+ * The hooks we setup do the following:
+ * on Windows:
+ * - commit: MEM_COMMIT
+ * - decommit: MEM_DECOMMIT
+ * on Mac:
+ * - purge: mmap new anonymous memory on top of the chunk
+ *
+ * We only set the above hooks, others are left with the default.
+ */
+#if defined(XP_WIN) || defined(XP_DARWIN)
+class JemallocInit {
+public:
+ JemallocInit()
+ {
+ chunk_hooks_t hooks;
+ size_t hooks_len;
+ unsigned narenas;
+ size_t mib[3];
+ size_t size;
+
+ size = sizeof(narenas);
+ je_(mallctl)("arenas.narenas", &narenas, &size, nullptr, 0);
+
+ size = sizeof(mib) / sizeof(mib[0]);
+ je_(mallctlnametomib)("arena.0.chunk_hooks", mib, &size);
+
+ /* Set the hooks on all the existing arenas. */
+ for (unsigned arena = 0; arena < narenas; arena++) {
+ mib[1] = arena;
+ hooks_len = sizeof(hooks);
+ je_(mallctlbymib)(mib, size, &hooks, &hooks_len, nullptr, 0);
+
+#ifdef XP_WIN
+ hooks.commit = CommitHook;
+ hooks.decommit = DecommitHook;
+#endif
+#ifdef XP_DARWIN
+ hooks.purge = PurgeHook;
+#endif
+
+ je_(mallctlbymib)(mib, size, nullptr, nullptr, &hooks, hooks_len);
+ }
+ }
+
+private:
+#ifdef XP_WIN
+ static bool
+ CommitHook(void* chunk, size_t size, size_t offset, size_t length,
+ unsigned arena_ind)
+ {
+ void* addr = reinterpret_cast<void*>(
+ reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset));
+
+ if (!VirtualAlloc(addr, length, MEM_COMMIT, PAGE_READWRITE))
+ return true;
+
+ return false;
+ }
+
+ static bool
+ DecommitHook(void* chunk, size_t size, size_t offset, size_t length,
+ unsigned arena_ind)
+ {
+ void* addr = reinterpret_cast<void*>(
+ reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset));
+
+ if (!VirtualFree(addr, length, MEM_DECOMMIT))
+ MOZ_CRASH();
+
+ return false;
+ }
+#endif
+
+#ifdef XP_DARWIN
+ static bool
+ PurgeHook(void* chunk, size_t size, size_t offset, size_t length,
+ unsigned arena_ind)
+ {
+ void* addr = reinterpret_cast<void*>(
+ reinterpret_cast<uintptr_t>(chunk) + static_cast<uintptr_t>(offset));
+
+ void* new_addr = mmap(addr, length, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0);
+ return (new_addr != addr);
+ }
+#endif
+};
+
+/* For the static constructor from the class above */
+JemallocInit gJemallocInit;
+#endif
+
+#else
+#include <mozilla/Assertions.h>
+#endif /* MOZ_JEMALLOC4 */
+
+/* Provide an abort function for use in jemalloc code */
+extern "C" void moz_abort() {
+ MOZ_CRASH();
+}
diff --git a/memory/build/malloc_decls.h b/memory/build/malloc_decls.h
new file mode 100644
index 000000000..f3f9570ac
--- /dev/null
+++ b/memory/build/malloc_decls.h
@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Helper header to declare all the supported malloc functions.
+ * MALLOC_DECL arguments are:
+ * - function name
+ * - return type
+ * - argument types
+ */
+
+#ifndef malloc_decls_h
+# define malloc_decls_h
+
+# include "jemalloc_types.h"
+
+#ifndef MALLOC_USABLE_SIZE_CONST_PTR
+#define MALLOC_USABLE_SIZE_CONST_PTR const
+#endif
+
+typedef MALLOC_USABLE_SIZE_CONST_PTR void * usable_ptr_t;
+
+# define MALLOC_FUNCS_MALLOC 1
+# define MALLOC_FUNCS_JEMALLOC 2
+# define MALLOC_FUNCS_INIT 4
+# define MALLOC_FUNCS_BRIDGE 8
+# define MALLOC_FUNCS_ALL (MALLOC_FUNCS_INIT | MALLOC_FUNCS_BRIDGE | \
+ MALLOC_FUNCS_MALLOC | MALLOC_FUNCS_JEMALLOC)
+
+#endif /* malloc_decls_h */
+
+#ifndef MALLOC_FUNCS
+# define MALLOC_FUNCS (MALLOC_FUNCS_MALLOC | MALLOC_FUNCS_JEMALLOC)
+#endif
+
+#ifdef MALLOC_DECL
+# ifndef MALLOC_DECL_VOID
+# define MALLOC_DECL_VOID(func, ...) MALLOC_DECL(func, void, __VA_ARGS__)
+# endif
+
+# if MALLOC_FUNCS & MALLOC_FUNCS_INIT
+MALLOC_DECL(init, void, const malloc_table_t *)
+# endif
+# if MALLOC_FUNCS & MALLOC_FUNCS_BRIDGE
+MALLOC_DECL(get_bridge, struct ReplaceMallocBridge*, void)
+# endif
+# if MALLOC_FUNCS & MALLOC_FUNCS_MALLOC
+MALLOC_DECL(malloc, void *, size_t)
+MALLOC_DECL(posix_memalign, int, void **, size_t, size_t)
+MALLOC_DECL(aligned_alloc, void *, size_t, size_t)
+MALLOC_DECL(calloc, void *, size_t, size_t)
+MALLOC_DECL(realloc, void *, void *, size_t)
+MALLOC_DECL_VOID(free, void *)
+MALLOC_DECL(memalign, void *, size_t, size_t)
+MALLOC_DECL(valloc, void *, size_t)
+MALLOC_DECL(malloc_usable_size, size_t, usable_ptr_t)
+MALLOC_DECL(malloc_good_size, size_t, size_t)
+# endif
+# if MALLOC_FUNCS & MALLOC_FUNCS_JEMALLOC
+MALLOC_DECL_VOID(jemalloc_stats, jemalloc_stats_t *)
+MALLOC_DECL_VOID(jemalloc_purge_freed_pages, void)
+MALLOC_DECL_VOID(jemalloc_free_dirty_pages, void)
+# endif
+
+# undef MALLOC_DECL_VOID
+#endif /* MALLOC_DECL */
+
+#undef MALLOC_DECL
+#undef MALLOC_FUNCS
diff --git a/memory/build/moz.build b/memory/build/moz.build
new file mode 100644
index 000000000..8d80561b1
--- /dev/null
+++ b/memory/build/moz.build
@@ -0,0 +1,55 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXPORTS += [
+ 'mozmemory.h',
+ 'mozmemory_wrap.h',
+]
+
+LIBRARY_DEFINES['MOZ_HAS_MOZGLUE'] = True
+DEFINES['MOZ_MEMORY_IMPL'] = True
+
+if CONFIG['MOZ_REPLACE_MALLOC']:
+ EXPORTS += [
+ 'malloc_decls.h',
+ 'replace_malloc.h',
+ 'replace_malloc_bridge.h',
+ ]
+
+SOURCES += [
+ 'jemalloc_config.cpp',
+ 'mozmemory_wrap.c',
+]
+
+if CONFIG['MOZ_JEMALLOC4']:
+ SOURCES += [
+ 'mozjemalloc_compat.c',
+ ]
+ LOCAL_INCLUDES += ['!../jemalloc/src/include']
+ if CONFIG['_MSC_VER']:
+ LOCAL_INCLUDES += ['/memory/jemalloc/src/include/msvc_compat']
+ if not CONFIG['HAVE_INTTYPES_H']:
+ LOCAL_INCLUDES += ['/memory/jemalloc/src/include/msvc_compat/C99']
+
+if CONFIG['MOZ_REPLACE_MALLOC']:
+ SOURCES += [
+ 'replace_malloc.c',
+ ]
+
+Library('memory')
+
+if CONFIG['MOZ_GLUE_IN_PROGRAM']:
+ SDK_LIBRARY = True
+ DIST_INSTALL = True
+
+# Keep jemalloc separated when mozglue is statically linked
+if CONFIG['MOZ_MEMORY'] and (CONFIG['OS_TARGET'] in ('WINNT', 'Darwin', 'Android') or
+ CONFIG['MOZ_SYSTEM_JEMALLOC']):
+ FINAL_LIBRARY = 'mozglue'
+
+if CONFIG['MOZ_REPLACE_MALLOC'] and CONFIG['OS_TARGET'] == 'Darwin':
+ # The zone allocator for OSX needs some jemalloc internal functions
+ LOCAL_INCLUDES += ['/memory/jemalloc/src/include']
diff --git a/memory/build/mozjemalloc_compat.c b/memory/build/mozjemalloc_compat.c
new file mode 100644
index 000000000..6591892c1
--- /dev/null
+++ b/memory/build/mozjemalloc_compat.c
@@ -0,0 +1,182 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZ_JEMALLOC4
+# error Should only compile this file when building with jemalloc 3
+#endif
+
+#define MOZ_JEMALLOC_IMPL
+
+#include "mozmemory_wrap.h"
+#include "jemalloc_types.h"
+#include "mozilla/Types.h"
+
+#include <stdbool.h>
+
+#if defined(MOZ_SYSTEM_JEMALLOC)
+# include MALLOC_H
+#else
+# include "jemalloc/jemalloc.h"
+#endif
+
+/*
+ * CTL_* macros are from memory/jemalloc/src/src/stats.c with changes:
+ * - drop `t' argument to avoid redundancy in calculating type size
+ * - require `i' argument for arena number explicitly
+ */
+
+#define CTL_GET(n, v) do { \
+ size_t sz = sizeof(v); \
+ je_(mallctl)(n, &v, &sz, NULL, 0); \
+} while (0)
+
+#define CTL_I_GET(n, v, i) do { \
+ size_t mib[6]; \
+ size_t miblen = sizeof(mib) / sizeof(mib[0]); \
+ size_t sz = sizeof(v); \
+ je_(mallctlnametomib)(n, mib, &miblen); \
+ mib[2] = i; \
+ je_(mallctlbymib)(mib, miblen, &v, &sz, NULL, 0); \
+} while (0)
+
+#define CTL_IJ_GET(n, v, i, j) do { \
+ size_t mib[6]; \
+ size_t miblen = sizeof(mib) / sizeof(mib[0]); \
+ size_t sz = sizeof(v); \
+ je_(mallctlnametomib)(n, mib, &miblen); \
+ mib[2] = i; \
+ mib[4] = j; \
+ je_(mallctlbymib)(mib, miblen, &v, &sz, NULL, 0); \
+} while (0)
+
+/*
+ * VARIABLE_ARRAY is copied from
+ * memory/jemalloc/src/include/jemalloc/internal/jemalloc_internal.h.in
+ */
+#if __STDC_VERSION__ < 199901L
+# ifdef _MSC_VER
+# include <malloc.h>
+# define alloca _alloca
+# else
+# ifdef HAVE_ALLOCA_H
+# include <alloca.h>
+# else
+# include <stdlib.h>
+# endif
+# endif
+# define VARIABLE_ARRAY(type, name, count) \
+ type *name = alloca(sizeof(type) * (count))
+#else
+# define VARIABLE_ARRAY(type, name, count) type name[(count)]
+#endif
+
+MOZ_MEMORY_API size_t
+malloc_good_size_impl(size_t size)
+{
+ /* je_nallocx crashes when given a size of 0. As
+ * malloc_usable_size(malloc(0)) and malloc_usable_size(malloc(1))
+ * return the same value, use a size of 1. */
+ if (size == 0)
+ size = 1;
+ return je_(nallocx)(size, 0);
+}
+
+static void
+compute_bin_unused_and_bookkeeping(jemalloc_stats_t *stats, unsigned int narenas)
+{
+ size_t bin_unused = 0;
+
+ uint32_t nregs; // number of regions per run in the j-th bin
+ size_t reg_size; // size of regions served by the j-th bin
+ size_t curruns; // number of runs belonging to a bin
+ size_t curregs; // number of allocated regions in a bin
+
+ unsigned int nbins; // number of bins per arena
+ unsigned int i, j;
+
+ size_t stats_metadata;
+ size_t stats_ametadata = 0; // total internal allocations in all arenas
+
+ // narenas also counts uninitialized arenas, and initialized arenas
+ // are not guaranteed to be adjacent
+ VARIABLE_ARRAY(bool, initialized, narenas);
+ size_t isz = sizeof(initialized) / sizeof(initialized[0]);
+
+ je_(mallctl)("arenas.initialized", initialized, &isz, NULL, 0);
+ CTL_GET("arenas.nbins", nbins);
+
+ for (j = 0; j < nbins; j++) {
+ CTL_I_GET("arenas.bin.0.nregs", nregs, j);
+ CTL_I_GET("arenas.bin.0.size", reg_size, j);
+
+ for (i = 0; i < narenas; i++) {
+ if (!initialized[i]) {
+ continue;
+ }
+
+ CTL_IJ_GET("stats.arenas.0.bins.0.curruns", curruns, i, j);
+ CTL_IJ_GET("stats.arenas.0.bins.0.curregs", curregs, i, j);
+
+ bin_unused += (nregs * curruns - curregs) * reg_size;
+ }
+ }
+
+ CTL_GET("stats.metadata", stats_metadata);
+
+ /* get the summation for all arenas, i == narenas */
+ CTL_I_GET("stats.arenas.0.metadata.allocated", stats_ametadata, narenas);
+
+ stats->bookkeeping = stats_metadata - stats_ametadata;
+ stats->bin_unused = bin_unused;
+}
+
+MOZ_JEMALLOC_API void
+jemalloc_stats_impl(jemalloc_stats_t *stats)
+{
+ unsigned narenas;
+ size_t active, allocated, mapped, page, pdirty;
+ size_t lg_chunk;
+
+ // Refresh jemalloc's stats by updating its epoch, see ctl_refresh in
+ // src/ctl.c
+ uint64_t epoch = 0;
+ size_t esz = sizeof(epoch);
+ je_(mallctl)("epoch", &epoch, &esz, &epoch, esz);
+
+ CTL_GET("arenas.narenas", narenas);
+ CTL_GET("arenas.page", page);
+ CTL_GET("stats.active", active);
+ CTL_GET("stats.allocated", allocated);
+ CTL_GET("stats.mapped", mapped);
+ CTL_GET("opt.lg_chunk", lg_chunk);
+
+ /* get the summation for all arenas, i == narenas */
+ CTL_I_GET("stats.arenas.0.pdirty", pdirty, narenas);
+
+ stats->chunksize = (size_t) 1 << lg_chunk;
+ stats->mapped = mapped;
+ stats->allocated = allocated;
+ stats->waste = active - allocated;
+ stats->page_cache = pdirty * page;
+ compute_bin_unused_and_bookkeeping(stats, narenas);
+ stats->waste -= stats->bin_unused;
+}
+
+MOZ_JEMALLOC_API void
+jemalloc_purge_freed_pages_impl()
+{
+}
+
+MOZ_JEMALLOC_API void
+jemalloc_free_dirty_pages_impl()
+{
+ unsigned narenas;
+ size_t mib[3];
+ size_t miblen = sizeof(mib) / sizeof(mib[0]);
+
+ CTL_GET("arenas.narenas", narenas);
+ je_(mallctlnametomib)("arena.0.purge", mib, &miblen);
+ mib[1] = narenas;
+ je_(mallctlbymib)(mib, miblen, NULL, NULL, NULL, 0);
+}
diff --git a/memory/build/mozmemory.h b/memory/build/mozmemory.h
new file mode 100644
index 000000000..84007fffb
--- /dev/null
+++ b/memory/build/mozmemory.h
@@ -0,0 +1,91 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozmemory_h
+#define mozmemory_h
+
+/*
+ * This header is meant to be used when the following functions are
+ * necessary:
+ * - malloc_good_size (used to be called je_malloc_usable_in_advance)
+ * - jemalloc_stats
+ * - jemalloc_purge_freed_pages
+ * - jemalloc_free_dirty_pages
+ */
+
+#ifndef MOZ_MEMORY
+# error Should not include mozmemory.h when MOZ_MEMORY is not set
+#endif
+
+#include "mozmemory_wrap.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Types.h"
+#include "jemalloc_types.h"
+
+MOZ_BEGIN_EXTERN_C
+
+/*
+ * On OSX, malloc/malloc.h contains the declaration for malloc_good_size,
+ * which will call back in jemalloc, through the zone allocator so just use it.
+ */
+#ifdef XP_DARWIN
+# include <malloc/malloc.h>
+#else
+MOZ_MEMORY_API size_t malloc_good_size_impl(size_t size);
+
+/* Note: the MOZ_GLUE_IN_PROGRAM ifdef below is there to avoid -Werror turning
+ * the protective if into errors. MOZ_GLUE_IN_PROGRAM is what triggers MFBT_API
+ * to use weak imports. */
+
+static inline size_t _malloc_good_size(size_t size) {
+# if defined(MOZ_GLUE_IN_PROGRAM) && !defined(IMPL_MFBT)
+ if (!malloc_good_size)
+ return size;
+# endif
+ return malloc_good_size_impl(size);
+}
+
+# define malloc_good_size _malloc_good_size
+#endif
+
+MOZ_JEMALLOC_API void jemalloc_stats(jemalloc_stats_t *stats);
+
+/*
+ * On some operating systems (Mac), we use madvise(MADV_FREE) to hand pages
+ * back to the operating system. On Mac, the operating system doesn't take
+ * this memory back immediately; instead, the OS takes it back only when the
+ * machine is running out of physical memory.
+ *
+ * This is great from the standpoint of efficiency, but it makes measuring our
+ * actual RSS difficult, because pages which we've MADV_FREE'd shouldn't count
+ * against our RSS.
+ *
+ * This function explicitly purges any MADV_FREE'd pages from physical memory,
+ * causing our reported RSS match the amount of memory we're actually using.
+ *
+ * Note that this call is expensive in two ways. First, it may be slow to
+ * execute, because it may make a number of slow syscalls to free memory. This
+ * function holds the big jemalloc locks, so basically all threads are blocked
+ * while this function runs.
+ *
+ * This function is also expensive in that the next time we go to access a page
+ * which we've just explicitly decommitted, the operating system has to attach
+ * to it a physical page! If we hadn't run this function, the OS would have
+ * less work to do.
+ *
+ * If MALLOC_DOUBLE_PURGE is not defined, this function does nothing.
+ */
+MOZ_JEMALLOC_API void jemalloc_purge_freed_pages();
+
+/*
+ * Free all unused dirty pages in all arenas. Calling this function will slow
+ * down subsequent allocations so it is recommended to use it only when
+ * memory needs to be reclaimed at all costs (see bug 805855). This function
+ * provides functionality similar to mallctl("arenas.purge") in jemalloc 3.
+ */
+MOZ_JEMALLOC_API void jemalloc_free_dirty_pages();
+
+MOZ_END_EXTERN_C
+
+#endif /* mozmemory_h */
diff --git a/memory/build/mozmemory_wrap.c b/memory/build/mozmemory_wrap.c
new file mode 100644
index 000000000..dba3ace56
--- /dev/null
+++ b/memory/build/mozmemory_wrap.c
@@ -0,0 +1,178 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <string.h>
+#include "mozmemory_wrap.h"
+#include "mozilla/Types.h"
+
+/* Declare malloc implementation functions with the right return and
+ * argument types. */
+#define MALLOC_DECL(name, return_type, ...) \
+ MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);
+#include "malloc_decls.h"
+
+#ifdef MOZ_WRAP_NEW_DELETE
+/* operator new(unsigned int) */
+MOZ_MEMORY_API void *
+mozmem_malloc_impl(_Znwj)(unsigned int size)
+{
+ return malloc_impl(size);
+}
+/* operator new[](unsigned int) */
+MOZ_MEMORY_API void *
+mozmem_malloc_impl(_Znaj)(unsigned int size)
+{
+ return malloc_impl(size);
+}
+/* operator delete(void*) */
+MOZ_MEMORY_API void
+mozmem_malloc_impl(_ZdlPv)(void *ptr)
+{
+ free_impl(ptr);
+}
+/* operator delete[](void*) */
+MOZ_MEMORY_API void
+mozmem_malloc_impl(_ZdaPv)(void *ptr)
+{
+ free_impl(ptr);
+}
+/*operator new(unsigned int, std::nothrow_t const&)*/
+MOZ_MEMORY_API void *
+mozmem_malloc_impl(_ZnwjRKSt9nothrow_t)(unsigned int size)
+{
+ return malloc_impl(size);
+}
+/*operator new[](unsigned int, std::nothrow_t const&)*/
+MOZ_MEMORY_API void *
+mozmem_malloc_impl(_ZnajRKSt9nothrow_t)(unsigned int size)
+{
+ return malloc_impl(size);
+}
+/* operator delete(void*, std::nothrow_t const&) */
+MOZ_MEMORY_API void
+mozmem_malloc_impl(_ZdlPvRKSt9nothrow_t)(void *ptr)
+{
+ free_impl(ptr);
+}
+/* operator delete[](void*, std::nothrow_t const&) */
+MOZ_MEMORY_API void
+mozmem_malloc_impl(_ZdaPvRKSt9nothrow_t)(void *ptr)
+{
+ free_impl(ptr);
+}
+#endif
+
+/* strndup and strdup may be defined as macros in string.h, which would
+ * clash with the definitions below. */
+#undef strndup
+#undef strdup
+
+#ifndef XP_DARWIN
+MOZ_MEMORY_API char *
+strndup_impl(const char *src, size_t len)
+{
+ char* dst = (char*) malloc_impl(len + 1);
+ if (dst) {
+ strncpy(dst, src, len);
+ dst[len] = '\0';
+ }
+ return dst;
+}
+
+MOZ_MEMORY_API char *
+strdup_impl(const char *src)
+{
+ size_t len = strlen(src);
+ return strndup_impl(src, len);
+}
+#endif /* XP_DARWIN */
+
+#ifdef ANDROID
+#include <stdarg.h>
+#include <stdio.h>
+
+MOZ_MEMORY_API int
+vasprintf_impl(char **str, const char *fmt, va_list ap)
+{
+ char* ptr, *_ptr;
+ int ret;
+
+ if (str == NULL || fmt == NULL) {
+ return -1;
+ }
+
+ ptr = (char*)malloc_impl(128);
+ if (ptr == NULL) {
+ *str = NULL;
+ return -1;
+ }
+
+ ret = vsnprintf(ptr, 128, fmt, ap);
+ if (ret < 0) {
+ free_impl(ptr);
+ *str = NULL;
+ return -1;
+ }
+
+ _ptr = realloc_impl(ptr, ret + 1);
+ if (_ptr == NULL) {
+ free_impl(ptr);
+ *str = NULL;
+ return -1;
+ }
+
+ *str = _ptr;
+
+ return ret;
+}
+
+MOZ_MEMORY_API int
+asprintf_impl(char **str, const char *fmt, ...)
+{
+ int ret;
+ va_list ap;
+ va_start(ap, fmt);
+
+ ret = vasprintf_impl(str, fmt, ap);
+
+ va_end(ap);
+
+ return ret;
+}
+#endif
+
+#ifdef XP_WIN
+/*
+ * There's a fun allocator mismatch in (at least) the VS 2010 CRT
+ * (see the giant comment in $(topsrcdir)/mozglue/build/Makefile.in)
+ * that gets redirected here to avoid a crash on shutdown.
+ */
+void
+dumb_free_thunk(void *ptr)
+{
+ return; /* shutdown leaks that we don't care about */
+}
+
+#include <wchar.h>
+
+/*
+ * We also need to provide our own impl of wcsdup so that we don't ask
+ * the CRT for memory from its heap (which will then be unfreeable).
+ */
+wchar_t *
+wcsdup_impl(const wchar_t *src)
+{
+ size_t len = wcslen(src);
+ wchar_t *dst = (wchar_t*) malloc_impl((len + 1) * sizeof(wchar_t));
+ if (dst)
+ wcsncpy(dst, src, len + 1);
+ return dst;
+}
+
+void *
+_aligned_malloc(size_t size, size_t alignment)
+{
+ return memalign_impl(alignment, size);
+}
+#endif /* XP_WIN */
diff --git a/memory/build/mozmemory_wrap.h b/memory/build/mozmemory_wrap.h
new file mode 100644
index 000000000..066d57782
--- /dev/null
+++ b/memory/build/mozmemory_wrap.h
@@ -0,0 +1,219 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozmemory_wrap_h
+#define mozmemory_wrap_h
+
+/*
+ * This header contains #defines which tweak the names of various memory
+ * allocation functions.
+ *
+ * There are several types of functions related to memory allocation
+ * that are meant to be used publicly by the Gecko codebase:
+ *
+ * - malloc implementation functions:
+ * - malloc
+ * - posix_memalign
+ * - aligned_alloc
+ * - calloc
+ * - realloc
+ * - free
+ * - memalign
+ * - valloc
+ * - malloc_usable_size
+ * - malloc_good_size
+ * Some of these functions are specific to some systems, but for
+ * convenience, they are treated as being cross-platform, and available
+ * as such.
+ *
+ * - duplication functions:
+ * - strndup
+ * - strdup
+ * - wcsdup (Windows only)
+ *
+ * - jemalloc specific functions:
+ * - jemalloc_stats
+ * - jemalloc_purge_freed_pages
+ * - jemalloc_free_dirty_pages
+ * (these functions are native to mozjemalloc, and have compatibility
+ * implementations for jemalloc3)
+ *
+ * These functions are all exported as part of libmozglue (see
+ * $(topsrcdir)/mozglue/build/Makefile.in), with a few implementation
+ * peculiarities:
+ *
+ * - On Windows, the malloc implementation functions are all prefixed with
+ * "je_", the duplication functions are prefixed with "wrap_", and jemalloc
+ * specific functions are left unprefixed. All these functions are however
+ * aliased when exporting them, such that the resulting mozglue.dll exports
+ * them unprefixed (see $(topsrcdir)/mozglue/build/mozglue.def.in). The
+ * prefixed malloc implementation and duplication functions are not
+ * exported.
+ *
+ * - On MacOSX, the system libc has a zone allocator, which allows us to
+ * hook custom malloc implementation functions without exporting them.
+ * The malloc implementation functions are all prefixed with "je_" and used
+ * this way from the custom zone allocator. They are not exported.
+ * Duplication functions are not included, since they will call the custom
+ * zone allocator anyways. Jemalloc-specific functions are also left
+ * unprefixed.
+ *
+ * - On Android and Gonk, all functions are left unprefixed. Additionally,
+ * C++ allocation functions (operator new/delete) are also exported and
+ * unprefixed.
+ *
+ * - On other systems (mostly Linux), all functions are left unprefixed.
+ *
+ * Only Android and Gonk add C++ allocation functions.
+ *
+ * Proper exporting of the various functions is done with the MOZ_MEMORY_API
+ * and MOZ_JEMALLOC_API macros. MOZ_MEMORY_API is meant to be used for malloc
+ * implementation and duplication functions, while MOZ_JEMALLOC_API is
+ * dedicated to jemalloc specific functions.
+ *
+ *
+ * All these functions are meant to be called with no prefix from Gecko code.
+ * In most cases, this is because that's how they are available at runtime.
+ * However, on Android, this relies on faulty.lib (the custom dynamic linker)
+ * resolving mozglue symbols before libc symbols, which is guaranteed by the
+ * way faulty.lib works (it respects the DT_NEEDED order, and libc always
+ * appears after mozglue ; which we double check when building anyways)
+ *
+ *
+ * Within libmozglue (when MOZ_MEMORY_IMPL is defined), all the functions
+ * should be suffixed with "_impl" both for declarations and use.
+ * That is, the implementation declaration for e.g. strdup would look like:
+ * char* strdup_impl(const char *)
+ * That implementation would call malloc by using "malloc_impl".
+ *
+ * While mozjemalloc uses these "_impl" suffixed helpers, jemalloc3, being
+ * third-party code, doesn't, but instead has an elaborate way to mangle
+ * individual functions. See under "Run jemalloc configure script" in
+ * $(topsrcdir)/configure.in.
+ *
+ *
+ * When building with replace-malloc support, the above still holds, but
+ * the malloc implementation and jemalloc specific functions are the
+ * replace-malloc functions from replace_malloc.c.
+ *
+ * The actual jemalloc/mozjemalloc implementation is prefixed with "je_".
+ *
+ * Thus, when MOZ_REPLACE_MALLOC is defined, the "_impl" suffixed macros
+ * expand to "je_" prefixed function when building mozjemalloc or
+ * jemalloc3/mozjemalloc_compat, where MOZ_JEMALLOC_IMPL is defined.
+ *
+ * In other cases, the "_impl" suffixed macros follow the original scheme,
+ * except on Windows and MacOSX, where they would expand to "je_" prefixed
+ * functions. Instead, they are left unmodified (malloc_impl expands to
+ * malloc_impl).
+ */
+
+#ifndef MOZ_MEMORY
+# error Should only include mozmemory_wrap.h when MOZ_MEMORY is set.
+#endif
+
+#if defined(MOZ_JEMALLOC_IMPL) && !defined(MOZ_MEMORY_IMPL)
+# define MOZ_MEMORY_IMPL
+#endif
+#if defined(MOZ_MEMORY_IMPL) && !defined(IMPL_MFBT)
+# ifdef MFBT_API /* mozilla/Types.h was already included */
+# error mozmemory_wrap.h has to be included before mozilla/Types.h when MOZ_MEMORY_IMPL is set and IMPL_MFBT is not.
+# endif
+# define IMPL_MFBT
+#endif
+
+#include "mozilla/Types.h"
+
+#if !defined(MOZ_SYSTEM_JEMALLOC)
+# ifdef MOZ_MEMORY_IMPL
+# if defined(MOZ_JEMALLOC_IMPL) && defined(MOZ_REPLACE_MALLOC) && !defined(MOZ_REPLACE_JEMALLOC)
+# define mozmem_malloc_impl(a) je_ ## a
+# define mozmem_jemalloc_impl(a) je_ ## a
+# else
+# define MOZ_JEMALLOC_API MFBT_API
+# ifdef MOZ_REPLACE_JEMALLOC
+# define MOZ_MEMORY_API MFBT_API
+# define mozmem_malloc_impl(a) replace_ ## a
+# define mozmem_jemalloc_impl(a) replace_ ## a
+# elif (defined(XP_WIN) || defined(XP_DARWIN))
+# if defined(MOZ_REPLACE_MALLOC)
+# define mozmem_malloc_impl(a) a ## _impl
+# else
+# define mozmem_malloc_impl(a) je_ ## a
+# endif
+# else
+# define MOZ_MEMORY_API MFBT_API
+# if defined(MOZ_WIDGET_ANDROID) || defined(MOZ_WIDGET_GONK)
+# define MOZ_WRAP_NEW_DELETE
+# endif
+# endif
+# endif
+# ifdef XP_WIN
+# define mozmem_dup_impl(a) wrap_ ## a
+# endif
+# endif
+
+/* All other jemalloc3 functions are prefixed with "je_", except when
+ * building against an unprefixed system jemalloc library */
+# define je_(a) je_ ## a
+#else /* defined(MOZ_SYSTEM_JEMALLOC) */
+# define je_(a) a
+#endif
+
+#if !defined(MOZ_MEMORY_IMPL) || defined(MOZ_SYSTEM_JEMALLOC)
+# define MOZ_MEMORY_API MFBT_API
+# define MOZ_JEMALLOC_API MFBT_API
+#endif
+
+#ifndef MOZ_MEMORY_API
+# define MOZ_MEMORY_API
+#endif
+#ifndef MOZ_JEMALLOC_API
+# define MOZ_JEMALLOC_API
+#endif
+
+#ifndef mozmem_malloc_impl
+# define mozmem_malloc_impl(a) a
+#endif
+#ifndef mozmem_dup_impl
+# define mozmem_dup_impl(a) a
+#endif
+#ifndef mozmem_jemalloc_impl
+# define mozmem_jemalloc_impl(a) a
+#endif
+
+/* Malloc implementation functions */
+#define malloc_impl mozmem_malloc_impl(malloc)
+#define posix_memalign_impl mozmem_malloc_impl(posix_memalign)
+#define aligned_alloc_impl mozmem_malloc_impl(aligned_alloc)
+#define calloc_impl mozmem_malloc_impl(calloc)
+#define realloc_impl mozmem_malloc_impl(realloc)
+#define free_impl mozmem_malloc_impl(free)
+#define memalign_impl mozmem_malloc_impl(memalign)
+#define valloc_impl mozmem_malloc_impl(valloc)
+#define malloc_usable_size_impl mozmem_malloc_impl(malloc_usable_size)
+#define malloc_good_size_impl mozmem_malloc_impl(malloc_good_size)
+
+/* Duplication functions */
+#define strndup_impl mozmem_dup_impl(strndup)
+#define strdup_impl mozmem_dup_impl(strdup)
+#ifdef XP_WIN
+# define wcsdup_impl mozmem_dup_impl(wcsdup)
+#endif
+
+/* String functions */
+#ifdef ANDROID
+/* Bug 801571 and Bug 879668, libstagefright uses vasprintf, causing malloc()/
+ * free() to be mismatched between bionic and mozglue implementation.
+ */
+#define vasprintf_impl mozmem_dup_impl(vasprintf)
+#define asprintf_impl mozmem_dup_impl(asprintf)
+#endif
+
+/* Jemalloc specific function */
+#define jemalloc_stats_impl mozmem_jemalloc_impl(jemalloc_stats)
+#define jemalloc_purge_freed_pages_impl mozmem_jemalloc_impl(jemalloc_purge_freed_pages)
+#define jemalloc_free_dirty_pages_impl mozmem_jemalloc_impl(jemalloc_free_dirty_pages)
+
+#endif /* mozmemory_wrap_h */
diff --git a/memory/build/replace_malloc.c b/memory/build/replace_malloc.c
new file mode 100644
index 000000000..88dfde33c
--- /dev/null
+++ b/memory/build/replace_malloc.c
@@ -0,0 +1,546 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZ_MEMORY
+# error Should not compile this file when MOZ_MEMORY is not set
+#endif
+
+#ifndef MOZ_REPLACE_MALLOC
+# error Should not compile this file when replace-malloc is disabled
+#endif
+
+#ifdef MOZ_SYSTEM_JEMALLOC
+# error Should not compile this file when we want to use native jemalloc
+#endif
+
+#include "mozmemory_wrap.h"
+
+/* Declare all je_* functions */
+#define MALLOC_DECL(name, return_type, ...) \
+ return_type je_ ## name(__VA_ARGS__);
+#include "malloc_decls.h"
+
+#include "mozilla/Likely.h"
+
+/*
+ * Windows doesn't come with weak imports as they are possible with
+ * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform,
+ * the replacement functions are defined as variable pointers to the
+ * function resolved with GetProcAddress() instead of weak definitions
+ * of functions. On Android, the same needs to happen as well, because
+ * the Android linker doesn't handle weak linking with non LD_PRELOADed
+ * libraries, but LD_PRELOADing is not very convenient on Android, with
+ * the zygote.
+ */
+#ifdef XP_DARWIN
+# define MOZ_REPLACE_WEAK __attribute__((weak_import))
+#elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID)
+# define MOZ_NO_REPLACE_FUNC_DECL
+#elif defined(__GNUC__)
+# define MOZ_REPLACE_WEAK __attribute__((weak))
+#endif
+
+#include "replace_malloc.h"
+
+#define MALLOC_DECL(name, return_type, ...) \
+ je_ ## name,
+
+static const malloc_table_t malloc_table = {
+#include "malloc_decls.h"
+};
+
+#ifdef MOZ_NO_REPLACE_FUNC_DECL
+# define MALLOC_DECL(name, return_type, ...) \
+ typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \
+ replace_ ## name ## _impl_t *replace_ ## name = NULL;
+# define MALLOC_FUNCS MALLOC_FUNCS_ALL
+# include "malloc_decls.h"
+
+# ifdef XP_WIN
+# include <windows.h>
+static void
+replace_malloc_init_funcs()
+{
+ char replace_malloc_lib[1024];
+ if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib,
+ sizeof(replace_malloc_lib)) > 0) {
+ HMODULE handle = LoadLibraryA(replace_malloc_lib);
+ if (handle) {
+#define MALLOC_DECL(name, ...) \
+ replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name);
+
+# define MALLOC_FUNCS MALLOC_FUNCS_ALL
+#include "malloc_decls.h"
+ }
+ }
+}
+# elif defined(MOZ_WIDGET_ANDROID)
+# include <dlfcn.h>
+# include <stdlib.h>
+static void
+replace_malloc_init_funcs()
+{
+ const char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB");
+ if (replace_malloc_lib && *replace_malloc_lib) {
+ void *handle = dlopen(replace_malloc_lib, RTLD_LAZY);
+ if (handle) {
+#define MALLOC_DECL(name, ...) \
+ replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name);
+
+# define MALLOC_FUNCS MALLOC_FUNCS_ALL
+#include "malloc_decls.h"
+ }
+ }
+}
+# else
+# error No implementation for replace_malloc_init_funcs()
+# endif
+
+#endif /* MOZ_NO_REPLACE_FUNC_DECL */
+
+/*
+ * Below is the malloc implementation overriding jemalloc and calling the
+ * replacement functions if they exist.
+ */
+
+/*
+ * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc
+ * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h
+ */
+#define MALLOC_DECL(name, return_type, ...) \
+ MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);
+#define MALLOC_FUNCS MALLOC_FUNCS_MALLOC
+#include "malloc_decls.h"
+
+#define MALLOC_DECL(name, return_type, ...) \
+ MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__);
+#define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC
+#include "malloc_decls.h"
+
+static int replace_malloc_initialized = 0;
+static void
+init()
+{
+#ifdef MOZ_NO_REPLACE_FUNC_DECL
+ replace_malloc_init_funcs();
+#endif
+ // Set this *before* calling replace_init, otherwise if replace_init calls
+ // malloc() we'll get an infinite loop.
+ replace_malloc_initialized = 1;
+ if (replace_init)
+ replace_init(&malloc_table);
+}
+
+MFBT_API struct ReplaceMallocBridge*
+get_bridge(void)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_get_bridge))
+ return NULL;
+ return replace_get_bridge();
+}
+
+void*
+malloc_impl(size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_malloc))
+ return je_malloc(size);
+ return replace_malloc(size);
+}
+
+int
+posix_memalign_impl(void **memptr, size_t alignment, size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_posix_memalign))
+ return je_posix_memalign(memptr, alignment, size);
+ return replace_posix_memalign(memptr, alignment, size);
+}
+
+void*
+aligned_alloc_impl(size_t alignment, size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_aligned_alloc))
+ return je_aligned_alloc(alignment, size);
+ return replace_aligned_alloc(alignment, size);
+}
+
+void*
+calloc_impl(size_t num, size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_calloc))
+ return je_calloc(num, size);
+ return replace_calloc(num, size);
+}
+
+void*
+realloc_impl(void *ptr, size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_realloc))
+ return je_realloc(ptr, size);
+ return replace_realloc(ptr, size);
+}
+
+void
+free_impl(void *ptr)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_free))
+ je_free(ptr);
+ else
+ replace_free(ptr);
+}
+
+void*
+memalign_impl(size_t alignment, size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_memalign))
+ return je_memalign(alignment, size);
+ return replace_memalign(alignment, size);
+}
+
+void*
+valloc_impl(size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_valloc))
+ return je_valloc(size);
+ return replace_valloc(size);
+}
+
+size_t
+malloc_usable_size_impl(usable_ptr_t ptr)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_malloc_usable_size))
+ return je_malloc_usable_size(ptr);
+ return replace_malloc_usable_size(ptr);
+}
+
+size_t
+malloc_good_size_impl(size_t size)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_malloc_good_size))
+ return je_malloc_good_size(size);
+ return replace_malloc_good_size(size);
+}
+
+void
+jemalloc_stats_impl(jemalloc_stats_t *stats)
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_jemalloc_stats))
+ je_jemalloc_stats(stats);
+ else
+ replace_jemalloc_stats(stats);
+}
+
+void
+jemalloc_purge_freed_pages_impl()
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages))
+ je_jemalloc_purge_freed_pages();
+ else
+ replace_jemalloc_purge_freed_pages();
+}
+
+void
+jemalloc_free_dirty_pages_impl()
+{
+ if (MOZ_UNLIKELY(!replace_malloc_initialized))
+ init();
+ if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages))
+ je_jemalloc_free_dirty_pages();
+ else
+ replace_jemalloc_free_dirty_pages();
+}
+
+/* The following comment and definitions are from jemalloc.c: */
+#if defined(__GLIBC__) && !defined(__UCLIBC__)
+
+/*
+ * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
+ * to inconsistently reference libc's malloc(3)-compatible functions
+ * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
+ *
+ * These definitions interpose hooks in glibc. The functions are actually
+ * passed an extra argument for the caller return address, which will be
+ * ignored.
+ */
+
+typedef void (* __free_hook_type)(void *ptr);
+typedef void *(* __malloc_hook_type)(size_t size);
+typedef void *(* __realloc_hook_type)(void *ptr, size_t size);
+typedef void *(* __memalign_hook_type)(size_t alignment, size_t size);
+
+MOZ_MEMORY_API __free_hook_type __free_hook = free_impl;
+MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl;
+MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl;
+MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl;
+
+#endif
+
+/*
+ * The following is a OSX zone allocator implementation.
+ * /!\ WARNING. It assumes the underlying malloc implementation's
+ * malloc_usable_size returns 0 when the given pointer is not owned by
+ * the allocator. Sadly, OSX does call zone_size with pointers not
+ * owned by the allocator.
+ */
+
+#ifdef XP_DARWIN
+#include <stdlib.h>
+#include <malloc/malloc.h>
+#include "mozilla/Assertions.h"
+
+static size_t
+zone_size(malloc_zone_t *zone, void *ptr)
+{
+ return malloc_usable_size_impl(ptr);
+}
+
+static void *
+zone_malloc(malloc_zone_t *zone, size_t size)
+{
+ return malloc_impl(size);
+}
+
+static void *
+zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
+{
+ return calloc_impl(num, size);
+}
+
+static void *
+zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
+{
+ if (malloc_usable_size_impl(ptr))
+ return realloc_impl(ptr, size);
+ return realloc(ptr, size);
+}
+
+static void
+zone_free(malloc_zone_t *zone, void *ptr)
+{
+ if (malloc_usable_size_impl(ptr)) {
+ free_impl(ptr);
+ return;
+ }
+ free(ptr);
+}
+
+static void
+zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
+{
+ size_t current_size = malloc_usable_size_impl(ptr);
+ if (current_size) {
+ MOZ_ASSERT(current_size == size);
+ free_impl(ptr);
+ return;
+ }
+ free(ptr);
+}
+
+static void *
+zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
+{
+ void *ptr;
+ if (posix_memalign_impl(&ptr, alignment, size) == 0)
+ return ptr;
+ return NULL;
+}
+
+static void *
+zone_valloc(malloc_zone_t *zone, size_t size)
+{
+ return valloc_impl(size);
+}
+
+static void *
+zone_destroy(malloc_zone_t *zone)
+{
+ /* This function should never be called. */
+ MOZ_CRASH();
+}
+
+static size_t
+zone_good_size(malloc_zone_t *zone, size_t size)
+{
+ return malloc_good_size_impl(size);
+}
+
+#ifdef MOZ_JEMALLOC
+
+#include "jemalloc/internal/jemalloc_internal.h"
+
+static void
+zone_force_lock(malloc_zone_t *zone)
+{
+ /* /!\ This calls into jemalloc. It works because we're linked in the
+ * same library. Stolen from jemalloc's zone.c. */
+ if (isthreaded)
+ jemalloc_prefork();
+}
+
+static void
+zone_force_unlock(malloc_zone_t *zone)
+{
+ /* /!\ This calls into jemalloc. It works because we're linked in the
+ * same library. Stolen from jemalloc's zone.c. */
+ if (isthreaded)
+ jemalloc_postfork_parent();
+}
+
+#else
+
+#define JEMALLOC_ZONE_VERSION 6
+
+/* Empty implementations are needed, because fork() calls zone->force_(un)lock
+ * unconditionally. */
+static void
+zone_force_lock(malloc_zone_t *zone)
+{
+}
+
+static void
+zone_force_unlock(malloc_zone_t *zone)
+{
+}
+
+#endif
+
+static malloc_zone_t zone;
+static struct malloc_introspection_t zone_introspect;
+
+static malloc_zone_t *get_default_zone()
+{
+ malloc_zone_t **zones = NULL;
+ unsigned int num_zones = 0;
+
+ /*
+ * On OSX 10.12, malloc_default_zone returns a special zone that is not
+ * present in the list of registered zones. That zone uses a "lite zone"
+ * if one is present (apparently enabled when malloc stack logging is
+ * enabled), or the first registered zone otherwise. In practice this
+ * means unless malloc stack logging is enabled, the first registered
+ * zone is the default.
+ * So get the list of zones to get the first one, instead of relying on
+ * malloc_default_zone.
+ */
+ if (KERN_SUCCESS != malloc_get_all_zones(0, NULL, (vm_address_t**) &zones,
+ &num_zones)) {
+ /* Reset the value in case the failure happened after it was set. */
+ num_zones = 0;
+ }
+ if (num_zones) {
+ return zones[0];
+ }
+ return malloc_default_zone();
+}
+
+
+__attribute__((constructor)) void
+register_zone(void)
+{
+ malloc_zone_t *default_zone = get_default_zone();
+
+ zone.size = (void *)zone_size;
+ zone.malloc = (void *)zone_malloc;
+ zone.calloc = (void *)zone_calloc;
+ zone.valloc = (void *)zone_valloc;
+ zone.free = (void *)zone_free;
+ zone.realloc = (void *)zone_realloc;
+ zone.destroy = (void *)zone_destroy;
+ zone.zone_name = "replace_malloc_zone";
+ zone.batch_malloc = NULL;
+ zone.batch_free = NULL;
+ zone.introspect = &zone_introspect;
+ zone.version = JEMALLOC_ZONE_VERSION;
+ zone.memalign = zone_memalign;
+ zone.free_definite_size = zone_free_definite_size;
+#if (JEMALLOC_ZONE_VERSION >= 8)
+ zone.pressure_relief = NULL;
+#endif
+ zone_introspect.enumerator = NULL;
+ zone_introspect.good_size = (void *)zone_good_size;
+ zone_introspect.check = NULL;
+ zone_introspect.print = NULL;
+ zone_introspect.log = NULL;
+ zone_introspect.force_lock = (void *)zone_force_lock;
+ zone_introspect.force_unlock = (void *)zone_force_unlock;
+ zone_introspect.statistics = NULL;
+ zone_introspect.zone_locked = NULL;
+#if (JEMALLOC_ZONE_VERSION >= 7)
+ zone_introspect.enable_discharge_checking = NULL;
+ zone_introspect.disable_discharge_checking = NULL;
+ zone_introspect.discharge = NULL;
+#ifdef __BLOCKS__
+ zone_introspect.enumerate_discharged_pointers = NULL;
+#else
+ zone_introspect.enumerate_unavailable_without_blocks = NULL;
+#endif
+#endif
+
+ /*
+ * The default purgeable zone is created lazily by OSX's libc. It uses
+ * the default zone when it is created for "small" allocations
+ * (< 15 KiB), but assumes the default zone is a scalable_zone. This
+ * obviously fails when the default zone is the jemalloc zone, so
+ * malloc_default_purgeable_zone is called beforehand so that the
+ * default purgeable zone is created when the default zone is still
+ * a scalable_zone.
+ */
+ malloc_zone_t *purgeable_zone = malloc_default_purgeable_zone();
+
+ /* Register the custom zone. At this point it won't be the default. */
+ malloc_zone_register(&zone);
+
+ do {
+ /*
+ * Unregister and reregister the default zone. On OSX >= 10.6,
+ * unregistering takes the last registered zone and places it at the
+ * location of the specified zone. Unregistering the default zone thus
+ * makes the last registered one the default. On OSX < 10.6,
+ * unregistering shifts all registered zones. The first registered zone
+ * then becomes the default.
+ */
+ malloc_zone_unregister(default_zone);
+ malloc_zone_register(default_zone);
+ /*
+ * On OSX 10.6, having the default purgeable zone appear before the default
+ * zone makes some things crash because it thinks it owns the default
+ * zone allocated pointers. We thus unregister/re-register it in order to
+ * ensure it's always after the default zone. On OSX < 10.6, as
+ * unregistering shifts registered zones, this simply removes the purgeable
+ * zone from the list and adds it back at the end, after the default zone.
+ * On OSX >= 10.6, unregistering replaces the purgeable zone with the last
+ * registered zone above, i.e the default zone. Registering it again then
+ * puts it at the end, obviously after the default zone.
+ */
+ malloc_zone_unregister(purgeable_zone);
+ malloc_zone_register(purgeable_zone);
+ default_zone = get_default_zone();
+ } while (default_zone != &zone);
+}
+#endif
diff --git a/memory/build/replace_malloc.h b/memory/build/replace_malloc.h
new file mode 100644
index 000000000..a61744f60
--- /dev/null
+++ b/memory/build/replace_malloc.h
@@ -0,0 +1,133 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef replace_malloc_h
+#define replace_malloc_h
+
+/*
+ * The replace_malloc facility allows an external library to replace or
+ * supplement the jemalloc implementation.
+ *
+ * The external library may be hooked by setting one of the following
+ * environment variables to the library path:
+ * - LD_PRELOAD on Linux,
+ * - DYLD_INSERT_LIBRARIES on OSX,
+ * - MOZ_REPLACE_MALLOC_LIB on Windows and Android.
+ *
+ * An initialization function is called before any malloc replacement
+ * function, and has the following declaration:
+ *
+ * void replace_init(const malloc_table_t *)
+ *
+ * The const malloc_table_t pointer given to that function is a table
+ * containing pointers to the original jemalloc implementation, so that
+ * replacement functions can call them back if they need to. The pointer
+ * itself can safely be kept around (no need to copy the table itself).
+ *
+ * The functions to be implemented in the external library are of the form:
+ *
+ * void *replace_malloc(size_t size)
+ * {
+ * // Fiddle with the size if necessary.
+ * // orig->malloc doesn't have to be called if the external library
+ * // provides its own allocator, but in this case it will have to
+ * // implement all functions.
+ * void *ptr = orig->malloc(size);
+ * // Do whatever you want with the ptr.
+ * return ptr;
+ * }
+ *
+ * where "orig" is the pointer obtained from replace_init.
+ *
+ * See malloc_decls.h for a list of functions that can be replaced this
+ * way. The implementations are all in the form:
+ * return_type replace_name(arguments [,...])
+ *
+ * They don't all need to be provided.
+ *
+ * Building a replace-malloc library is like rocket science. It can end up
+ * with things blowing up, especially when trying to use complex types, and
+ * even more especially when these types come from XPCOM or other parts of the
+ * Mozilla codebase.
+ * It is recommended to add the following to a replace-malloc implementation's
+ * moz.build:
+ * DISABLE_STL_WRAPPING = True # Avoid STL wrapping
+ *
+ * If your replace-malloc implementation lives under memory/replace, these
+ * are taken care of by memory/replace/defs.mk.
+ */
+
+#ifdef replace_malloc_bridge_h
+#error Do not include replace_malloc_bridge.h before replace_malloc.h. \
+ In fact, you only need the latter.
+#endif
+
+#define REPLACE_MALLOC_IMPL
+
+#include "replace_malloc_bridge.h"
+
+/* Implementing a replace-malloc library is incompatible with using mozalloc. */
+#define MOZ_NO_MOZALLOC 1
+
+#include "mozilla/Types.h"
+
+MOZ_BEGIN_EXTERN_C
+
+/* MOZ_NO_REPLACE_FUNC_DECL and MOZ_REPLACE_WEAK are only defined in
+ * replace_malloc.c. Normally including this header will add function
+ * definitions. */
+#ifndef MOZ_NO_REPLACE_FUNC_DECL
+
+# ifndef MOZ_REPLACE_WEAK
+# define MOZ_REPLACE_WEAK
+# endif
+
+# define MALLOC_DECL(name, return_type, ...) \
+ MOZ_EXPORT return_type replace_ ## name(__VA_ARGS__) MOZ_REPLACE_WEAK;
+
+# define MALLOC_FUNCS MALLOC_FUNCS_ALL
+# include "malloc_decls.h"
+
+#endif /* MOZ_NO_REPLACE_FUNC_DECL */
+
+/*
+ * posix_memalign, aligned_alloc, memalign and valloc all implement some
+ * kind of aligned memory allocation. For convenience, replace_posix_memalign,
+ * replace_aligned_alloc and replace_valloc can be automatically derived from
+ * memalign when MOZ_REPLACE_ONLY_MEMALIGN is defined before including this
+ * header. PAGE_SIZE also needs to be defined to the appropriate expression.
+ */
+#ifdef MOZ_REPLACE_ONLY_MEMALIGN
+#include <errno.h>
+
+int replace_posix_memalign(void **ptr, size_t alignment, size_t size)
+{
+ if (size == 0) {
+ *ptr = NULL;
+ return 0;
+ }
+ /* alignment must be a power of two and a multiple of sizeof(void *) */
+ if (((alignment - 1) & alignment) != 0 || (alignment % sizeof(void *)))
+ return EINVAL;
+ *ptr = replace_memalign(alignment, size);
+ return *ptr ? 0 : ENOMEM;
+}
+
+void *replace_aligned_alloc(size_t alignment, size_t size)
+{
+ /* size should be a multiple of alignment */
+ if (size % alignment)
+ return NULL;
+ return replace_memalign(alignment, size);
+}
+
+void *replace_valloc(size_t size)
+{
+ return replace_memalign(PAGE_SIZE, size);
+}
+#endif
+
+MOZ_END_EXTERN_C
+
+#endif /* replace_malloc_h */
diff --git a/memory/build/replace_malloc_bridge.h b/memory/build/replace_malloc_bridge.h
new file mode 100644
index 000000000..301b165eb
--- /dev/null
+++ b/memory/build/replace_malloc_bridge.h
@@ -0,0 +1,202 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef replace_malloc_bridge_h
+#define replace_malloc_bridge_h
+
+/*
+ * The replace-malloc bridge allows bidirectional method calls between
+ * a program and the replace-malloc library that has been loaded for it.
+ * In Firefox, this is used to allow method calls between code in libxul
+ * and code in the replace-malloc library, without libxul needing to link
+ * against that library or vice-versa.
+ *
+ * Subsystems can add methods for their own need. Replace-malloc libraries
+ * can decide to implement those methods or not.
+ *
+ * Replace-malloc libraries can provide such a bridge by implementing
+ * a ReplaceMallocBridge-derived class, and a replace_get_bridge function
+ * returning an instance of that class. The default methods in
+ * ReplaceMallocBridge are expected to return values that callers would
+ * understand as "the bridge doesn't implement this method", so that a
+ * replace-malloc library doesn't have to implement all methods.
+ *
+ * The ReplaceMallocBridge class contains definitions for methods for
+ * all replace-malloc libraries. Each library picks the methods it wants
+ * to reply to in its ReplaceMallocBridge-derived class instance.
+ * All methods of ReplaceMallocBridge must be virtual. Similarly,
+ * anything passed as an argument to those methods must be plain data, or
+ * an instance of a class with only virtual methods.
+ *
+ * Binary compatibility is expected to be maintained, such that a newer
+ * Firefox can be used with an old replace-malloc library, or an old
+ * Firefox can be used with a newer replace-malloc library. As such, only
+ * new virtual methods should be added to ReplaceMallocBridge, and
+ * each change should have a corresponding bump of the mVersion value.
+ * At the same time, each virtual method should have a corresponding
+ * wrapper calling the virtual method on the instance from
+ * ReplaceMallocBridge::Get(), giving it the version the virtual method
+ * was added.
+ *
+ * Parts that are not relevant to the replace-malloc library end of the
+ * bridge are hidden when REPLACE_MALLOC_IMPL is not defined, which is
+ * the case when including replace_malloc.h.
+ */
+
+struct ReplaceMallocBridge;
+
+#include "mozilla/Types.h"
+
+MOZ_BEGIN_EXTERN_C
+
+#ifndef REPLACE_MALLOC_IMPL
+/* Returns the replace-malloc bridge if there is one to be returned. */
+MFBT_API ReplaceMallocBridge* get_bridge();
+#endif
+
+/* Table of malloc functions.
+ * e.g. void* (*malloc)(size_t), etc.
+ */
+#define MALLOC_DECL(name, return_type, ...) \
+ typedef return_type(name ## _impl_t)(__VA_ARGS__);
+
+#include "malloc_decls.h"
+
+#define MALLOC_DECL(name, return_type, ...) \
+ name ## _impl_t * name;
+
+typedef struct {
+#include "malloc_decls.h"
+} malloc_table_t;
+
+
+/* Table of malloc hook functions.
+ * Those functions are called with the arguments and results of malloc
+ * functions after they are called.
+ * e.g. void* (*malloc_hook)(void*, size_t), etc.
+ * They can either return the result they're given, or alter it before
+ * returning it.
+ * The hooks corresponding to functions, like free(void*), that return no
+ * value, don't take an extra argument.
+ * The table must at least contain a pointer for malloc_hook and free_hook
+ * functions. They will be used as fallback if no pointer is given for
+ * other allocation functions, like calloc_hook.
+ */
+#define MALLOC_DECL(name, return_type, ...) \
+ return_type (*name ## _hook)(return_type, __VA_ARGS__);
+#define MALLOC_DECL_VOID(name, ...) \
+ void (*name ## _hook)(__VA_ARGS__);
+
+typedef struct {
+#include "malloc_decls.h"
+ /* Like free_hook, but called before realloc_hook. free_hook is called
+ * instead of not given. */
+ void (*realloc_hook_before)(void* aPtr);
+} malloc_hook_table_t;
+
+MOZ_END_EXTERN_C
+
+#ifdef __cplusplus
+
+namespace mozilla {
+namespace dmd {
+struct DMDFuncs;
+} // namespace dmd
+
+/* Callbacks to register debug file handles for Poison IO interpose.
+ * See Mozilla(|Un)RegisterDebugHandle in xpcom/build/PoisonIOInterposer.h */
+struct DebugFdRegistry
+{
+ virtual void RegisterHandle(intptr_t aFd);
+
+ virtual void UnRegisterHandle(intptr_t aFd);
+};
+
+} // namespace mozilla
+
+struct ReplaceMallocBridge
+{
+ ReplaceMallocBridge() : mVersion(3) {}
+
+ /* This method was added in version 1 of the bridge. */
+ virtual mozilla::dmd::DMDFuncs* GetDMDFuncs() { return nullptr; }
+
+ /* Send a DebugFdRegistry instance to the replace-malloc library so that
+ * it can register/unregister file descriptors whenever needed. The
+ * instance is valid until the process dies.
+ * This method was added in version 2 of the bridge. */
+ virtual void InitDebugFd(mozilla::DebugFdRegistry&) {}
+
+ /* Register a list of malloc functions and hook functions to the
+ * replace-malloc library so that it can choose to dispatch to them
+ * when needed. The details of what is dispatched when is left to the
+ * replace-malloc library.
+ * Passing a nullptr for either table will unregister a previously
+ * registered table under the same name.
+ * Returns nullptr if registration failed.
+ * If registration succeeded, a table of "pure" malloc functions is
+ * returned. Those "pure" malloc functions won't call hooks.
+ * /!\ Do not rely on registration/unregistration to be instantaneous.
+ * Functions from a previously registered table may still be called for
+ * a brief time after RegisterHook returns.
+ * This method was added in version 3 of the bridge. */
+ virtual const malloc_table_t*
+ RegisterHook(const char* aName, const malloc_table_t* aTable,
+ const malloc_hook_table_t* aHookTable) { return nullptr; }
+
+#ifndef REPLACE_MALLOC_IMPL
+ /* Returns the replace-malloc bridge if its version is at least the
+ * requested one. */
+ static ReplaceMallocBridge* Get(int aMinimumVersion) {
+ static ReplaceMallocBridge* sSingleton = get_bridge();
+ return (sSingleton && sSingleton->mVersion >= aMinimumVersion)
+ ? sSingleton : nullptr;
+ }
+#endif
+
+protected:
+ const int mVersion;
+};
+
+#ifndef REPLACE_MALLOC_IMPL
+/* Class containing wrappers for calls to ReplaceMallocBridge methods.
+ * Those wrappers need to be static methods in a class because compilers
+ * complain about unused static global functions, and linkers complain
+ * about multiple definitions of non-static global functions.
+ * Using a separate class from ReplaceMallocBridge allows the function
+ * names to be identical. */
+struct ReplaceMalloc
+{
+ /* Don't call this method from performance critical code. Use
+ * mozilla::dmd::DMDFuncs::Get() instead, it has less overhead. */
+ static mozilla::dmd::DMDFuncs* GetDMDFuncs()
+ {
+ auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 1);
+ return singleton ? singleton->GetDMDFuncs() : nullptr;
+ }
+
+ static void InitDebugFd(mozilla::DebugFdRegistry& aRegistry)
+ {
+ auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 2);
+ if (singleton) {
+ singleton->InitDebugFd(aRegistry);
+ }
+ }
+
+ static const malloc_table_t*
+ RegisterHook(const char* aName, const malloc_table_t* aTable,
+ const malloc_hook_table_t* aHookTable)
+ {
+ auto singleton = ReplaceMallocBridge::Get(/* minimumVersion */ 3);
+ return singleton ? singleton->RegisterHook(aName, aTable, aHookTable)
+ : nullptr;
+ }
+};
+#endif
+
+#endif /* __cplusplus */
+
+#endif /* replace_malloc_bridge_h */