summaryrefslogtreecommitdiffstats
path: root/js/src/jsgc.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jsgc.cpp')
-rw-r--r--js/src/jsgc.cpp7722
1 files changed, 7722 insertions, 0 deletions
diff --git a/js/src/jsgc.cpp b/js/src/jsgc.cpp
new file mode 100644
index 000000000..42d10283c
--- /dev/null
+++ b/js/src/jsgc.cpp
@@ -0,0 +1,7722 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This code implements an incremental mark-and-sweep garbage collector, with
+ * most sweeping carried out in the background on a parallel thread.
+ *
+ * Full vs. zone GC
+ * ----------------
+ *
+ * The collector can collect all zones at once, or a subset. These types of
+ * collection are referred to as a full GC and a zone GC respectively.
+ *
+ * The atoms zone is only collected in a full GC since objects in any zone may
+ * have pointers to atoms, and these are not recorded in the cross compartment
+ * pointer map. Also, the atoms zone is not collected if any thread has an
+ * AutoKeepAtoms instance on the stack, or there are any exclusive threads using
+ * the runtime.
+ *
+ * It is possible for an incremental collection that started out as a full GC to
+ * become a zone GC if new zones are created during the course of the
+ * collection.
+ *
+ * Incremental collection
+ * ----------------------
+ *
+ * For a collection to be carried out incrementally the following conditions
+ * must be met:
+ * - the collection must be run by calling js::GCSlice() rather than js::GC()
+ * - the GC mode must have been set to JSGC_MODE_INCREMENTAL with
+ * JS_SetGCParameter()
+ * - no thread may have an AutoKeepAtoms instance on the stack
+ *
+ * The last condition is an engine-internal mechanism to ensure that incremental
+ * collection is not carried out without the correct barriers being implemented.
+ * For more information see 'Incremental marking' below.
+ *
+ * If the collection is not incremental, all foreground activity happens inside
+ * a single call to GC() or GCSlice(). However the collection is not complete
+ * until the background sweeping activity has finished.
+ *
+ * An incremental collection proceeds as a series of slices, interleaved with
+ * mutator activity, i.e. running JavaScript code. Slices are limited by a time
+ * budget. The slice finishes as soon as possible after the requested time has
+ * passed.
+ *
+ * Collector states
+ * ----------------
+ *
+ * The collector proceeds through the following states, the current state being
+ * held in JSRuntime::gcIncrementalState:
+ *
+ * - MarkRoots - marks the stack and other roots
+ * - Mark - incrementally marks reachable things
+ * - Sweep - sweeps zones in groups and continues marking unswept zones
+ * - Finalize - performs background finalization, concurrent with mutator
+ * - Compact - incrementally compacts by zone
+ * - Decommit - performs background decommit and chunk removal
+ *
+ * The MarkRoots activity always takes place in the first slice. The next two
+ * states can take place over one or more slices.
+ *
+ * In other words an incremental collection proceeds like this:
+ *
+ * Slice 1: MarkRoots: Roots pushed onto the mark stack.
+ * Mark: The mark stack is processed by popping an element,
+ * marking it, and pushing its children.
+ *
+ * ... JS code runs ...
+ *
+ * Slice 2: Mark: More mark stack processing.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n-1: Mark: More mark stack processing.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n: Mark: Mark stack is completely drained.
+ * Sweep: Select first group of zones to sweep and sweep them.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n+1: Sweep: Mark objects in unswept zones that were newly
+ * identified as alive (see below). Then sweep more zone
+ * groups.
+ *
+ * ... JS code runs ...
+ *
+ * Slice n+2: Sweep: Mark objects in unswept zones that were newly
+ * identified as alive. Then sweep more zone groups.
+ *
+ * ... JS code runs ...
+ *
+ * Slice m: Sweep: Sweeping is finished, and background sweeping
+ * started on the helper thread.
+ *
+ * ... JS code runs, remaining sweeping done on background thread ...
+ *
+ * When background sweeping finishes the GC is complete.
+ *
+ * Incremental marking
+ * -------------------
+ *
+ * Incremental collection requires close collaboration with the mutator (i.e.,
+ * JS code) to guarantee correctness.
+ *
+ * - During an incremental GC, if a memory location (except a root) is written
+ * to, then the value it previously held must be marked. Write barriers
+ * ensure this.
+ *
+ * - Any object that is allocated during incremental GC must start out marked.
+ *
+ * - Roots are marked in the first slice and hence don't need write barriers.
+ * Roots are things like the C stack and the VM stack.
+ *
+ * The problem that write barriers solve is that between slices the mutator can
+ * change the object graph. We must ensure that it cannot do this in such a way
+ * that makes us fail to mark a reachable object (marking an unreachable object
+ * is tolerable).
+ *
+ * We use a snapshot-at-the-beginning algorithm to do this. This means that we
+ * promise to mark at least everything that is reachable at the beginning of
+ * collection. To implement it we mark the old contents of every non-root memory
+ * location written to by the mutator while the collection is in progress, using
+ * write barriers. This is described in gc/Barrier.h.
+ *
+ * Incremental sweeping
+ * --------------------
+ *
+ * Sweeping is difficult to do incrementally because object finalizers must be
+ * run at the start of sweeping, before any mutator code runs. The reason is
+ * that some objects use their finalizers to remove themselves from caches. If
+ * mutator code was allowed to run after the start of sweeping, it could observe
+ * the state of the cache and create a new reference to an object that was just
+ * about to be destroyed.
+ *
+ * Sweeping all finalizable objects in one go would introduce long pauses, so
+ * instead sweeping broken up into groups of zones. Zones which are not yet
+ * being swept are still marked, so the issue above does not apply.
+ *
+ * The order of sweeping is restricted by cross compartment pointers - for
+ * example say that object |a| from zone A points to object |b| in zone B and
+ * neither object was marked when we transitioned to the Sweep phase. Imagine we
+ * sweep B first and then return to the mutator. It's possible that the mutator
+ * could cause |a| to become alive through a read barrier (perhaps it was a
+ * shape that was accessed via a shape table). Then we would need to mark |b|,
+ * which |a| points to, but |b| has already been swept.
+ *
+ * So if there is such a pointer then marking of zone B must not finish before
+ * marking of zone A. Pointers which form a cycle between zones therefore
+ * restrict those zones to being swept at the same time, and these are found
+ * using Tarjan's algorithm for finding the strongly connected components of a
+ * graph.
+ *
+ * GC things without finalizers, and things with finalizers that are able to run
+ * in the background, are swept on the background thread. This accounts for most
+ * of the sweeping work.
+ *
+ * Reset
+ * -----
+ *
+ * During incremental collection it is possible, although unlikely, for
+ * conditions to change such that incremental collection is no longer safe. In
+ * this case, the collection is 'reset' by ResetIncrementalGC(). If we are in
+ * the mark state, this just stops marking, but if we have started sweeping
+ * already, we continue until we have swept the current zone group. Following a
+ * reset, a new non-incremental collection is started.
+ *
+ * Compacting GC
+ * -------------
+ *
+ * Compacting GC happens at the end of a major GC as part of the last slice.
+ * There are three parts:
+ *
+ * - Arenas are selected for compaction.
+ * - The contents of those arenas are moved to new arenas.
+ * - All references to moved things are updated.
+ */
+
+#include "jsgcinlines.h"
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MacroForEach.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/Move.h"
+#include "mozilla/ScopeExit.h"
+
+#include <ctype.h>
+#include <string.h>
+#ifndef XP_WIN
+# include <sys/mman.h>
+# include <unistd.h>
+#endif
+
+#include "jsapi.h"
+#include "jsatom.h"
+#include "jscntxt.h"
+#include "jscompartment.h"
+#include "jsfriendapi.h"
+#include "jsobj.h"
+#include "jsprf.h"
+#include "jsscript.h"
+#include "jstypes.h"
+#include "jsutil.h"
+#include "jswatchpoint.h"
+#include "jsweakmap.h"
+#ifdef XP_WIN
+# include "jswin.h"
+#endif
+
+#include "gc/FindSCCs.h"
+#include "gc/GCInternals.h"
+#include "gc/GCTrace.h"
+#include "gc/Marking.h"
+#include "gc/Memory.h"
+#include "gc/Policy.h"
+#include "jit/BaselineJIT.h"
+#include "jit/IonCode.h"
+#include "jit/JitcodeMap.h"
+#include "js/SliceBudget.h"
+#include "proxy/DeadObjectProxy.h"
+#include "vm/Debugger.h"
+#include "vm/ProxyObject.h"
+#include "vm/Shape.h"
+#include "vm/SPSProfiler.h"
+#include "vm/String.h"
+#include "vm/Symbol.h"
+#include "vm/Time.h"
+#include "vm/TraceLogging.h"
+#include "vm/WrapperObject.h"
+
+#include "jsobjinlines.h"
+#include "jsscriptinlines.h"
+
+#include "vm/Stack-inl.h"
+#include "vm/String-inl.h"
+
+using namespace js;
+using namespace js::gc;
+
+using mozilla::ArrayLength;
+using mozilla::Get;
+using mozilla::HashCodeScrambler;
+using mozilla::Maybe;
+using mozilla::Swap;
+
+using JS::AutoGCRooter;
+
+/* Increase the IGC marking slice time if we are in highFrequencyGC mode. */
+static const int IGC_MARK_SLICE_MULTIPLIER = 2;
+
+const AllocKind gc::slotsToThingKind[] = {
+ /* 0 */ AllocKind::OBJECT0, AllocKind::OBJECT2, AllocKind::OBJECT2, AllocKind::OBJECT4,
+ /* 4 */ AllocKind::OBJECT4, AllocKind::OBJECT8, AllocKind::OBJECT8, AllocKind::OBJECT8,
+ /* 8 */ AllocKind::OBJECT8, AllocKind::OBJECT12, AllocKind::OBJECT12, AllocKind::OBJECT12,
+ /* 12 */ AllocKind::OBJECT12, AllocKind::OBJECT16, AllocKind::OBJECT16, AllocKind::OBJECT16,
+ /* 16 */ AllocKind::OBJECT16
+};
+
+static_assert(JS_ARRAY_LENGTH(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT,
+ "We have defined a slot count for each kind.");
+
+#define CHECK_THING_SIZE(allocKind, traceKind, type, sizedType) \
+ static_assert(sizeof(sizedType) >= SortedArenaList::MinThingSize, \
+ #sizedType " is smaller than SortedArenaList::MinThingSize!"); \
+ static_assert(sizeof(sizedType) >= sizeof(FreeSpan), \
+ #sizedType " is smaller than FreeSpan"); \
+ static_assert(sizeof(sizedType) % CellSize == 0, \
+ "Size of " #sizedType " is not a multiple of CellSize");
+FOR_EACH_ALLOCKIND(CHECK_THING_SIZE);
+#undef CHECK_THING_SIZE
+
+const uint32_t Arena::ThingSizes[] = {
+#define EXPAND_THING_SIZE(allocKind, traceKind, type, sizedType) \
+ sizeof(sizedType),
+FOR_EACH_ALLOCKIND(EXPAND_THING_SIZE)
+#undef EXPAND_THING_SIZE
+};
+
+FreeSpan ArenaLists::placeholder;
+
+#undef CHECK_THING_SIZE_INNER
+#undef CHECK_THING_SIZE
+
+#define OFFSET(type) uint32_t(ArenaHeaderSize + (ArenaSize - ArenaHeaderSize) % sizeof(type))
+
+const uint32_t Arena::FirstThingOffsets[] = {
+#define EXPAND_FIRST_THING_OFFSET(allocKind, traceKind, type, sizedType) \
+ OFFSET(sizedType),
+FOR_EACH_ALLOCKIND(EXPAND_FIRST_THING_OFFSET)
+#undef EXPAND_FIRST_THING_OFFSET
+};
+
+#undef OFFSET
+
+#define COUNT(type) uint32_t((ArenaSize - ArenaHeaderSize) / sizeof(type))
+
+const uint32_t Arena::ThingsPerArena[] = {
+#define EXPAND_THINGS_PER_ARENA(allocKind, traceKind, type, sizedType) \
+ COUNT(sizedType),
+FOR_EACH_ALLOCKIND(EXPAND_THINGS_PER_ARENA)
+#undef EXPAND_THINGS_PER_ARENA
+};
+
+#undef COUNT
+
+struct js::gc::FinalizePhase
+{
+ gcstats::Phase statsPhase;
+ AllocKinds kinds;
+};
+
+/*
+ * Finalization order for GC things swept incrementally on the main thrad.
+ */
+static const FinalizePhase IncrementalFinalizePhases[] = {
+ {
+ gcstats::PHASE_SWEEP_STRING, {
+ AllocKind::EXTERNAL_STRING
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_SCRIPT, {
+ AllocKind::SCRIPT
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_JITCODE, {
+ AllocKind::JITCODE
+ }
+ }
+};
+
+/*
+ * Finalization order for GC things swept on the background thread.
+ */
+static const FinalizePhase BackgroundFinalizePhases[] = {
+ {
+ gcstats::PHASE_SWEEP_SCRIPT, {
+ AllocKind::LAZY_SCRIPT
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_OBJECT, {
+ AllocKind::FUNCTION,
+ AllocKind::FUNCTION_EXTENDED,
+ AllocKind::OBJECT0_BACKGROUND,
+ AllocKind::OBJECT2_BACKGROUND,
+ AllocKind::OBJECT4_BACKGROUND,
+ AllocKind::OBJECT8_BACKGROUND,
+ AllocKind::OBJECT12_BACKGROUND,
+ AllocKind::OBJECT16_BACKGROUND
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_SCOPE, {
+ AllocKind::SCOPE
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_STRING, {
+ AllocKind::FAT_INLINE_STRING,
+ AllocKind::STRING,
+ AllocKind::FAT_INLINE_ATOM,
+ AllocKind::ATOM,
+ AllocKind::SYMBOL
+ }
+ },
+ {
+ gcstats::PHASE_SWEEP_SHAPE, {
+ AllocKind::SHAPE,
+ AllocKind::ACCESSOR_SHAPE,
+ AllocKind::BASE_SHAPE,
+ AllocKind::OBJECT_GROUP
+ }
+ }
+};
+
+template<>
+JSObject*
+ArenaCellIterImpl::get<JSObject>() const
+{
+ MOZ_ASSERT(!done());
+ return reinterpret_cast<JSObject*>(getCell());
+}
+
+void
+Arena::unmarkAll()
+{
+ uintptr_t* word = chunk()->bitmap.arenaBits(this);
+ memset(word, 0, ArenaBitmapWords * sizeof(uintptr_t));
+}
+
+/* static */ void
+Arena::staticAsserts()
+{
+ static_assert(size_t(AllocKind::LIMIT) <= 255,
+ "We must be able to fit the allockind into uint8_t.");
+ static_assert(JS_ARRAY_LENGTH(ThingSizes) == size_t(AllocKind::LIMIT),
+ "We haven't defined all thing sizes.");
+ static_assert(JS_ARRAY_LENGTH(FirstThingOffsets) == size_t(AllocKind::LIMIT),
+ "We haven't defined all offsets.");
+ static_assert(JS_ARRAY_LENGTH(ThingsPerArena) == size_t(AllocKind::LIMIT),
+ "We haven't defined all counts.");
+}
+
+template<typename T>
+inline size_t
+Arena::finalize(FreeOp* fop, AllocKind thingKind, size_t thingSize)
+{
+ /* Enforce requirements on size of T. */
+ MOZ_ASSERT(thingSize % CellSize == 0);
+ MOZ_ASSERT(thingSize <= 255);
+
+ MOZ_ASSERT(allocated());
+ MOZ_ASSERT(thingKind == getAllocKind());
+ MOZ_ASSERT(thingSize == getThingSize());
+ MOZ_ASSERT(!hasDelayedMarking);
+ MOZ_ASSERT(!markOverflow);
+ MOZ_ASSERT(!allocatedDuringIncremental);
+
+ uint_fast16_t firstThing = firstThingOffset(thingKind);
+ uint_fast16_t firstThingOrSuccessorOfLastMarkedThing = firstThing;
+ uint_fast16_t lastThing = ArenaSize - thingSize;
+
+ FreeSpan newListHead;
+ FreeSpan* newListTail = &newListHead;
+ size_t nmarked = 0;
+
+ if (MOZ_UNLIKELY(MemProfiler::enabled())) {
+ for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) {
+ T* t = i.get<T>();
+ if (t->asTenured().isMarked())
+ MemProfiler::MarkTenured(reinterpret_cast<void*>(t));
+ }
+ }
+
+ for (ArenaCellIterUnderFinalize i(this); !i.done(); i.next()) {
+ T* t = i.get<T>();
+ if (t->asTenured().isMarked()) {
+ uint_fast16_t thing = uintptr_t(t) & ArenaMask;
+ if (thing != firstThingOrSuccessorOfLastMarkedThing) {
+ // We just finished passing over one or more free things,
+ // so record a new FreeSpan.
+ newListTail->initBounds(firstThingOrSuccessorOfLastMarkedThing,
+ thing - thingSize, this);
+ newListTail = newListTail->nextSpanUnchecked(this);
+ }
+ firstThingOrSuccessorOfLastMarkedThing = thing + thingSize;
+ nmarked++;
+ } else {
+ t->finalize(fop);
+ JS_POISON(t, JS_SWEPT_TENURED_PATTERN, thingSize);
+ TraceTenuredFinalize(t);
+ }
+ }
+
+ if (nmarked == 0) {
+ // Do nothing. The caller will update the arena appropriately.
+ MOZ_ASSERT(newListTail == &newListHead);
+ JS_EXTRA_POISON(data, JS_SWEPT_TENURED_PATTERN, sizeof(data));
+ return nmarked;
+ }
+
+ MOZ_ASSERT(firstThingOrSuccessorOfLastMarkedThing != firstThing);
+ uint_fast16_t lastMarkedThing = firstThingOrSuccessorOfLastMarkedThing - thingSize;
+ if (lastThing == lastMarkedThing) {
+ // If the last thing was marked, we will have already set the bounds of
+ // the final span, and we just need to terminate the list.
+ newListTail->initAsEmpty();
+ } else {
+ // Otherwise, end the list with a span that covers the final stretch of free things.
+ newListTail->initFinal(firstThingOrSuccessorOfLastMarkedThing, lastThing, this);
+ }
+
+ firstFreeSpan = newListHead;
+#ifdef DEBUG
+ size_t nfree = numFreeThings(thingSize);
+ MOZ_ASSERT(nfree + nmarked == thingsPerArena(thingKind));
+#endif
+ return nmarked;
+}
+
+// Finalize arenas from src list, releasing empty arenas if keepArenas wasn't
+// specified and inserting the others into the appropriate destination size
+// bins.
+template<typename T>
+static inline bool
+FinalizeTypedArenas(FreeOp* fop,
+ Arena** src,
+ SortedArenaList& dest,
+ AllocKind thingKind,
+ SliceBudget& budget,
+ ArenaLists::KeepArenasEnum keepArenas)
+{
+ // When operating in the foreground, take the lock at the top.
+ Maybe<AutoLockGC> maybeLock;
+ if (fop->onMainThread())
+ maybeLock.emplace(fop->runtime());
+
+ // During background sweeping free arenas are released later on in
+ // sweepBackgroundThings().
+ MOZ_ASSERT_IF(!fop->onMainThread(), keepArenas == ArenaLists::KEEP_ARENAS);
+
+ size_t thingSize = Arena::thingSize(thingKind);
+ size_t thingsPerArena = Arena::thingsPerArena(thingKind);
+
+ while (Arena* arena = *src) {
+ *src = arena->next;
+ size_t nmarked = arena->finalize<T>(fop, thingKind, thingSize);
+ size_t nfree = thingsPerArena - nmarked;
+
+ if (nmarked)
+ dest.insertAt(arena, nfree);
+ else if (keepArenas == ArenaLists::KEEP_ARENAS)
+ arena->chunk()->recycleArena(arena, dest, thingsPerArena);
+ else
+ fop->runtime()->gc.releaseArena(arena, maybeLock.ref());
+
+ budget.step(thingsPerArena);
+ if (budget.isOverBudget())
+ return false;
+ }
+
+ return true;
+}
+
+/*
+ * Finalize the list. On return, |al|'s cursor points to the first non-empty
+ * arena in the list (which may be null if all arenas are full).
+ */
+static bool
+FinalizeArenas(FreeOp* fop,
+ Arena** src,
+ SortedArenaList& dest,
+ AllocKind thingKind,
+ SliceBudget& budget,
+ ArenaLists::KeepArenasEnum keepArenas)
+{
+ switch (thingKind) {
+#define EXPAND_CASE(allocKind, traceKind, type, sizedType) \
+ case AllocKind::allocKind: \
+ return FinalizeTypedArenas<type>(fop, src, dest, thingKind, budget, keepArenas);
+FOR_EACH_ALLOCKIND(EXPAND_CASE)
+#undef EXPAND_CASE
+
+ default:
+ MOZ_CRASH("Invalid alloc kind");
+ }
+}
+
+Chunk*
+ChunkPool::pop()
+{
+ MOZ_ASSERT(bool(head_) == bool(count_));
+ if (!count_)
+ return nullptr;
+ return remove(head_);
+}
+
+void
+ChunkPool::push(Chunk* chunk)
+{
+ MOZ_ASSERT(!chunk->info.next);
+ MOZ_ASSERT(!chunk->info.prev);
+
+ chunk->info.next = head_;
+ if (head_)
+ head_->info.prev = chunk;
+ head_ = chunk;
+ ++count_;
+
+ MOZ_ASSERT(verify());
+}
+
+Chunk*
+ChunkPool::remove(Chunk* chunk)
+{
+ MOZ_ASSERT(count_ > 0);
+ MOZ_ASSERT(contains(chunk));
+
+ if (head_ == chunk)
+ head_ = chunk->info.next;
+ if (chunk->info.prev)
+ chunk->info.prev->info.next = chunk->info.next;
+ if (chunk->info.next)
+ chunk->info.next->info.prev = chunk->info.prev;
+ chunk->info.next = chunk->info.prev = nullptr;
+ --count_;
+
+ MOZ_ASSERT(verify());
+ return chunk;
+}
+
+#ifdef DEBUG
+bool
+ChunkPool::contains(Chunk* chunk) const
+{
+ verify();
+ for (Chunk* cursor = head_; cursor; cursor = cursor->info.next) {
+ if (cursor == chunk)
+ return true;
+ }
+ return false;
+}
+
+bool
+ChunkPool::verify() const
+{
+ MOZ_ASSERT(bool(head_) == bool(count_));
+ uint32_t count = 0;
+ for (Chunk* cursor = head_; cursor; cursor = cursor->info.next, ++count) {
+ MOZ_ASSERT_IF(cursor->info.prev, cursor->info.prev->info.next == cursor);
+ MOZ_ASSERT_IF(cursor->info.next, cursor->info.next->info.prev == cursor);
+ }
+ MOZ_ASSERT(count_ == count);
+ return true;
+}
+#endif
+
+void
+ChunkPool::Iter::next()
+{
+ MOZ_ASSERT(!done());
+ current_ = current_->info.next;
+}
+
+ChunkPool
+GCRuntime::expireEmptyChunkPool(const AutoLockGC& lock)
+{
+ MOZ_ASSERT(emptyChunks(lock).verify());
+ MOZ_ASSERT(tunables.minEmptyChunkCount(lock) <= tunables.maxEmptyChunkCount());
+
+ ChunkPool expired;
+ while (emptyChunks(lock).count() > tunables.minEmptyChunkCount(lock)) {
+ Chunk* chunk = emptyChunks(lock).pop();
+ prepareToFreeChunk(chunk->info);
+ expired.push(chunk);
+ }
+
+ MOZ_ASSERT(expired.verify());
+ MOZ_ASSERT(emptyChunks(lock).verify());
+ MOZ_ASSERT(emptyChunks(lock).count() <= tunables.maxEmptyChunkCount());
+ MOZ_ASSERT(emptyChunks(lock).count() <= tunables.minEmptyChunkCount(lock));
+ return expired;
+}
+
+static void
+FreeChunkPool(JSRuntime* rt, ChunkPool& pool)
+{
+ for (ChunkPool::Iter iter(pool); !iter.done();) {
+ Chunk* chunk = iter.get();
+ iter.next();
+ pool.remove(chunk);
+ MOZ_ASSERT(!chunk->info.numArenasFreeCommitted);
+ UnmapPages(static_cast<void*>(chunk), ChunkSize);
+ }
+ MOZ_ASSERT(pool.count() == 0);
+}
+
+void
+GCRuntime::freeEmptyChunks(JSRuntime* rt, const AutoLockGC& lock)
+{
+ FreeChunkPool(rt, emptyChunks(lock));
+}
+
+inline void
+GCRuntime::prepareToFreeChunk(ChunkInfo& info)
+{
+ MOZ_ASSERT(numArenasFreeCommitted >= info.numArenasFreeCommitted);
+ numArenasFreeCommitted -= info.numArenasFreeCommitted;
+ stats.count(gcstats::STAT_DESTROY_CHUNK);
+#ifdef DEBUG
+ /*
+ * Let FreeChunkPool detect a missing prepareToFreeChunk call before it
+ * frees chunk.
+ */
+ info.numArenasFreeCommitted = 0;
+#endif
+}
+
+inline void
+GCRuntime::updateOnArenaFree(const ChunkInfo& info)
+{
+ ++numArenasFreeCommitted;
+}
+
+void
+Chunk::addArenaToFreeList(JSRuntime* rt, Arena* arena)
+{
+ MOZ_ASSERT(!arena->allocated());
+ arena->next = info.freeArenasHead;
+ info.freeArenasHead = arena;
+ ++info.numArenasFreeCommitted;
+ ++info.numArenasFree;
+ rt->gc.updateOnArenaFree(info);
+}
+
+void
+Chunk::addArenaToDecommittedList(JSRuntime* rt, const Arena* arena)
+{
+ ++info.numArenasFree;
+ decommittedArenas.set(Chunk::arenaIndex(arena->address()));
+}
+
+void
+Chunk::recycleArena(Arena* arena, SortedArenaList& dest, size_t thingsPerArena)
+{
+ arena->setAsFullyUnused();
+ dest.insertAt(arena, thingsPerArena);
+}
+
+void
+Chunk::releaseArena(JSRuntime* rt, Arena* arena, const AutoLockGC& lock)
+{
+ MOZ_ASSERT(arena->allocated());
+ MOZ_ASSERT(!arena->hasDelayedMarking);
+
+ arena->setAsNotAllocated();
+ addArenaToFreeList(rt, arena);
+ updateChunkListAfterFree(rt, lock);
+}
+
+bool
+Chunk::decommitOneFreeArena(JSRuntime* rt, AutoLockGC& lock)
+{
+ MOZ_ASSERT(info.numArenasFreeCommitted > 0);
+ Arena* arena = fetchNextFreeArena(rt);
+ updateChunkListAfterAlloc(rt, lock);
+
+ bool ok;
+ {
+ AutoUnlockGC unlock(lock);
+ ok = MarkPagesUnused(arena, ArenaSize);
+ }
+
+ if (ok)
+ addArenaToDecommittedList(rt, arena);
+ else
+ addArenaToFreeList(rt, arena);
+ updateChunkListAfterFree(rt, lock);
+
+ return ok;
+}
+
+void
+Chunk::decommitAllArenasWithoutUnlocking(const AutoLockGC& lock)
+{
+ for (size_t i = 0; i < ArenasPerChunk; ++i) {
+ if (decommittedArenas.get(i) || arenas[i].allocated())
+ continue;
+
+ if (MarkPagesUnused(&arenas[i], ArenaSize)) {
+ info.numArenasFreeCommitted--;
+ decommittedArenas.set(i);
+ }
+ }
+}
+
+void
+Chunk::updateChunkListAfterAlloc(JSRuntime* rt, const AutoLockGC& lock)
+{
+ if (MOZ_UNLIKELY(!hasAvailableArenas())) {
+ rt->gc.availableChunks(lock).remove(this);
+ rt->gc.fullChunks(lock).push(this);
+ }
+}
+
+void
+Chunk::updateChunkListAfterFree(JSRuntime* rt, const AutoLockGC& lock)
+{
+ if (info.numArenasFree == 1) {
+ rt->gc.fullChunks(lock).remove(this);
+ rt->gc.availableChunks(lock).push(this);
+ } else if (!unused()) {
+ MOZ_ASSERT(!rt->gc.fullChunks(lock).contains(this));
+ MOZ_ASSERT(rt->gc.availableChunks(lock).contains(this));
+ MOZ_ASSERT(!rt->gc.emptyChunks(lock).contains(this));
+ } else {
+ MOZ_ASSERT(unused());
+ rt->gc.availableChunks(lock).remove(this);
+ decommitAllArenas(rt);
+ MOZ_ASSERT(info.numArenasFreeCommitted == 0);
+ rt->gc.recycleChunk(this, lock);
+ }
+}
+
+void
+GCRuntime::releaseArena(Arena* arena, const AutoLockGC& lock)
+{
+ arena->zone->usage.removeGCArena();
+ if (isBackgroundSweeping())
+ arena->zone->threshold.updateForRemovedArena(tunables);
+ return arena->chunk()->releaseArena(rt, arena, lock);
+}
+
+GCRuntime::GCRuntime(JSRuntime* rt) :
+ rt(rt),
+ systemZone(nullptr),
+ nursery(rt),
+ storeBuffer(rt, nursery),
+ stats(rt),
+ marker(rt),
+ usage(nullptr),
+ mMemProfiler(rt),
+ maxMallocBytes(0),
+ nextCellUniqueId_(LargestTaggedNullCellPointer + 1), // Ensure disjoint from null tagged pointers.
+ numArenasFreeCommitted(0),
+ verifyPreData(nullptr),
+ chunkAllocationSinceLastGC(false),
+ lastGCTime(PRMJ_Now()),
+ mode(JSGC_MODE_INCREMENTAL),
+ numActiveZoneIters(0),
+ cleanUpEverything(false),
+ grayBufferState(GCRuntime::GrayBufferState::Unused),
+ majorGCTriggerReason(JS::gcreason::NO_REASON),
+ minorGCTriggerReason(JS::gcreason::NO_REASON),
+ fullGCForAtomsRequested_(false),
+ minorGCNumber(0),
+ majorGCNumber(0),
+ jitReleaseNumber(0),
+ number(0),
+ startNumber(0),
+ isFull(false),
+#ifdef DEBUG
+ disableStrictProxyCheckingCount(0),
+#endif
+ incrementalState(gc::State::NotActive),
+ lastMarkSlice(false),
+ sweepOnBackgroundThread(false),
+ blocksToFreeAfterSweeping(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+ blocksToFreeAfterMinorGC(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+ zoneGroupIndex(0),
+ zoneGroups(nullptr),
+ currentZoneGroup(nullptr),
+ sweepZone(nullptr),
+ sweepKind(AllocKind::FIRST),
+ abortSweepAfterCurrentGroup(false),
+ arenasAllocatedDuringSweep(nullptr),
+ startedCompacting(false),
+ relocatedArenasToRelease(nullptr),
+#ifdef JS_GC_ZEAL
+ markingValidator(nullptr),
+#endif
+ interFrameGC(false),
+ defaultTimeBudget_(SliceBudget::UnlimitedTimeBudget),
+ incrementalAllowed(true),
+ generationalDisabled(0),
+ compactingEnabled(true),
+ compactingDisabledCount(0),
+ manipulatingDeadZones(false),
+ objectsMarkedInDeadZones(0),
+ poked(false),
+#ifdef JS_GC_ZEAL
+ zealModeBits(0),
+ zealFrequency(0),
+ nextScheduled(0),
+ deterministicOnly(false),
+ incrementalLimit(0),
+#endif
+ fullCompartmentChecks(false),
+ mallocBytesUntilGC(0),
+ mallocGCTriggered(false),
+ alwaysPreserveCode(false),
+ inUnsafeRegion(0),
+#ifdef DEBUG
+ noGCOrAllocationCheck(0),
+ noNurseryAllocationCheck(0),
+ arenasEmptyAtShutdown(true),
+#endif
+ lock(mutexid::GCLock),
+ allocTask(rt, emptyChunks_),
+ decommitTask(rt),
+ helperState(rt)
+{
+ setGCMode(JSGC_MODE_GLOBAL);
+}
+
+#ifdef JS_GC_ZEAL
+
+void
+GCRuntime::getZealBits(uint32_t* zealBits, uint32_t* frequency, uint32_t* scheduled)
+{
+ *zealBits = zealModeBits;
+ *frequency = zealFrequency;
+ *scheduled = nextScheduled;
+}
+
+const char* gc::ZealModeHelpText =
+ " Specifies how zealous the garbage collector should be. Some of these modes can\n"
+ " be set simultaneously, by passing multiple level options, e.g. \"2;4\" will activate\n"
+ " both modes 2 and 4. Modes can be specified by name or number.\n"
+ " \n"
+ " Values:\n"
+ " 0: (None) Normal amount of collection (resets all modes)\n"
+ " 1: (Poke) Collect when roots are added or removed\n"
+ " 2: (Alloc) Collect when every N allocations (default: 100)\n"
+ " 3: (FrameGC) Collect when the window paints (browser only)\n"
+ " 4: (VerifierPre) Verify pre write barriers between instructions\n"
+ " 5: (FrameVerifierPre) Verify pre write barriers between paints\n"
+ " 6: (StackRooting) Verify stack rooting\n"
+ " 7: (GenerationalGC) Collect the nursery every N nursery allocations\n"
+ " 8: (IncrementalRootsThenFinish) Incremental GC in two slices: 1) mark roots 2) finish collection\n"
+ " 9: (IncrementalMarkAllThenFinish) Incremental GC in two slices: 1) mark all 2) new marking and finish\n"
+ " 10: (IncrementalMultipleSlices) Incremental GC in multiple slices\n"
+ " 11: (IncrementalMarkingValidator) Verify incremental marking\n"
+ " 12: (ElementsBarrier) Always use the individual element post-write barrier, regardless of elements size\n"
+ " 13: (CheckHashTablesOnMinorGC) Check internal hashtables on minor GC\n"
+ " 14: (Compact) Perform a shrinking collection every N allocations\n"
+ " 15: (CheckHeapAfterGC) Walk the heap to check its integrity after every GC\n"
+ " 16: (CheckNursery) Check nursery integrity on minor GC\n";
+
+void
+GCRuntime::setZeal(uint8_t zeal, uint32_t frequency)
+{
+ MOZ_ASSERT(zeal <= unsigned(ZealMode::Limit));
+
+ if (verifyPreData)
+ VerifyBarriers(rt, PreBarrierVerifier);
+
+ if (zeal == 0 && hasZealMode(ZealMode::GenerationalGC)) {
+ evictNursery(JS::gcreason::DEBUG_GC);
+ nursery.leaveZealMode();
+ }
+
+ ZealMode zealMode = ZealMode(zeal);
+ if (zealMode == ZealMode::GenerationalGC)
+ nursery.enterZealMode();
+
+ // Zeal modes 8-10 are mutually exclusive. If we're setting one of those,
+ // we first reset all of them.
+ if (zealMode >= ZealMode::IncrementalRootsThenFinish &&
+ zealMode <= ZealMode::IncrementalMultipleSlices)
+ {
+ clearZealMode(ZealMode::IncrementalRootsThenFinish);
+ clearZealMode(ZealMode::IncrementalMarkAllThenFinish);
+ clearZealMode(ZealMode::IncrementalMultipleSlices);
+ }
+
+ bool schedule = zealMode >= ZealMode::Alloc;
+ if (zeal != 0)
+ zealModeBits |= 1 << unsigned(zeal);
+ else
+ zealModeBits = 0;
+ zealFrequency = frequency;
+ nextScheduled = schedule ? frequency : 0;
+}
+
+void
+GCRuntime::setNextScheduled(uint32_t count)
+{
+ nextScheduled = count;
+}
+
+bool
+GCRuntime::parseAndSetZeal(const char* str)
+{
+ int frequency = -1;
+ bool foundFrequency = false;
+ mozilla::Vector<int, 0, SystemAllocPolicy> zeals;
+
+ static const struct {
+ const char* const zealMode;
+ size_t length;
+ uint32_t zeal;
+ } zealModes[] = {
+#define ZEAL_MODE(name, value) {#name, sizeof(#name) - 1, value},
+ JS_FOR_EACH_ZEAL_MODE(ZEAL_MODE)
+#undef ZEAL_MODE
+ {"None", 4, 0}
+ };
+
+ do {
+ int zeal = -1;
+
+ const char* p = nullptr;
+ if (isdigit(str[0])) {
+ zeal = atoi(str);
+
+ size_t offset = strspn(str, "0123456789");
+ p = str + offset;
+ } else {
+ for (auto z : zealModes) {
+ if (!strncmp(str, z.zealMode, z.length)) {
+ zeal = z.zeal;
+ p = str + z.length;
+ break;
+ }
+ }
+ }
+ if (p) {
+ if (!*p || *p == ';') {
+ frequency = JS_DEFAULT_ZEAL_FREQ;
+ } else if (*p == ',') {
+ frequency = atoi(p + 1);
+ foundFrequency = true;
+ }
+ }
+
+ if (zeal < 0 || zeal > int(ZealMode::Limit) || frequency <= 0) {
+ fprintf(stderr, "Format: JS_GC_ZEAL=level(;level)*[,N]\n");
+ fputs(ZealModeHelpText, stderr);
+ return false;
+ }
+
+ if (!zeals.emplaceBack(zeal)) {
+ return false;
+ }
+ } while (!foundFrequency &&
+ (str = strchr(str, ';')) != nullptr &&
+ str++);
+
+ for (auto z : zeals)
+ setZeal(z, frequency);
+ return true;
+}
+
+#endif
+
+/*
+ * Lifetime in number of major GCs for type sets attached to scripts containing
+ * observed types.
+ */
+static const uint64_t JIT_SCRIPT_RELEASE_TYPES_PERIOD = 20;
+
+bool
+GCRuntime::init(uint32_t maxbytes, uint32_t maxNurseryBytes)
+{
+ InitMemorySubsystem();
+
+ if (!rootsHash.init(256))
+ return false;
+
+ {
+ AutoLockGC lock(rt);
+
+ /*
+ * Separate gcMaxMallocBytes from gcMaxBytes but initialize to maxbytes
+ * for default backward API compatibility.
+ */
+ MOZ_ALWAYS_TRUE(tunables.setParameter(JSGC_MAX_BYTES, maxbytes, lock));
+ setMaxMallocBytes(maxbytes);
+
+ const char* size = getenv("JSGC_MARK_STACK_LIMIT");
+ if (size)
+ setMarkStackLimit(atoi(size), lock);
+
+ jitReleaseNumber = majorGCNumber + JIT_SCRIPT_RELEASE_TYPES_PERIOD;
+
+ if (!nursery.init(maxNurseryBytes, lock))
+ return false;
+
+ if (!nursery.isEnabled()) {
+ MOZ_ASSERT(nursery.nurserySize() == 0);
+ ++rt->gc.generationalDisabled;
+ } else {
+ MOZ_ASSERT(nursery.nurserySize() > 0);
+ }
+ }
+
+#ifdef JS_GC_ZEAL
+ const char* zealSpec = getenv("JS_GC_ZEAL");
+ if (zealSpec && zealSpec[0] && !parseAndSetZeal(zealSpec))
+ return false;
+#endif
+
+ if (!InitTrace(*this))
+ return false;
+
+ if (!marker.init(mode))
+ return false;
+
+ return true;
+}
+
+void
+GCRuntime::finish()
+{
+ /* Wait for the nursery sweeping to end. */
+ if (nursery.isEnabled())
+ nursery.waitBackgroundFreeEnd();
+
+ /*
+ * Wait until the background finalization and allocation stops and the
+ * helper thread shuts down before we forcefully release any remaining GC
+ * memory.
+ */
+ helperState.finish();
+ allocTask.cancel(GCParallelTask::CancelAndWait);
+ decommitTask.cancel(GCParallelTask::CancelAndWait);
+
+#ifdef JS_GC_ZEAL
+ /* Free memory associated with GC verification. */
+ finishVerifier();
+#endif
+
+ /* Delete all remaining zones. */
+ if (rt->gcInitialized) {
+ AutoSetThreadIsSweeping threadIsSweeping;
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+ js_delete(comp.get());
+ js_delete(zone.get());
+ }
+ }
+
+ zones.clear();
+
+ FreeChunkPool(rt, fullChunks_);
+ FreeChunkPool(rt, availableChunks_);
+ FreeChunkPool(rt, emptyChunks_);
+
+ FinishTrace();
+
+ nursery.printTotalProfileTimes();
+ stats.printTotalProfileTimes();
+}
+
+bool
+GCRuntime::setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock)
+{
+ switch (key) {
+ case JSGC_MAX_MALLOC_BYTES:
+ setMaxMallocBytes(value);
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ zone->setGCMaxMallocBytes(maxMallocBytesAllocated() * 0.9);
+ break;
+ case JSGC_SLICE_TIME_BUDGET:
+ defaultTimeBudget_ = value ? value : SliceBudget::UnlimitedTimeBudget;
+ break;
+ case JSGC_MARK_STACK_LIMIT:
+ if (value == 0)
+ return false;
+ setMarkStackLimit(value, lock);
+ break;
+ case JSGC_MODE:
+ if (mode != JSGC_MODE_GLOBAL &&
+ mode != JSGC_MODE_ZONE &&
+ mode != JSGC_MODE_INCREMENTAL)
+ {
+ return false;
+ }
+ mode = JSGCMode(value);
+ break;
+ case JSGC_COMPACTING_ENABLED:
+ compactingEnabled = value != 0;
+ break;
+ default:
+ if (!tunables.setParameter(key, value, lock))
+ return false;
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ zone->threshold.updateAfterGC(zone->usage.gcBytes(), GC_NORMAL, tunables,
+ schedulingState, lock);
+ }
+ }
+
+ return true;
+}
+
+bool
+GCSchedulingTunables::setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock)
+{
+ // Limit heap growth factor to one hundred times size of current heap.
+ const double MaxHeapGrowthFactor = 100;
+
+ switch(key) {
+ case JSGC_MAX_BYTES:
+ gcMaxBytes_ = value;
+ break;
+ case JSGC_HIGH_FREQUENCY_TIME_LIMIT:
+ highFrequencyThresholdUsec_ = value * PRMJ_USEC_PER_MSEC;
+ break;
+ case JSGC_HIGH_FREQUENCY_LOW_LIMIT: {
+ uint64_t newLimit = (uint64_t)value * 1024 * 1024;
+ if (newLimit == UINT64_MAX)
+ return false;
+ highFrequencyLowLimitBytes_ = newLimit;
+ if (highFrequencyLowLimitBytes_ >= highFrequencyHighLimitBytes_)
+ highFrequencyHighLimitBytes_ = highFrequencyLowLimitBytes_ + 1;
+ MOZ_ASSERT(highFrequencyHighLimitBytes_ > highFrequencyLowLimitBytes_);
+ break;
+ }
+ case JSGC_HIGH_FREQUENCY_HIGH_LIMIT: {
+ uint64_t newLimit = (uint64_t)value * 1024 * 1024;
+ if (newLimit == 0)
+ return false;
+ highFrequencyHighLimitBytes_ = newLimit;
+ if (highFrequencyHighLimitBytes_ <= highFrequencyLowLimitBytes_)
+ highFrequencyLowLimitBytes_ = highFrequencyHighLimitBytes_ - 1;
+ MOZ_ASSERT(highFrequencyHighLimitBytes_ > highFrequencyLowLimitBytes_);
+ break;
+ }
+ case JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MAX: {
+ double newGrowth = value / 100.0;
+ if (newGrowth <= 0.85 || newGrowth > MaxHeapGrowthFactor)
+ return false;
+ highFrequencyHeapGrowthMax_ = newGrowth;
+ MOZ_ASSERT(highFrequencyHeapGrowthMax_ / 0.85 > 1.0);
+ break;
+ }
+ case JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MIN: {
+ double newGrowth = value / 100.0;
+ if (newGrowth <= 0.85 || newGrowth > MaxHeapGrowthFactor)
+ return false;
+ highFrequencyHeapGrowthMin_ = newGrowth;
+ MOZ_ASSERT(highFrequencyHeapGrowthMin_ / 0.85 > 1.0);
+ break;
+ }
+ case JSGC_LOW_FREQUENCY_HEAP_GROWTH: {
+ double newGrowth = value / 100.0;
+ if (newGrowth <= 0.9 || newGrowth > MaxHeapGrowthFactor)
+ return false;
+ lowFrequencyHeapGrowth_ = newGrowth;
+ MOZ_ASSERT(lowFrequencyHeapGrowth_ / 0.9 > 1.0);
+ break;
+ }
+ case JSGC_DYNAMIC_HEAP_GROWTH:
+ dynamicHeapGrowthEnabled_ = value != 0;
+ break;
+ case JSGC_DYNAMIC_MARK_SLICE:
+ dynamicMarkSliceEnabled_ = value != 0;
+ break;
+ case JSGC_ALLOCATION_THRESHOLD:
+ gcZoneAllocThresholdBase_ = value * 1024 * 1024;
+ break;
+ case JSGC_MIN_EMPTY_CHUNK_COUNT:
+ minEmptyChunkCount_ = value;
+ if (minEmptyChunkCount_ > maxEmptyChunkCount_)
+ maxEmptyChunkCount_ = minEmptyChunkCount_;
+ MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_);
+ break;
+ case JSGC_MAX_EMPTY_CHUNK_COUNT:
+ maxEmptyChunkCount_ = value;
+ if (minEmptyChunkCount_ > maxEmptyChunkCount_)
+ minEmptyChunkCount_ = maxEmptyChunkCount_;
+ MOZ_ASSERT(maxEmptyChunkCount_ >= minEmptyChunkCount_);
+ break;
+ case JSGC_REFRESH_FRAME_SLICES_ENABLED:
+ refreshFrameSlicesEnabled_ = value != 0;
+ break;
+ default:
+ MOZ_CRASH("Unknown GC parameter.");
+ }
+
+ return true;
+}
+
+uint32_t
+GCRuntime::getParameter(JSGCParamKey key, const AutoLockGC& lock)
+{
+ switch (key) {
+ case JSGC_MAX_BYTES:
+ return uint32_t(tunables.gcMaxBytes());
+ case JSGC_MAX_MALLOC_BYTES:
+ return maxMallocBytes;
+ case JSGC_BYTES:
+ return uint32_t(usage.gcBytes());
+ case JSGC_MODE:
+ return uint32_t(mode);
+ case JSGC_UNUSED_CHUNKS:
+ return uint32_t(emptyChunks(lock).count());
+ case JSGC_TOTAL_CHUNKS:
+ return uint32_t(fullChunks(lock).count() +
+ availableChunks(lock).count() +
+ emptyChunks(lock).count());
+ case JSGC_SLICE_TIME_BUDGET:
+ if (defaultTimeBudget_ == SliceBudget::UnlimitedTimeBudget) {
+ return 0;
+ } else {
+ MOZ_RELEASE_ASSERT(defaultTimeBudget_ >= 0);
+ MOZ_RELEASE_ASSERT(defaultTimeBudget_ <= UINT32_MAX);
+ return uint32_t(defaultTimeBudget_);
+ }
+ case JSGC_MARK_STACK_LIMIT:
+ return marker.maxCapacity();
+ case JSGC_HIGH_FREQUENCY_TIME_LIMIT:
+ return tunables.highFrequencyThresholdUsec() / PRMJ_USEC_PER_MSEC;
+ case JSGC_HIGH_FREQUENCY_LOW_LIMIT:
+ return tunables.highFrequencyLowLimitBytes() / 1024 / 1024;
+ case JSGC_HIGH_FREQUENCY_HIGH_LIMIT:
+ return tunables.highFrequencyHighLimitBytes() / 1024 / 1024;
+ case JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MAX:
+ return uint32_t(tunables.highFrequencyHeapGrowthMax() * 100);
+ case JSGC_HIGH_FREQUENCY_HEAP_GROWTH_MIN:
+ return uint32_t(tunables.highFrequencyHeapGrowthMin() * 100);
+ case JSGC_LOW_FREQUENCY_HEAP_GROWTH:
+ return uint32_t(tunables.lowFrequencyHeapGrowth() * 100);
+ case JSGC_DYNAMIC_HEAP_GROWTH:
+ return tunables.isDynamicHeapGrowthEnabled();
+ case JSGC_DYNAMIC_MARK_SLICE:
+ return tunables.isDynamicMarkSliceEnabled();
+ case JSGC_ALLOCATION_THRESHOLD:
+ return tunables.gcZoneAllocThresholdBase() / 1024 / 1024;
+ case JSGC_MIN_EMPTY_CHUNK_COUNT:
+ return tunables.minEmptyChunkCount(lock);
+ case JSGC_MAX_EMPTY_CHUNK_COUNT:
+ return tunables.maxEmptyChunkCount();
+ case JSGC_COMPACTING_ENABLED:
+ return compactingEnabled;
+ case JSGC_REFRESH_FRAME_SLICES_ENABLED:
+ return tunables.areRefreshFrameSlicesEnabled();
+ default:
+ MOZ_ASSERT(key == JSGC_NUMBER);
+ return uint32_t(number);
+ }
+}
+
+void
+GCRuntime::setMarkStackLimit(size_t limit, AutoLockGC& lock)
+{
+ MOZ_ASSERT(!rt->isHeapBusy());
+ AutoUnlockGC unlock(lock);
+ AutoStopVerifyingBarriers pauseVerification(rt, false);
+ marker.setMaxCapacity(limit);
+}
+
+bool
+GCRuntime::addBlackRootsTracer(JSTraceDataOp traceOp, void* data)
+{
+ AssertHeapIsIdle(rt);
+ return !!blackRootTracers.append(Callback<JSTraceDataOp>(traceOp, data));
+}
+
+void
+GCRuntime::removeBlackRootsTracer(JSTraceDataOp traceOp, void* data)
+{
+ // Can be called from finalizers
+ for (size_t i = 0; i < blackRootTracers.length(); i++) {
+ Callback<JSTraceDataOp>* e = &blackRootTracers[i];
+ if (e->op == traceOp && e->data == data) {
+ blackRootTracers.erase(e);
+ }
+ }
+}
+
+void
+GCRuntime::setGrayRootsTracer(JSTraceDataOp traceOp, void* data)
+{
+ AssertHeapIsIdle(rt);
+ grayRootTracer.op = traceOp;
+ grayRootTracer.data = data;
+}
+
+void
+GCRuntime::setGCCallback(JSGCCallback callback, void* data)
+{
+ gcCallback.op = callback;
+ gcCallback.data = data;
+}
+
+void
+GCRuntime::callGCCallback(JSGCStatus status) const
+{
+ if (gcCallback.op)
+ gcCallback.op(rt->contextFromMainThread(), status, gcCallback.data);
+}
+
+void
+GCRuntime::setObjectsTenuredCallback(JSObjectsTenuredCallback callback,
+ void* data)
+{
+ tenuredCallback.op = callback;
+ tenuredCallback.data = data;
+}
+
+void
+GCRuntime::callObjectsTenuredCallback()
+{
+ if (tenuredCallback.op)
+ tenuredCallback.op(rt->contextFromMainThread(), tenuredCallback.data);
+}
+
+namespace {
+
+class AutoNotifyGCActivity {
+ public:
+ explicit AutoNotifyGCActivity(GCRuntime& gc) : gc_(gc) {
+ if (!gc_.isIncrementalGCInProgress()) {
+ gcstats::AutoPhase ap(gc_.stats, gcstats::PHASE_GC_BEGIN);
+ gc_.callGCCallback(JSGC_BEGIN);
+ }
+ }
+ ~AutoNotifyGCActivity() {
+ if (!gc_.isIncrementalGCInProgress()) {
+ gcstats::AutoPhase ap(gc_.stats, gcstats::PHASE_GC_END);
+ gc_.callGCCallback(JSGC_END);
+ }
+ }
+
+ private:
+ GCRuntime& gc_;
+};
+
+} // (anon)
+
+bool
+GCRuntime::addFinalizeCallback(JSFinalizeCallback callback, void* data)
+{
+ return finalizeCallbacks.append(Callback<JSFinalizeCallback>(callback, data));
+}
+
+void
+GCRuntime::removeFinalizeCallback(JSFinalizeCallback callback)
+{
+ for (Callback<JSFinalizeCallback>* p = finalizeCallbacks.begin();
+ p < finalizeCallbacks.end(); p++)
+ {
+ if (p->op == callback) {
+ finalizeCallbacks.erase(p);
+ break;
+ }
+ }
+}
+
+void
+GCRuntime::callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const
+{
+ for (auto& p : finalizeCallbacks)
+ p.op(fop, status, !isFull, p.data);
+}
+
+bool
+GCRuntime::addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, void* data)
+{
+ return updateWeakPointerZoneGroupCallbacks.append(
+ Callback<JSWeakPointerZoneGroupCallback>(callback, data));
+}
+
+void
+GCRuntime::removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback)
+{
+ for (auto& p : updateWeakPointerZoneGroupCallbacks) {
+ if (p.op == callback) {
+ updateWeakPointerZoneGroupCallbacks.erase(&p);
+ break;
+ }
+ }
+}
+
+void
+GCRuntime::callWeakPointerZoneGroupCallbacks() const
+{
+ for (auto const& p : updateWeakPointerZoneGroupCallbacks)
+ p.op(rt->contextFromMainThread(), p.data);
+}
+
+bool
+GCRuntime::addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback, void* data)
+{
+ return updateWeakPointerCompartmentCallbacks.append(
+ Callback<JSWeakPointerCompartmentCallback>(callback, data));
+}
+
+void
+GCRuntime::removeWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback)
+{
+ for (auto& p : updateWeakPointerCompartmentCallbacks) {
+ if (p.op == callback) {
+ updateWeakPointerCompartmentCallbacks.erase(&p);
+ break;
+ }
+ }
+}
+
+void
+GCRuntime::callWeakPointerCompartmentCallbacks(JSCompartment* comp) const
+{
+ for (auto const& p : updateWeakPointerCompartmentCallbacks)
+ p.op(rt->contextFromMainThread(), comp, p.data);
+}
+
+JS::GCSliceCallback
+GCRuntime::setSliceCallback(JS::GCSliceCallback callback) {
+ return stats.setSliceCallback(callback);
+}
+
+JS::GCNurseryCollectionCallback
+GCRuntime::setNurseryCollectionCallback(JS::GCNurseryCollectionCallback callback) {
+ return stats.setNurseryCollectionCallback(callback);
+}
+
+JS::DoCycleCollectionCallback
+GCRuntime::setDoCycleCollectionCallback(JS::DoCycleCollectionCallback callback)
+{
+ auto prior = gcDoCycleCollectionCallback;
+ gcDoCycleCollectionCallback = Callback<JS::DoCycleCollectionCallback>(callback, nullptr);
+ return prior.op;
+}
+
+void
+GCRuntime::callDoCycleCollectionCallback(JSContext* cx)
+{
+ if (gcDoCycleCollectionCallback.op)
+ gcDoCycleCollectionCallback.op(cx);
+}
+
+bool
+GCRuntime::addRoot(Value* vp, const char* name)
+{
+ /*
+ * Sometimes Firefox will hold weak references to objects and then convert
+ * them to strong references by calling AddRoot (e.g., via PreserveWrapper,
+ * or ModifyBusyCount in workers). We need a read barrier to cover these
+ * cases.
+ */
+ if (isIncrementalGCInProgress())
+ GCPtrValue::writeBarrierPre(*vp);
+
+ return rootsHash.put(vp, name);
+}
+
+void
+GCRuntime::removeRoot(Value* vp)
+{
+ rootsHash.remove(vp);
+ poke();
+}
+
+extern JS_FRIEND_API(bool)
+js::AddRawValueRoot(JSContext* cx, Value* vp, const char* name)
+{
+ MOZ_ASSERT(vp);
+ MOZ_ASSERT(name);
+ bool ok = cx->runtime()->gc.addRoot(vp, name);
+ if (!ok)
+ JS_ReportOutOfMemory(cx);
+ return ok;
+}
+
+extern JS_FRIEND_API(void)
+js::RemoveRawValueRoot(JSContext* cx, Value* vp)
+{
+ cx->runtime()->gc.removeRoot(vp);
+}
+
+void
+GCRuntime::setMaxMallocBytes(size_t value)
+{
+ /*
+ * For compatibility treat any value that exceeds PTRDIFF_T_MAX to
+ * mean that value.
+ */
+ maxMallocBytes = (ptrdiff_t(value) >= 0) ? value : size_t(-1) >> 1;
+ resetMallocBytes();
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ zone->setGCMaxMallocBytes(value);
+}
+
+void
+GCRuntime::resetMallocBytes()
+{
+ mallocBytesUntilGC = ptrdiff_t(maxMallocBytes);
+ mallocGCTriggered = false;
+}
+
+void
+GCRuntime::updateMallocCounter(JS::Zone* zone, size_t nbytes)
+{
+ mallocBytesUntilGC -= ptrdiff_t(nbytes);
+ if (MOZ_UNLIKELY(isTooMuchMalloc()))
+ onTooMuchMalloc();
+ else if (zone)
+ zone->updateMallocCounter(nbytes);
+}
+
+void
+GCRuntime::onTooMuchMalloc()
+{
+ if (!mallocGCTriggered)
+ mallocGCTriggered = triggerGC(JS::gcreason::TOO_MUCH_MALLOC);
+}
+
+double
+ZoneHeapThreshold::allocTrigger(bool highFrequencyGC) const
+{
+ return (highFrequencyGC ? 0.85 : 0.9) * gcTriggerBytes();
+}
+
+/* static */ double
+ZoneHeapThreshold::computeZoneHeapGrowthFactorForHeapSize(size_t lastBytes,
+ const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state)
+{
+ if (!tunables.isDynamicHeapGrowthEnabled())
+ return 3.0;
+
+ // For small zones, our collection heuristics do not matter much: favor
+ // something simple in this case.
+ if (lastBytes < 1 * 1024 * 1024)
+ return tunables.lowFrequencyHeapGrowth();
+
+ // If GC's are not triggering in rapid succession, use a lower threshold so
+ // that we will collect garbage sooner.
+ if (!state.inHighFrequencyGCMode())
+ return tunables.lowFrequencyHeapGrowth();
+
+ // The heap growth factor depends on the heap size after a GC and the GC
+ // frequency. For low frequency GCs (more than 1sec between GCs) we let
+ // the heap grow to 150%. For high frequency GCs we let the heap grow
+ // depending on the heap size:
+ // lastBytes < highFrequencyLowLimit: 300%
+ // lastBytes > highFrequencyHighLimit: 150%
+ // otherwise: linear interpolation between 300% and 150% based on lastBytes
+
+ // Use shorter names to make the operation comprehensible.
+ double minRatio = tunables.highFrequencyHeapGrowthMin();
+ double maxRatio = tunables.highFrequencyHeapGrowthMax();
+ double lowLimit = tunables.highFrequencyLowLimitBytes();
+ double highLimit = tunables.highFrequencyHighLimitBytes();
+
+ if (lastBytes <= lowLimit)
+ return maxRatio;
+
+ if (lastBytes >= highLimit)
+ return minRatio;
+
+ double factor = maxRatio - ((maxRatio - minRatio) * ((lastBytes - lowLimit) /
+ (highLimit - lowLimit)));
+ MOZ_ASSERT(factor >= minRatio);
+ MOZ_ASSERT(factor <= maxRatio);
+ return factor;
+}
+
+/* static */ size_t
+ZoneHeapThreshold::computeZoneTriggerBytes(double growthFactor, size_t lastBytes,
+ JSGCInvocationKind gckind,
+ const GCSchedulingTunables& tunables,
+ const AutoLockGC& lock)
+{
+ size_t base = gckind == GC_SHRINK
+ ? Max(lastBytes, tunables.minEmptyChunkCount(lock) * ChunkSize)
+ : Max(lastBytes, tunables.gcZoneAllocThresholdBase());
+ double trigger = double(base) * growthFactor;
+ return size_t(Min(double(tunables.gcMaxBytes()), trigger));
+}
+
+void
+ZoneHeapThreshold::updateAfterGC(size_t lastBytes, JSGCInvocationKind gckind,
+ const GCSchedulingTunables& tunables,
+ const GCSchedulingState& state, const AutoLockGC& lock)
+{
+ gcHeapGrowthFactor_ = computeZoneHeapGrowthFactorForHeapSize(lastBytes, tunables, state);
+ gcTriggerBytes_ = computeZoneTriggerBytes(gcHeapGrowthFactor_, lastBytes, gckind, tunables,
+ lock);
+}
+
+void
+ZoneHeapThreshold::updateForRemovedArena(const GCSchedulingTunables& tunables)
+{
+ size_t amount = ArenaSize * gcHeapGrowthFactor_;
+ MOZ_ASSERT(amount > 0);
+
+ if ((gcTriggerBytes_ < amount) ||
+ (gcTriggerBytes_ - amount < tunables.gcZoneAllocThresholdBase() * gcHeapGrowthFactor_))
+ {
+ return;
+ }
+
+ gcTriggerBytes_ -= amount;
+}
+
+void
+GCMarker::delayMarkingArena(Arena* arena)
+{
+ if (arena->hasDelayedMarking) {
+ /* Arena already scheduled to be marked later */
+ return;
+ }
+ arena->setNextDelayedMarking(unmarkedArenaStackTop);
+ unmarkedArenaStackTop = arena;
+#ifdef DEBUG
+ markLaterArenas++;
+#endif
+}
+
+void
+GCMarker::delayMarkingChildren(const void* thing)
+{
+ const TenuredCell* cell = TenuredCell::fromPointer(thing);
+ cell->arena()->markOverflow = 1;
+ delayMarkingArena(cell->arena());
+}
+
+inline void
+ArenaLists::prepareForIncrementalGC(JSRuntime* rt)
+{
+ for (auto i : AllAllocKinds()) {
+ FreeSpan* span = freeLists[i];
+ if (span != &placeholder) {
+ if (!span->isEmpty()) {
+ Arena* arena = span->getArena();
+ arena->allocatedDuringIncremental = true;
+ rt->gc.marker.delayMarkingArena(arena);
+ } else {
+ freeLists[i] = &placeholder;
+ }
+ }
+ }
+}
+
+/* Compacting GC */
+
+bool
+GCRuntime::shouldCompact()
+{
+ // Compact on shrinking GC if enabled, but skip compacting in incremental
+ // GCs if we are currently animating.
+ return invocationKind == GC_SHRINK && isCompactingGCEnabled() &&
+ (!isIncremental || rt->lastAnimationTime + PRMJ_USEC_PER_SEC < PRMJ_Now());
+}
+
+void
+GCRuntime::disableCompactingGC()
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ ++compactingDisabledCount;
+}
+
+void
+GCRuntime::enableCompactingGC()
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(compactingDisabledCount > 0);
+ --compactingDisabledCount;
+}
+
+bool
+GCRuntime::isCompactingGCEnabled() const
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ return compactingEnabled && compactingDisabledCount == 0;
+}
+
+AutoDisableCompactingGC::AutoDisableCompactingGC(JSContext* cx)
+ : gc(cx->gc)
+{
+ gc.disableCompactingGC();
+ if (gc.isIncrementalGCInProgress() && gc.isCompactingGc())
+ FinishGC(cx);
+}
+
+AutoDisableCompactingGC::~AutoDisableCompactingGC()
+{
+ gc.enableCompactingGC();
+}
+
+static bool
+CanRelocateZone(Zone* zone)
+{
+ return !zone->isAtomsZone() && !zone->isSelfHostingZone();
+}
+
+static const AllocKind AllocKindsToRelocate[] = {
+ AllocKind::FUNCTION,
+ AllocKind::FUNCTION_EXTENDED,
+ AllocKind::OBJECT0,
+ AllocKind::OBJECT0_BACKGROUND,
+ AllocKind::OBJECT2,
+ AllocKind::OBJECT2_BACKGROUND,
+ AllocKind::OBJECT4,
+ AllocKind::OBJECT4_BACKGROUND,
+ AllocKind::OBJECT8,
+ AllocKind::OBJECT8_BACKGROUND,
+ AllocKind::OBJECT12,
+ AllocKind::OBJECT12_BACKGROUND,
+ AllocKind::OBJECT16,
+ AllocKind::OBJECT16_BACKGROUND,
+ AllocKind::SCRIPT,
+ AllocKind::LAZY_SCRIPT,
+ AllocKind::SCOPE,
+ AllocKind::SHAPE,
+ AllocKind::ACCESSOR_SHAPE,
+ AllocKind::BASE_SHAPE,
+ AllocKind::FAT_INLINE_STRING,
+ AllocKind::STRING,
+ AllocKind::EXTERNAL_STRING,
+ AllocKind::FAT_INLINE_ATOM,
+ AllocKind::ATOM
+};
+
+Arena*
+ArenaList::removeRemainingArenas(Arena** arenap)
+{
+ // This is only ever called to remove arenas that are after the cursor, so
+ // we don't need to update it.
+#ifdef DEBUG
+ for (Arena* arena = *arenap; arena; arena = arena->next)
+ MOZ_ASSERT(cursorp_ != &arena->next);
+#endif
+ Arena* remainingArenas = *arenap;
+ *arenap = nullptr;
+ check();
+ return remainingArenas;
+}
+
+static bool
+ShouldRelocateAllArenas(JS::gcreason::Reason reason)
+{
+ return reason == JS::gcreason::DEBUG_GC;
+}
+
+/*
+ * Choose which arenas to relocate all cells from. Return an arena cursor that
+ * can be passed to removeRemainingArenas().
+ */
+Arena**
+ArenaList::pickArenasToRelocate(size_t& arenaTotalOut, size_t& relocTotalOut)
+{
+ // Relocate the greatest number of arenas such that the number of used cells
+ // in relocated arenas is less than or equal to the number of free cells in
+ // unrelocated arenas. In other words we only relocate cells we can move
+ // into existing arenas, and we choose the least full areans to relocate.
+ //
+ // This is made easier by the fact that the arena list has been sorted in
+ // descending order of number of used cells, so we will always relocate a
+ // tail of the arena list. All we need to do is find the point at which to
+ // start relocating.
+
+ check();
+
+ if (isCursorAtEnd())
+ return nullptr;
+
+ Arena** arenap = cursorp_; // Next arena to consider for relocation.
+ size_t previousFreeCells = 0; // Count of free cells before arenap.
+ size_t followingUsedCells = 0; // Count of used cells after arenap.
+ size_t fullArenaCount = 0; // Number of full arenas (not relocated).
+ size_t nonFullArenaCount = 0; // Number of non-full arenas (considered for relocation).
+ size_t arenaIndex = 0; // Index of the next arena to consider.
+
+ for (Arena* arena = head_; arena != *cursorp_; arena = arena->next)
+ fullArenaCount++;
+
+ for (Arena* arena = *cursorp_; arena; arena = arena->next) {
+ followingUsedCells += arena->countUsedCells();
+ nonFullArenaCount++;
+ }
+
+ mozilla::DebugOnly<size_t> lastFreeCells(0);
+ size_t cellsPerArena = Arena::thingsPerArena((*arenap)->getAllocKind());
+
+ while (*arenap) {
+ Arena* arena = *arenap;
+ if (followingUsedCells <= previousFreeCells)
+ break;
+
+ size_t freeCells = arena->countFreeCells();
+ size_t usedCells = cellsPerArena - freeCells;
+ followingUsedCells -= usedCells;
+#ifdef DEBUG
+ MOZ_ASSERT(freeCells >= lastFreeCells);
+ lastFreeCells = freeCells;
+#endif
+ previousFreeCells += freeCells;
+ arenap = &arena->next;
+ arenaIndex++;
+ }
+
+ size_t relocCount = nonFullArenaCount - arenaIndex;
+ MOZ_ASSERT(relocCount < nonFullArenaCount);
+ MOZ_ASSERT((relocCount == 0) == (!*arenap));
+ arenaTotalOut += fullArenaCount + nonFullArenaCount;
+ relocTotalOut += relocCount;
+
+ return arenap;
+}
+
+#ifdef DEBUG
+inline bool
+PtrIsInRange(const void* ptr, const void* start, size_t length)
+{
+ return uintptr_t(ptr) - uintptr_t(start) < length;
+}
+#endif
+
+static TenuredCell*
+AllocRelocatedCell(Zone* zone, AllocKind thingKind, size_t thingSize)
+{
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ void* dstAlloc = zone->arenas.allocateFromFreeList(thingKind, thingSize);
+ if (!dstAlloc)
+ dstAlloc = GCRuntime::refillFreeListInGC(zone, thingKind);
+ if (!dstAlloc) {
+ // This can only happen in zeal mode or debug builds as we don't
+ // otherwise relocate more cells than we have existing free space
+ // for.
+ oomUnsafe.crash("Could not allocate new arena while compacting");
+ }
+ return TenuredCell::fromPointer(dstAlloc);
+}
+
+static void
+RelocateCell(Zone* zone, TenuredCell* src, AllocKind thingKind, size_t thingSize)
+{
+ JS::AutoSuppressGCAnalysis nogc(zone->contextFromMainThread());
+
+ // Allocate a new cell.
+ MOZ_ASSERT(zone == src->zone());
+ TenuredCell* dst = AllocRelocatedCell(zone, thingKind, thingSize);
+
+ // Copy source cell contents to destination.
+ memcpy(dst, src, thingSize);
+
+ // Move any uid attached to the object.
+ src->zone()->transferUniqueId(dst, src);
+
+ if (IsObjectAllocKind(thingKind)) {
+ JSObject* srcObj = static_cast<JSObject*>(static_cast<Cell*>(src));
+ JSObject* dstObj = static_cast<JSObject*>(static_cast<Cell*>(dst));
+
+ if (srcObj->isNative()) {
+ NativeObject* srcNative = &srcObj->as<NativeObject>();
+ NativeObject* dstNative = &dstObj->as<NativeObject>();
+
+ // Fixup the pointer to inline object elements if necessary.
+ if (srcNative->hasFixedElements())
+ dstNative->setFixedElements();
+
+ // For copy-on-write objects that own their elements, fix up the
+ // owner pointer to point to the relocated object.
+ if (srcNative->denseElementsAreCopyOnWrite()) {
+ GCPtrNativeObject& owner = dstNative->getElementsHeader()->ownerObject();
+ if (owner == srcNative)
+ owner = dstNative;
+ }
+ }
+
+ // Call object moved hook if present.
+ if (JSObjectMovedOp op = srcObj->getClass()->extObjectMovedOp())
+ op(dstObj, srcObj);
+
+ MOZ_ASSERT_IF(dstObj->isNative(),
+ !PtrIsInRange((const Value*)dstObj->as<NativeObject>().getDenseElements(),
+ src, thingSize));
+ }
+
+ // Copy the mark bits.
+ dst->copyMarkBitsFrom(src);
+
+ // Mark source cell as forwarded and leave a pointer to the destination.
+ RelocationOverlay* overlay = RelocationOverlay::fromCell(src);
+ overlay->forwardTo(dst);
+}
+
+static void
+RelocateArena(Arena* arena, SliceBudget& sliceBudget)
+{
+ MOZ_ASSERT(arena->allocated());
+ MOZ_ASSERT(!arena->hasDelayedMarking);
+ MOZ_ASSERT(!arena->markOverflow);
+ MOZ_ASSERT(!arena->allocatedDuringIncremental);
+ MOZ_ASSERT(arena->bufferedCells->isEmpty());
+
+ Zone* zone = arena->zone;
+
+ AllocKind thingKind = arena->getAllocKind();
+ size_t thingSize = arena->getThingSize();
+
+ for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) {
+ RelocateCell(zone, i.getCell(), thingKind, thingSize);
+ sliceBudget.step();
+ }
+
+#ifdef DEBUG
+ for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) {
+ TenuredCell* src = i.getCell();
+ MOZ_ASSERT(RelocationOverlay::isCellForwarded(src));
+ TenuredCell* dest = Forwarded(src);
+ MOZ_ASSERT(src->isMarked(BLACK) == dest->isMarked(BLACK));
+ MOZ_ASSERT(src->isMarked(GRAY) == dest->isMarked(GRAY));
+ }
+#endif
+}
+
+static inline bool
+ShouldProtectRelocatedArenas(JS::gcreason::Reason reason)
+{
+ // For zeal mode collections we don't release the relocated arenas
+ // immediately. Instead we protect them and keep them around until the next
+ // collection so we can catch any stray accesses to them.
+#ifdef DEBUG
+ return reason == JS::gcreason::DEBUG_GC;
+#else
+ return false;
+#endif
+}
+
+/*
+ * Relocate all arenas identified by pickArenasToRelocate: for each arena,
+ * relocate each cell within it, then add it to a list of relocated arenas.
+ */
+Arena*
+ArenaList::relocateArenas(Arena* toRelocate, Arena* relocated, SliceBudget& sliceBudget,
+ gcstats::Statistics& stats)
+{
+ check();
+
+ while (Arena* arena = toRelocate) {
+ toRelocate = arena->next;
+ RelocateArena(arena, sliceBudget);
+ // Prepend to list of relocated arenas
+ arena->next = relocated;
+ relocated = arena;
+ stats.count(gcstats::STAT_ARENA_RELOCATED);
+ }
+
+ check();
+
+ return relocated;
+}
+
+// Skip compacting zones unless we can free a certain proportion of their GC
+// heap memory.
+static const double MIN_ZONE_RECLAIM_PERCENT = 2.0;
+
+static bool
+ShouldRelocateZone(size_t arenaCount, size_t relocCount, JS::gcreason::Reason reason)
+{
+ if (relocCount == 0)
+ return false;
+
+ if (IsOOMReason(reason))
+ return true;
+
+ return (relocCount * 100.0) / arenaCount >= MIN_ZONE_RECLAIM_PERCENT;
+}
+
+bool
+ArenaLists::relocateArenas(Zone* zone, Arena*& relocatedListOut, JS::gcreason::Reason reason,
+ SliceBudget& sliceBudget, gcstats::Statistics& stats)
+{
+ // This is only called from the main thread while we are doing a GC, so
+ // there is no need to lock.
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_));
+ MOZ_ASSERT(runtime_->gc.isHeapCompacting());
+ MOZ_ASSERT(!runtime_->gc.isBackgroundSweeping());
+
+ // Clear all the free lists.
+ purge();
+
+ if (ShouldRelocateAllArenas(reason)) {
+ zone->prepareForCompacting();
+ for (auto kind : AllocKindsToRelocate) {
+ ArenaList& al = arenaLists[kind];
+ Arena* allArenas = al.head();
+ al.clear();
+ relocatedListOut = al.relocateArenas(allArenas, relocatedListOut, sliceBudget, stats);
+ }
+ } else {
+ size_t arenaCount = 0;
+ size_t relocCount = 0;
+ AllAllocKindArray<Arena**> toRelocate;
+
+ for (auto kind : AllocKindsToRelocate)
+ toRelocate[kind] = arenaLists[kind].pickArenasToRelocate(arenaCount, relocCount);
+
+ if (!ShouldRelocateZone(arenaCount, relocCount, reason))
+ return false;
+
+ zone->prepareForCompacting();
+ for (auto kind : AllocKindsToRelocate) {
+ if (toRelocate[kind]) {
+ ArenaList& al = arenaLists[kind];
+ Arena* arenas = al.removeRemainingArenas(toRelocate[kind]);
+ relocatedListOut = al.relocateArenas(arenas, relocatedListOut, sliceBudget, stats);
+ }
+ }
+ }
+
+ return true;
+}
+
+bool
+GCRuntime::relocateArenas(Zone* zone, JS::gcreason::Reason reason, Arena*& relocatedListOut,
+ SliceBudget& sliceBudget)
+{
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT_MOVE);
+
+ MOZ_ASSERT(!zone->isPreservingCode());
+ MOZ_ASSERT(CanRelocateZone(zone));
+
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Compact);
+
+ if (!zone->arenas.relocateArenas(zone, relocatedListOut, reason, sliceBudget, stats))
+ return false;
+
+#ifdef DEBUG
+ // Check that we did as much compaction as we should have. There
+ // should always be less than one arena's worth of free cells.
+ for (auto i : AllocKindsToRelocate) {
+ ArenaList& al = zone->arenas.arenaLists[i];
+ size_t freeCells = 0;
+ for (Arena* arena = al.arenaAfterCursor(); arena; arena = arena->next)
+ freeCells += arena->countFreeCells();
+ MOZ_ASSERT(freeCells < Arena::thingsPerArena(i));
+ }
+#endif
+
+ return true;
+}
+
+void
+MovingTracer::onObjectEdge(JSObject** objp)
+{
+ JSObject* obj = *objp;
+ if (obj->runtimeFromAnyThread() == runtime() && IsForwarded(obj))
+ *objp = Forwarded(obj);
+}
+
+void
+MovingTracer::onShapeEdge(Shape** shapep)
+{
+ Shape* shape = *shapep;
+ if (shape->runtimeFromAnyThread() == runtime() && IsForwarded(shape))
+ *shapep = Forwarded(shape);
+}
+
+void
+MovingTracer::onStringEdge(JSString** stringp)
+{
+ JSString* string = *stringp;
+ if (string->runtimeFromAnyThread() == runtime() && IsForwarded(string))
+ *stringp = Forwarded(string);
+}
+
+void
+MovingTracer::onScriptEdge(JSScript** scriptp)
+{
+ JSScript* script = *scriptp;
+ if (script->runtimeFromAnyThread() == runtime() && IsForwarded(script))
+ *scriptp = Forwarded(script);
+}
+
+void
+MovingTracer::onLazyScriptEdge(LazyScript** lazyp)
+{
+ LazyScript* lazy = *lazyp;
+ if (lazy->runtimeFromAnyThread() == runtime() && IsForwarded(lazy))
+ *lazyp = Forwarded(lazy);
+}
+
+void
+MovingTracer::onBaseShapeEdge(BaseShape** basep)
+{
+ BaseShape* base = *basep;
+ if (base->runtimeFromAnyThread() == runtime() && IsForwarded(base))
+ *basep = Forwarded(base);
+}
+
+void
+MovingTracer::onScopeEdge(Scope** scopep)
+{
+ Scope* scope = *scopep;
+ if (scope->runtimeFromAnyThread() == runtime() && IsForwarded(scope))
+ *scopep = Forwarded(scope);
+}
+
+void
+Zone::prepareForCompacting()
+{
+ FreeOp* fop = runtimeFromMainThread()->defaultFreeOp();
+ discardJitCode(fop);
+}
+
+void
+GCRuntime::sweepTypesAfterCompacting(Zone* zone)
+{
+ FreeOp* fop = rt->defaultFreeOp();
+ zone->beginSweepTypes(fop, rt->gc.releaseObservedTypes && !zone->isPreservingCode());
+
+ AutoClearTypeInferenceStateOnOOM oom(zone);
+
+ for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next())
+ script->maybeSweepTypes(&oom);
+ for (auto group = zone->cellIter<ObjectGroup>(); !group.done(); group.next())
+ group->maybeSweep(&oom);
+
+ zone->types.endSweep(rt);
+}
+
+void
+GCRuntime::sweepZoneAfterCompacting(Zone* zone)
+{
+ MOZ_ASSERT(zone->isCollecting());
+ FreeOp* fop = rt->defaultFreeOp();
+ sweepTypesAfterCompacting(zone);
+ zone->sweepBreakpoints(fop);
+ zone->sweepWeakMaps();
+ for (auto* cache : zone->weakCaches_)
+ cache->sweep();
+
+ for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
+ c->objectGroups.sweep(fop);
+ c->sweepRegExps();
+ c->sweepSavedStacks();
+ c->sweepGlobalObject(fop);
+ c->sweepSelfHostingScriptSource();
+ c->sweepDebugEnvironments();
+ c->sweepJitCompartment(fop);
+ c->sweepNativeIterators();
+ c->sweepTemplateObjects();
+ }
+}
+
+template <typename T>
+static inline void
+UpdateCellPointers(MovingTracer* trc, T* cell)
+{
+ cell->fixupAfterMovingGC();
+ cell->traceChildren(trc);
+}
+
+template <typename T>
+static void
+UpdateArenaPointersTyped(MovingTracer* trc, Arena* arena, JS::TraceKind traceKind)
+{
+ for (ArenaCellIterUnderGC i(arena); !i.done(); i.next())
+ UpdateCellPointers(trc, reinterpret_cast<T*>(i.getCell()));
+}
+
+/*
+ * Update the internal pointers for all cells in an arena.
+ */
+static void
+UpdateArenaPointers(MovingTracer* trc, Arena* arena)
+{
+ AllocKind kind = arena->getAllocKind();
+
+ switch (kind) {
+#define EXPAND_CASE(allocKind, traceKind, type, sizedType) \
+ case AllocKind::allocKind: \
+ UpdateArenaPointersTyped<type>(trc, arena, JS::TraceKind::traceKind); \
+ return;
+FOR_EACH_ALLOCKIND(EXPAND_CASE)
+#undef EXPAND_CASE
+
+ default:
+ MOZ_CRASH("Invalid alloc kind for UpdateArenaPointers");
+ }
+}
+
+namespace js {
+namespace gc {
+
+struct ArenaListSegment
+{
+ Arena* begin;
+ Arena* end;
+};
+
+struct ArenasToUpdate
+{
+ ArenasToUpdate(Zone* zone, AllocKinds kinds);
+ bool done() { return kind == AllocKind::LIMIT; }
+ ArenaListSegment getArenasToUpdate(AutoLockHelperThreadState& lock, unsigned maxLength);
+
+ private:
+ AllocKinds kinds; // Selects which thing kinds to update
+ Zone* zone; // Zone to process
+ AllocKind kind; // Current alloc kind to process
+ Arena* arena; // Next arena to process
+
+ AllocKind nextAllocKind(AllocKind i) { return AllocKind(uint8_t(i) + 1); }
+ bool shouldProcessKind(AllocKind kind);
+ Arena* next(AutoLockHelperThreadState& lock);
+};
+
+ArenasToUpdate::ArenasToUpdate(Zone* zone, AllocKinds kinds)
+ : kinds(kinds), zone(zone), kind(AllocKind::FIRST), arena(nullptr)
+{
+ MOZ_ASSERT(zone->isGCCompacting());
+}
+
+Arena*
+ArenasToUpdate::next(AutoLockHelperThreadState& lock)
+{
+ // Find the next arena to update.
+ //
+ // This iterates through the GC thing kinds filtered by shouldProcessKind(),
+ // and then through thea arenas of that kind. All state is held in the
+ // object and we just return when we find an arena.
+
+ for (; kind < AllocKind::LIMIT; kind = nextAllocKind(kind)) {
+ if (kinds.contains(kind)) {
+ if (!arena)
+ arena = zone->arenas.getFirstArena(kind);
+ else
+ arena = arena->next;
+ if (arena)
+ return arena;
+ }
+ }
+
+ MOZ_ASSERT(!arena);
+ MOZ_ASSERT(done());
+ return nullptr;
+}
+
+ArenaListSegment
+ArenasToUpdate::getArenasToUpdate(AutoLockHelperThreadState& lock, unsigned maxLength)
+{
+ Arena* begin = next(lock);
+ if (!begin)
+ return { nullptr, nullptr };
+
+ Arena* last = begin;
+ unsigned count = 1;
+ while (last->next && count < maxLength) {
+ last = last->next;
+ count++;
+ }
+
+ arena = last;
+ return { begin, last->next };
+}
+
+struct UpdatePointersTask : public GCParallelTask
+{
+ // Maximum number of arenas to update in one block.
+#ifdef DEBUG
+ static const unsigned MaxArenasToProcess = 16;
+#else
+ static const unsigned MaxArenasToProcess = 256;
+#endif
+
+ UpdatePointersTask(JSRuntime* rt, ArenasToUpdate* source, AutoLockHelperThreadState& lock)
+ : rt_(rt), source_(source)
+ {
+ arenas_.begin = nullptr;
+ arenas_.end = nullptr;
+ }
+
+ ~UpdatePointersTask() override { join(); }
+
+ private:
+ JSRuntime* rt_;
+ ArenasToUpdate* source_;
+ ArenaListSegment arenas_;
+
+ virtual void run() override;
+ bool getArenasToUpdate();
+ void updateArenas();
+};
+
+bool
+UpdatePointersTask::getArenasToUpdate()
+{
+ AutoLockHelperThreadState lock;
+ arenas_ = source_->getArenasToUpdate(lock, MaxArenasToProcess);
+ return arenas_.begin != nullptr;
+}
+
+void
+UpdatePointersTask::updateArenas()
+{
+ MovingTracer trc(rt_);
+ for (Arena* arena = arenas_.begin; arena != arenas_.end; arena = arena->next)
+ UpdateArenaPointers(&trc, arena);
+}
+
+/* virtual */ void
+UpdatePointersTask::run()
+{
+ while (getArenasToUpdate())
+ updateArenas();
+}
+
+} // namespace gc
+} // namespace js
+
+static const size_t MinCellUpdateBackgroundTasks = 2;
+static const size_t MaxCellUpdateBackgroundTasks = 8;
+
+static size_t
+CellUpdateBackgroundTaskCount()
+{
+ if (!CanUseExtraThreads())
+ return 0;
+
+ size_t targetTaskCount = HelperThreadState().cpuCount / 2;
+ return Min(Max(targetTaskCount, MinCellUpdateBackgroundTasks), MaxCellUpdateBackgroundTasks);
+}
+
+static bool
+CanUpdateKindInBackground(AllocKind kind) {
+ // We try to update as many GC things in parallel as we can, but there are
+ // kinds for which this might not be safe:
+ // - we assume JSObjects that are foreground finalized are not safe to
+ // update in parallel
+ // - updating a shape touches child shapes in fixupShapeTreeAfterMovingGC()
+ if (!js::gc::IsBackgroundFinalized(kind) || IsShapeAllocKind(kind))
+ return false;
+
+ return true;
+}
+
+static AllocKinds
+ForegroundUpdateKinds(AllocKinds kinds)
+{
+ AllocKinds result;
+ for (AllocKind kind : kinds) {
+ if (!CanUpdateKindInBackground(kind))
+ result += kind;
+ }
+ return result;
+}
+
+void
+GCRuntime::updateTypeDescrObjects(MovingTracer* trc, Zone* zone)
+{
+ zone->typeDescrObjects.sweep();
+ for (auto r = zone->typeDescrObjects.all(); !r.empty(); r.popFront())
+ UpdateCellPointers(trc, r.front().get());
+}
+
+void
+GCRuntime::updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds, size_t bgTaskCount)
+{
+ AllocKinds fgKinds = bgTaskCount == 0 ? kinds : ForegroundUpdateKinds(kinds);
+ AllocKinds bgKinds = kinds - fgKinds;
+
+ ArenasToUpdate fgArenas(zone, fgKinds);
+ ArenasToUpdate bgArenas(zone, bgKinds);
+ Maybe<UpdatePointersTask> fgTask;
+ Maybe<UpdatePointersTask> bgTasks[MaxCellUpdateBackgroundTasks];
+
+ size_t tasksStarted = 0;
+
+ {
+ AutoLockHelperThreadState lock;
+
+ fgTask.emplace(rt, &fgArenas, lock);
+
+ for (size_t i = 0; i < bgTaskCount && !bgArenas.done(); i++) {
+ bgTasks[i].emplace(rt, &bgArenas, lock);
+ startTask(*bgTasks[i], gcstats::PHASE_COMPACT_UPDATE_CELLS, lock);
+ tasksStarted = i;
+ }
+ }
+
+ fgTask->runFromMainThread(rt);
+
+ {
+ AutoLockHelperThreadState lock;
+
+ for (size_t i = 0; i < tasksStarted; i++)
+ joinTask(*bgTasks[i], gcstats::PHASE_COMPACT_UPDATE_CELLS, lock);
+ }
+}
+
+// After cells have been relocated any pointers to a cell's old locations must
+// be updated to point to the new location. This happens by iterating through
+// all cells in heap and tracing their children (non-recursively) to update
+// them.
+//
+// This is complicated by the fact that updating a GC thing sometimes depends on
+// making use of other GC things. After a moving GC these things may not be in
+// a valid state since they may contain pointers which have not been updated
+// yet.
+//
+// The main dependencies are:
+//
+// - Updating a JSObject makes use of its shape
+// - Updating a typed object makes use of its type descriptor object
+//
+// This means we require at least three phases for update:
+//
+// 1) shapes
+// 2) typed object type descriptor objects
+// 3) all other objects
+//
+// Since we want to minimize the number of phases, we put everything else into
+// the first phase and label it the 'misc' phase.
+
+static const AllocKinds UpdatePhaseMisc {
+ AllocKind::SCRIPT,
+ AllocKind::LAZY_SCRIPT,
+ AllocKind::BASE_SHAPE,
+ AllocKind::SHAPE,
+ AllocKind::ACCESSOR_SHAPE,
+ AllocKind::OBJECT_GROUP,
+ AllocKind::STRING,
+ AllocKind::JITCODE,
+ AllocKind::SCOPE
+};
+
+static const AllocKinds UpdatePhaseObjects {
+ AllocKind::FUNCTION,
+ AllocKind::FUNCTION_EXTENDED,
+ AllocKind::OBJECT0,
+ AllocKind::OBJECT0_BACKGROUND,
+ AllocKind::OBJECT2,
+ AllocKind::OBJECT2_BACKGROUND,
+ AllocKind::OBJECT4,
+ AllocKind::OBJECT4_BACKGROUND,
+ AllocKind::OBJECT8,
+ AllocKind::OBJECT8_BACKGROUND,
+ AllocKind::OBJECT12,
+ AllocKind::OBJECT12_BACKGROUND,
+ AllocKind::OBJECT16,
+ AllocKind::OBJECT16_BACKGROUND
+};
+
+void
+GCRuntime::updateAllCellPointers(MovingTracer* trc, Zone* zone)
+{
+ AutoDisableProxyCheck noProxyCheck(rt); // These checks assert when run in parallel.
+
+ size_t bgTaskCount = CellUpdateBackgroundTaskCount();
+
+ updateCellPointers(trc, zone, UpdatePhaseMisc, bgTaskCount);
+
+ // Update TypeDescrs before all other objects as typed objects access these
+ // objects when we trace them.
+ updateTypeDescrObjects(trc, zone);
+
+ updateCellPointers(trc, zone, UpdatePhaseObjects, bgTaskCount);
+}
+
+/*
+ * Update pointers to relocated cells by doing a full heap traversal and sweep.
+ *
+ * The latter is necessary to update weak references which are not marked as
+ * part of the traversal.
+ */
+void
+GCRuntime::updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess& lock)
+{
+ MOZ_ASSERT(!rt->isBeingDestroyed());
+ MOZ_ASSERT(zone->isGCCompacting());
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT_UPDATE);
+ MovingTracer trc(rt);
+
+ zone->fixupAfterMovingGC();
+
+ // Fixup compartment global pointers as these get accessed during marking.
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+ comp->fixupAfterMovingGC();
+ JSCompartment::fixupCrossCompartmentWrappersAfterMovingGC(&trc);
+ rt->spsProfiler.fixupStringsMapAfterMovingGC();
+
+ // Iterate through all cells that can contain relocatable pointers to update
+ // them. Since updating each cell is independent we try to parallelize this
+ // as much as possible.
+ updateAllCellPointers(&trc, zone);
+
+ // Mark roots to update them.
+ {
+ traceRuntimeForMajorGC(&trc, lock);
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS);
+ Debugger::markAll(&trc);
+ Debugger::markIncomingCrossCompartmentEdges(&trc);
+
+ WeakMapBase::markAll(zone, &trc);
+ for (CompartmentsInZoneIter c(zone); !c.done(); c.next()) {
+ c->trace(&trc);
+ if (c->watchpointMap)
+ c->watchpointMap->markAll(&trc);
+ }
+
+ // Mark all gray roots, making sure we call the trace callback to get the
+ // current set.
+ if (JSTraceDataOp op = grayRootTracer.op)
+ (*op)(&trc, grayRootTracer.data);
+ }
+
+ // Sweep everything to fix up weak pointers
+ WatchpointMap::sweepAll(rt);
+ Debugger::sweepAll(rt->defaultFreeOp());
+ jit::JitRuntime::SweepJitcodeGlobalTable(rt);
+ rt->gc.sweepZoneAfterCompacting(zone);
+
+ // Type inference may put more blocks here to free.
+ blocksToFreeAfterSweeping.freeAll();
+
+ // Call callbacks to get the rest of the system to fixup other untraced pointers.
+ callWeakPointerZoneGroupCallbacks();
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+ callWeakPointerCompartmentCallbacks(comp);
+ if (rt->sweepZoneCallback)
+ rt->sweepZoneCallback(zone);
+}
+
+void
+GCRuntime::protectAndHoldArenas(Arena* arenaList)
+{
+ for (Arena* arena = arenaList; arena; ) {
+ MOZ_ASSERT(arena->allocated());
+ Arena* next = arena->next;
+ if (!next) {
+ // Prepend to hold list before we protect the memory.
+ arena->next = relocatedArenasToRelease;
+ relocatedArenasToRelease = arenaList;
+ }
+ ProtectPages(arena, ArenaSize);
+ arena = next;
+ }
+}
+
+void
+GCRuntime::unprotectHeldRelocatedArenas()
+{
+ for (Arena* arena = relocatedArenasToRelease; arena; arena = arena->next) {
+ UnprotectPages(arena, ArenaSize);
+ MOZ_ASSERT(arena->allocated());
+ }
+}
+
+void
+GCRuntime::releaseRelocatedArenas(Arena* arenaList)
+{
+ AutoLockGC lock(rt);
+ releaseRelocatedArenasWithoutUnlocking(arenaList, lock);
+}
+
+void
+GCRuntime::releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, const AutoLockGC& lock)
+{
+ // Release the relocated arenas, now containing only forwarding pointers
+ unsigned count = 0;
+ while (arenaList) {
+ Arena* arena = arenaList;
+ arenaList = arenaList->next;
+
+ // Clear the mark bits
+ arena->unmarkAll();
+
+ // Mark arena as empty
+ arena->setAsFullyUnused();
+
+#if defined(JS_CRASH_DIAGNOSTICS) || defined(JS_GC_ZEAL)
+ JS_POISON(reinterpret_cast<void*>(arena->thingsStart()),
+ JS_MOVED_TENURED_PATTERN, arena->getThingsSpan());
+#endif
+
+ releaseArena(arena, lock);
+ ++count;
+ }
+}
+
+// In debug mode we don't always release relocated arenas straight away.
+// Sometimes protect them instead and hold onto them until the next GC sweep
+// phase to catch any pointers to them that didn't get forwarded.
+
+void
+GCRuntime::releaseHeldRelocatedArenas()
+{
+#ifdef DEBUG
+ unprotectHeldRelocatedArenas();
+ Arena* arenas = relocatedArenasToRelease;
+ relocatedArenasToRelease = nullptr;
+ releaseRelocatedArenas(arenas);
+#endif
+}
+
+void
+GCRuntime::releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock)
+{
+#ifdef DEBUG
+ unprotectHeldRelocatedArenas();
+ releaseRelocatedArenasWithoutUnlocking(relocatedArenasToRelease, lock);
+ relocatedArenasToRelease = nullptr;
+#endif
+}
+
+void
+ReleaseArenaList(JSRuntime* rt, Arena* arena, const AutoLockGC& lock)
+{
+ Arena* next;
+ for (; arena; arena = next) {
+ next = arena->next;
+ rt->gc.releaseArena(arena, lock);
+ }
+}
+
+ArenaLists::~ArenaLists()
+{
+ AutoLockGC lock(runtime_);
+
+ for (auto i : AllAllocKinds()) {
+ /*
+ * We can only call this during the shutdown after the last GC when
+ * the background finalization is disabled.
+ */
+ MOZ_ASSERT(backgroundFinalizeState[i] == BFS_DONE);
+ ReleaseArenaList(runtime_, arenaLists[i].head(), lock);
+ }
+ ReleaseArenaList(runtime_, incrementalSweptArenas.head(), lock);
+
+ for (auto i : ObjectAllocKinds())
+ ReleaseArenaList(runtime_, savedObjectArenas[i].head(), lock);
+ ReleaseArenaList(runtime_, savedEmptyObjectArenas, lock);
+}
+
+void
+ArenaLists::finalizeNow(FreeOp* fop, const FinalizePhase& phase)
+{
+ gcstats::AutoPhase ap(fop->runtime()->gc.stats, phase.statsPhase);
+ for (auto kind : phase.kinds)
+ finalizeNow(fop, kind, RELEASE_ARENAS, nullptr);
+}
+
+void
+ArenaLists::finalizeNow(FreeOp* fop, AllocKind thingKind, KeepArenasEnum keepArenas, Arena** empty)
+{
+ MOZ_ASSERT(!IsBackgroundFinalized(thingKind));
+ forceFinalizeNow(fop, thingKind, keepArenas, empty);
+}
+
+void
+ArenaLists::forceFinalizeNow(FreeOp* fop, AllocKind thingKind,
+ KeepArenasEnum keepArenas, Arena** empty)
+{
+ MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
+
+ Arena* arenas = arenaLists[thingKind].head();
+ if (!arenas)
+ return;
+ arenaLists[thingKind].clear();
+
+ size_t thingsPerArena = Arena::thingsPerArena(thingKind);
+ SortedArenaList finalizedSorted(thingsPerArena);
+
+ auto unlimited = SliceBudget::unlimited();
+ FinalizeArenas(fop, &arenas, finalizedSorted, thingKind, unlimited, keepArenas);
+ MOZ_ASSERT(!arenas);
+
+ if (empty) {
+ MOZ_ASSERT(keepArenas == KEEP_ARENAS);
+ finalizedSorted.extractEmpty(empty);
+ }
+
+ arenaLists[thingKind] = finalizedSorted.toArenaList();
+}
+
+void
+ArenaLists::queueForForegroundSweep(FreeOp* fop, const FinalizePhase& phase)
+{
+ gcstats::AutoPhase ap(fop->runtime()->gc.stats, phase.statsPhase);
+ for (auto kind : phase.kinds)
+ queueForForegroundSweep(fop, kind);
+}
+
+void
+ArenaLists::queueForForegroundSweep(FreeOp* fop, AllocKind thingKind)
+{
+ MOZ_ASSERT(!IsBackgroundFinalized(thingKind));
+ MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
+ MOZ_ASSERT(!arenaListsToSweep[thingKind]);
+
+ arenaListsToSweep[thingKind] = arenaLists[thingKind].head();
+ arenaLists[thingKind].clear();
+}
+
+void
+ArenaLists::queueForBackgroundSweep(FreeOp* fop, const FinalizePhase& phase)
+{
+ gcstats::AutoPhase ap(fop->runtime()->gc.stats, phase.statsPhase);
+ for (auto kind : phase.kinds)
+ queueForBackgroundSweep(fop, kind);
+}
+
+inline void
+ArenaLists::queueForBackgroundSweep(FreeOp* fop, AllocKind thingKind)
+{
+ MOZ_ASSERT(IsBackgroundFinalized(thingKind));
+
+ ArenaList* al = &arenaLists[thingKind];
+ if (al->isEmpty()) {
+ MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
+ return;
+ }
+
+ MOZ_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
+
+ arenaListsToSweep[thingKind] = al->head();
+ al->clear();
+ backgroundFinalizeState[thingKind] = BFS_RUN;
+}
+
+/*static*/ void
+ArenaLists::backgroundFinalize(FreeOp* fop, Arena* listHead, Arena** empty)
+{
+ MOZ_ASSERT(listHead);
+ MOZ_ASSERT(empty);
+
+ AllocKind thingKind = listHead->getAllocKind();
+ Zone* zone = listHead->zone;
+
+ size_t thingsPerArena = Arena::thingsPerArena(thingKind);
+ SortedArenaList finalizedSorted(thingsPerArena);
+
+ auto unlimited = SliceBudget::unlimited();
+ FinalizeArenas(fop, &listHead, finalizedSorted, thingKind, unlimited, KEEP_ARENAS);
+ MOZ_ASSERT(!listHead);
+
+ finalizedSorted.extractEmpty(empty);
+
+ // When arenas are queued for background finalization, all arenas are moved
+ // to arenaListsToSweep[], leaving the arenaLists[] empty. However, new
+ // arenas may be allocated before background finalization finishes; now that
+ // finalization is complete, we want to merge these lists back together.
+ ArenaLists* lists = &zone->arenas;
+ ArenaList* al = &lists->arenaLists[thingKind];
+
+ // Flatten |finalizedSorted| into a regular ArenaList.
+ ArenaList finalized = finalizedSorted.toArenaList();
+
+ // We must take the GC lock to be able to safely modify the ArenaList;
+ // however, this does not by itself make the changes visible to all threads,
+ // as not all threads take the GC lock to read the ArenaLists.
+ // That safety is provided by the ReleaseAcquire memory ordering of the
+ // background finalize state, which we explicitly set as the final step.
+ {
+ AutoLockGC lock(lists->runtime_);
+ MOZ_ASSERT(lists->backgroundFinalizeState[thingKind] == BFS_RUN);
+
+ // Join |al| and |finalized| into a single list.
+ *al = finalized.insertListWithCursorAtEnd(*al);
+
+ lists->arenaListsToSweep[thingKind] = nullptr;
+ }
+
+ lists->backgroundFinalizeState[thingKind] = BFS_DONE;
+}
+
+void
+ArenaLists::queueForegroundObjectsForSweep(FreeOp* fop)
+{
+ gcstats::AutoPhase ap(fop->runtime()->gc.stats, gcstats::PHASE_SWEEP_OBJECT);
+
+#ifdef DEBUG
+ for (auto i : ObjectAllocKinds())
+ MOZ_ASSERT(savedObjectArenas[i].isEmpty());
+ MOZ_ASSERT(savedEmptyObjectArenas == nullptr);
+#endif
+
+ // Foreground finalized objects must be finalized at the beginning of the
+ // sweep phase, before control can return to the mutator. Otherwise,
+ // mutator behavior can resurrect certain objects whose references would
+ // otherwise have been erased by the finalizer.
+ finalizeNow(fop, AllocKind::OBJECT0, KEEP_ARENAS, &savedEmptyObjectArenas);
+ finalizeNow(fop, AllocKind::OBJECT2, KEEP_ARENAS, &savedEmptyObjectArenas);
+ finalizeNow(fop, AllocKind::OBJECT4, KEEP_ARENAS, &savedEmptyObjectArenas);
+ finalizeNow(fop, AllocKind::OBJECT8, KEEP_ARENAS, &savedEmptyObjectArenas);
+ finalizeNow(fop, AllocKind::OBJECT12, KEEP_ARENAS, &savedEmptyObjectArenas);
+ finalizeNow(fop, AllocKind::OBJECT16, KEEP_ARENAS, &savedEmptyObjectArenas);
+
+ // Prevent the arenas from having new objects allocated into them. We need
+ // to know which objects are marked while we incrementally sweep dead
+ // references from type information.
+ savedObjectArenas[AllocKind::OBJECT0] = arenaLists[AllocKind::OBJECT0].copyAndClear();
+ savedObjectArenas[AllocKind::OBJECT2] = arenaLists[AllocKind::OBJECT2].copyAndClear();
+ savedObjectArenas[AllocKind::OBJECT4] = arenaLists[AllocKind::OBJECT4].copyAndClear();
+ savedObjectArenas[AllocKind::OBJECT8] = arenaLists[AllocKind::OBJECT8].copyAndClear();
+ savedObjectArenas[AllocKind::OBJECT12] = arenaLists[AllocKind::OBJECT12].copyAndClear();
+ savedObjectArenas[AllocKind::OBJECT16] = arenaLists[AllocKind::OBJECT16].copyAndClear();
+}
+
+void
+ArenaLists::mergeForegroundSweptObjectArenas()
+{
+ AutoLockGC lock(runtime_);
+ ReleaseArenaList(runtime_, savedEmptyObjectArenas, lock);
+ savedEmptyObjectArenas = nullptr;
+
+ mergeSweptArenas(AllocKind::OBJECT0);
+ mergeSweptArenas(AllocKind::OBJECT2);
+ mergeSweptArenas(AllocKind::OBJECT4);
+ mergeSweptArenas(AllocKind::OBJECT8);
+ mergeSweptArenas(AllocKind::OBJECT12);
+ mergeSweptArenas(AllocKind::OBJECT16);
+}
+
+inline void
+ArenaLists::mergeSweptArenas(AllocKind thingKind)
+{
+ ArenaList* al = &arenaLists[thingKind];
+ ArenaList* saved = &savedObjectArenas[thingKind];
+
+ *al = saved->insertListWithCursorAtEnd(*al);
+ saved->clear();
+}
+
+void
+ArenaLists::queueForegroundThingsForSweep(FreeOp* fop)
+{
+ gcShapeArenasToUpdate = arenaListsToSweep[AllocKind::SHAPE];
+ gcAccessorShapeArenasToUpdate = arenaListsToSweep[AllocKind::ACCESSOR_SHAPE];
+ gcObjectGroupArenasToUpdate = arenaListsToSweep[AllocKind::OBJECT_GROUP];
+ gcScriptArenasToUpdate = arenaListsToSweep[AllocKind::SCRIPT];
+}
+
+SliceBudget::SliceBudget()
+ : timeBudget(UnlimitedTimeBudget), workBudget(UnlimitedWorkBudget)
+{
+ makeUnlimited();
+}
+
+SliceBudget::SliceBudget(TimeBudget time)
+ : timeBudget(time), workBudget(UnlimitedWorkBudget)
+{
+ if (time.budget < 0) {
+ makeUnlimited();
+ } else {
+ // Note: TimeBudget(0) is equivalent to WorkBudget(CounterReset).
+ deadline = PRMJ_Now() + time.budget * PRMJ_USEC_PER_MSEC;
+ counter = CounterReset;
+ }
+}
+
+SliceBudget::SliceBudget(WorkBudget work)
+ : timeBudget(UnlimitedTimeBudget), workBudget(work)
+{
+ if (work.budget < 0) {
+ makeUnlimited();
+ } else {
+ deadline = 0;
+ counter = work.budget;
+ }
+}
+
+int
+SliceBudget::describe(char* buffer, size_t maxlen) const
+{
+ if (isUnlimited())
+ return snprintf(buffer, maxlen, "unlimited");
+ else if (isWorkBudget())
+ return snprintf(buffer, maxlen, "work(%" PRId64 ")", workBudget.budget);
+ else
+ return snprintf(buffer, maxlen, "%" PRId64 "ms", timeBudget.budget);
+}
+
+bool
+SliceBudget::checkOverBudget()
+{
+ bool over = PRMJ_Now() >= deadline;
+ if (!over)
+ counter = CounterReset;
+ return over;
+}
+
+void
+js::MarkCompartmentActive(InterpreterFrame* fp)
+{
+ fp->script()->compartment()->zone()->active = true;
+}
+
+void
+GCRuntime::requestMajorGC(JS::gcreason::Reason reason)
+{
+ MOZ_ASSERT(!CurrentThreadIsPerformingGC());
+
+ if (majorGCRequested())
+ return;
+
+ majorGCTriggerReason = reason;
+
+ // There's no need to use RequestInterruptUrgent here. It's slower because
+ // it has to interrupt (looping) Ion code, but loops in Ion code that
+ // affect GC will have an explicit interrupt check.
+ rt->requestInterrupt(JSRuntime::RequestInterruptCanWait);
+}
+
+void
+GCRuntime::requestMinorGC(JS::gcreason::Reason reason)
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(!CurrentThreadIsPerformingGC());
+
+ if (minorGCRequested())
+ return;
+
+ minorGCTriggerReason = reason;
+
+ // See comment in requestMajorGC.
+ rt->requestInterrupt(JSRuntime::RequestInterruptCanWait);
+}
+
+bool
+GCRuntime::triggerGC(JS::gcreason::Reason reason)
+{
+ /*
+ * Don't trigger GCs if this is being called off the main thread from
+ * onTooMuchMalloc().
+ */
+ if (!CurrentThreadCanAccessRuntime(rt))
+ return false;
+
+ /* GC is already running. */
+ if (rt->isHeapCollecting())
+ return false;
+
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+ requestMajorGC(reason);
+ return true;
+}
+
+void
+GCRuntime::maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock)
+{
+ size_t usedBytes = zone->usage.gcBytes();
+ size_t thresholdBytes = zone->threshold.gcTriggerBytes();
+ size_t igcThresholdBytes = thresholdBytes * tunables.zoneAllocThresholdFactor();
+
+ if (usedBytes >= thresholdBytes) {
+ // The threshold has been surpassed, immediately trigger a GC,
+ // which will be done non-incrementally.
+ triggerZoneGC(zone, JS::gcreason::ALLOC_TRIGGER);
+ } else if (usedBytes >= igcThresholdBytes) {
+ // Reduce the delay to the start of the next incremental slice.
+ if (zone->gcDelayBytes < ArenaSize)
+ zone->gcDelayBytes = 0;
+ else
+ zone->gcDelayBytes -= ArenaSize;
+
+ if (!zone->gcDelayBytes) {
+ // Start or continue an in progress incremental GC. We do this
+ // to try to avoid performing non-incremental GCs on zones
+ // which allocate a lot of data, even when incremental slices
+ // can't be triggered via scheduling in the event loop.
+ triggerZoneGC(zone, JS::gcreason::ALLOC_TRIGGER);
+
+ // Delay the next slice until a certain amount of allocation
+ // has been performed.
+ zone->gcDelayBytes = tunables.zoneAllocDelayBytes();
+ }
+ }
+}
+
+bool
+GCRuntime::triggerZoneGC(Zone* zone, JS::gcreason::Reason reason)
+{
+ /* Zones in use by a thread with an exclusive context can't be collected. */
+ if (!CurrentThreadCanAccessRuntime(rt)) {
+ MOZ_ASSERT(zone->usedByExclusiveThread || zone->isAtomsZone());
+ return false;
+ }
+
+ /* GC is already running. */
+ if (rt->isHeapCollecting())
+ return false;
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::Alloc)) {
+ MOZ_RELEASE_ASSERT(triggerGC(reason));
+ return true;
+ }
+#endif
+
+ if (zone->isAtomsZone()) {
+ /* We can't do a zone GC of the atoms compartment. */
+ if (rt->keepAtoms()) {
+ /* Skip GC and retrigger later, since atoms zone won't be collected
+ * if keepAtoms is true. */
+ fullGCForAtomsRequested_ = true;
+ return false;
+ }
+ MOZ_RELEASE_ASSERT(triggerGC(reason));
+ return true;
+ }
+
+ PrepareZoneForGC(zone);
+ requestMajorGC(reason);
+ return true;
+}
+
+void
+GCRuntime::maybeGC(Zone* zone)
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::Alloc) || hasZealMode(ZealMode::Poke)) {
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+ gc(GC_NORMAL, JS::gcreason::DEBUG_GC);
+ return;
+ }
+#endif
+
+ if (gcIfRequested())
+ return;
+
+ if (zone->usage.gcBytes() > 1024 * 1024 &&
+ zone->usage.gcBytes() >= zone->threshold.allocTrigger(schedulingState.inHighFrequencyGCMode()) &&
+ !isIncrementalGCInProgress() &&
+ !isBackgroundSweeping())
+ {
+ PrepareZoneForGC(zone);
+ startGC(GC_NORMAL, JS::gcreason::EAGER_ALLOC_TRIGGER);
+ }
+}
+
+// Do all possible decommit immediately from the current thread without
+// releasing the GC lock or allocating any memory.
+void
+GCRuntime::decommitAllWithoutUnlocking(const AutoLockGC& lock)
+{
+ MOZ_ASSERT(emptyChunks(lock).count() == 0);
+ for (ChunkPool::Iter chunk(availableChunks(lock)); !chunk.done(); chunk.next())
+ chunk->decommitAllArenasWithoutUnlocking(lock);
+ MOZ_ASSERT(availableChunks(lock).verify());
+}
+
+void
+GCRuntime::startDecommit()
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ MOZ_ASSERT(!decommitTask.isRunning());
+
+ // If we are allocating heavily enough to trigger "high freqency" GC, then
+ // skip decommit so that we do not compete with the mutator.
+ if (schedulingState.inHighFrequencyGCMode())
+ return;
+
+ BackgroundDecommitTask::ChunkVector toDecommit;
+ {
+ AutoLockGC lock(rt);
+
+ // Verify that all entries in the empty chunks pool are already decommitted.
+ for (ChunkPool::Iter chunk(emptyChunks(lock)); !chunk.done(); chunk.next())
+ MOZ_ASSERT(!chunk->info.numArenasFreeCommitted);
+
+ // Since we release the GC lock while doing the decommit syscall below,
+ // it is dangerous to iterate the available list directly, as the main
+ // thread could modify it concurrently. Instead, we build and pass an
+ // explicit Vector containing the Chunks we want to visit.
+ MOZ_ASSERT(availableChunks(lock).verify());
+ for (ChunkPool::Iter iter(availableChunks(lock)); !iter.done(); iter.next()) {
+ if (!toDecommit.append(iter.get())) {
+ // The OOM handler does a full, immediate decommit.
+ return onOutOfMallocMemory(lock);
+ }
+ }
+ }
+ decommitTask.setChunksToScan(toDecommit);
+
+ if (sweepOnBackgroundThread && decommitTask.start())
+ return;
+
+ decommitTask.runFromMainThread(rt);
+}
+
+void
+js::gc::BackgroundDecommitTask::setChunksToScan(ChunkVector &chunks)
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime));
+ MOZ_ASSERT(!isRunning());
+ MOZ_ASSERT(toDecommit.empty());
+ Swap(toDecommit, chunks);
+}
+
+/* virtual */ void
+js::gc::BackgroundDecommitTask::run()
+{
+ AutoLockGC lock(runtime);
+
+ for (Chunk* chunk : toDecommit) {
+
+ // The arena list is not doubly-linked, so we have to work in the free
+ // list order and not in the natural order.
+ while (chunk->info.numArenasFreeCommitted) {
+ bool ok = chunk->decommitOneFreeArena(runtime, lock);
+
+ // If we are low enough on memory that we can't update the page
+ // tables, or if we need to return for any other reason, break out
+ // of the loop.
+ if (cancel_ || !ok)
+ break;
+ }
+ }
+ toDecommit.clearAndFree();
+
+ ChunkPool toFree = runtime->gc.expireEmptyChunkPool(lock);
+ if (toFree.count()) {
+ AutoUnlockGC unlock(lock);
+ FreeChunkPool(runtime, toFree);
+ }
+}
+
+void
+GCRuntime::sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks)
+{
+ freeBlocks.freeAll();
+
+ if (zones.isEmpty())
+ return;
+
+ // We must finalize thing kinds in the order specified by BackgroundFinalizePhases.
+ Arena* emptyArenas = nullptr;
+ FreeOp fop(nullptr);
+ for (unsigned phase = 0 ; phase < ArrayLength(BackgroundFinalizePhases) ; ++phase) {
+ for (Zone* zone = zones.front(); zone; zone = zone->nextZone()) {
+ for (auto kind : BackgroundFinalizePhases[phase].kinds) {
+ Arena* arenas = zone->arenas.arenaListsToSweep[kind];
+ MOZ_RELEASE_ASSERT(uintptr_t(arenas) != uintptr_t(-1));
+ if (arenas)
+ ArenaLists::backgroundFinalize(&fop, arenas, &emptyArenas);
+ }
+ }
+ }
+
+ AutoLockGC lock(rt);
+
+ // Release swept arenas, dropping and reaquiring the lock every so often to
+ // avoid blocking the main thread from allocating chunks.
+ static const size_t LockReleasePeriod = 32;
+ size_t releaseCount = 0;
+ Arena* next;
+ for (Arena* arena = emptyArenas; arena; arena = next) {
+ next = arena->next;
+ rt->gc.releaseArena(arena, lock);
+ releaseCount++;
+ if (releaseCount % LockReleasePeriod == 0) {
+ lock.unlock();
+ lock.lock();
+ }
+ }
+
+ while (!zones.isEmpty())
+ zones.removeFront();
+}
+
+void
+GCRuntime::assertBackgroundSweepingFinished()
+{
+#ifdef DEBUG
+ MOZ_ASSERT(backgroundSweepZones.isEmpty());
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT(!zone->arenas.arenaListsToSweep[i]);
+ MOZ_ASSERT(zone->arenas.doneBackgroundFinalize(i));
+ }
+ }
+ MOZ_ASSERT(blocksToFreeAfterSweeping.computedSizeOfExcludingThis() == 0);
+#endif
+}
+
+unsigned
+js::GetCPUCount()
+{
+ static unsigned ncpus = 0;
+ if (ncpus == 0) {
+# ifdef XP_WIN
+ SYSTEM_INFO sysinfo;
+ GetSystemInfo(&sysinfo);
+ ncpus = unsigned(sysinfo.dwNumberOfProcessors);
+# else
+ long n = sysconf(_SC_NPROCESSORS_ONLN);
+ ncpus = (n > 0) ? unsigned(n) : 1;
+# endif
+ }
+ return ncpus;
+}
+
+void
+GCHelperState::finish()
+{
+ // Wait for any lingering background sweeping to finish.
+ waitBackgroundSweepEnd();
+}
+
+GCHelperState::State
+GCHelperState::state(const AutoLockGC&)
+{
+ return state_;
+}
+
+void
+GCHelperState::setState(State state, const AutoLockGC&)
+{
+ state_ = state;
+}
+
+void
+GCHelperState::startBackgroundThread(State newState, const AutoLockGC& lock,
+ const AutoLockHelperThreadState& helperLock)
+{
+ MOZ_ASSERT(!thread && state(lock) == IDLE && newState != IDLE);
+ setState(newState, lock);
+
+ {
+ AutoEnterOOMUnsafeRegion noOOM;
+ if (!HelperThreadState().gcHelperWorklist(helperLock).append(this))
+ noOOM.crash("Could not add to pending GC helpers list");
+ }
+
+ HelperThreadState().notifyAll(GlobalHelperThreadState::PRODUCER, helperLock);
+}
+
+void
+GCHelperState::waitForBackgroundThread(js::AutoLockGC& lock)
+{
+ done.wait(lock.guard());
+}
+
+void
+GCHelperState::work()
+{
+ MOZ_ASSERT(CanUseExtraThreads());
+
+ AutoLockGC lock(rt);
+
+ MOZ_ASSERT(thread.isNothing());
+ thread = mozilla::Some(ThisThread::GetId());
+
+ TraceLoggerThread* logger = TraceLoggerForCurrentThread();
+
+ switch (state(lock)) {
+
+ case IDLE:
+ MOZ_CRASH("GC helper triggered on idle state");
+ break;
+
+ case SWEEPING: {
+ AutoTraceLog logSweeping(logger, TraceLogger_GCSweeping);
+ doSweep(lock);
+ MOZ_ASSERT(state(lock) == SWEEPING);
+ break;
+ }
+
+ }
+
+ setState(IDLE, lock);
+ thread.reset();
+
+ done.notify_all();
+}
+
+void
+GCRuntime::queueZonesForBackgroundSweep(ZoneList& zones)
+{
+ AutoLockHelperThreadState helperLock;
+ AutoLockGC lock(rt);
+ backgroundSweepZones.transferFrom(zones);
+ helperState.maybeStartBackgroundSweep(lock, helperLock);
+}
+
+void
+GCRuntime::freeUnusedLifoBlocksAfterSweeping(LifoAlloc* lifo)
+{
+ MOZ_ASSERT(rt->isHeapBusy());
+ AutoLockGC lock(rt);
+ blocksToFreeAfterSweeping.transferUnusedFrom(lifo);
+}
+
+void
+GCRuntime::freeAllLifoBlocksAfterSweeping(LifoAlloc* lifo)
+{
+ MOZ_ASSERT(rt->isHeapBusy());
+ AutoLockGC lock(rt);
+ blocksToFreeAfterSweeping.transferFrom(lifo);
+}
+
+void
+GCRuntime::freeAllLifoBlocksAfterMinorGC(LifoAlloc* lifo)
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ blocksToFreeAfterMinorGC.transferFrom(lifo);
+}
+
+void
+GCHelperState::maybeStartBackgroundSweep(const AutoLockGC& lock,
+ const AutoLockHelperThreadState& helperLock)
+{
+ MOZ_ASSERT(CanUseExtraThreads());
+
+ if (state(lock) == IDLE)
+ startBackgroundThread(SWEEPING, lock, helperLock);
+}
+
+void
+GCHelperState::waitBackgroundSweepEnd()
+{
+ AutoLockGC lock(rt);
+ while (state(lock) == SWEEPING)
+ waitForBackgroundThread(lock);
+ if (!rt->gc.isIncrementalGCInProgress())
+ rt->gc.assertBackgroundSweepingFinished();
+}
+
+void
+GCHelperState::doSweep(AutoLockGC& lock)
+{
+ // The main thread may call queueZonesForBackgroundSweep() while this is
+ // running so we must check there is no more work to do before exiting.
+
+ do {
+ while (!rt->gc.backgroundSweepZones.isEmpty()) {
+ AutoSetThreadIsSweeping threadIsSweeping;
+
+ ZoneList zones;
+ zones.transferFrom(rt->gc.backgroundSweepZones);
+ LifoAlloc freeLifoAlloc(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE);
+ freeLifoAlloc.transferFrom(&rt->gc.blocksToFreeAfterSweeping);
+
+ AutoUnlockGC unlock(lock);
+ rt->gc.sweepBackgroundThings(zones, freeLifoAlloc);
+ }
+ } while (!rt->gc.backgroundSweepZones.isEmpty());
+}
+
+bool
+GCHelperState::onBackgroundThread()
+{
+ return thread.isSome() && *thread == ThisThread::GetId();
+}
+
+bool
+GCRuntime::shouldReleaseObservedTypes()
+{
+ bool releaseTypes = false;
+
+#ifdef JS_GC_ZEAL
+ if (zealModeBits != 0)
+ releaseTypes = true;
+#endif
+
+ /* We may miss the exact target GC due to resets. */
+ if (majorGCNumber >= jitReleaseNumber)
+ releaseTypes = true;
+
+ if (releaseTypes)
+ jitReleaseNumber = majorGCNumber + JIT_SCRIPT_RELEASE_TYPES_PERIOD;
+
+ return releaseTypes;
+}
+
+struct IsAboutToBeFinalizedFunctor {
+ template <typename T> bool operator()(Cell** t) {
+ mozilla::DebugOnly<const Cell*> prior = *t;
+ bool result = IsAboutToBeFinalizedUnbarriered(reinterpret_cast<T**>(t));
+ // Sweep should not have to deal with moved pointers, since moving GC
+ // handles updating the UID table manually.
+ MOZ_ASSERT(*t == prior);
+ return result;
+ }
+};
+
+/* static */ bool
+UniqueIdGCPolicy::needsSweep(Cell** cell, uint64_t*)
+{
+ return DispatchTraceKindTyped(IsAboutToBeFinalizedFunctor(), (*cell)->getTraceKind(), cell);
+}
+
+void
+JS::Zone::sweepUniqueIds(js::FreeOp* fop)
+{
+ uniqueIds_.sweep();
+}
+
+/*
+ * It's simpler if we preserve the invariant that every zone has at least one
+ * compartment. If we know we're deleting the entire zone, then
+ * SweepCompartments is allowed to delete all compartments. In this case,
+ * |keepAtleastOne| is false. If some objects remain in the zone so that it
+ * cannot be deleted, then we set |keepAtleastOne| to true, which prohibits
+ * SweepCompartments from deleting every compartment. Instead, it preserves an
+ * arbitrary compartment in the zone.
+ */
+void
+Zone::sweepCompartments(FreeOp* fop, bool keepAtleastOne, bool destroyingRuntime)
+{
+ JSRuntime* rt = runtimeFromMainThread();
+ JSDestroyCompartmentCallback callback = rt->destroyCompartmentCallback;
+
+ JSCompartment** read = compartments.begin();
+ JSCompartment** end = compartments.end();
+ JSCompartment** write = read;
+ bool foundOne = false;
+ while (read < end) {
+ JSCompartment* comp = *read++;
+ MOZ_ASSERT(!rt->isAtomsCompartment(comp));
+
+ /*
+ * Don't delete the last compartment if all the ones before it were
+ * deleted and keepAtleastOne is true.
+ */
+ bool dontDelete = read == end && !foundOne && keepAtleastOne;
+ if ((!comp->marked && !dontDelete) || destroyingRuntime) {
+ if (callback)
+ callback(fop, comp);
+ if (comp->principals())
+ JS_DropPrincipals(rt->contextFromMainThread(), comp->principals());
+ js_delete(comp);
+ rt->gc.stats.sweptCompartment();
+ } else {
+ *write++ = comp;
+ foundOne = true;
+ }
+ }
+ compartments.shrinkTo(write - compartments.begin());
+ MOZ_ASSERT_IF(keepAtleastOne, !compartments.empty());
+}
+
+void
+GCRuntime::sweepZones(FreeOp* fop, bool destroyingRuntime)
+{
+ MOZ_ASSERT_IF(destroyingRuntime, numActiveZoneIters == 0);
+ MOZ_ASSERT_IF(destroyingRuntime, arenasEmptyAtShutdown);
+
+ if (rt->gc.numActiveZoneIters)
+ return;
+
+ assertBackgroundSweepingFinished();
+
+ JSZoneCallback callback = rt->destroyZoneCallback;
+
+ /* Skip the atomsCompartment zone. */
+ Zone** read = zones.begin() + 1;
+ Zone** end = zones.end();
+ Zone** write = read;
+ MOZ_ASSERT(zones.length() >= 1);
+ MOZ_ASSERT(zones[0]->isAtomsZone());
+
+ while (read < end) {
+ Zone* zone = *read++;
+
+ if (zone->wasGCStarted()) {
+ MOZ_ASSERT(!zone->isQueuedForBackgroundSweep());
+ const bool zoneIsDead = zone->arenas.arenaListsAreEmpty() &&
+ !zone->hasMarkedCompartments();
+ if (zoneIsDead || destroyingRuntime)
+ {
+ // We have just finished sweeping, so we should have freed any
+ // empty arenas back to their Chunk for future allocation.
+ zone->arenas.checkEmptyFreeLists();
+
+ // We are about to delete the Zone; this will leave the Zone*
+ // in the arena header dangling if there are any arenas
+ // remaining at this point.
+#ifdef DEBUG
+ if (!zone->arenas.checkEmptyArenaLists())
+ arenasEmptyAtShutdown = false;
+#endif
+
+ if (callback)
+ callback(zone);
+
+ zone->sweepCompartments(fop, false, destroyingRuntime);
+ MOZ_ASSERT(zone->compartments.empty());
+ MOZ_ASSERT_IF(arenasEmptyAtShutdown, zone->typeDescrObjects.empty());
+ fop->delete_(zone);
+ stats.sweptZone();
+ continue;
+ }
+ zone->sweepCompartments(fop, true, destroyingRuntime);
+ }
+ *write++ = zone;
+ }
+ zones.shrinkTo(write - zones.begin());
+}
+
+#ifdef DEBUG
+static const char*
+AllocKindToAscii(AllocKind kind)
+{
+ switch(kind) {
+#define MAKE_CASE(allocKind, traceKind, type, sizedType) \
+ case AllocKind:: allocKind: return #allocKind;
+FOR_EACH_ALLOCKIND(MAKE_CASE)
+#undef MAKE_CASE
+
+ default:
+ MOZ_CRASH("Unknown AllocKind in AllocKindToAscii");
+ }
+}
+#endif // DEBUG
+
+bool
+ArenaLists::checkEmptyArenaList(AllocKind kind)
+{
+ size_t num_live = 0;
+#ifdef DEBUG
+ if (!arenaLists[kind].isEmpty()) {
+ size_t max_cells = 20;
+ char *env = getenv("JS_GC_MAX_LIVE_CELLS");
+ if (env && *env)
+ max_cells = atol(env);
+ for (Arena* current = arenaLists[kind].head(); current; current = current->next) {
+ for (ArenaCellIterUnderGC i(current); !i.done(); i.next()) {
+ TenuredCell* t = i.getCell();
+ MOZ_ASSERT(t->isMarked(), "unmarked cells should have been finalized");
+ if (++num_live <= max_cells) {
+ fprintf(stderr, "ERROR: GC found live Cell %p of kind %s at shutdown\n",
+ t, AllocKindToAscii(kind));
+ }
+ }
+ }
+ fprintf(stderr, "ERROR: GC found %" PRIuSIZE " live Cells at shutdown\n", num_live);
+ }
+#endif // DEBUG
+ return num_live == 0;
+}
+
+void
+GCRuntime::purgeRuntime(AutoLockForExclusiveAccess& lock)
+{
+ for (GCCompartmentsIter comp(rt); !comp.done(); comp.next())
+ comp->purge();
+
+ freeUnusedLifoBlocksAfterSweeping(&rt->tempLifoAlloc);
+
+ rt->interpreterStack().purge(rt);
+
+ JSContext* cx = rt->contextFromMainThread();
+ cx->caches.gsnCache.purge();
+ cx->caches.envCoordinateNameCache.purge();
+ cx->caches.newObjectCache.purge();
+ cx->caches.nativeIterCache.purge();
+ cx->caches.uncompressedSourceCache.purge();
+ if (cx->caches.evalCache.initialized())
+ cx->caches.evalCache.clear();
+
+ rt->mainThread.frontendCollectionPool.purge();
+
+ if (auto cache = rt->maybeThisRuntimeSharedImmutableStrings())
+ cache->purge();
+
+ rt->promiseTasksToDestroy.lock()->clear();
+}
+
+bool
+GCRuntime::shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime,
+ JS::gcreason::Reason reason, bool canAllocateMoreCode)
+{
+ if (cleanUpEverything)
+ return false;
+ if (!canAllocateMoreCode)
+ return false;
+
+ if (alwaysPreserveCode)
+ return true;
+ if (comp->preserveJitCode())
+ return true;
+ if (comp->lastAnimationTime + PRMJ_USEC_PER_SEC >= currentTime)
+ return true;
+ if (reason == JS::gcreason::DEBUG_GC)
+ return true;
+
+ return false;
+}
+
+#ifdef DEBUG
+class CompartmentCheckTracer : public JS::CallbackTracer
+{
+ void onChild(const JS::GCCellPtr& thing) override;
+
+ public:
+ explicit CompartmentCheckTracer(JSRuntime* rt)
+ : JS::CallbackTracer(rt), src(nullptr), zone(nullptr), compartment(nullptr)
+ {}
+
+ Cell* src;
+ JS::TraceKind srcKind;
+ Zone* zone;
+ JSCompartment* compartment;
+};
+
+namespace {
+struct IsDestComparatorFunctor {
+ JS::GCCellPtr dst_;
+ explicit IsDestComparatorFunctor(JS::GCCellPtr dst) : dst_(dst) {}
+
+ template <typename T> bool operator()(T* t) { return (*t) == dst_.asCell(); }
+};
+} // namespace (anonymous)
+
+static bool
+InCrossCompartmentMap(JSObject* src, JS::GCCellPtr dst)
+{
+ JSCompartment* srccomp = src->compartment();
+
+ if (dst.is<JSObject>()) {
+ Value key = ObjectValue(dst.as<JSObject>());
+ if (WrapperMap::Ptr p = srccomp->lookupWrapper(key)) {
+ if (*p->value().unsafeGet() == ObjectValue(*src))
+ return true;
+ }
+ }
+
+ /*
+ * If the cross-compartment edge is caused by the debugger, then we don't
+ * know the right hashtable key, so we have to iterate.
+ */
+ for (JSCompartment::WrapperEnum e(srccomp); !e.empty(); e.popFront()) {
+ if (e.front().mutableKey().applyToWrapped(IsDestComparatorFunctor(dst)) &&
+ ToMarkable(e.front().value().unbarrieredGet()) == src)
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+struct MaybeCompartmentFunctor {
+ template <typename T> JSCompartment* operator()(T* t) { return t->maybeCompartment(); }
+};
+
+void
+CompartmentCheckTracer::onChild(const JS::GCCellPtr& thing)
+{
+ JSCompartment* comp = DispatchTyped(MaybeCompartmentFunctor(), thing);
+ if (comp && compartment) {
+ MOZ_ASSERT(comp == compartment || runtime()->isAtomsCompartment(comp) ||
+ (srcKind == JS::TraceKind::Object &&
+ InCrossCompartmentMap(static_cast<JSObject*>(src), thing)));
+ } else {
+ TenuredCell* tenured = TenuredCell::fromPointer(thing.asCell());
+ Zone* thingZone = tenured->zoneFromAnyThread();
+ MOZ_ASSERT(thingZone == zone || thingZone->isAtomsZone());
+ }
+}
+
+void
+GCRuntime::checkForCompartmentMismatches()
+{
+ if (disableStrictProxyCheckingCount)
+ return;
+
+ CompartmentCheckTracer trc(rt);
+ AutoAssertEmptyNursery empty(rt);
+ for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
+ trc.zone = zone;
+ for (auto thingKind : AllAllocKinds()) {
+ for (auto i = zone->cellIter<TenuredCell>(thingKind, empty); !i.done(); i.next()) {
+ trc.src = i.getCell();
+ trc.srcKind = MapAllocToTraceKind(thingKind);
+ trc.compartment = DispatchTraceKindTyped(MaybeCompartmentFunctor(),
+ trc.src, trc.srcKind);
+ js::TraceChildren(&trc, trc.src, trc.srcKind);
+ }
+ }
+ }
+}
+#endif
+
+static void
+RelazifyFunctions(Zone* zone, AllocKind kind)
+{
+ MOZ_ASSERT(kind == AllocKind::FUNCTION ||
+ kind == AllocKind::FUNCTION_EXTENDED);
+
+ JSRuntime* rt = zone->runtimeFromMainThread();
+ AutoAssertEmptyNursery empty(rt);
+
+ for (auto i = zone->cellIter<JSObject>(kind, empty); !i.done(); i.next()) {
+ JSFunction* fun = &i->as<JSFunction>();
+ if (fun->hasScript())
+ fun->maybeRelazify(rt);
+ }
+}
+
+bool
+GCRuntime::beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock)
+{
+ int64_t currentTime = PRMJ_Now();
+
+#ifdef DEBUG
+ if (fullCompartmentChecks)
+ checkForCompartmentMismatches();
+#endif
+
+ isFull = true;
+ bool any = false;
+
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ /* Assert that zone state is as we expect */
+ MOZ_ASSERT(!zone->isCollecting());
+ MOZ_ASSERT(!zone->compartments.empty());
+#ifdef DEBUG
+ for (auto i : AllAllocKinds())
+ MOZ_ASSERT(!zone->arenas.arenaListsToSweep[i]);
+#endif
+
+ /* Set up which zones will be collected. */
+ if (zone->isGCScheduled()) {
+ if (!zone->isAtomsZone()) {
+ any = true;
+ zone->setGCState(Zone::Mark);
+ }
+ } else {
+ isFull = false;
+ }
+
+ zone->setPreservingCode(false);
+ }
+
+ // Discard JIT code more aggressively if the process is approaching its
+ // executable code limit.
+ bool canAllocateMoreCode = jit::CanLikelyAllocateMoreExecutableMemory();
+
+ for (CompartmentsIter c(rt, WithAtoms); !c.done(); c.next()) {
+ c->marked = false;
+ c->scheduledForDestruction = false;
+ c->maybeAlive = false;
+ if (shouldPreserveJITCode(c, currentTime, reason, canAllocateMoreCode))
+ c->zone()->setPreservingCode(true);
+ }
+
+ if (!rt->gc.cleanUpEverything && canAllocateMoreCode) {
+ if (JSCompartment* comp = jit::TopmostIonActivationCompartment(rt))
+ comp->zone()->setPreservingCode(true);
+ }
+
+ /*
+ * Atoms are not in the cross-compartment map. So if there are any
+ * zones that are not being collected, we are not allowed to collect
+ * atoms. Otherwise, the non-collected zones could contain pointers
+ * to atoms that we would miss.
+ *
+ * keepAtoms() will only change on the main thread, which we are currently
+ * on. If the value of keepAtoms() changes between GC slices, then we'll
+ * cancel the incremental GC. See IsIncrementalGCSafe.
+ */
+ if (isFull && !rt->keepAtoms()) {
+ Zone* atomsZone = rt->atomsCompartment(lock)->zone();
+ if (atomsZone->isGCScheduled()) {
+ MOZ_ASSERT(!atomsZone->isCollecting());
+ atomsZone->setGCState(Zone::Mark);
+ any = true;
+ }
+ }
+
+ /* Check that at least one zone is scheduled for collection. */
+ if (!any)
+ return false;
+
+ /*
+ * At the end of each incremental slice, we call prepareForIncrementalGC,
+ * which marks objects in all arenas that we're currently allocating
+ * into. This can cause leaks if unreachable objects are in these
+ * arenas. This purge call ensures that we only mark arenas that have had
+ * allocations after the incremental GC started.
+ */
+ if (isIncremental) {
+ for (GCZonesIter zone(rt); !zone.done(); zone.next())
+ zone->arenas.purge();
+ }
+
+ MemProfiler::MarkTenuredStart(rt);
+ marker.start();
+ GCMarker* gcmarker = &marker;
+
+ /* For non-incremental GC the following sweep discards the jit code. */
+ if (isIncremental) {
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Mark);
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_DISCARD_CODE);
+ zone->discardJitCode(rt->defaultFreeOp());
+ }
+ }
+
+ /*
+ * Relazify functions after discarding JIT code (we can't relazify
+ * functions with JIT code) and before the actual mark phase, so that
+ * the current GC can collect the JSScripts we're unlinking here.
+ * We do this only when we're performing a shrinking GC, as too much
+ * relazification can cause performance issues when we have to reparse
+ * the same functions over and over.
+ */
+ if (invocationKind == GC_SHRINK) {
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_RELAZIFY_FUNCTIONS);
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ if (zone->isSelfHostingZone())
+ continue;
+ RelazifyFunctions(zone, AllocKind::FUNCTION);
+ RelazifyFunctions(zone, AllocKind::FUNCTION_EXTENDED);
+ }
+ }
+
+ /* Purge ShapeTables. */
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_PURGE_SHAPE_TABLES);
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ if (zone->keepShapeTables() || zone->isSelfHostingZone())
+ continue;
+ for (auto baseShape = zone->cellIter<BaseShape>(); !baseShape.done(); baseShape.next())
+ baseShape->maybePurgeTable();
+ }
+ }
+
+ startNumber = number;
+
+ /*
+ * We must purge the runtime at the beginning of an incremental GC. The
+ * danger if we purge later is that the snapshot invariant of incremental
+ * GC will be broken, as follows. If some object is reachable only through
+ * some cache (say the dtoaCache) then it will not be part of the snapshot.
+ * If we purge after root marking, then the mutator could obtain a pointer
+ * to the object and start using it. This object might never be marked, so
+ * a GC hazard would exist.
+ */
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_PURGE);
+ purgeRuntime(lock);
+ }
+
+ /*
+ * Mark phase.
+ */
+ gcstats::AutoPhase ap1(stats, gcstats::PHASE_MARK);
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_UNMARK);
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ /* Unmark everything in the zones being collected. */
+ zone->arenas.unmarkAll();
+ }
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ /* Unmark all weak maps in the zones being collected. */
+ WeakMapBase::unmarkZone(zone);
+ }
+ }
+
+ traceRuntimeForMajorGC(gcmarker, lock);
+
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_MARK_ROOTS);
+
+ if (isIncremental) {
+ gcstats::AutoPhase ap3(stats, gcstats::PHASE_BUFFER_GRAY_ROOTS);
+ bufferGrayRoots();
+ }
+
+ markCompartments();
+
+ return true;
+}
+
+void
+GCRuntime::markCompartments()
+{
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_COMPARTMENTS);
+
+ /*
+ * This code ensures that if a compartment is "dead", then it will be
+ * collected in this GC. A compartment is considered dead if its maybeAlive
+ * flag is false. The maybeAlive flag is set if:
+ * (1) the compartment has incoming cross-compartment edges, or
+ * (2) an object in the compartment was marked during root marking, either
+ * as a black root or a gray root.
+ * If the maybeAlive is false, then we set the scheduledForDestruction flag.
+ * At the end of the GC, we look for compartments where
+ * scheduledForDestruction is true. These are compartments that were somehow
+ * "revived" during the incremental GC. If any are found, we do a special,
+ * non-incremental GC of those compartments to try to collect them.
+ *
+ * Compartments can be revived for a variety of reasons. On reason is bug
+ * 811587, where a reflector that was dead can be revived by DOM code that
+ * still refers to the underlying DOM node.
+ *
+ * Read barriers and allocations can also cause revival. This might happen
+ * during a function like JS_TransplantObject, which iterates over all
+ * compartments, live or dead, and operates on their objects. See bug 803376
+ * for details on this problem. To avoid the problem, we try to avoid
+ * allocation and read barriers during JS_TransplantObject and the like.
+ */
+
+ /* Set the maybeAlive flag based on cross-compartment edges. */
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ for (JSCompartment::WrapperEnum e(c); !e.empty(); e.popFront()) {
+ if (e.front().key().is<JSString*>())
+ continue;
+ JSCompartment* dest = e.front().mutableKey().compartment();
+ if (dest)
+ dest->maybeAlive = true;
+ }
+ }
+
+ /*
+ * For black roots, code in gc/Marking.cpp will already have set maybeAlive
+ * during MarkRuntime.
+ */
+
+ /* Propogate maybeAlive to scheduleForDestruction. */
+ for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
+ if (!c->maybeAlive && !rt->isAtomsCompartment(c))
+ c->scheduledForDestruction = true;
+ }
+}
+
+template <class ZoneIterT>
+void
+GCRuntime::markWeakReferences(gcstats::Phase phase)
+{
+ MOZ_ASSERT(marker.isDrained());
+
+ gcstats::AutoPhase ap1(stats, phase);
+
+ marker.enterWeakMarkingMode();
+
+ // TODO bug 1167452: Make weak marking incremental
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(marker.drainMarkStack(unlimited));
+
+ for (;;) {
+ bool markedAny = false;
+ if (!marker.isWeakMarkingTracer()) {
+ for (ZoneIterT zone(rt); !zone.done(); zone.next())
+ markedAny |= WeakMapBase::markZoneIteratively(zone, &marker);
+ }
+ for (CompartmentsIterT<ZoneIterT> c(rt); !c.done(); c.next()) {
+ if (c->watchpointMap)
+ markedAny |= c->watchpointMap->markIteratively(&marker);
+ }
+ markedAny |= Debugger::markAllIteratively(&marker);
+ markedAny |= jit::JitRuntime::MarkJitcodeGlobalTableIteratively(&marker);
+
+ if (!markedAny)
+ break;
+
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(marker.drainMarkStack(unlimited));
+ }
+ MOZ_ASSERT(marker.isDrained());
+
+ marker.leaveWeakMarkingMode();
+}
+
+void
+GCRuntime::markWeakReferencesInCurrentGroup(gcstats::Phase phase)
+{
+ markWeakReferences<GCZoneGroupIter>(phase);
+}
+
+template <class ZoneIterT, class CompartmentIterT>
+void
+GCRuntime::markGrayReferences(gcstats::Phase phase)
+{
+ gcstats::AutoPhase ap(stats, phase);
+ if (hasBufferedGrayRoots()) {
+ for (ZoneIterT zone(rt); !zone.done(); zone.next())
+ markBufferedGrayRoots(zone);
+ } else {
+ MOZ_ASSERT(!isIncremental);
+ if (JSTraceDataOp op = grayRootTracer.op)
+ (*op)(&marker, grayRootTracer.data);
+ }
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(marker.drainMarkStack(unlimited));
+}
+
+void
+GCRuntime::markGrayReferencesInCurrentGroup(gcstats::Phase phase)
+{
+ markGrayReferences<GCZoneGroupIter, GCCompartmentGroupIter>(phase);
+}
+
+void
+GCRuntime::markAllWeakReferences(gcstats::Phase phase)
+{
+ markWeakReferences<GCZonesIter>(phase);
+}
+
+void
+GCRuntime::markAllGrayReferences(gcstats::Phase phase)
+{
+ markGrayReferences<GCZonesIter, GCCompartmentsIter>(phase);
+}
+
+#ifdef JS_GC_ZEAL
+
+struct GCChunkHasher {
+ typedef gc::Chunk* Lookup;
+
+ /*
+ * Strip zeros for better distribution after multiplying by the golden
+ * ratio.
+ */
+ static HashNumber hash(gc::Chunk* chunk) {
+ MOZ_ASSERT(!(uintptr_t(chunk) & gc::ChunkMask));
+ return HashNumber(uintptr_t(chunk) >> gc::ChunkShift);
+ }
+
+ static bool match(gc::Chunk* k, gc::Chunk* l) {
+ MOZ_ASSERT(!(uintptr_t(k) & gc::ChunkMask));
+ MOZ_ASSERT(!(uintptr_t(l) & gc::ChunkMask));
+ return k == l;
+ }
+};
+
+class js::gc::MarkingValidator
+{
+ public:
+ explicit MarkingValidator(GCRuntime* gc);
+ ~MarkingValidator();
+ void nonIncrementalMark(AutoLockForExclusiveAccess& lock);
+ void validate();
+
+ private:
+ GCRuntime* gc;
+ bool initialized;
+
+ typedef HashMap<Chunk*, ChunkBitmap*, GCChunkHasher, SystemAllocPolicy> BitmapMap;
+ BitmapMap map;
+};
+
+js::gc::MarkingValidator::MarkingValidator(GCRuntime* gc)
+ : gc(gc),
+ initialized(false)
+{}
+
+js::gc::MarkingValidator::~MarkingValidator()
+{
+ if (!map.initialized())
+ return;
+
+ for (BitmapMap::Range r(map.all()); !r.empty(); r.popFront())
+ js_delete(r.front().value());
+}
+
+void
+js::gc::MarkingValidator::nonIncrementalMark(AutoLockForExclusiveAccess& lock)
+{
+ /*
+ * Perform a non-incremental mark for all collecting zones and record
+ * the results for later comparison.
+ *
+ * Currently this does not validate gray marking.
+ */
+
+ if (!map.init())
+ return;
+
+ JSRuntime* runtime = gc->rt;
+ GCMarker* gcmarker = &gc->marker;
+
+ gc->waitBackgroundSweepEnd();
+
+ /* Save existing mark bits. */
+ for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next()) {
+ ChunkBitmap* bitmap = &chunk->bitmap;
+ ChunkBitmap* entry = js_new<ChunkBitmap>();
+ if (!entry)
+ return;
+
+ memcpy((void*)entry->bitmap, (void*)bitmap->bitmap, sizeof(bitmap->bitmap));
+ if (!map.putNew(chunk, entry))
+ return;
+ }
+
+ /*
+ * Temporarily clear the weakmaps' mark flags for the compartments we are
+ * collecting.
+ */
+
+ WeakMapSet markedWeakMaps;
+ if (!markedWeakMaps.init())
+ return;
+
+ /*
+ * For saving, smush all of the keys into one big table and split them back
+ * up into per-zone tables when restoring.
+ */
+ gc::WeakKeyTable savedWeakKeys(SystemAllocPolicy(), runtime->randomHashCodeScrambler());
+ if (!savedWeakKeys.init())
+ return;
+
+ for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
+ if (!WeakMapBase::saveZoneMarkedWeakMaps(zone, markedWeakMaps))
+ return;
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ for (gc::WeakKeyTable::Range r = zone->gcWeakKeys.all(); !r.empty(); r.popFront()) {
+ if (!savedWeakKeys.put(Move(r.front().key), Move(r.front().value)))
+ oomUnsafe.crash("saving weak keys table for validator");
+ }
+
+ if (!zone->gcWeakKeys.clear())
+ oomUnsafe.crash("clearing weak keys table for validator");
+ }
+
+ /*
+ * After this point, the function should run to completion, so we shouldn't
+ * do anything fallible.
+ */
+ initialized = true;
+
+ /* Re-do all the marking, but non-incrementally. */
+ js::gc::State state = gc->incrementalState;
+ gc->incrementalState = State::MarkRoots;
+
+ {
+ gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_MARK);
+
+ {
+ gcstats::AutoPhase ap(gc->stats, gcstats::PHASE_UNMARK);
+
+ for (GCZonesIter zone(runtime); !zone.done(); zone.next())
+ WeakMapBase::unmarkZone(zone);
+
+ MOZ_ASSERT(gcmarker->isDrained());
+ gcmarker->reset();
+
+ for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next())
+ chunk->bitmap.clear();
+ }
+
+ gc->traceRuntimeForMajorGC(gcmarker, lock);
+
+ gc->incrementalState = State::Mark;
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(gc->marker.drainMarkStack(unlimited));
+ }
+
+ gc->incrementalState = State::Sweep;
+ {
+ gcstats::AutoPhase ap1(gc->stats, gcstats::PHASE_SWEEP);
+ gcstats::AutoPhase ap2(gc->stats, gcstats::PHASE_SWEEP_MARK);
+
+ gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_WEAK);
+
+ /* Update zone state for gray marking. */
+ for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarkingBlack());
+ zone->setGCState(Zone::MarkGray);
+ }
+ gc->marker.setMarkColorGray();
+
+ gc->markAllGrayReferences(gcstats::PHASE_SWEEP_MARK_GRAY);
+ gc->markAllWeakReferences(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
+
+ /* Restore zone state. */
+ for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarkingGray());
+ zone->setGCState(Zone::Mark);
+ }
+ MOZ_ASSERT(gc->marker.isDrained());
+ gc->marker.setMarkColorBlack();
+ }
+
+ /* Take a copy of the non-incremental mark state and restore the original. */
+ for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next()) {
+ ChunkBitmap* bitmap = &chunk->bitmap;
+ ChunkBitmap* entry = map.lookup(chunk)->value();
+ Swap(*entry, *bitmap);
+ }
+
+ for (GCZonesIter zone(runtime); !zone.done(); zone.next()) {
+ WeakMapBase::unmarkZone(zone);
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcWeakKeys.clear())
+ oomUnsafe.crash("clearing weak keys table for validator");
+ }
+
+ WeakMapBase::restoreMarkedWeakMaps(markedWeakMaps);
+
+ for (gc::WeakKeyTable::Range r = savedWeakKeys.all(); !r.empty(); r.popFront()) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Zone* zone = gc::TenuredCell::fromPointer(r.front().key.asCell())->zone();
+ if (!zone->gcWeakKeys.put(Move(r.front().key), Move(r.front().value)))
+ oomUnsafe.crash("restoring weak keys table for validator");
+ }
+
+ gc->incrementalState = state;
+}
+
+void
+js::gc::MarkingValidator::validate()
+{
+ /*
+ * Validates the incremental marking for a single compartment by comparing
+ * the mark bits to those previously recorded for a non-incremental mark.
+ */
+
+ if (!initialized)
+ return;
+
+ gc->waitBackgroundSweepEnd();
+
+ for (auto chunk = gc->allNonEmptyChunks(); !chunk.done(); chunk.next()) {
+ BitmapMap::Ptr ptr = map.lookup(chunk);
+ if (!ptr)
+ continue; /* Allocated after we did the non-incremental mark. */
+
+ ChunkBitmap* bitmap = ptr->value();
+ ChunkBitmap* incBitmap = &chunk->bitmap;
+
+ for (size_t i = 0; i < ArenasPerChunk; i++) {
+ if (chunk->decommittedArenas.get(i))
+ continue;
+ Arena* arena = &chunk->arenas[i];
+ if (!arena->allocated())
+ continue;
+ if (!arena->zone->isGCSweeping())
+ continue;
+ if (arena->allocatedDuringIncremental)
+ continue;
+
+ AllocKind kind = arena->getAllocKind();
+ uintptr_t thing = arena->thingsStart();
+ uintptr_t end = arena->thingsEnd();
+ while (thing < end) {
+ Cell* cell = (Cell*)thing;
+
+ /*
+ * If a non-incremental GC wouldn't have collected a cell, then
+ * an incremental GC won't collect it.
+ */
+ MOZ_ASSERT_IF(bitmap->isMarked(cell, BLACK), incBitmap->isMarked(cell, BLACK));
+
+ /*
+ * If the cycle collector isn't allowed to collect an object
+ * after a non-incremental GC has run, then it isn't allowed to
+ * collected it after an incremental GC.
+ */
+ MOZ_ASSERT_IF(!bitmap->isMarked(cell, GRAY), !incBitmap->isMarked(cell, GRAY));
+
+ thing += Arena::thingSize(kind);
+ }
+ }
+ }
+}
+
+#endif // JS_GC_ZEAL
+
+void
+GCRuntime::computeNonIncrementalMarkingForValidation(AutoLockForExclusiveAccess& lock)
+{
+#ifdef JS_GC_ZEAL
+ MOZ_ASSERT(!markingValidator);
+ if (isIncremental && hasZealMode(ZealMode::IncrementalMarkingValidator))
+ markingValidator = js_new<MarkingValidator>(this);
+ if (markingValidator)
+ markingValidator->nonIncrementalMark(lock);
+#endif
+}
+
+void
+GCRuntime::validateIncrementalMarking()
+{
+#ifdef JS_GC_ZEAL
+ if (markingValidator)
+ markingValidator->validate();
+#endif
+}
+
+void
+GCRuntime::finishMarkingValidation()
+{
+#ifdef JS_GC_ZEAL
+ js_delete(markingValidator);
+ markingValidator = nullptr;
+#endif
+}
+
+static void
+DropStringWrappers(JSRuntime* rt)
+{
+ /*
+ * String "wrappers" are dropped on GC because their presence would require
+ * us to sweep the wrappers in all compartments every time we sweep a
+ * compartment group.
+ */
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ for (JSCompartment::WrapperEnum e(c); !e.empty(); e.popFront()) {
+ if (e.front().key().is<JSString*>())
+ e.removeFront();
+ }
+ }
+}
+
+/*
+ * Group zones that must be swept at the same time.
+ *
+ * If compartment A has an edge to an unmarked object in compartment B, then we
+ * must not sweep A in a later slice than we sweep B. That's because a write
+ * barrier in A could lead to the unmarked object in B becoming marked.
+ * However, if we had already swept that object, we would be in trouble.
+ *
+ * If we consider these dependencies as a graph, then all the compartments in
+ * any strongly-connected component of this graph must be swept in the same
+ * slice.
+ *
+ * Tarjan's algorithm is used to calculate the components.
+ */
+namespace {
+struct AddOutgoingEdgeFunctor {
+ bool needsEdge_;
+ ZoneComponentFinder& finder_;
+
+ AddOutgoingEdgeFunctor(bool needsEdge, ZoneComponentFinder& finder)
+ : needsEdge_(needsEdge), finder_(finder)
+ {}
+
+ template <typename T>
+ void operator()(T tp) {
+ TenuredCell& other = (*tp)->asTenured();
+
+ /*
+ * Add edge to wrapped object compartment if wrapped object is not
+ * marked black to indicate that wrapper compartment not be swept
+ * after wrapped compartment.
+ */
+ if (needsEdge_) {
+ JS::Zone* zone = other.zone();
+ if (zone->isGCMarking())
+ finder_.addEdgeTo(zone);
+ }
+ }
+};
+} // namespace (anonymous)
+
+void
+JSCompartment::findOutgoingEdges(ZoneComponentFinder& finder)
+{
+ for (js::WrapperMap::Enum e(crossCompartmentWrappers); !e.empty(); e.popFront()) {
+ CrossCompartmentKey& key = e.front().mutableKey();
+ MOZ_ASSERT(!key.is<JSString*>());
+ bool needsEdge = true;
+ if (key.is<JSObject*>()) {
+ TenuredCell& other = key.as<JSObject*>()->asTenured();
+ needsEdge = !other.isMarked(BLACK) || other.isMarked(GRAY);
+ }
+ key.applyToWrapped(AddOutgoingEdgeFunctor(needsEdge, finder));
+ }
+}
+
+void
+Zone::findOutgoingEdges(ZoneComponentFinder& finder)
+{
+ /*
+ * Any compartment may have a pointer to an atom in the atoms
+ * compartment, and these aren't in the cross compartment map.
+ */
+ JSRuntime* rt = runtimeFromMainThread();
+ Zone* atomsZone = rt->atomsCompartment(finder.lock)->zone();
+ if (atomsZone->isGCMarking())
+ finder.addEdgeTo(atomsZone);
+
+ for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next())
+ comp->findOutgoingEdges(finder);
+
+ for (ZoneSet::Range r = gcZoneGroupEdges.all(); !r.empty(); r.popFront()) {
+ if (r.front()->isGCMarking())
+ finder.addEdgeTo(r.front());
+ }
+
+ Debugger::findZoneEdges(this, finder);
+}
+
+bool
+GCRuntime::findInterZoneEdges()
+{
+ /*
+ * Weakmaps which have keys with delegates in a different zone introduce the
+ * need for zone edges from the delegate's zone to the weakmap zone.
+ *
+ * Since the edges point into and not away from the zone the weakmap is in
+ * we must find these edges in advance and store them in a set on the Zone.
+ * If we run out of memory, we fall back to sweeping everything in one
+ * group.
+ */
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ if (!WeakMapBase::findInterZoneEdges(zone))
+ return false;
+ }
+
+ return true;
+}
+
+void
+GCRuntime::findZoneGroups(AutoLockForExclusiveAccess& lock)
+{
+#ifdef DEBUG
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ MOZ_ASSERT(zone->gcZoneGroupEdges.empty());
+#endif
+
+ JSContext* cx = rt->contextFromMainThread();
+ ZoneComponentFinder finder(cx->nativeStackLimit[StackForSystemCode], lock);
+ if (!isIncremental || !findInterZoneEdges())
+ finder.useOneComponent();
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarking());
+ finder.addNode(zone);
+ }
+ zoneGroups = finder.getResultsList();
+ currentZoneGroup = zoneGroups;
+ zoneGroupIndex = 0;
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next())
+ zone->gcZoneGroupEdges.clear();
+
+#ifdef DEBUG
+ for (Zone* head = currentZoneGroup; head; head = head->nextGroup()) {
+ for (Zone* zone = head; zone; zone = zone->nextNodeInGroup())
+ MOZ_ASSERT(zone->isGCMarking());
+ }
+
+ MOZ_ASSERT_IF(!isIncremental, !currentZoneGroup->nextGroup());
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ MOZ_ASSERT(zone->gcZoneGroupEdges.empty());
+#endif
+}
+
+static void
+ResetGrayList(JSCompartment* comp);
+
+void
+GCRuntime::getNextZoneGroup()
+{
+ currentZoneGroup = currentZoneGroup->nextGroup();
+ ++zoneGroupIndex;
+ if (!currentZoneGroup) {
+ abortSweepAfterCurrentGroup = false;
+ return;
+ }
+
+ for (Zone* zone = currentZoneGroup; zone; zone = zone->nextNodeInGroup()) {
+ MOZ_ASSERT(zone->isGCMarking());
+ MOZ_ASSERT(!zone->isQueuedForBackgroundSweep());
+ }
+
+ if (!isIncremental)
+ ZoneComponentFinder::mergeGroups(currentZoneGroup);
+
+ if (abortSweepAfterCurrentGroup) {
+ MOZ_ASSERT(!isIncremental);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(!zone->gcNextGraphComponent);
+ MOZ_ASSERT(zone->isGCMarking());
+ zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit);
+ zone->setGCState(Zone::NoGC);
+ zone->gcGrayRoots.clearAndFree();
+ }
+
+ for (GCCompartmentGroupIter comp(rt); !comp.done(); comp.next())
+ ResetGrayList(comp);
+
+ abortSweepAfterCurrentGroup = false;
+ currentZoneGroup = nullptr;
+ }
+}
+
+/*
+ * Gray marking:
+ *
+ * At the end of collection, anything reachable from a gray root that has not
+ * otherwise been marked black must be marked gray.
+ *
+ * This means that when marking things gray we must not allow marking to leave
+ * the current compartment group, as that could result in things being marked
+ * grey when they might subsequently be marked black. To achieve this, when we
+ * find a cross compartment pointer we don't mark the referent but add it to a
+ * singly-linked list of incoming gray pointers that is stored with each
+ * compartment.
+ *
+ * The list head is stored in JSCompartment::gcIncomingGrayPointers and contains
+ * cross compartment wrapper objects. The next pointer is stored in the second
+ * extra slot of the cross compartment wrapper.
+ *
+ * The list is created during gray marking when one of the
+ * MarkCrossCompartmentXXX functions is called for a pointer that leaves the
+ * current compartent group. This calls DelayCrossCompartmentGrayMarking to
+ * push the referring object onto the list.
+ *
+ * The list is traversed and then unlinked in
+ * MarkIncomingCrossCompartmentPointers.
+ */
+
+static bool
+IsGrayListObject(JSObject* obj)
+{
+ MOZ_ASSERT(obj);
+ return obj->is<CrossCompartmentWrapperObject>() && !IsDeadProxyObject(obj);
+}
+
+/* static */ unsigned
+ProxyObject::grayLinkExtraSlot(JSObject* obj)
+{
+ MOZ_ASSERT(IsGrayListObject(obj));
+ return 1;
+}
+
+#ifdef DEBUG
+static void
+AssertNotOnGrayList(JSObject* obj)
+{
+ MOZ_ASSERT_IF(IsGrayListObject(obj),
+ GetProxyExtra(obj, ProxyObject::grayLinkExtraSlot(obj)).isUndefined());
+}
+#endif
+
+static void
+AssertNoWrappersInGrayList(JSRuntime* rt)
+{
+#ifdef DEBUG
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ MOZ_ASSERT(!c->gcIncomingGrayPointers);
+ for (JSCompartment::WrapperEnum e(c); !e.empty(); e.popFront()) {
+ if (!e.front().key().is<JSString*>())
+ AssertNotOnGrayList(&e.front().value().unbarrieredGet().toObject());
+ }
+ }
+#endif
+}
+
+static JSObject*
+CrossCompartmentPointerReferent(JSObject* obj)
+{
+ MOZ_ASSERT(IsGrayListObject(obj));
+ return &obj->as<ProxyObject>().private_().toObject();
+}
+
+static JSObject*
+NextIncomingCrossCompartmentPointer(JSObject* prev, bool unlink)
+{
+ unsigned slot = ProxyObject::grayLinkExtraSlot(prev);
+ JSObject* next = GetProxyExtra(prev, slot).toObjectOrNull();
+ MOZ_ASSERT_IF(next, IsGrayListObject(next));
+
+ if (unlink)
+ SetProxyExtra(prev, slot, UndefinedValue());
+
+ return next;
+}
+
+void
+js::DelayCrossCompartmentGrayMarking(JSObject* src)
+{
+ MOZ_ASSERT(IsGrayListObject(src));
+
+ /* Called from MarkCrossCompartmentXXX functions. */
+ unsigned slot = ProxyObject::grayLinkExtraSlot(src);
+ JSObject* dest = CrossCompartmentPointerReferent(src);
+ JSCompartment* comp = dest->compartment();
+
+ if (GetProxyExtra(src, slot).isUndefined()) {
+ SetProxyExtra(src, slot, ObjectOrNullValue(comp->gcIncomingGrayPointers));
+ comp->gcIncomingGrayPointers = src;
+ } else {
+ MOZ_ASSERT(GetProxyExtra(src, slot).isObjectOrNull());
+ }
+
+#ifdef DEBUG
+ /*
+ * Assert that the object is in our list, also walking the list to check its
+ * integrity.
+ */
+ JSObject* obj = comp->gcIncomingGrayPointers;
+ bool found = false;
+ while (obj) {
+ if (obj == src)
+ found = true;
+ obj = NextIncomingCrossCompartmentPointer(obj, false);
+ }
+ MOZ_ASSERT(found);
+#endif
+}
+
+static void
+MarkIncomingCrossCompartmentPointers(JSRuntime* rt, const uint32_t color)
+{
+ MOZ_ASSERT(color == BLACK || color == GRAY);
+
+ static const gcstats::Phase statsPhases[] = {
+ gcstats::PHASE_SWEEP_MARK_INCOMING_BLACK,
+ gcstats::PHASE_SWEEP_MARK_INCOMING_GRAY
+ };
+ gcstats::AutoPhase ap1(rt->gc.stats, statsPhases[color]);
+
+ bool unlinkList = color == GRAY;
+
+ for (GCCompartmentGroupIter c(rt); !c.done(); c.next()) {
+ MOZ_ASSERT_IF(color == GRAY, c->zone()->isGCMarkingGray());
+ MOZ_ASSERT_IF(color == BLACK, c->zone()->isGCMarkingBlack());
+ MOZ_ASSERT_IF(c->gcIncomingGrayPointers, IsGrayListObject(c->gcIncomingGrayPointers));
+
+ for (JSObject* src = c->gcIncomingGrayPointers;
+ src;
+ src = NextIncomingCrossCompartmentPointer(src, unlinkList))
+ {
+ JSObject* dst = CrossCompartmentPointerReferent(src);
+ MOZ_ASSERT(dst->compartment() == c);
+
+ if (color == GRAY) {
+ if (IsMarkedUnbarriered(rt, &src) && src->asTenured().isMarked(GRAY))
+ TraceManuallyBarrieredEdge(&rt->gc.marker, &dst,
+ "cross-compartment gray pointer");
+ } else {
+ if (IsMarkedUnbarriered(rt, &src) && !src->asTenured().isMarked(GRAY))
+ TraceManuallyBarrieredEdge(&rt->gc.marker, &dst,
+ "cross-compartment black pointer");
+ }
+ }
+
+ if (unlinkList)
+ c->gcIncomingGrayPointers = nullptr;
+ }
+
+ auto unlimited = SliceBudget::unlimited();
+ MOZ_RELEASE_ASSERT(rt->gc.marker.drainMarkStack(unlimited));
+}
+
+static bool
+RemoveFromGrayList(JSObject* wrapper)
+{
+ if (!IsGrayListObject(wrapper))
+ return false;
+
+ unsigned slot = ProxyObject::grayLinkExtraSlot(wrapper);
+ if (GetProxyExtra(wrapper, slot).isUndefined())
+ return false; /* Not on our list. */
+
+ JSObject* tail = GetProxyExtra(wrapper, slot).toObjectOrNull();
+ SetProxyExtra(wrapper, slot, UndefinedValue());
+
+ JSCompartment* comp = CrossCompartmentPointerReferent(wrapper)->compartment();
+ JSObject* obj = comp->gcIncomingGrayPointers;
+ if (obj == wrapper) {
+ comp->gcIncomingGrayPointers = tail;
+ return true;
+ }
+
+ while (obj) {
+ unsigned slot = ProxyObject::grayLinkExtraSlot(obj);
+ JSObject* next = GetProxyExtra(obj, slot).toObjectOrNull();
+ if (next == wrapper) {
+ SetProxyExtra(obj, slot, ObjectOrNullValue(tail));
+ return true;
+ }
+ obj = next;
+ }
+
+ MOZ_CRASH("object not found in gray link list");
+}
+
+static void
+ResetGrayList(JSCompartment* comp)
+{
+ JSObject* src = comp->gcIncomingGrayPointers;
+ while (src)
+ src = NextIncomingCrossCompartmentPointer(src, true);
+ comp->gcIncomingGrayPointers = nullptr;
+}
+
+void
+js::NotifyGCNukeWrapper(JSObject* obj)
+{
+ /*
+ * References to target of wrapper are being removed, we no longer have to
+ * remember to mark it.
+ */
+ RemoveFromGrayList(obj);
+}
+
+enum {
+ JS_GC_SWAP_OBJECT_A_REMOVED = 1 << 0,
+ JS_GC_SWAP_OBJECT_B_REMOVED = 1 << 1
+};
+
+unsigned
+js::NotifyGCPreSwap(JSObject* a, JSObject* b)
+{
+ /*
+ * Two objects in the same compartment are about to have had their contents
+ * swapped. If either of them are in our gray pointer list, then we remove
+ * them from the lists, returning a bitset indicating what happened.
+ */
+ return (RemoveFromGrayList(a) ? JS_GC_SWAP_OBJECT_A_REMOVED : 0) |
+ (RemoveFromGrayList(b) ? JS_GC_SWAP_OBJECT_B_REMOVED : 0);
+}
+
+void
+js::NotifyGCPostSwap(JSObject* a, JSObject* b, unsigned removedFlags)
+{
+ /*
+ * Two objects in the same compartment have had their contents swapped. If
+ * either of them were in our gray pointer list, we re-add them again.
+ */
+ if (removedFlags & JS_GC_SWAP_OBJECT_A_REMOVED)
+ DelayCrossCompartmentGrayMarking(b);
+ if (removedFlags & JS_GC_SWAP_OBJECT_B_REMOVED)
+ DelayCrossCompartmentGrayMarking(a);
+}
+
+void
+GCRuntime::endMarkingZoneGroup()
+{
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MARK);
+
+ /*
+ * Mark any incoming black pointers from previously swept compartments
+ * whose referents are not marked. This can occur when gray cells become
+ * black by the action of UnmarkGray.
+ */
+ MarkIncomingCrossCompartmentPointers(rt, BLACK);
+ markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_WEAK);
+
+ /*
+ * Change state of current group to MarkGray to restrict marking to this
+ * group. Note that there may be pointers to the atoms compartment, and
+ * these will be marked through, as they are not marked with
+ * MarkCrossCompartmentXXX.
+ */
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarkingBlack());
+ zone->setGCState(Zone::MarkGray);
+ }
+ marker.setMarkColorGray();
+
+ /* Mark incoming gray pointers from previously swept compartments. */
+ MarkIncomingCrossCompartmentPointers(rt, GRAY);
+
+ /* Mark gray roots and mark transitively inside the current compartment group. */
+ markGrayReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY);
+ markWeakReferencesInCurrentGroup(gcstats::PHASE_SWEEP_MARK_GRAY_WEAK);
+
+ /* Restore marking state. */
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarkingGray());
+ zone->setGCState(Zone::Mark);
+ }
+ MOZ_ASSERT(marker.isDrained());
+ marker.setMarkColorBlack();
+}
+
+class GCSweepTask : public GCParallelTask
+{
+ GCSweepTask(const GCSweepTask&) = delete;
+
+ protected:
+ JSRuntime* runtime;
+
+ public:
+ explicit GCSweepTask(JSRuntime* rt) : runtime(rt) {}
+ GCSweepTask(GCSweepTask&& other)
+ : GCParallelTask(mozilla::Move(other)),
+ runtime(other.runtime)
+ {}
+};
+
+// Causes the given WeakCache to be swept when run.
+class SweepWeakCacheTask : public GCSweepTask
+{
+ JS::WeakCache<void*>& cache;
+
+ SweepWeakCacheTask(const SweepWeakCacheTask&) = delete;
+
+ public:
+ SweepWeakCacheTask(JSRuntime* rt, JS::WeakCache<void*>& wc) : GCSweepTask(rt), cache(wc) {}
+ SweepWeakCacheTask(SweepWeakCacheTask&& other)
+ : GCSweepTask(mozilla::Move(other)), cache(other.cache)
+ {}
+
+ void run() override {
+ cache.sweep();
+ }
+};
+
+#define MAKE_GC_SWEEP_TASK(name) \
+ class name : public GCSweepTask { \
+ void run() override; \
+ public: \
+ explicit name (JSRuntime* rt) : GCSweepTask(rt) {} \
+ }
+MAKE_GC_SWEEP_TASK(SweepAtomsTask);
+MAKE_GC_SWEEP_TASK(SweepCCWrappersTask);
+MAKE_GC_SWEEP_TASK(SweepBaseShapesTask);
+MAKE_GC_SWEEP_TASK(SweepInitialShapesTask);
+MAKE_GC_SWEEP_TASK(SweepObjectGroupsTask);
+MAKE_GC_SWEEP_TASK(SweepRegExpsTask);
+MAKE_GC_SWEEP_TASK(SweepMiscTask);
+#undef MAKE_GC_SWEEP_TASK
+
+/* virtual */ void
+SweepAtomsTask::run()
+{
+ runtime->sweepAtoms();
+ for (CompartmentsIter comp(runtime, SkipAtoms); !comp.done(); comp.next())
+ comp->sweepVarNames();
+}
+
+/* virtual */ void
+SweepCCWrappersTask::run()
+{
+ for (GCCompartmentGroupIter c(runtime); !c.done(); c.next())
+ c->sweepCrossCompartmentWrappers();
+}
+
+/* virtual */ void
+SweepObjectGroupsTask::run()
+{
+ for (GCCompartmentGroupIter c(runtime); !c.done(); c.next())
+ c->objectGroups.sweep(runtime->defaultFreeOp());
+}
+
+/* virtual */ void
+SweepRegExpsTask::run()
+{
+ for (GCCompartmentGroupIter c(runtime); !c.done(); c.next())
+ c->sweepRegExps();
+}
+
+/* virtual */ void
+SweepMiscTask::run()
+{
+ for (GCCompartmentGroupIter c(runtime); !c.done(); c.next()) {
+ c->sweepSavedStacks();
+ c->sweepSelfHostingScriptSource();
+ c->sweepNativeIterators();
+ }
+}
+
+void
+GCRuntime::startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked)
+{
+ if (!task.startWithLockHeld(locked)) {
+ AutoUnlockHelperThreadState unlock(locked);
+ gcstats::AutoPhase ap(stats, phase);
+ task.runFromMainThread(rt);
+ }
+}
+
+void
+GCRuntime::joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked)
+{
+ gcstats::AutoPhase ap(stats, task, phase);
+ task.joinWithLockHeld(locked);
+}
+
+using WeakCacheTaskVector = mozilla::Vector<SweepWeakCacheTask, 0, SystemAllocPolicy>;
+
+static void
+SweepWeakCachesFromMainThread(JSRuntime* rt)
+{
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ for (JS::WeakCache<void*>* cache : zone->weakCaches_) {
+ SweepWeakCacheTask task(rt, *cache);
+ task.runFromMainThread(rt);
+ }
+ }
+}
+
+static WeakCacheTaskVector
+PrepareWeakCacheTasks(JSRuntime* rt)
+{
+ WeakCacheTaskVector out;
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ for (JS::WeakCache<void*>* cache : zone->weakCaches_) {
+ if (!out.append(SweepWeakCacheTask(rt, *cache))) {
+ SweepWeakCachesFromMainThread(rt);
+ return WeakCacheTaskVector();
+ }
+ }
+ }
+ return out;
+}
+
+void
+GCRuntime::beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock)
+{
+ /*
+ * Begin sweeping the group of zones in gcCurrentZoneGroup,
+ * performing actions that must be done before yielding to caller.
+ */
+
+ bool sweepingAtoms = false;
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ /* Set the GC state to sweeping. */
+ MOZ_ASSERT(zone->isGCMarking());
+ zone->setGCState(Zone::Sweep);
+
+ /* Purge the ArenaLists before sweeping. */
+ zone->arenas.purge();
+
+ if (zone->isAtomsZone())
+ sweepingAtoms = true;
+
+ if (rt->sweepZoneCallback)
+ rt->sweepZoneCallback(zone);
+
+#ifdef DEBUG
+ zone->gcLastZoneGroupIndex = zoneGroupIndex;
+#endif
+ }
+
+ validateIncrementalMarking();
+
+ FreeOp fop(rt);
+ SweepAtomsTask sweepAtomsTask(rt);
+ SweepCCWrappersTask sweepCCWrappersTask(rt);
+ SweepObjectGroupsTask sweepObjectGroupsTask(rt);
+ SweepRegExpsTask sweepRegExpsTask(rt);
+ SweepMiscTask sweepMiscTask(rt);
+ WeakCacheTaskVector sweepCacheTasks = PrepareWeakCacheTasks(rt);
+
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ /* Clear all weakrefs that point to unmarked things. */
+ for (auto edge : zone->gcWeakRefs) {
+ /* Edges may be present multiple times, so may already be nulled. */
+ if (*edge && IsAboutToBeFinalizedDuringSweep(**edge))
+ *edge = nullptr;
+ }
+ zone->gcWeakRefs.clear();
+
+ /* No need to look up any more weakmap keys from this zone group. */
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!zone->gcWeakKeys.clear())
+ oomUnsafe.crash("clearing weak keys in beginSweepingZoneGroup()");
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_START);
+ callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_START);
+ {
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_ZONEGROUP_CALLBACK);
+ callWeakPointerZoneGroupCallbacks();
+ }
+ {
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_WEAK_COMPARTMENT_CALLBACK);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+ callWeakPointerCompartmentCallbacks(comp);
+ }
+ }
+ }
+
+ if (sweepingAtoms) {
+ AutoLockHelperThreadState helperLock;
+ startTask(sweepAtomsTask, gcstats::PHASE_SWEEP_ATOMS, helperLock);
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_COMPARTMENTS);
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+
+ {
+ AutoLockHelperThreadState helperLock;
+ startTask(sweepCCWrappersTask, gcstats::PHASE_SWEEP_CC_WRAPPER, helperLock);
+ startTask(sweepObjectGroupsTask, gcstats::PHASE_SWEEP_TYPE_OBJECT, helperLock);
+ startTask(sweepRegExpsTask, gcstats::PHASE_SWEEP_REGEXP, helperLock);
+ startTask(sweepMiscTask, gcstats::PHASE_SWEEP_MISC, helperLock);
+ for (auto& task : sweepCacheTasks)
+ startTask(task, gcstats::PHASE_SWEEP_MISC, helperLock);
+ }
+
+ // The remainder of the of the tasks run in parallel on the main
+ // thread until we join, below.
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_MISC);
+
+ // Cancel any active or pending off thread compilations.
+ js::CancelOffThreadIonCompile(rt, JS::Zone::Sweep);
+
+ for (GCCompartmentGroupIter c(rt); !c.done(); c.next()) {
+ c->sweepGlobalObject(&fop);
+ c->sweepDebugEnvironments();
+ c->sweepJitCompartment(&fop);
+ c->sweepTemplateObjects();
+ }
+
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zone->sweepWeakMaps();
+
+ // Bug 1071218: the following two methods have not yet been
+ // refactored to work on a single zone-group at once.
+
+ // Collect watch points associated with unreachable objects.
+ WatchpointMap::sweepAll(rt);
+
+ // Detach unreachable debuggers and global objects from each other.
+ Debugger::sweepAll(&fop);
+
+ // Sweep entries containing about-to-be-finalized JitCode and
+ // update relocated TypeSet::Types inside the JitcodeGlobalTable.
+ jit::JitRuntime::SweepJitcodeGlobalTable(rt);
+ }
+
+ {
+ gcstats::AutoPhase apdc(stats, gcstats::PHASE_SWEEP_DISCARD_CODE);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zone->discardJitCode(&fop);
+ }
+
+ {
+ gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_TYPES);
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_SWEEP_TYPES_BEGIN);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zone->beginSweepTypes(&fop, releaseObservedTypes && !zone->isPreservingCode());
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_BREAKPOINT);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zone->sweepBreakpoints(&fop);
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_BREAKPOINT);
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zone->sweepUniqueIds(&fop);
+ }
+ }
+
+ if (sweepingAtoms) {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_SYMBOL_REGISTRY);
+ rt->symbolRegistry(lock).sweep();
+ }
+
+ // Rejoin our off-main-thread tasks.
+ if (sweepingAtoms) {
+ AutoLockHelperThreadState helperLock;
+ joinTask(sweepAtomsTask, gcstats::PHASE_SWEEP_ATOMS, helperLock);
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_COMPARTMENTS);
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+
+ AutoLockHelperThreadState helperLock;
+ joinTask(sweepCCWrappersTask, gcstats::PHASE_SWEEP_CC_WRAPPER, helperLock);
+ joinTask(sweepObjectGroupsTask, gcstats::PHASE_SWEEP_TYPE_OBJECT, helperLock);
+ joinTask(sweepRegExpsTask, gcstats::PHASE_SWEEP_REGEXP, helperLock);
+ joinTask(sweepMiscTask, gcstats::PHASE_SWEEP_MISC, helperLock);
+ for (auto& task : sweepCacheTasks)
+ joinTask(task, gcstats::PHASE_SWEEP_MISC, helperLock);
+ }
+
+ /*
+ * Queue all GC things in all zones for sweeping, either in the
+ * foreground or on the background thread.
+ *
+ * Note that order is important here for the background case.
+ *
+ * Objects are finalized immediately but this may change in the future.
+ */
+
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+ zone->arenas.queueForegroundObjectsForSweep(&fop);
+ }
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+ for (unsigned i = 0; i < ArrayLength(IncrementalFinalizePhases); ++i)
+ zone->arenas.queueForForegroundSweep(&fop, IncrementalFinalizePhases[i]);
+ }
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+ for (unsigned i = 0; i < ArrayLength(BackgroundFinalizePhases); ++i)
+ zone->arenas.queueForBackgroundSweep(&fop, BackgroundFinalizePhases[i]);
+ }
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ gcstats::AutoSCC scc(stats, zoneGroupIndex);
+ zone->arenas.queueForegroundThingsForSweep(&fop);
+ }
+
+ sweepingTypes = true;
+
+ finalizePhase = 0;
+ sweepZone = currentZoneGroup;
+ sweepKind = AllocKind::FIRST;
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_END);
+ callFinalizeCallbacks(&fop, JSFINALIZE_GROUP_END);
+ }
+}
+
+void
+GCRuntime::endSweepingZoneGroup()
+{
+ /* Update the GC state for zones we have swept. */
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCSweeping());
+ AutoLockGC lock(rt);
+ zone->setGCState(Zone::Finished);
+ zone->threshold.updateAfterGC(zone->usage.gcBytes(), invocationKind, tunables,
+ schedulingState, lock);
+ }
+
+ /* Start background thread to sweep zones if required. */
+ ZoneList zones;
+ for (GCZoneGroupIter zone(rt); !zone.done(); zone.next())
+ zones.append(zone);
+ if (sweepOnBackgroundThread)
+ queueZonesForBackgroundSweep(zones);
+ else
+ sweepBackgroundThings(zones, blocksToFreeAfterSweeping);
+
+ /* Reset the list of arenas marked as being allocated during sweep phase. */
+ while (Arena* arena = arenasAllocatedDuringSweep) {
+ arenasAllocatedDuringSweep = arena->getNextAllocDuringSweep();
+ arena->unsetAllocDuringSweep();
+ }
+}
+
+void
+GCRuntime::beginSweepPhase(bool destroyingRuntime, AutoLockForExclusiveAccess& lock)
+{
+ /*
+ * Sweep phase.
+ *
+ * Finalize as we sweep, outside of lock but with rt->isHeapBusy()
+ * true so that any attempt to allocate a GC-thing from a finalizer will
+ * fail, rather than nest badly and leave the unmarked newborn to be swept.
+ */
+
+ MOZ_ASSERT(!abortSweepAfterCurrentGroup);
+
+ AutoSetThreadIsSweeping threadIsSweeping;
+
+ releaseHeldRelocatedArenas();
+
+ computeNonIncrementalMarkingForValidation(lock);
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP);
+
+ sweepOnBackgroundThread =
+ !destroyingRuntime && !TraceEnabled() && CanUseExtraThreads();
+
+ releaseObservedTypes = shouldReleaseObservedTypes();
+
+ AssertNoWrappersInGrayList(rt);
+ DropStringWrappers(rt);
+
+ findZoneGroups(lock);
+ endMarkingZoneGroup();
+ beginSweepingZoneGroup(lock);
+}
+
+bool
+ArenaLists::foregroundFinalize(FreeOp* fop, AllocKind thingKind, SliceBudget& sliceBudget,
+ SortedArenaList& sweepList)
+{
+ if (!arenaListsToSweep[thingKind] && incrementalSweptArenas.isEmpty())
+ return true;
+
+ if (!FinalizeArenas(fop, &arenaListsToSweep[thingKind], sweepList,
+ thingKind, sliceBudget, RELEASE_ARENAS))
+ {
+ incrementalSweptArenaKind = thingKind;
+ incrementalSweptArenas = sweepList.toArenaList();
+ return false;
+ }
+
+ // Clear any previous incremental sweep state we may have saved.
+ incrementalSweptArenas.clear();
+
+ // Join |arenaLists[thingKind]| and |sweepList| into a single list.
+ ArenaList finalized = sweepList.toArenaList();
+ arenaLists[thingKind] =
+ finalized.insertListWithCursorAtEnd(arenaLists[thingKind]);
+
+ return true;
+}
+
+GCRuntime::IncrementalProgress
+GCRuntime::drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase)
+{
+ /* Run a marking slice and return whether the stack is now empty. */
+ gcstats::AutoPhase ap(stats, phase);
+ return marker.drainMarkStack(sliceBudget) ? Finished : NotFinished;
+}
+
+static void
+SweepThing(Shape* shape)
+{
+ if (!shape->isMarked())
+ shape->sweep();
+}
+
+static void
+SweepThing(JSScript* script, AutoClearTypeInferenceStateOnOOM* oom)
+{
+ script->maybeSweepTypes(oom);
+}
+
+static void
+SweepThing(ObjectGroup* group, AutoClearTypeInferenceStateOnOOM* oom)
+{
+ group->maybeSweep(oom);
+}
+
+template <typename T, typename... Args>
+static bool
+SweepArenaList(Arena** arenasToSweep, SliceBudget& sliceBudget, Args... args)
+{
+ while (Arena* arena = *arenasToSweep) {
+ for (ArenaCellIterUnderGC i(arena); !i.done(); i.next())
+ SweepThing(i.get<T>(), args...);
+
+ *arenasToSweep = (*arenasToSweep)->next;
+ AllocKind kind = MapTypeToFinalizeKind<T>::kind;
+ sliceBudget.step(Arena::thingsPerArena(kind));
+ if (sliceBudget.isOverBudget())
+ return false;
+ }
+
+ return true;
+}
+
+GCRuntime::IncrementalProgress
+GCRuntime::sweepPhase(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock)
+{
+ AutoSetThreadIsSweeping threadIsSweeping;
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP);
+ FreeOp fop(rt);
+
+ if (drainMarkStack(sliceBudget, gcstats::PHASE_SWEEP_MARK) == NotFinished)
+ return NotFinished;
+
+
+ for (;;) {
+ // Sweep dead type information stored in scripts and object groups, but
+ // don't finalize them yet. We have to sweep dead information from both
+ // live and dead scripts and object groups, so that no dead references
+ // remain in them. Type inference can end up crawling these zones
+ // again, such as for TypeCompartment::markSetsUnknown, and if this
+ // happens after sweeping for the zone group finishes we won't be able
+ // to determine which things in the zone are live.
+ if (sweepingTypes) {
+ gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP_COMPARTMENTS);
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_SWEEP_TYPES);
+
+ for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
+ ArenaLists& al = sweepZone->arenas;
+
+ AutoClearTypeInferenceStateOnOOM oom(sweepZone);
+
+ if (!SweepArenaList<JSScript>(&al.gcScriptArenasToUpdate, sliceBudget, &oom))
+ return NotFinished;
+
+ if (!SweepArenaList<ObjectGroup>(
+ &al.gcObjectGroupArenasToUpdate, sliceBudget, &oom))
+ {
+ return NotFinished;
+ }
+
+ // Finish sweeping type information in the zone.
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_TYPES_END);
+ sweepZone->types.endSweep(rt);
+ }
+
+ // Foreground finalized objects have already been finalized,
+ // and now their arenas can be reclaimed by freeing empty ones
+ // and making non-empty ones available for allocation.
+ al.mergeForegroundSweptObjectArenas();
+ }
+
+ sweepZone = currentZoneGroup;
+ sweepingTypes = false;
+ }
+
+ /* Finalize foreground finalized things. */
+ for (; finalizePhase < ArrayLength(IncrementalFinalizePhases) ; ++finalizePhase) {
+ gcstats::AutoPhase ap(stats, IncrementalFinalizePhases[finalizePhase].statsPhase);
+
+ for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
+ Zone* zone = sweepZone;
+
+ for (auto kind : SomeAllocKinds(sweepKind, AllocKind::LIMIT)) {
+ if (!IncrementalFinalizePhases[finalizePhase].kinds.contains(kind))
+ continue;
+
+ /* Set the number of things per arena for this AllocKind. */
+ size_t thingsPerArena = Arena::thingsPerArena(kind);
+ incrementalSweepList.setThingsPerArena(thingsPerArena);
+
+ if (!zone->arenas.foregroundFinalize(&fop, kind, sliceBudget,
+ incrementalSweepList))
+ {
+ sweepKind = kind;
+ return NotFinished;
+ }
+
+ /* Reset the slots of the sweep list that we used. */
+ incrementalSweepList.reset(thingsPerArena);
+ }
+ sweepKind = AllocKind::FIRST;
+ }
+ sweepZone = currentZoneGroup;
+ }
+
+ /* Remove dead shapes from the shape tree, but don't finalize them yet. */
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP_SHAPE);
+
+ for (; sweepZone; sweepZone = sweepZone->nextNodeInGroup()) {
+ ArenaLists& al = sweepZone->arenas;
+
+ if (!SweepArenaList<Shape>(&al.gcShapeArenasToUpdate, sliceBudget))
+ return NotFinished;
+
+ if (!SweepArenaList<AccessorShape>(&al.gcAccessorShapeArenasToUpdate, sliceBudget))
+ return NotFinished;
+ }
+ }
+
+ endSweepingZoneGroup();
+ getNextZoneGroup();
+ if (!currentZoneGroup)
+ return Finished;
+
+ endMarkingZoneGroup();
+ beginSweepingZoneGroup(lock);
+ }
+}
+
+void
+GCRuntime::endSweepPhase(bool destroyingRuntime, AutoLockForExclusiveAccess& lock)
+{
+ AutoSetThreadIsSweeping threadIsSweeping;
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_SWEEP);
+ FreeOp fop(rt);
+
+ MOZ_ASSERT_IF(destroyingRuntime, !sweepOnBackgroundThread);
+
+ /*
+ * Recalculate whether GC was full or not as this may have changed due to
+ * newly created zones. Can only change from full to not full.
+ */
+ if (isFull) {
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ if (!zone->isCollecting()) {
+ isFull = false;
+ break;
+ }
+ }
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_DESTROY);
+
+ /*
+ * Sweep script filenames after sweeping functions in the generic loop
+ * above. In this way when a scripted function's finalizer destroys the
+ * script and calls rt->destroyScriptHook, the hook can still access the
+ * script's filename. See bug 323267.
+ */
+ SweepScriptData(rt, lock);
+
+ /* Clear out any small pools that we're hanging on to. */
+ if (jit::JitRuntime* jitRuntime = rt->jitRuntime()) {
+ jitRuntime->execAlloc().purge();
+ jitRuntime->backedgeExecAlloc().purge();
+ }
+ }
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_FINALIZE_END);
+ callFinalizeCallbacks(&fop, JSFINALIZE_COLLECTION_END);
+
+ /* If we finished a full GC, then the gray bits are correct. */
+ if (isFull)
+ rt->setGCGrayBitsValid(true);
+ }
+
+ finishMarkingValidation();
+
+#ifdef DEBUG
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ for (auto i : AllAllocKinds()) {
+ MOZ_ASSERT_IF(!IsBackgroundFinalized(i) ||
+ !sweepOnBackgroundThread,
+ !zone->arenas.arenaListsToSweep[i]);
+ }
+ }
+#endif
+
+ AssertNoWrappersInGrayList(rt);
+}
+
+void
+GCRuntime::beginCompactPhase()
+{
+ MOZ_ASSERT(!isBackgroundSweeping());
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT);
+
+ MOZ_ASSERT(zonesToMaybeCompact.isEmpty());
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ if (CanRelocateZone(zone))
+ zonesToMaybeCompact.append(zone);
+ }
+
+ MOZ_ASSERT(!relocatedArenasToRelease);
+ startedCompacting = true;
+}
+
+GCRuntime::IncrementalProgress
+GCRuntime::compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget,
+ AutoLockForExclusiveAccess& lock)
+{
+ MOZ_ASSERT(rt->gc.nursery.isEmpty());
+ assertBackgroundSweepingFinished();
+ MOZ_ASSERT(startedCompacting);
+
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_COMPACT);
+
+ Arena* relocatedArenas = nullptr;
+ while (!zonesToMaybeCompact.isEmpty()) {
+ // TODO: JSScripts can move. If the sampler interrupts the GC in the
+ // middle of relocating an arena, invalid JSScript pointers may be
+ // accessed. Suppress all sampling until a finer-grained solution can be
+ // found. See bug 1295775.
+ AutoSuppressProfilerSampling suppressSampling(rt);
+
+ Zone* zone = zonesToMaybeCompact.front();
+ MOZ_ASSERT(zone->isGCFinished());
+ zone->setGCState(Zone::Compact);
+ if (relocateArenas(zone, reason, relocatedArenas, sliceBudget))
+ updatePointersToRelocatedCells(zone, lock);
+ zone->setGCState(Zone::Finished);
+ zonesToMaybeCompact.removeFront();
+ if (sliceBudget.isOverBudget())
+ break;
+ }
+
+ if (ShouldProtectRelocatedArenas(reason))
+ protectAndHoldArenas(relocatedArenas);
+ else
+ releaseRelocatedArenas(relocatedArenas);
+
+ // Clear caches that can contain cell pointers.
+ JSContext* cx = rt->contextFromMainThread();
+ cx->caches.newObjectCache.purge();
+ cx->caches.nativeIterCache.purge();
+ if (cx->caches.evalCache.initialized())
+ cx->caches.evalCache.clear();
+
+#ifdef DEBUG
+ CheckHashTablesAfterMovingGC(rt);
+#endif
+
+ return zonesToMaybeCompact.isEmpty() ? Finished : NotFinished;
+}
+
+void
+GCRuntime::endCompactPhase(JS::gcreason::Reason reason)
+{
+ startedCompacting = false;
+}
+
+void
+GCRuntime::finishCollection(JS::gcreason::Reason reason)
+{
+ assertBackgroundSweepingFinished();
+ MOZ_ASSERT(marker.isDrained());
+ marker.stop();
+ clearBufferedGrayRoots();
+ MemProfiler::SweepTenured(rt);
+
+ uint64_t currentTime = PRMJ_Now();
+ schedulingState.updateHighFrequencyMode(lastGCTime, currentTime, tunables);
+
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isCollecting()) {
+ MOZ_ASSERT(zone->isGCFinished());
+ zone->setGCState(Zone::NoGC);
+ zone->active = false;
+ }
+
+ MOZ_ASSERT(!zone->isCollecting());
+ MOZ_ASSERT(!zone->wasGCStarted());
+ }
+
+ MOZ_ASSERT(zonesToMaybeCompact.isEmpty());
+
+ lastGCTime = currentTime;
+}
+
+static const char*
+HeapStateToLabel(JS::HeapState heapState)
+{
+ switch (heapState) {
+ case JS::HeapState::MinorCollecting:
+ return "js::Nursery::collect";
+ case JS::HeapState::MajorCollecting:
+ return "js::GCRuntime::collect";
+ case JS::HeapState::Tracing:
+ return "JS_IterateCompartments";
+ case JS::HeapState::Idle:
+ case JS::HeapState::CycleCollecting:
+ MOZ_CRASH("Should never have an Idle or CC heap state when pushing GC pseudo frames!");
+ }
+ MOZ_ASSERT_UNREACHABLE("Should have exhausted every JS::HeapState variant!");
+ return nullptr;
+}
+
+/* Start a new heap session. */
+AutoTraceSession::AutoTraceSession(JSRuntime* rt, JS::HeapState heapState)
+ : lock(rt),
+ runtime(rt),
+ prevState(rt->heapState()),
+ pseudoFrame(rt, HeapStateToLabel(heapState), ProfileEntry::Category::GC)
+{
+ MOZ_ASSERT(prevState == JS::HeapState::Idle);
+ MOZ_ASSERT(heapState != JS::HeapState::Idle);
+ MOZ_ASSERT_IF(heapState == JS::HeapState::MajorCollecting, rt->gc.nursery.isEmpty());
+ rt->setHeapState(heapState);
+}
+
+AutoTraceSession::~AutoTraceSession()
+{
+ MOZ_ASSERT(runtime->isHeapBusy());
+ runtime->setHeapState(prevState);
+}
+
+void
+GCRuntime::resetIncrementalGC(gc::AbortReason reason, AutoLockForExclusiveAccess& lock)
+{
+ MOZ_ASSERT(reason != gc::AbortReason::None);
+
+ switch (incrementalState) {
+ case State::NotActive:
+ return;
+
+ case State::MarkRoots:
+ MOZ_CRASH("resetIncrementalGC did not expect MarkRoots state");
+ break;
+
+ case State::Mark: {
+ /* Cancel any ongoing marking. */
+ marker.reset();
+ marker.stop();
+ clearBufferedGrayRoots();
+
+ for (GCCompartmentsIter c(rt); !c.done(); c.next())
+ ResetGrayList(c);
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ MOZ_ASSERT(zone->isGCMarking());
+ zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit);
+ zone->setGCState(Zone::NoGC);
+ }
+
+ blocksToFreeAfterSweeping.freeAll();
+
+ incrementalState = State::NotActive;
+
+ MOZ_ASSERT(!marker.shouldCheckCompartments());
+
+ break;
+ }
+
+ case State::Sweep: {
+ marker.reset();
+
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next())
+ c->scheduledForDestruction = false;
+
+ /* Finish sweeping the current zone group, then abort. */
+ abortSweepAfterCurrentGroup = true;
+
+ /* Don't perform any compaction after sweeping. */
+ bool wasCompacting = isCompacting;
+ isCompacting = false;
+
+ auto unlimited = SliceBudget::unlimited();
+ incrementalCollectSlice(unlimited, JS::gcreason::RESET, lock);
+
+ isCompacting = wasCompacting;
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
+ rt->gc.waitBackgroundSweepOrAllocEnd();
+ }
+ break;
+ }
+
+ case State::Finalize: {
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
+ rt->gc.waitBackgroundSweepOrAllocEnd();
+ }
+
+ bool wasCompacting = isCompacting;
+ isCompacting = false;
+
+ auto unlimited = SliceBudget::unlimited();
+ incrementalCollectSlice(unlimited, JS::gcreason::RESET, lock);
+
+ isCompacting = wasCompacting;
+
+ break;
+ }
+
+ case State::Compact: {
+ bool wasCompacting = isCompacting;
+
+ isCompacting = true;
+ startedCompacting = true;
+ zonesToMaybeCompact.clear();
+
+ auto unlimited = SliceBudget::unlimited();
+ incrementalCollectSlice(unlimited, JS::gcreason::RESET, lock);
+
+ isCompacting = wasCompacting;
+ break;
+ }
+
+ case State::Decommit: {
+ auto unlimited = SliceBudget::unlimited();
+ incrementalCollectSlice(unlimited, JS::gcreason::RESET, lock);
+ break;
+ }
+ }
+
+ stats.reset(reason);
+
+#ifdef DEBUG
+ assertBackgroundSweepingFinished();
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ MOZ_ASSERT(!zone->isCollecting());
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ MOZ_ASSERT(!zone->isOnList());
+ }
+ MOZ_ASSERT(zonesToMaybeCompact.isEmpty());
+ MOZ_ASSERT(incrementalState == State::NotActive);
+#endif
+}
+
+namespace {
+
+class AutoGCSlice {
+ public:
+ explicit AutoGCSlice(JSRuntime* rt);
+ ~AutoGCSlice();
+
+ private:
+ JSRuntime* runtime;
+ AutoSetThreadIsPerformingGC performingGC;
+};
+
+} /* anonymous namespace */
+
+AutoGCSlice::AutoGCSlice(JSRuntime* rt)
+ : runtime(rt)
+{
+ /*
+ * During incremental GC, the compartment's active flag determines whether
+ * there are stack frames active for any of its scripts. Normally this flag
+ * is set at the beginning of the mark phase. During incremental GC, we also
+ * set it at the start of every phase.
+ */
+ for (ActivationIterator iter(rt); !iter.done(); ++iter)
+ iter->compartment()->zone()->active = true;
+
+ for (GCZonesIter zone(rt); !zone.done(); zone.next()) {
+ /*
+ * Clear needsIncrementalBarrier early so we don't do any write
+ * barriers during GC. We don't need to update the Ion barriers (which
+ * is expensive) because Ion code doesn't run during GC. If need be,
+ * we'll update the Ion barriers in ~AutoGCSlice.
+ */
+ if (zone->isGCMarking()) {
+ MOZ_ASSERT(zone->needsIncrementalBarrier());
+ zone->setNeedsIncrementalBarrier(false, Zone::DontUpdateJit);
+ } else {
+ MOZ_ASSERT(!zone->needsIncrementalBarrier());
+ }
+ }
+}
+
+AutoGCSlice::~AutoGCSlice()
+{
+ /* We can't use GCZonesIter if this is the end of the last slice. */
+ for (ZonesIter zone(runtime, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isGCMarking()) {
+ zone->setNeedsIncrementalBarrier(true, Zone::UpdateJit);
+ zone->arenas.prepareForIncrementalGC(runtime);
+ } else {
+ zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit);
+ }
+ }
+}
+
+void
+GCRuntime::pushZealSelectedObjects()
+{
+#ifdef JS_GC_ZEAL
+ /* Push selected objects onto the mark stack and clear the list. */
+ for (JSObject** obj = selectedForMarking.begin(); obj != selectedForMarking.end(); obj++)
+ TraceManuallyBarrieredEdge(&marker, obj, "selected obj");
+#endif
+}
+
+static bool
+IsShutdownGC(JS::gcreason::Reason reason)
+{
+ return reason == JS::gcreason::SHUTDOWN_CC || reason == JS::gcreason::DESTROY_RUNTIME;
+}
+
+static bool
+ShouldCleanUpEverything(JS::gcreason::Reason reason, JSGCInvocationKind gckind)
+{
+ // During shutdown, we must clean everything up, for the sake of leak
+ // detection. When a runtime has no contexts, or we're doing a GC before a
+ // shutdown CC, those are strong indications that we're shutting down.
+ return IsShutdownGC(reason) || gckind == GC_SHRINK;
+}
+
+void
+GCRuntime::incrementalCollectSlice(SliceBudget& budget, JS::gcreason::Reason reason,
+ AutoLockForExclusiveAccess& lock)
+{
+ AutoGCSlice slice(rt);
+
+ bool destroyingRuntime = (reason == JS::gcreason::DESTROY_RUNTIME);
+
+ gc::State initialState = incrementalState;
+
+ bool useZeal = false;
+#ifdef JS_GC_ZEAL
+ if (reason == JS::gcreason::DEBUG_GC && !budget.isUnlimited()) {
+ /*
+ * Do the incremental collection type specified by zeal mode if the
+ * collection was triggered by runDebugGC() and incremental GC has not
+ * been cancelled by resetIncrementalGC().
+ */
+ useZeal = true;
+ }
+#endif
+
+ MOZ_ASSERT_IF(isIncrementalGCInProgress(), isIncremental);
+ isIncremental = !budget.isUnlimited();
+
+ if (useZeal && (hasZealMode(ZealMode::IncrementalRootsThenFinish) ||
+ hasZealMode(ZealMode::IncrementalMarkAllThenFinish)))
+ {
+ /*
+ * Yields between slices occurs at predetermined points in these modes;
+ * the budget is not used.
+ */
+ budget.makeUnlimited();
+ }
+
+ switch (incrementalState) {
+ case State::NotActive:
+ initialReason = reason;
+ cleanUpEverything = ShouldCleanUpEverything(reason, invocationKind);
+ isCompacting = shouldCompact();
+ lastMarkSlice = false;
+
+ incrementalState = State::MarkRoots;
+
+ MOZ_FALLTHROUGH;
+
+ case State::MarkRoots:
+ if (!beginMarkPhase(reason, lock)) {
+ incrementalState = State::NotActive;
+ return;
+ }
+
+ if (!destroyingRuntime)
+ pushZealSelectedObjects();
+
+ incrementalState = State::Mark;
+
+ if (isIncremental && useZeal && hasZealMode(ZealMode::IncrementalRootsThenFinish))
+ break;
+
+ MOZ_FALLTHROUGH;
+
+ case State::Mark:
+ AutoGCRooter::traceAllWrappers(&marker);
+
+ /* If we needed delayed marking for gray roots, then collect until done. */
+ if (!hasBufferedGrayRoots()) {
+ budget.makeUnlimited();
+ isIncremental = false;
+ }
+
+ if (drainMarkStack(budget, gcstats::PHASE_MARK) == NotFinished)
+ break;
+
+ MOZ_ASSERT(marker.isDrained());
+
+ /*
+ * In incremental GCs where we have already performed more than once
+ * slice we yield after marking with the aim of starting the sweep in
+ * the next slice, since the first slice of sweeping can be expensive.
+ *
+ * This is modified by the various zeal modes. We don't yield in
+ * IncrementalRootsThenFinish mode and we always yield in
+ * IncrementalMarkAllThenFinish mode.
+ *
+ * We will need to mark anything new on the stack when we resume, so
+ * we stay in Mark state.
+ */
+ if (!lastMarkSlice && isIncremental &&
+ ((initialState == State::Mark &&
+ !(useZeal && hasZealMode(ZealMode::IncrementalRootsThenFinish))) ||
+ (useZeal && hasZealMode(ZealMode::IncrementalMarkAllThenFinish))))
+ {
+ lastMarkSlice = true;
+ break;
+ }
+
+ incrementalState = State::Sweep;
+
+ /*
+ * This runs to completion, but we don't continue if the budget is
+ * now exhasted.
+ */
+ beginSweepPhase(destroyingRuntime, lock);
+ if (budget.isOverBudget())
+ break;
+
+ /*
+ * Always yield here when running in incremental multi-slice zeal
+ * mode, so RunDebugGC can reset the slice buget.
+ */
+ if (isIncremental && useZeal && hasZealMode(ZealMode::IncrementalMultipleSlices))
+ break;
+
+ MOZ_FALLTHROUGH;
+
+ case State::Sweep:
+ if (sweepPhase(budget, lock) == NotFinished)
+ break;
+
+ endSweepPhase(destroyingRuntime, lock);
+
+ incrementalState = State::Finalize;
+
+ MOZ_FALLTHROUGH;
+
+ case State::Finalize:
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
+
+ // Yield until background finalization is done.
+ if (isIncremental) {
+ // Poll for end of background sweeping
+ AutoLockGC lock(rt);
+ if (isBackgroundSweeping())
+ break;
+ } else {
+ waitBackgroundSweepEnd();
+ }
+ }
+
+ {
+ // Re-sweep the zones list, now that background finalization is
+ // finished to actually remove and free dead zones.
+ gcstats::AutoPhase ap1(stats, gcstats::PHASE_SWEEP);
+ gcstats::AutoPhase ap2(stats, gcstats::PHASE_DESTROY);
+ AutoSetThreadIsSweeping threadIsSweeping;
+ FreeOp fop(rt);
+ sweepZones(&fop, destroyingRuntime);
+ }
+
+ MOZ_ASSERT(!startedCompacting);
+ incrementalState = State::Compact;
+
+ // Always yield before compacting since it is not incremental.
+ if (isCompacting && isIncremental)
+ break;
+
+ MOZ_FALLTHROUGH;
+
+ case State::Compact:
+ if (isCompacting) {
+ if (!startedCompacting)
+ beginCompactPhase();
+
+ if (compactPhase(reason, budget, lock) == NotFinished)
+ break;
+
+ endCompactPhase(reason);
+ }
+
+ startDecommit();
+ incrementalState = State::Decommit;
+
+ MOZ_FALLTHROUGH;
+
+ case State::Decommit:
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
+
+ // Yield until background decommit is done.
+ if (isIncremental && decommitTask.isRunning())
+ break;
+
+ decommitTask.join();
+ }
+
+ finishCollection(reason);
+ incrementalState = State::NotActive;
+ break;
+ }
+}
+
+gc::AbortReason
+gc::IsIncrementalGCUnsafe(JSRuntime* rt)
+{
+ MOZ_ASSERT(!rt->mainThread.suppressGC);
+
+ if (rt->keepAtoms())
+ return gc::AbortReason::KeepAtomsSet;
+
+ if (!rt->gc.isIncrementalGCAllowed())
+ return gc::AbortReason::IncrementalDisabled;
+
+ return gc::AbortReason::None;
+}
+
+void
+GCRuntime::budgetIncrementalGC(SliceBudget& budget, AutoLockForExclusiveAccess& lock)
+{
+ AbortReason unsafeReason = IsIncrementalGCUnsafe(rt);
+ if (unsafeReason != AbortReason::None) {
+ resetIncrementalGC(unsafeReason, lock);
+ budget.makeUnlimited();
+ stats.nonincremental(unsafeReason);
+ return;
+ }
+
+ if (mode != JSGC_MODE_INCREMENTAL) {
+ resetIncrementalGC(AbortReason::ModeChange, lock);
+ budget.makeUnlimited();
+ stats.nonincremental(AbortReason::ModeChange);
+ return;
+ }
+
+ if (isTooMuchMalloc()) {
+ budget.makeUnlimited();
+ stats.nonincremental(AbortReason::MallocBytesTrigger);
+ }
+
+ bool reset = false;
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->usage.gcBytes() >= zone->threshold.gcTriggerBytes()) {
+ budget.makeUnlimited();
+ stats.nonincremental(AbortReason::GCBytesTrigger);
+ }
+
+ if (isIncrementalGCInProgress() && zone->isGCScheduled() != zone->wasGCStarted())
+ reset = true;
+
+ if (zone->isTooMuchMalloc()) {
+ budget.makeUnlimited();
+ stats.nonincremental(AbortReason::MallocBytesTrigger);
+ }
+ }
+
+ if (reset)
+ resetIncrementalGC(AbortReason::ZoneChange, lock);
+}
+
+namespace {
+
+class AutoScheduleZonesForGC
+{
+ JSRuntime* rt_;
+
+ public:
+ explicit AutoScheduleZonesForGC(JSRuntime* rt) : rt_(rt) {
+ for (ZonesIter zone(rt_, WithAtoms); !zone.done(); zone.next()) {
+ if (rt->gc.gcMode() == JSGC_MODE_GLOBAL)
+ zone->scheduleGC();
+
+ /* This is a heuristic to avoid resets. */
+ if (rt->gc.isIncrementalGCInProgress() && zone->needsIncrementalBarrier())
+ zone->scheduleGC();
+
+ /* This is a heuristic to reduce the total number of collections. */
+ if (zone->usage.gcBytes() >=
+ zone->threshold.allocTrigger(rt->gc.schedulingState.inHighFrequencyGCMode()))
+ {
+ zone->scheduleGC();
+ }
+ }
+ }
+
+ ~AutoScheduleZonesForGC() {
+ for (ZonesIter zone(rt_, WithAtoms); !zone.done(); zone.next())
+ zone->unscheduleGC();
+ }
+};
+
+/*
+ * An invariant of our GC/CC interaction is that there must not ever be any
+ * black to gray edges in the system. It is possible to violate this with
+ * simple compartmental GC. For example, in GC[n], we collect in both
+ * compartmentA and compartmentB, and mark both sides of the cross-compartment
+ * edge gray. Later in GC[n+1], we only collect compartmentA, but this time
+ * mark it black. Now we are violating the invariants and must fix it somehow.
+ *
+ * To prevent this situation, we explicitly detect the black->gray state when
+ * marking cross-compartment edges -- see ShouldMarkCrossCompartment -- adding
+ * each violating edges to foundBlackGrayEdges. After we leave the trace
+ * session for each GC slice, we "ExposeToActiveJS" on each of these edges
+ * (which we cannot do safely from the guts of the GC).
+ */
+class AutoExposeLiveCrossZoneEdges
+{
+ BlackGrayEdgeVector* edges;
+
+ public:
+ explicit AutoExposeLiveCrossZoneEdges(BlackGrayEdgeVector* edgesPtr) : edges(edgesPtr) {
+ MOZ_ASSERT(edges->empty());
+ }
+ ~AutoExposeLiveCrossZoneEdges() {
+ for (auto& target : *edges) {
+ MOZ_ASSERT(target);
+ MOZ_ASSERT(!target->zone()->isCollecting());
+ UnmarkGrayCellRecursively(target, target->getTraceKind());
+ }
+ edges->clear();
+ }
+};
+
+} /* anonymous namespace */
+
+/*
+ * Run one GC "cycle" (either a slice of incremental GC or an entire
+ * non-incremental GC. We disable inlining to ensure that the bottom of the
+ * stack with possible GC roots recorded in MarkRuntime excludes any pointers we
+ * use during the marking implementation.
+ *
+ * Returns true if we "reset" an existing incremental GC, which would force us
+ * to run another cycle.
+ */
+MOZ_NEVER_INLINE bool
+GCRuntime::gcCycle(bool nonincrementalByAPI, SliceBudget& budget, JS::gcreason::Reason reason)
+{
+ // Note that the following is allowed to re-enter GC in the finalizer.
+ AutoNotifyGCActivity notify(*this);
+
+ gcstats::AutoGCSlice agc(stats, scanZonesBeforeGC(), invocationKind, budget, reason);
+
+ AutoExposeLiveCrossZoneEdges aelcze(&foundBlackGrayEdges);
+
+ evictNursery(reason);
+
+ AutoTraceSession session(rt, JS::HeapState::MajorCollecting);
+
+ majorGCTriggerReason = JS::gcreason::NO_REASON;
+ interFrameGC = true;
+
+ number++;
+ if (!isIncrementalGCInProgress())
+ incMajorGcNumber();
+
+ // It's ok if threads other than the main thread have suppressGC set, as
+ // they are operating on zones which will not be collected from here.
+ MOZ_ASSERT(!rt->mainThread.suppressGC);
+
+ // Assert if this is a GC unsafe region.
+ verifyIsSafeToGC();
+
+ {
+ gcstats::AutoPhase ap(stats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
+
+ // Background finalization and decommit are finished by defininition
+ // before we can start a new GC session.
+ if (!isIncrementalGCInProgress()) {
+ assertBackgroundSweepingFinished();
+ MOZ_ASSERT(!decommitTask.isRunning());
+ }
+
+ // We must also wait for background allocation to finish so we can
+ // avoid taking the GC lock when manipulating the chunks during the GC.
+ // The background alloc task can run between slices, so we must wait
+ // for it at the start of every slice.
+ allocTask.cancel(GCParallelTask::CancelAndWait);
+ }
+
+ State prevState = incrementalState;
+
+ if (nonincrementalByAPI) {
+ // Reset any in progress incremental GC if this was triggered via the
+ // API. This isn't required for correctness, but sometimes during tests
+ // the caller expects this GC to collect certain objects, and we need
+ // to make sure to collect everything possible.
+ if (reason != JS::gcreason::ALLOC_TRIGGER)
+ resetIncrementalGC(gc::AbortReason::NonIncrementalRequested, session.lock);
+
+ stats.nonincremental(gc::AbortReason::NonIncrementalRequested);
+ budget.makeUnlimited();
+ } else {
+ budgetIncrementalGC(budget, session.lock);
+ }
+
+ /* The GC was reset, so we need a do-over. */
+ if (prevState != State::NotActive && !isIncrementalGCInProgress())
+ return true;
+
+ TraceMajorGCStart();
+
+ incrementalCollectSlice(budget, reason, session.lock);
+
+ chunkAllocationSinceLastGC = false;
+
+#ifdef JS_GC_ZEAL
+ /* Keeping these around after a GC is dangerous. */
+ clearSelectedForMarking();
+#endif
+
+ /* Clear gcMallocBytes for all zones. */
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ zone->resetGCMallocBytes();
+
+ resetMallocBytes();
+
+ TraceMajorGCEnd();
+
+ return false;
+}
+
+#ifdef JS_GC_ZEAL
+static bool
+IsDeterministicGCReason(JS::gcreason::Reason reason)
+{
+ if (reason > JS::gcreason::DEBUG_GC &&
+ reason != JS::gcreason::CC_FORCED && reason != JS::gcreason::SHUTDOWN_CC)
+ {
+ return false;
+ }
+
+ if (reason == JS::gcreason::EAGER_ALLOC_TRIGGER)
+ return false;
+
+ return true;
+}
+#endif
+
+gcstats::ZoneGCStats
+GCRuntime::scanZonesBeforeGC()
+{
+ gcstats::ZoneGCStats zoneStats;
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ zoneStats.zoneCount++;
+ if (zone->isGCScheduled()) {
+ zoneStats.collectedZoneCount++;
+ zoneStats.collectedCompartmentCount += zone->compartments.length();
+ }
+ }
+
+ for (CompartmentsIter c(rt, WithAtoms); !c.done(); c.next())
+ zoneStats.compartmentCount++;
+
+ return zoneStats;
+}
+
+// The GC can only clean up scheduledForDestruction compartments that were
+// marked live by a barrier (e.g. by RemapWrappers from a navigation event).
+// It is also common to have compartments held live because they are part of a
+// cycle in gecko, e.g. involving the HTMLDocument wrapper. In this case, we
+// need to run the CycleCollector in order to remove these edges before the
+// compartment can be freed.
+void
+GCRuntime::maybeDoCycleCollection()
+{
+ const static double ExcessiveGrayCompartments = 0.8;
+ const static size_t LimitGrayCompartments = 200;
+
+ size_t compartmentsTotal = 0;
+ size_t compartmentsGray = 0;
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ ++compartmentsTotal;
+ GlobalObject* global = c->unsafeUnbarrieredMaybeGlobal();
+ if (global && global->asTenured().isMarked(GRAY))
+ ++compartmentsGray;
+ }
+ double grayFraction = double(compartmentsGray) / double(compartmentsTotal);
+ if (grayFraction > ExcessiveGrayCompartments || compartmentsGray > LimitGrayCompartments)
+ callDoCycleCollectionCallback(rt->contextFromMainThread());
+}
+
+void
+GCRuntime::checkCanCallAPI()
+{
+ MOZ_RELEASE_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ /* If we attempt to invoke the GC while we are running in the GC, assert. */
+ MOZ_RELEASE_ASSERT(!rt->isHeapBusy());
+
+ MOZ_ASSERT(isAllocAllowed());
+}
+
+bool
+GCRuntime::checkIfGCAllowedInCurrentState(JS::gcreason::Reason reason)
+{
+ if (rt->mainThread.suppressGC)
+ return false;
+
+ // Only allow shutdown GCs when we're destroying the runtime. This keeps
+ // the GC callback from triggering a nested GC and resetting global state.
+ if (rt->isBeingDestroyed() && !IsShutdownGC(reason))
+ return false;
+
+#ifdef JS_GC_ZEAL
+ if (deterministicOnly && !IsDeterministicGCReason(reason))
+ return false;
+#endif
+
+ return true;
+}
+
+void
+GCRuntime::collect(bool nonincrementalByAPI, SliceBudget budget, JS::gcreason::Reason reason)
+{
+ // Checks run for each request, even if we do not actually GC.
+ checkCanCallAPI();
+
+ // Check if we are allowed to GC at this time before proceeding.
+ if (!checkIfGCAllowedInCurrentState(reason))
+ return;
+
+ AutoTraceLog logGC(TraceLoggerForMainThread(rt), TraceLogger_GC);
+ AutoStopVerifyingBarriers av(rt, IsShutdownGC(reason));
+ AutoEnqueuePendingParseTasksAfterGC aept(*this);
+ AutoScheduleZonesForGC asz(rt);
+
+ bool repeat = false;
+ do {
+ poked = false;
+ bool wasReset = gcCycle(nonincrementalByAPI, budget, reason);
+
+ /* Need to re-schedule all zones for GC. */
+ if (poked && cleanUpEverything)
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+
+ /*
+ * This code makes an extra effort to collect compartments that we
+ * thought were dead at the start of the GC. See the large comment in
+ * beginMarkPhase.
+ */
+ bool repeatForDeadZone = false;
+ if (!nonincrementalByAPI && !isIncrementalGCInProgress()) {
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ if (c->scheduledForDestruction) {
+ nonincrementalByAPI = true;
+ repeatForDeadZone = true;
+ reason = JS::gcreason::COMPARTMENT_REVIVED;
+ c->zone()->scheduleGC();
+ }
+ }
+ }
+
+ /*
+ * If we reset an existing GC, we need to start a new one. Also, we
+ * repeat GCs that happen during shutdown (the gcShouldCleanUpEverything
+ * case) until we can be sure that no additional garbage is created
+ * (which typically happens if roots are dropped during finalizers).
+ */
+ repeat = (poked && cleanUpEverything) || wasReset || repeatForDeadZone;
+ } while (repeat);
+
+ if (reason == JS::gcreason::COMPARTMENT_REVIVED)
+ maybeDoCycleCollection();
+
+#ifdef JS_GC_ZEAL
+ if (rt->hasZealMode(ZealMode::CheckHeapAfterGC)) {
+ gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_TRACE_HEAP);
+ CheckHeapAfterGC(rt);
+ }
+#endif
+}
+
+js::AutoEnqueuePendingParseTasksAfterGC::~AutoEnqueuePendingParseTasksAfterGC()
+{
+ if (!OffThreadParsingMustWaitForGC(gc_.rt))
+ EnqueuePendingParseTasksAfterGC(gc_.rt);
+}
+
+SliceBudget
+GCRuntime::defaultBudget(JS::gcreason::Reason reason, int64_t millis)
+{
+ if (millis == 0) {
+ if (reason == JS::gcreason::ALLOC_TRIGGER)
+ millis = defaultSliceBudget();
+ else if (schedulingState.inHighFrequencyGCMode() && tunables.isDynamicMarkSliceEnabled())
+ millis = defaultSliceBudget() * IGC_MARK_SLICE_MULTIPLIER;
+ else
+ millis = defaultSliceBudget();
+ }
+
+ return SliceBudget(TimeBudget(millis));
+}
+
+void
+GCRuntime::gc(JSGCInvocationKind gckind, JS::gcreason::Reason reason)
+{
+ invocationKind = gckind;
+ collect(true, SliceBudget::unlimited(), reason);
+}
+
+void
+GCRuntime::startGC(JSGCInvocationKind gckind, JS::gcreason::Reason reason, int64_t millis)
+{
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ if (!JS::IsIncrementalGCEnabled(rt->contextFromMainThread())) {
+ gc(gckind, reason);
+ return;
+ }
+ invocationKind = gckind;
+ collect(false, defaultBudget(reason, millis), reason);
+}
+
+void
+GCRuntime::gcSlice(JS::gcreason::Reason reason, int64_t millis)
+{
+ MOZ_ASSERT(isIncrementalGCInProgress());
+ collect(false, defaultBudget(reason, millis), reason);
+}
+
+void
+GCRuntime::finishGC(JS::gcreason::Reason reason)
+{
+ MOZ_ASSERT(isIncrementalGCInProgress());
+
+ // If we're not collecting because we're out of memory then skip the
+ // compacting phase if we need to finish an ongoing incremental GC
+ // non-incrementally to avoid janking the browser.
+ if (!IsOOMReason(initialReason)) {
+ if (incrementalState == State::Compact) {
+ abortGC();
+ return;
+ }
+
+ isCompacting = false;
+ }
+
+ collect(false, SliceBudget::unlimited(), reason);
+}
+
+void
+GCRuntime::abortGC()
+{
+ checkCanCallAPI();
+ MOZ_ASSERT(!rt->mainThread.suppressGC);
+
+ AutoStopVerifyingBarriers av(rt, false);
+ AutoEnqueuePendingParseTasksAfterGC aept(*this);
+
+ gcstats::AutoGCSlice agc(stats, scanZonesBeforeGC(), invocationKind,
+ SliceBudget::unlimited(), JS::gcreason::ABORT_GC);
+
+ evictNursery(JS::gcreason::ABORT_GC);
+ AutoTraceSession session(rt, JS::HeapState::MajorCollecting);
+
+ number++;
+ resetIncrementalGC(gc::AbortReason::AbortRequested, session.lock);
+}
+
+void
+GCRuntime::notifyDidPaint()
+{
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+#ifdef JS_GC_ZEAL
+ if (hasZealMode(ZealMode::FrameVerifierPre))
+ verifyPreBarriers();
+
+ if (hasZealMode(ZealMode::FrameGC)) {
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+ gc(GC_NORMAL, JS::gcreason::REFRESH_FRAME);
+ return;
+ }
+#endif
+
+ if (isIncrementalGCInProgress() && !interFrameGC && tunables.areRefreshFrameSlicesEnabled()) {
+ JS::PrepareForIncrementalGC(rt->contextFromMainThread());
+ gcSlice(JS::gcreason::REFRESH_FRAME);
+ }
+
+ interFrameGC = false;
+}
+
+static bool
+ZonesSelected(JSRuntime* rt)
+{
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isGCScheduled())
+ return true;
+ }
+ return false;
+}
+
+void
+GCRuntime::startDebugGC(JSGCInvocationKind gckind, SliceBudget& budget)
+{
+ MOZ_ASSERT(!isIncrementalGCInProgress());
+ if (!ZonesSelected(rt))
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+ invocationKind = gckind;
+ collect(false, budget, JS::gcreason::DEBUG_GC);
+}
+
+void
+GCRuntime::debugGCSlice(SliceBudget& budget)
+{
+ MOZ_ASSERT(isIncrementalGCInProgress());
+ if (!ZonesSelected(rt))
+ JS::PrepareForIncrementalGC(rt->contextFromMainThread());
+ collect(false, budget, JS::gcreason::DEBUG_GC);
+}
+
+/* Schedule a full GC unless a zone will already be collected. */
+void
+js::PrepareForDebugGC(JSRuntime* rt)
+{
+ if (!ZonesSelected(rt))
+ JS::PrepareForFullGC(rt->contextFromMainThread());
+}
+
+void
+GCRuntime::onOutOfMallocMemory()
+{
+ // Stop allocating new chunks.
+ allocTask.cancel(GCParallelTask::CancelAndWait);
+
+ // Make sure we release anything queued for release.
+ decommitTask.join();
+
+ // Wait for background free of nursery huge slots to finish.
+ nursery.waitBackgroundFreeEnd();
+
+ AutoLockGC lock(rt);
+ onOutOfMallocMemory(lock);
+}
+
+void
+GCRuntime::onOutOfMallocMemory(const AutoLockGC& lock)
+{
+ // Release any relocated arenas we may be holding on to, without releasing
+ // the GC lock.
+ releaseHeldRelocatedArenasWithoutUnlocking(lock);
+
+ // Throw away any excess chunks we have lying around.
+ freeEmptyChunks(rt, lock);
+
+ // Immediately decommit as many arenas as possible in the hopes that this
+ // might let the OS scrape together enough pages to satisfy the failing
+ // malloc request.
+ decommitAllWithoutUnlocking(lock);
+}
+
+void
+GCRuntime::minorGC(JS::gcreason::Reason reason, gcstats::Phase phase)
+{
+ MOZ_ASSERT(!rt->isHeapBusy());
+
+ if (rt->mainThread.suppressGC)
+ return;
+
+ gcstats::AutoPhase ap(stats, phase);
+
+ minorGCTriggerReason = JS::gcreason::NO_REASON;
+ TraceLoggerThread* logger = TraceLoggerForMainThread(rt);
+ AutoTraceLog logMinorGC(logger, TraceLogger_MinorGC);
+ nursery.collect(rt, reason);
+ MOZ_ASSERT(nursery.isEmpty());
+
+ blocksToFreeAfterMinorGC.freeAll();
+
+#ifdef JS_GC_ZEAL
+ if (rt->hasZealMode(ZealMode::CheckHeapAfterGC))
+ CheckHeapAfterGC(rt);
+#endif
+
+ {
+ AutoLockGC lock(rt);
+ for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next())
+ maybeAllocTriggerZoneGC(zone, lock);
+ }
+}
+
+void
+GCRuntime::disableGenerationalGC()
+{
+ if (isGenerationalGCEnabled()) {
+ evictNursery(JS::gcreason::API);
+ nursery.disable();
+ }
+ ++rt->gc.generationalDisabled;
+}
+
+void
+GCRuntime::enableGenerationalGC()
+{
+ MOZ_ASSERT(generationalDisabled > 0);
+ --generationalDisabled;
+ if (generationalDisabled == 0)
+ nursery.enable();
+}
+
+bool
+GCRuntime::gcIfRequested()
+{
+ // This method returns whether a major GC was performed.
+
+ if (minorGCRequested())
+ minorGC(minorGCTriggerReason);
+
+ if (majorGCRequested()) {
+ if (!isIncrementalGCInProgress())
+ startGC(GC_NORMAL, majorGCTriggerReason);
+ else
+ gcSlice(majorGCTriggerReason);
+ return true;
+ }
+
+ return false;
+}
+
+void
+js::gc::FinishGC(JSContext* cx)
+{
+ if (JS::IsIncrementalGCInProgress(cx)) {
+ JS::PrepareForIncrementalGC(cx);
+ JS::FinishIncrementalGC(cx, JS::gcreason::API);
+ }
+
+ cx->gc.nursery.waitBackgroundFreeEnd();
+}
+
+AutoPrepareForTracing::AutoPrepareForTracing(JSContext* cx, ZoneSelector selector)
+{
+ js::gc::FinishGC(cx);
+ session_.emplace(cx);
+}
+
+JSCompartment*
+js::NewCompartment(JSContext* cx, Zone* zone, JSPrincipals* principals,
+ const JS::CompartmentOptions& options)
+{
+ JSRuntime* rt = cx->runtime();
+ JS_AbortIfWrongThread(cx);
+
+ ScopedJSDeletePtr<Zone> zoneHolder;
+ if (!zone) {
+ zone = cx->new_<Zone>(rt);
+ if (!zone)
+ return nullptr;
+
+ zoneHolder.reset(zone);
+
+ const JSPrincipals* trusted = rt->trustedPrincipals();
+ bool isSystem = principals && principals == trusted;
+ if (!zone->init(isSystem)) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+ }
+
+ ScopedJSDeletePtr<JSCompartment> compartment(cx->new_<JSCompartment>(zone, options));
+ if (!compartment || !compartment->init(cx))
+ return nullptr;
+
+ // Set up the principals.
+ JS_SetCompartmentPrincipals(compartment, principals);
+
+ AutoLockGC lock(rt);
+
+ if (!zone->compartments.append(compartment.get())) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ if (zoneHolder && !rt->gc.zones.append(zone)) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ zoneHolder.forget();
+ return compartment.forget();
+}
+
+void
+gc::MergeCompartments(JSCompartment* source, JSCompartment* target)
+{
+ // The source compartment must be specifically flagged as mergable. This
+ // also implies that the compartment is not visible to the debugger.
+ MOZ_ASSERT(source->creationOptions_.mergeable());
+ MOZ_ASSERT(source->creationOptions_.invisibleToDebugger());
+
+ MOZ_ASSERT(source->creationOptions().addonIdOrNull() ==
+ target->creationOptions().addonIdOrNull());
+
+ JSContext* cx = source->contextFromMainThread();
+
+ AutoPrepareForTracing prepare(cx, SkipAtoms);
+
+ // Cleanup tables and other state in the source compartment that will be
+ // meaningless after merging into the target compartment.
+
+ source->clearTables();
+ source->zone()->clearTables();
+ source->unsetIsDebuggee();
+
+ // The delazification flag indicates the presence of LazyScripts in a
+ // compartment for the Debugger API, so if the source compartment created
+ // LazyScripts, the flag must be propagated to the target compartment.
+ if (source->needsDelazificationForDebugger())
+ target->scheduleDelazificationForDebugger();
+
+ // Release any relocated arenas which we may be holding on to as they might
+ // be in the source zone
+ cx->gc.releaseHeldRelocatedArenas();
+
+ // Fixup compartment pointers in source to refer to target, and make sure
+ // type information generations are in sync.
+
+ for (auto script = source->zone()->cellIter<JSScript>(); !script.done(); script.next()) {
+ MOZ_ASSERT(script->compartment() == source);
+ script->compartment_ = target;
+ script->setTypesGeneration(target->zone()->types.generation);
+ }
+
+ for (auto group = source->zone()->cellIter<ObjectGroup>(); !group.done(); group.next()) {
+ group->setGeneration(target->zone()->types.generation);
+ group->compartment_ = target;
+
+ // Remove any unboxed layouts from the list in the off thread
+ // compartment. These do not need to be reinserted in the target
+ // compartment's list, as the list is not required to be complete.
+ if (UnboxedLayout* layout = group->maybeUnboxedLayoutDontCheckGeneration())
+ layout->detachFromCompartment();
+ }
+
+ // Fixup zone pointers in source's zone to refer to target's zone.
+
+ for (auto thingKind : AllAllocKinds()) {
+ for (ArenaIter aiter(source->zone(), thingKind); !aiter.done(); aiter.next()) {
+ Arena* arena = aiter.get();
+ arena->zone = target->zone();
+ }
+ }
+
+ // The source should be the only compartment in its zone.
+ for (CompartmentsInZoneIter c(source->zone()); !c.done(); c.next())
+ MOZ_ASSERT(c.get() == source);
+
+ // Merge the allocator, stats and UIDs in source's zone into target's zone.
+ target->zone()->arenas.adoptArenas(cx, &source->zone()->arenas);
+ target->zone()->usage.adopt(source->zone()->usage);
+ target->zone()->adoptUniqueIds(source->zone());
+
+ // Merge other info in source's zone into target's zone.
+ target->zone()->types.typeLifoAlloc.transferFrom(&source->zone()->types.typeLifoAlloc);
+}
+
+void
+GCRuntime::runDebugGC()
+{
+#ifdef JS_GC_ZEAL
+ if (rt->mainThread.suppressGC)
+ return;
+
+ if (hasZealMode(ZealMode::GenerationalGC))
+ return minorGC(JS::gcreason::DEBUG_GC);
+
+ PrepareForDebugGC(rt);
+
+ auto budget = SliceBudget::unlimited();
+ if (hasZealMode(ZealMode::IncrementalRootsThenFinish) ||
+ hasZealMode(ZealMode::IncrementalMarkAllThenFinish) ||
+ hasZealMode(ZealMode::IncrementalMultipleSlices))
+ {
+ js::gc::State initialState = incrementalState;
+ if (hasZealMode(ZealMode::IncrementalMultipleSlices)) {
+ /*
+ * Start with a small slice limit and double it every slice. This
+ * ensure that we get multiple slices, and collection runs to
+ * completion.
+ */
+ if (!isIncrementalGCInProgress())
+ incrementalLimit = zealFrequency / 2;
+ else
+ incrementalLimit *= 2;
+ budget = SliceBudget(WorkBudget(incrementalLimit));
+ } else {
+ // This triggers incremental GC but is actually ignored by IncrementalMarkSlice.
+ budget = SliceBudget(WorkBudget(1));
+ }
+
+ if (!isIncrementalGCInProgress())
+ invocationKind = GC_SHRINK;
+ collect(false, budget, JS::gcreason::DEBUG_GC);
+
+ /*
+ * For multi-slice zeal, reset the slice size when we get to the sweep
+ * or compact phases.
+ */
+ if (hasZealMode(ZealMode::IncrementalMultipleSlices)) {
+ if ((initialState == State::Mark && incrementalState == State::Sweep) ||
+ (initialState == State::Sweep && incrementalState == State::Compact))
+ {
+ incrementalLimit = zealFrequency / 2;
+ }
+ }
+ } else if (hasZealMode(ZealMode::Compact)) {
+ gc(GC_SHRINK, JS::gcreason::DEBUG_GC);
+ } else {
+ gc(GC_NORMAL, JS::gcreason::DEBUG_GC);
+ }
+
+#endif
+}
+
+void
+GCRuntime::setFullCompartmentChecks(bool enabled)
+{
+ MOZ_ASSERT(!rt->isHeapMajorCollecting());
+ fullCompartmentChecks = enabled;
+}
+
+#ifdef JS_GC_ZEAL
+bool
+GCRuntime::selectForMarking(JSObject* object)
+{
+ MOZ_ASSERT(!rt->isHeapMajorCollecting());
+ return selectedForMarking.append(object);
+}
+
+void
+GCRuntime::clearSelectedForMarking()
+{
+ selectedForMarking.clearAndFree();
+}
+
+void
+GCRuntime::setDeterministic(bool enabled)
+{
+ MOZ_ASSERT(!rt->isHeapMajorCollecting());
+ deterministicOnly = enabled;
+}
+#endif
+
+#ifdef DEBUG
+
+/* Should only be called manually under gdb */
+void PreventGCDuringInteractiveDebug()
+{
+ TlsPerThreadData.get()->suppressGC++;
+}
+
+#endif
+
+void
+js::ReleaseAllJITCode(FreeOp* fop)
+{
+ js::CancelOffThreadIonCompile(fop->runtime());
+ for (ZonesIter zone(fop->runtime(), SkipAtoms); !zone.done(); zone.next()) {
+ zone->setPreservingCode(false);
+ zone->discardJitCode(fop);
+ }
+}
+
+void
+js::PurgeJITCaches(Zone* zone)
+{
+ /* Discard Ion caches. */
+ for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next())
+ jit::PurgeCaches(script);
+}
+
+void
+ArenaLists::normalizeBackgroundFinalizeState(AllocKind thingKind)
+{
+ ArenaLists::BackgroundFinalizeState* bfs = &backgroundFinalizeState[thingKind];
+ switch (*bfs) {
+ case BFS_DONE:
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Background finalization in progress, but it should not be.");
+ break;
+ }
+}
+
+void
+ArenaLists::adoptArenas(JSRuntime* rt, ArenaLists* fromArenaLists)
+{
+ // GC should be inactive, but still take the lock as a kind of read fence.
+ AutoLockGC lock(rt);
+
+ fromArenaLists->purge();
+
+ for (auto thingKind : AllAllocKinds()) {
+ // When we enter a parallel section, we join the background
+ // thread, and we do not run GC while in the parallel section,
+ // so no finalizer should be active!
+ normalizeBackgroundFinalizeState(thingKind);
+ fromArenaLists->normalizeBackgroundFinalizeState(thingKind);
+
+ ArenaList* fromList = &fromArenaLists->arenaLists[thingKind];
+ ArenaList* toList = &arenaLists[thingKind];
+ fromList->check();
+ toList->check();
+ Arena* next;
+ for (Arena* fromArena = fromList->head(); fromArena; fromArena = next) {
+ // Copy fromArena->next before releasing/reinserting.
+ next = fromArena->next;
+
+ MOZ_ASSERT(!fromArena->isEmpty());
+ toList->insertAtCursor(fromArena);
+ }
+ fromList->clear();
+ toList->check();
+ }
+}
+
+bool
+ArenaLists::containsArena(JSRuntime* rt, Arena* needle)
+{
+ AutoLockGC lock(rt);
+ ArenaList& list = arenaLists[needle->getAllocKind()];
+ for (Arena* arena = list.head(); arena; arena = arena->next) {
+ if (arena == needle)
+ return true;
+ }
+ return false;
+}
+
+
+AutoSuppressGC::AutoSuppressGC(ExclusiveContext* cx)
+ : suppressGC_(cx->perThreadData->suppressGC)
+{
+ suppressGC_++;
+}
+
+AutoSuppressGC::AutoSuppressGC(JSCompartment* comp)
+ : suppressGC_(comp->runtimeFromMainThread()->mainThread.suppressGC)
+{
+ suppressGC_++;
+}
+
+AutoSuppressGC::AutoSuppressGC(JSContext* cx)
+ : suppressGC_(cx->mainThread().suppressGC)
+{
+ suppressGC_++;
+}
+
+bool
+js::UninlinedIsInsideNursery(const gc::Cell* cell)
+{
+ return IsInsideNursery(cell);
+}
+
+#ifdef DEBUG
+AutoDisableProxyCheck::AutoDisableProxyCheck(JSRuntime* rt)
+ : gc(rt->gc)
+{
+ gc.disableStrictProxyChecking();
+}
+
+AutoDisableProxyCheck::~AutoDisableProxyCheck()
+{
+ gc.enableStrictProxyChecking();
+}
+
+JS_FRIEND_API(void)
+JS::AssertGCThingMustBeTenured(JSObject* obj)
+{
+ MOZ_ASSERT(obj->isTenured() &&
+ (!IsNurseryAllocable(obj->asTenured().getAllocKind()) ||
+ obj->getClass()->hasFinalize()));
+}
+
+JS_FRIEND_API(void)
+JS::AssertGCThingIsNotAnObjectSubclass(Cell* cell)
+{
+ MOZ_ASSERT(cell);
+ MOZ_ASSERT(cell->getTraceKind() != JS::TraceKind::Object);
+}
+
+JS_FRIEND_API(void)
+js::gc::AssertGCThingHasType(js::gc::Cell* cell, JS::TraceKind kind)
+{
+ if (!cell)
+ MOZ_ASSERT(kind == JS::TraceKind::Null);
+ else if (IsInsideNursery(cell))
+ MOZ_ASSERT(kind == JS::TraceKind::Object);
+ else
+ MOZ_ASSERT(MapAllocToTraceKind(cell->asTenured().getAllocKind()) == kind);
+}
+
+JS_PUBLIC_API(size_t)
+JS::GetGCNumber()
+{
+ JSRuntime* rt = js::TlsPerThreadData.get()->runtimeFromMainThread();
+ if (!rt)
+ return 0;
+ return rt->gc.gcNumber();
+}
+#endif
+
+JS::AutoAssertNoGC::AutoAssertNoGC()
+ : gc(nullptr), gcNumber(0)
+{
+ js::PerThreadData* data = js::TlsPerThreadData.get();
+ if (data) {
+ /*
+ * GC's from off-thread will always assert, so off-thread is implicitly
+ * AutoAssertNoGC. We still need to allow AutoAssertNoGC to be used in
+ * code that works from both threads, however. We also use this to
+ * annotate the off thread run loops.
+ */
+ JSRuntime* runtime = data->runtimeIfOnOwnerThread();
+ if (runtime) {
+ gc = &runtime->gc;
+ gcNumber = gc->gcNumber();
+ gc->enterUnsafeRegion();
+ }
+ }
+}
+
+JS::AutoAssertNoGC::AutoAssertNoGC(JSRuntime* rt)
+ : gc(&rt->gc), gcNumber(rt->gc.gcNumber())
+{
+ gc->enterUnsafeRegion();
+}
+
+JS::AutoAssertNoGC::AutoAssertNoGC(JSContext* cx)
+ : gc(&cx->gc), gcNumber(cx->gc.gcNumber())
+{
+ gc->enterUnsafeRegion();
+}
+
+JS::AutoAssertNoGC::~AutoAssertNoGC()
+{
+ if (gc) {
+ gc->leaveUnsafeRegion();
+
+ /*
+ * The following backstop assertion should never fire: if we bumped the
+ * gcNumber, we should have asserted because inUnsafeRegion was true.
+ */
+ MOZ_ASSERT(gcNumber == gc->gcNumber(), "GC ran inside an AutoAssertNoGC scope.");
+ }
+}
+
+JS::AutoAssertOnBarrier::AutoAssertOnBarrier(JSContext* cx)
+ : context(cx),
+ prev(cx->runtime()->allowGCBarriers())
+{
+ context->runtime()->allowGCBarriers_ = false;
+}
+
+JS::AutoAssertOnBarrier::~AutoAssertOnBarrier()
+{
+ MOZ_ASSERT(!context->runtime()->allowGCBarriers_);
+ context->runtime()->allowGCBarriers_ = prev;
+}
+
+#ifdef DEBUG
+JS::AutoAssertNoAlloc::AutoAssertNoAlloc(JSContext* cx)
+ : gc(nullptr)
+{
+ disallowAlloc(cx);
+}
+
+void JS::AutoAssertNoAlloc::disallowAlloc(JSRuntime* rt)
+{
+ MOZ_ASSERT(!gc);
+ gc = &rt->gc;
+ gc->disallowAlloc();
+}
+
+JS::AutoAssertNoAlloc::~AutoAssertNoAlloc()
+{
+ if (gc)
+ gc->allowAlloc();
+}
+
+AutoAssertNoNurseryAlloc::AutoAssertNoNurseryAlloc(JSRuntime* rt)
+ : gc(rt->gc)
+{
+ gc.disallowNurseryAlloc();
+}
+
+AutoAssertNoNurseryAlloc::~AutoAssertNoNurseryAlloc()
+{
+ gc.allowNurseryAlloc();
+}
+
+JS::AutoEnterCycleCollection::AutoEnterCycleCollection(JSContext* cx)
+ : runtime(cx->runtime())
+{
+ MOZ_ASSERT(!runtime->isHeapBusy());
+ runtime->setHeapState(HeapState::CycleCollecting);
+}
+
+JS::AutoEnterCycleCollection::~AutoEnterCycleCollection()
+{
+ MOZ_ASSERT(runtime->isCycleCollecting());
+ runtime->setHeapState(HeapState::Idle);
+}
+#endif
+
+JS::AutoAssertGCCallback::AutoAssertGCCallback(JSObject* obj)
+ : AutoSuppressGCAnalysis()
+{
+ MOZ_ASSERT(obj->runtimeFromMainThread()->isHeapCollecting());
+}
+
+JS_FRIEND_API(const char*)
+JS::GCTraceKindToAscii(JS::TraceKind kind)
+{
+ switch(kind) {
+#define MAP_NAME(name, _0, _1) case JS::TraceKind::name: return #name;
+JS_FOR_EACH_TRACEKIND(MAP_NAME);
+#undef MAP_NAME
+ default: return "Invalid";
+ }
+}
+
+JS::GCCellPtr::GCCellPtr(const Value& v)
+ : ptr(0)
+{
+ if (v.isString())
+ ptr = checkedCast(v.toString(), JS::TraceKind::String);
+ else if (v.isObject())
+ ptr = checkedCast(&v.toObject(), JS::TraceKind::Object);
+ else if (v.isSymbol())
+ ptr = checkedCast(v.toSymbol(), JS::TraceKind::Symbol);
+ else if (v.isPrivateGCThing())
+ ptr = checkedCast(v.toGCThing(), v.toGCThing()->getTraceKind());
+ else
+ ptr = checkedCast(nullptr, JS::TraceKind::Null);
+}
+
+JS::TraceKind
+JS::GCCellPtr::outOfLineKind() const
+{
+ MOZ_ASSERT((ptr & OutOfLineTraceKindMask) == OutOfLineTraceKindMask);
+ MOZ_ASSERT(asCell()->isTenured());
+ return MapAllocToTraceKind(asCell()->asTenured().getAllocKind());
+}
+
+bool
+JS::GCCellPtr::mayBeOwnedByOtherRuntime() const
+{
+ return (is<JSString>() && as<JSString>().isPermanentAtom()) ||
+ (is<Symbol>() && as<Symbol>().isWellKnownSymbol());
+}
+
+#ifdef JSGC_HASH_TABLE_CHECKS
+void
+js::gc::CheckHashTablesAfterMovingGC(JSRuntime* rt)
+{
+ /*
+ * Check that internal hash tables no longer have any pointers to things
+ * that have been moved.
+ */
+ rt->spsProfiler.checkStringsMapAfterMovingGC();
+ for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
+ zone->checkUniqueIdTableAfterMovingGC();
+ zone->checkInitialShapesTableAfterMovingGC();
+ zone->checkBaseShapeTableAfterMovingGC();
+
+ JS::AutoCheckCannotGC nogc;
+ for (auto baseShape = zone->cellIter<BaseShape>(); !baseShape.done(); baseShape.next()) {
+ if (ShapeTable* table = baseShape->maybeTable(nogc))
+ table->checkAfterMovingGC();
+ }
+ }
+ for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) {
+ c->objectGroups.checkTablesAfterMovingGC();
+ c->dtoaCache.checkCacheAfterMovingGC();
+ c->checkWrapperMapAfterMovingGC();
+ c->checkScriptMapsAfterMovingGC();
+ if (c->debugEnvs)
+ c->debugEnvs->checkHashTablesAfterMovingGC(rt);
+ }
+}
+#endif
+
+JS_PUBLIC_API(void)
+JS::PrepareZoneForGC(Zone* zone)
+{
+ zone->scheduleGC();
+}
+
+JS_PUBLIC_API(void)
+JS::PrepareForFullGC(JSContext* cx)
+{
+ for (ZonesIter zone(cx, WithAtoms); !zone.done(); zone.next())
+ zone->scheduleGC();
+}
+
+JS_PUBLIC_API(void)
+JS::PrepareForIncrementalGC(JSContext* cx)
+{
+ if (!JS::IsIncrementalGCInProgress(cx))
+ return;
+
+ for (ZonesIter zone(cx, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->wasGCStarted())
+ PrepareZoneForGC(zone);
+ }
+}
+
+JS_PUBLIC_API(bool)
+JS::IsGCScheduled(JSContext* cx)
+{
+ for (ZonesIter zone(cx, WithAtoms); !zone.done(); zone.next()) {
+ if (zone->isGCScheduled())
+ return true;
+ }
+
+ return false;
+}
+
+JS_PUBLIC_API(void)
+JS::SkipZoneForGC(Zone* zone)
+{
+ zone->unscheduleGC();
+}
+
+JS_PUBLIC_API(void)
+JS::GCForReason(JSContext* cx, JSGCInvocationKind gckind, gcreason::Reason reason)
+{
+ MOZ_ASSERT(gckind == GC_NORMAL || gckind == GC_SHRINK);
+ cx->gc.gc(gckind, reason);
+}
+
+JS_PUBLIC_API(void)
+JS::StartIncrementalGC(JSContext* cx, JSGCInvocationKind gckind, gcreason::Reason reason, int64_t millis)
+{
+ MOZ_ASSERT(gckind == GC_NORMAL || gckind == GC_SHRINK);
+ cx->gc.startGC(gckind, reason, millis);
+}
+
+JS_PUBLIC_API(void)
+JS::IncrementalGCSlice(JSContext* cx, gcreason::Reason reason, int64_t millis)
+{
+ cx->gc.gcSlice(reason, millis);
+}
+
+JS_PUBLIC_API(void)
+JS::FinishIncrementalGC(JSContext* cx, gcreason::Reason reason)
+{
+ cx->gc.finishGC(reason);
+}
+
+JS_PUBLIC_API(void)
+JS::AbortIncrementalGC(JSContext* cx)
+{
+ cx->gc.abortGC();
+}
+
+char16_t*
+JS::GCDescription::formatSliceMessage(JSContext* cx) const
+{
+ UniqueChars cstr = cx->gc.stats.formatCompactSliceMessage();
+
+ size_t nchars = strlen(cstr.get());
+ UniqueTwoByteChars out(js_pod_malloc<char16_t>(nchars + 1));
+ if (!out)
+ return nullptr;
+ out.get()[nchars] = 0;
+
+ CopyAndInflateChars(out.get(), cstr.get(), nchars);
+ return out.release();
+}
+
+char16_t*
+JS::GCDescription::formatSummaryMessage(JSContext* cx) const
+{
+ UniqueChars cstr = cx->gc.stats.formatCompactSummaryMessage();
+
+ size_t nchars = strlen(cstr.get());
+ UniqueTwoByteChars out(js_pod_malloc<char16_t>(nchars + 1));
+ if (!out)
+ return nullptr;
+ out.get()[nchars] = 0;
+
+ CopyAndInflateChars(out.get(), cstr.get(), nchars);
+ return out.release();
+}
+
+JS::dbg::GarbageCollectionEvent::Ptr
+JS::GCDescription::toGCEvent(JSContext* cx) const
+{
+ return JS::dbg::GarbageCollectionEvent::Create(cx, cx->gc.stats, cx->gc.majorGCCount());
+}
+
+char16_t*
+JS::GCDescription::formatJSON(JSContext* cx, uint64_t timestamp) const
+{
+ UniqueChars cstr = cx->gc.stats.formatJsonMessage(timestamp);
+
+ size_t nchars = strlen(cstr.get());
+ UniqueTwoByteChars out(js_pod_malloc<char16_t>(nchars + 1));
+ if (!out)
+ return nullptr;
+ out.get()[nchars] = 0;
+
+ CopyAndInflateChars(out.get(), cstr.get(), nchars);
+ return out.release();
+}
+
+JS_PUBLIC_API(JS::GCSliceCallback)
+JS::SetGCSliceCallback(JSContext* cx, GCSliceCallback callback)
+{
+ return cx->gc.setSliceCallback(callback);
+}
+
+JS_PUBLIC_API(JS::DoCycleCollectionCallback)
+JS::SetDoCycleCollectionCallback(JSContext* cx, JS::DoCycleCollectionCallback callback)
+{
+ return cx->gc.setDoCycleCollectionCallback(callback);
+}
+
+JS_PUBLIC_API(JS::GCNurseryCollectionCallback)
+JS::SetGCNurseryCollectionCallback(JSContext* cx, GCNurseryCollectionCallback callback)
+{
+ return cx->gc.setNurseryCollectionCallback(callback);
+}
+
+JS_PUBLIC_API(void)
+JS::DisableIncrementalGC(JSContext* cx)
+{
+ cx->gc.disallowIncrementalGC();
+}
+
+JS_PUBLIC_API(bool)
+JS::IsIncrementalGCEnabled(JSContext* cx)
+{
+ return cx->gc.isIncrementalGCEnabled();
+}
+
+JS_PUBLIC_API(bool)
+JS::IsIncrementalGCInProgress(JSContext* cx)
+{
+ return cx->gc.isIncrementalGCInProgress() && !cx->gc.isVerifyPreBarriersEnabled();
+}
+
+JS_PUBLIC_API(bool)
+JS::IsIncrementalBarrierNeeded(JSContext* cx)
+{
+ if (cx->isHeapBusy())
+ return false;
+
+ auto state = cx->gc.state();
+ return state != gc::State::NotActive && state <= gc::State::Sweep;
+}
+
+struct IncrementalReferenceBarrierFunctor {
+ template <typename T> void operator()(T* t) { T::writeBarrierPre(t); }
+};
+
+JS_PUBLIC_API(void)
+JS::IncrementalReferenceBarrier(GCCellPtr thing)
+{
+ if (!thing)
+ return;
+
+ DispatchTyped(IncrementalReferenceBarrierFunctor(), thing);
+}
+
+JS_PUBLIC_API(void)
+JS::IncrementalValueBarrier(const Value& v)
+{
+ js::GCPtrValue::writeBarrierPre(v);
+}
+
+JS_PUBLIC_API(void)
+JS::IncrementalObjectBarrier(JSObject* obj)
+{
+ if (!obj)
+ return;
+
+ MOZ_ASSERT(!obj->zone()->runtimeFromMainThread()->isHeapMajorCollecting());
+
+ JSObject::writeBarrierPre(obj);
+}
+
+JS_PUBLIC_API(bool)
+JS::WasIncrementalGC(JSContext* cx)
+{
+ return cx->gc.isIncrementalGc();
+}
+
+JS::AutoDisableGenerationalGC::AutoDisableGenerationalGC(JSRuntime* rt)
+ : gc(&rt->gc)
+{
+ gc->disableGenerationalGC();
+}
+
+JS::AutoDisableGenerationalGC::~AutoDisableGenerationalGC()
+{
+ gc->enableGenerationalGC();
+}
+
+JS_PUBLIC_API(bool)
+JS::IsGenerationalGCEnabled(JSRuntime* rt)
+{
+ return rt->gc.isGenerationalGCEnabled();
+}
+
+uint64_t
+js::gc::NextCellUniqueId(JSRuntime* rt)
+{
+ return rt->gc.nextCellUniqueId();
+}
+
+namespace js {
+namespace gc {
+namespace MemInfo {
+
+static bool
+GCBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.usage.gcBytes()));
+ return true;
+}
+
+static bool
+GCMaxBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.tunables.gcMaxBytes()));
+ return true;
+}
+
+static bool
+MallocBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.getMallocBytes()));
+ return true;
+}
+
+static bool
+MaxMallocGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.maxMallocBytesAllocated()));
+ return true;
+}
+
+static bool
+GCHighFreqGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setBoolean(cx->runtime()->gc.schedulingState.inHighFrequencyGCMode());
+ return true;
+}
+
+static bool
+GCNumberGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.gcNumber()));
+ return true;
+}
+
+static bool
+MajorGCCountGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.majorGCCount()));
+ return true;
+}
+
+static bool
+MinorGCCountGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->runtime()->gc.minorGCCount()));
+ return true;
+}
+
+static bool
+ZoneGCBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->usage.gcBytes()));
+ return true;
+}
+
+static bool
+ZoneGCTriggerBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->threshold.gcTriggerBytes()));
+ return true;
+}
+
+static bool
+ZoneGCAllocTriggerGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->threshold.allocTrigger(cx->runtime()->gc.schedulingState.inHighFrequencyGCMode())));
+ return true;
+}
+
+static bool
+ZoneMallocBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcMallocBytes));
+ return true;
+}
+
+static bool
+ZoneMaxMallocGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcMaxMallocBytes));
+ return true;
+}
+
+static bool
+ZoneGCDelayBytesGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcDelayBytes));
+ return true;
+}
+
+static bool
+ZoneGCHeapGrowthFactorGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(cx->zone()->threshold.gcHeapGrowthFactor());
+ return true;
+}
+
+static bool
+ZoneGCNumberGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setNumber(double(cx->zone()->gcNumber()));
+ return true;
+}
+
+#ifdef JS_MORE_DETERMINISTIC
+static bool
+DummyGetter(JSContext* cx, unsigned argc, Value* vp)
+{
+ CallArgs args = CallArgsFromVp(argc, vp);
+ args.rval().setUndefined();
+ return true;
+}
+#endif
+
+} /* namespace MemInfo */
+
+JSObject*
+NewMemoryInfoObject(JSContext* cx)
+{
+ RootedObject obj(cx, JS_NewObject(cx, nullptr));
+ if (!obj)
+ return nullptr;
+
+ using namespace MemInfo;
+ struct NamedGetter {
+ const char* name;
+ JSNative getter;
+ } getters[] = {
+ { "gcBytes", GCBytesGetter },
+ { "gcMaxBytes", GCMaxBytesGetter },
+ { "mallocBytesRemaining", MallocBytesGetter },
+ { "maxMalloc", MaxMallocGetter },
+ { "gcIsHighFrequencyMode", GCHighFreqGetter },
+ { "gcNumber", GCNumberGetter },
+ { "majorGCCount", MajorGCCountGetter },
+ { "minorGCCount", MinorGCCountGetter }
+ };
+
+ for (auto pair : getters) {
+#ifdef JS_MORE_DETERMINISTIC
+ JSNative getter = DummyGetter;
+#else
+ JSNative getter = pair.getter;
+#endif
+ if (!JS_DefineProperty(cx, obj, pair.name, UndefinedHandleValue,
+ JSPROP_ENUMERATE | JSPROP_SHARED,
+ getter, nullptr))
+ {
+ return nullptr;
+ }
+ }
+
+ RootedObject zoneObj(cx, JS_NewObject(cx, nullptr));
+ if (!zoneObj)
+ return nullptr;
+
+ if (!JS_DefineProperty(cx, obj, "zone", zoneObj, JSPROP_ENUMERATE))
+ return nullptr;
+
+ struct NamedZoneGetter {
+ const char* name;
+ JSNative getter;
+ } zoneGetters[] = {
+ { "gcBytes", ZoneGCBytesGetter },
+ { "gcTriggerBytes", ZoneGCTriggerBytesGetter },
+ { "gcAllocTrigger", ZoneGCAllocTriggerGetter },
+ { "mallocBytesRemaining", ZoneMallocBytesGetter },
+ { "maxMalloc", ZoneMaxMallocGetter },
+ { "delayBytes", ZoneGCDelayBytesGetter },
+ { "heapGrowthFactor", ZoneGCHeapGrowthFactorGetter },
+ { "gcNumber", ZoneGCNumberGetter }
+ };
+
+ for (auto pair : zoneGetters) {
+ #ifdef JS_MORE_DETERMINISTIC
+ JSNative getter = DummyGetter;
+#else
+ JSNative getter = pair.getter;
+#endif
+ if (!JS_DefineProperty(cx, zoneObj, pair.name, UndefinedHandleValue,
+ JSPROP_ENUMERATE | JSPROP_SHARED,
+ getter, nullptr))
+ {
+ return nullptr;
+ }
+ }
+
+ return obj;
+}
+
+const char*
+StateName(State state)
+{
+ switch(state) {
+#define MAKE_CASE(name) case State::name: return #name;
+ GCSTATES(MAKE_CASE)
+#undef MAKE_CASE
+ }
+ MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("invalide gc::State enum value");
+}
+
+void
+AutoAssertHeapBusy::checkCondition(JSRuntime *rt)
+{
+ this->rt = rt;
+ MOZ_ASSERT(rt->isHeapBusy());
+}
+
+void
+AutoAssertEmptyNursery::checkCondition(JSRuntime *rt) {
+ if (!noAlloc)
+ noAlloc.emplace(rt);
+ this->rt = rt;
+ MOZ_ASSERT(rt->gc.nursery.isEmpty());
+}
+
+AutoEmptyNursery::AutoEmptyNursery(JSRuntime *rt)
+ : AutoAssertEmptyNursery()
+{
+ MOZ_ASSERT(!rt->mainThread.suppressGC);
+ rt->gc.stats.suspendPhases();
+ rt->gc.evictNursery();
+ rt->gc.stats.resumePhases();
+ checkCondition(rt);
+}
+
+} /* namespace gc */
+} /* namespace js */
+
+#ifdef DEBUG
+void
+js::gc::Cell::dump(FILE* fp) const
+{
+ switch (getTraceKind()) {
+ case JS::TraceKind::Object:
+ reinterpret_cast<const JSObject*>(this)->dump(fp);
+ break;
+
+ case JS::TraceKind::String:
+ js::DumpString(reinterpret_cast<JSString*>(const_cast<Cell*>(this)), fp);
+ break;
+
+ case JS::TraceKind::Shape:
+ reinterpret_cast<const Shape*>(this)->dump(fp);
+ break;
+
+ default:
+ fprintf(fp, "%s(%p)\n", JS::GCTraceKindToAscii(getTraceKind()), (void*) this);
+ }
+}
+
+// For use in a debugger.
+void
+js::gc::Cell::dump() const
+{
+ dump(stderr);
+}
+#endif
+
+JS_PUBLIC_API(bool)
+js::gc::detail::CellIsMarkedGrayIfKnown(const Cell* cell)
+{
+ MOZ_ASSERT(cell);
+ if (!cell->isTenured())
+ return false;
+
+ // We ignore the gray marking state of cells and return false in two cases:
+ //
+ // 1) When OOM has caused us to clear the gcGrayBitsValid_ flag.
+ //
+ // 2) When we are in an incremental GC and examine a cell that is in a zone
+ // that is not being collected. Gray targets of CCWs that are marked black
+ // by a barrier will eventually be marked black in the next GC slice.
+ auto tc = &cell->asTenured();
+ auto rt = tc->runtimeFromMainThread();
+ if (!rt->areGCGrayBitsValid() ||
+ (rt->gc.isIncrementalGCInProgress() && !tc->zone()->wasGCStarted()))
+ {
+ return false;
+ }
+
+ return detail::CellIsMarkedGray(tc);
+}