/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef gc_GCRuntime_h #define gc_GCRuntime_h #include "mozilla/Atomics.h" #include "mozilla/EnumSet.h" #include "jsfriendapi.h" #include "jsgc.h" #include "gc/Heap.h" #include "gc/Nursery.h" #include "gc/Statistics.h" #include "gc/StoreBuffer.h" #include "gc/Tracer.h" #include "js/GCAnnotations.h" namespace js { class AutoLockGC; class AutoLockHelperThreadState; class VerifyPreTracer; namespace gc { typedef Vector ZoneVector; using BlackGrayEdgeVector = Vector; class AutoMaybeStartBackgroundAllocation; class MarkingValidator; class AutoTraceSession; struct MovingTracer; class ChunkPool { Chunk* head_; size_t count_; public: ChunkPool() : head_(nullptr), count_(0) {} size_t count() const { return count_; } Chunk* head() { MOZ_ASSERT(head_); return head_; } Chunk* pop(); void push(Chunk* chunk); Chunk* remove(Chunk* chunk); #ifdef DEBUG bool contains(Chunk* chunk) const; bool verify() const; #endif // Pool mutation does not invalidate an Iter unless the mutation // is of the Chunk currently being visited by the Iter. class Iter { public: explicit Iter(ChunkPool& pool) : current_(pool.head_) {} bool done() const { return !current_; } void next(); Chunk* get() const { return current_; } operator Chunk*() const { return get(); } Chunk* operator->() const { return get(); } private: Chunk* current_; }; }; // Performs extra allocation off the main thread so that when memory is // required on the main thread it will already be available and waiting. class BackgroundAllocTask : public GCParallelTask { // Guarded by the GC lock. JSRuntime* runtime; ChunkPool& chunkPool_; const bool enabled_; public: BackgroundAllocTask(JSRuntime* rt, ChunkPool& pool); bool enabled() const { return enabled_; } protected: void run() override; }; // Search the provided Chunks for free arenas and decommit them. class BackgroundDecommitTask : public GCParallelTask { public: using ChunkVector = mozilla::Vector; explicit BackgroundDecommitTask(JSRuntime *rt) : runtime(rt) {} void setChunksToScan(ChunkVector &chunks); protected: void run() override; private: JSRuntime* runtime; ChunkVector toDecommit; }; /* * Encapsulates all of the GC tunables. These are effectively constant and * should only be modified by setParameter. */ class GCSchedulingTunables { /* * Soft limit on the number of bytes we are allowed to allocate in the GC * heap. Attempts to allocate gcthings over this limit will return null and * subsequently invoke the standard OOM machinery, independent of available * physical memory. */ size_t gcMaxBytes_; /* * The base value used to compute zone->trigger.gcBytes(). When * usage.gcBytes() surpasses threshold.gcBytes() for a zone, the zone may * be scheduled for a GC, depending on the exact circumstances. */ size_t gcZoneAllocThresholdBase_; /* Fraction of threshold.gcBytes() which triggers an incremental GC. */ double zoneAllocThresholdFactor_; /* * Number of bytes to allocate between incremental slices in GCs triggered * by the zone allocation threshold. */ size_t zoneAllocDelayBytes_; /* * Totally disables |highFrequencyGC|, the HeapGrowthFactor, and other * tunables that make GC non-deterministic. */ bool dynamicHeapGrowthEnabled_; /* * We enter high-frequency mode if we GC a twice within this many * microseconds. This value is stored directly in microseconds. */ uint64_t highFrequencyThresholdUsec_; /* * When in the |highFrequencyGC| mode, these parameterize the per-zone * "HeapGrowthFactor" computation. */ uint64_t highFrequencyLowLimitBytes_; uint64_t highFrequencyHighLimitBytes_; double highFrequencyHeapGrowthMax_; double highFrequencyHeapGrowthMin_; /* * When not in |highFrequencyGC| mode, this is the global (stored per-zone) * "HeapGrowthFactor". */ double lowFrequencyHeapGrowth_; /* * Doubles the length of IGC slices when in the |highFrequencyGC| mode. */ bool dynamicMarkSliceEnabled_; /* * Controls whether painting can trigger IGC slices. */ bool refreshFrameSlicesEnabled_; /* * Controls the number of empty chunks reserved for future allocation. */ uint32_t minEmptyChunkCount_; uint32_t maxEmptyChunkCount_; public: GCSchedulingTunables() : gcMaxBytes_(0), gcZoneAllocThresholdBase_(30 * 1024 * 1024), zoneAllocThresholdFactor_(0.9), zoneAllocDelayBytes_(1024 * 1024), dynamicHeapGrowthEnabled_(false), highFrequencyThresholdUsec_(1000 * 1000), highFrequencyLowLimitBytes_(100 * 1024 * 1024), highFrequencyHighLimitBytes_(500 * 1024 * 1024), highFrequencyHeapGrowthMax_(3.0), highFrequencyHeapGrowthMin_(1.5), lowFrequencyHeapGrowth_(1.5), dynamicMarkSliceEnabled_(false), refreshFrameSlicesEnabled_(true), minEmptyChunkCount_(1), maxEmptyChunkCount_(30) {} size_t gcMaxBytes() const { return gcMaxBytes_; } size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; } double zoneAllocThresholdFactor() const { return zoneAllocThresholdFactor_; } size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; } bool isDynamicHeapGrowthEnabled() const { return dynamicHeapGrowthEnabled_; } uint64_t highFrequencyThresholdUsec() const { return highFrequencyThresholdUsec_; } uint64_t highFrequencyLowLimitBytes() const { return highFrequencyLowLimitBytes_; } uint64_t highFrequencyHighLimitBytes() const { return highFrequencyHighLimitBytes_; } double highFrequencyHeapGrowthMax() const { return highFrequencyHeapGrowthMax_; } double highFrequencyHeapGrowthMin() const { return highFrequencyHeapGrowthMin_; } double lowFrequencyHeapGrowth() const { return lowFrequencyHeapGrowth_; } bool isDynamicMarkSliceEnabled() const { return dynamicMarkSliceEnabled_; } bool areRefreshFrameSlicesEnabled() const { return refreshFrameSlicesEnabled_; } unsigned minEmptyChunkCount(const AutoLockGC&) const { return minEmptyChunkCount_; } unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; } MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock); }; /* * GC Scheduling Overview * ====================== * * Scheduling GC's in SpiderMonkey/Firefox is tremendously complicated because * of the large number of subtle, cross-cutting, and widely dispersed factors * that must be taken into account. A summary of some of the more important * factors follows. * * Cost factors: * * * GC too soon and we'll revisit an object graph almost identical to the * one we just visited; since we are unlikely to find new garbage, the * traversal will be largely overhead. We rely heavily on external factors * to signal us that we are likely to find lots of garbage: e.g. "a tab * just got closed". * * * GC too late and we'll run out of memory to allocate (e.g. Out-Of-Memory, * hereafter simply abbreviated to OOM). If this happens inside * SpiderMonkey we may be able to recover, but most embedder allocations * will simply crash on OOM, even if the GC has plenty of free memory it * could surrender. * * * Memory fragmentation: if we fill the process with GC allocations, a * request for a large block of contiguous memory may fail because no * contiguous block is free, despite having enough memory available to * service the request. * * * Management overhead: if our GC heap becomes large, we create extra * overhead when managing the GC's structures, even if the allocations are * mostly unused. * * Heap Management Factors: * * * GC memory: The GC has its own allocator that it uses to make fixed size * allocations for GC managed things. In cases where the GC thing requires * larger or variable sized memory to implement itself, it is responsible * for using the system heap. * * * C Heap Memory: Rather than allowing for large or variable allocations, * the SpiderMonkey GC allows GC things to hold pointers to C heap memory. * It is the responsibility of the thing to free this memory with a custom * finalizer (with the sole exception of NativeObject, which knows about * slots and elements for performance reasons). C heap memory has different * performance and overhead tradeoffs than GC internal memory, which need * to be considered with scheduling a GC. * * Application Factors: * * * Most applications allocate heavily at startup, then enter a processing * stage where memory utilization remains roughly fixed with a slower * allocation rate. This is not always the case, however, so while we may * optimize for this pattern, we must be able to handle arbitrary * allocation patterns. * * Other factors: * * * Other memory: This is memory allocated outside the purview of the GC. * Data mapped by the system for code libraries, data allocated by those * libraries, data in the JSRuntime that is used to manage the engine, * memory used by the embedding that is not attached to a GC thing, memory * used by unrelated processes running on the hardware that use space we * could otherwise use for allocation, etc. While we don't have to manage * it, we do have to take it into account when scheduling since it affects * when we will OOM. * * * Physical Reality: All real machines have limits on the number of bits * that they are physically able to store. While modern operating systems * can generally make additional space available with swapping, at some * point there are simply no more bits to allocate. There is also the * factor of address space limitations, particularly on 32bit machines. * * * Platform Factors: Each OS makes use of wildly different memory * management techniques. These differences result in different performance * tradeoffs, different fragmentation patterns, and different hard limits * on the amount of physical and/or virtual memory that we can use before * OOMing. * * * Reasons for scheduling GC * ------------------------- * * While code generally takes the above factors into account in only an ad-hoc * fashion, the API forces the user to pick a "reason" for the GC. We have a * bunch of JS::gcreason reasons in GCAPI.h. These fall into a few categories * that generally coincide with one or more of the above factors. * * Embedding reasons: * * 1) Do a GC now because the embedding knows something useful about the * zone's memory retention state. These are gcreasons like LOAD_END, * PAGE_HIDE, SET_NEW_DOCUMENT, DOM_UTILS. Mostly, Gecko uses these to * indicate that a significant fraction of the scheduled zone's memory is * probably reclaimable. * * 2) Do some known amount of GC work now because the embedding knows now is * a good time to do a long, unblockable operation of a known duration. * These are INTER_SLICE_GC and REFRESH_FRAME. * * Correctness reasons: * * 3) Do a GC now because correctness depends on some GC property. For * example, CC_WAITING is where the embedding requires the mark bits * to be set correct. Also, EVICT_NURSERY where we need to work on the tenured * heap. * * 4) Do a GC because we are shutting down: e.g. SHUTDOWN_CC or DESTROY_*. * * 5) Do a GC because a compartment was accessed between GC slices when we * would have otherwise discarded it. We have to do a second GC to clean * it up: e.g. COMPARTMENT_REVIVED. * * Emergency Reasons: * * 6) Do an all-zones, non-incremental GC now because the embedding knows it * cannot wait: e.g. MEM_PRESSURE. * * 7) OOM when fetching a new Chunk results in a LAST_DITCH GC. * * Heap Size Limitation Reasons: * * 8) Do an incremental, zonal GC with reason MAYBEGC when we discover that * the gc's allocated size is approaching the current trigger. This is * called MAYBEGC because we make this check in the MaybeGC function. * MaybeGC gets called at the top of the main event loop. Normally, it is * expected that this callback will keep the heap size limited. It is * relatively inexpensive, because it is invoked with no JS running and * thus few stack roots to scan. For this reason, the GC's "trigger" bytes * is less than the GC's "max" bytes as used by the trigger below. * * 9) Do an incremental, zonal GC with reason MAYBEGC when we go to allocate * a new GC thing and find that the GC heap size has grown beyond the * configured maximum (JSGC_MAX_BYTES). We trigger this GC by returning * nullptr and then calling maybeGC at the top level of the allocator. * This is then guaranteed to fail the "size greater than trigger" check * above, since trigger is always less than max. After performing the GC, * the allocator unconditionally returns nullptr to force an OOM exception * is raised by the script. * * Note that this differs from a LAST_DITCH GC where we actually run out * of memory (i.e., a call to a system allocator fails) when trying to * allocate. Unlike above, LAST_DITCH GC only happens when we are really * out of memory, not just when we cross an arbitrary trigger; despite * this, it may still return an allocation at the end and allow the script * to continue, if the LAST_DITCH GC was able to free up enough memory. * * 10) Do a GC under reason ALLOC_TRIGGER when we are over the GC heap trigger * limit, but in the allocator rather than in a random call to maybeGC. * This occurs if we allocate too much before returning to the event loop * and calling maybeGC; this is extremely common in benchmarks and * long-running Worker computations. Note that this uses a wildly * different mechanism from the above in that it sets the interrupt flag * and does the GC at the next loop head, before the next alloc, or * maybeGC. The reason for this is that this check is made after the * allocation and we cannot GC with an uninitialized thing in the heap. * * 11) Do an incremental, zonal GC with reason TOO_MUCH_MALLOC when we have * malloced more than JSGC_MAX_MALLOC_BYTES in a zone since the last GC. * * * Size Limitation Triggers Explanation * ------------------------------------ * * The GC internally is entirely unaware of the context of the execution of * the mutator. It sees only: * * A) Allocated size: this is the amount of memory currently requested by the * mutator. This quantity is monotonically increasing: i.e. the allocation * rate is always >= 0. It is also easy for the system to track. * * B) Retained size: this is the amount of memory that the mutator can * currently reach. Said another way, it is the size of the heap * immediately after a GC (modulo background sweeping). This size is very * costly to know exactly and also extremely hard to estimate with any * fidelity. * * For reference, a common allocated vs. retained graph might look like: * * | ** ** * | ** ** * ** * | ** * ** * ** * | * ** * ** * ** * | ** ** * ** * ** * s| * * ** ** + + ** * i| * * * + + + + + * z| * * * + + + + + * e| * **+ * | * + * | * + * | * + * | * + * | * + * |*+ * +-------------------------------------------------- * time * *** = allocated * +++ = retained * * Note that this is a bit of a simplification * because in reality we track malloc and GC heap * sizes separately and have a different level of * granularity and accuracy on each heap. * * This presents some obvious implications for Mark-and-Sweep collectors. * Namely: * -> t[marking] ~= size[retained] * -> t[sweeping] ~= size[allocated] - size[retained] * * In a non-incremental collector, maintaining low latency and high * responsiveness requires that total GC times be as low as possible. Thus, * in order to stay responsive when we did not have a fully incremental * collector, our GC triggers were focused on minimizing collection time. * Furthermore, since size[retained] is not under control of the GC, all the * GC could do to control collection times was reduce sweep times by * minimizing size[allocated], per the equation above. * * The result of the above is GC triggers that focus on size[allocated] to * the exclusion of other important factors and default heuristics that are * not optimal for a fully incremental collector. On the other hand, this is * not all bad: minimizing size[allocated] also minimizes the chance of OOM * and sweeping remains one of the hardest areas to further incrementalize. * * EAGER_ALLOC_TRIGGER * ------------------- * Occurs when we return to the event loop and find our heap is getting * largish, but before t[marking] OR t[sweeping] is too large for a * responsive non-incremental GC. This is intended to be the common case * in normal web applications: e.g. we just finished an event handler and * the few objects we allocated when computing the new whatzitz have * pushed us slightly over the limit. After this GC we rescale the new * EAGER_ALLOC_TRIGGER trigger to 150% of size[retained] so that our * non-incremental GC times will always be proportional to this size * rather than being dominated by sweeping. * * As a concession to mutators that allocate heavily during their startup * phase, we have a highFrequencyGCMode that ups the growth rate to 300% * of the current size[retained] so that we'll do fewer longer GCs at the * end of the mutator startup rather than more, smaller GCs. * * Assumptions: * -> Responsiveness is proportional to t[marking] + t[sweeping]. * -> size[retained] is proportional only to GC allocations. * * ALLOC_TRIGGER (non-incremental) * ------------------------------- * If we do not return to the event loop before getting all the way to our * gc trigger bytes then MAYBEGC will never fire. To avoid OOMing, we * succeed the current allocation and set the script interrupt so that we * will (hopefully) do a GC before we overflow our max and have to raise * an OOM exception for the script. * * Assumptions: * -> Common web scripts will return to the event loop before using * 10% of the current gcTriggerBytes worth of GC memory. * * ALLOC_TRIGGER (incremental) * --------------------------- * In practice the above trigger is rough: if a website is just on the * cusp, sometimes it will trigger a non-incremental GC moments before * returning to the event loop, where it could have done an incremental * GC. Thus, we recently added an incremental version of the above with a * substantially lower threshold, so that we have a soft limit here. If * IGC can collect faster than the allocator generates garbage, even if * the allocator does not return to the event loop frequently, we should * not have to fall back to a non-incremental GC. * * INCREMENTAL_TOO_SLOW * -------------------- * Do a full, non-incremental GC if we overflow ALLOC_TRIGGER during an * incremental GC. When in the middle of an incremental GC, we suppress * our other triggers, so we need a way to backstop the IGC if the * mutator allocates faster than the IGC can clean things up. * * TOO_MUCH_MALLOC * --------------- * Performs a GC before size[allocated] - size[retained] gets too large * for non-incremental sweeping to be fast in the case that we have * significantly more malloc allocation than GC allocation. This is meant * to complement MAYBEGC triggers. We track this by counting malloced * bytes; the counter gets reset at every GC since we do not always have a * size at the time we call free. Because of this, the malloc heuristic * is, unfortunatly, not usefully able to augment our other GC heap * triggers and is limited to this singular heuristic. * * Assumptions: * -> EITHER size[allocated_by_malloc] ~= size[allocated_by_GC] * OR time[sweeping] ~= size[allocated_by_malloc] * -> size[retained] @ t0 ~= size[retained] @ t1 * i.e. That the mutator is in steady-state operation. * * LAST_DITCH_GC * ------------- * Does a GC because we are out of memory. * * Assumptions: * -> size[retained] < size[available_memory] */ class GCSchedulingState { /* * Influences how we schedule and run GC's in several subtle ways. The most * important factor is in how it controls the "HeapGrowthFactor". The * growth factor is a measure of how large (as a percentage of the last GC) * the heap is allowed to grow before we try to schedule another GC. */ bool inHighFrequencyGCMode_; public: GCSchedulingState() : inHighFrequencyGCMode_(false) {} bool inHighFrequencyGCMode() const { return inHighFrequencyGCMode_; } void updateHighFrequencyMode(uint64_t lastGCTime, uint64_t currentTime, const GCSchedulingTunables& tunables) { inHighFrequencyGCMode_ = tunables.isDynamicHeapGrowthEnabled() && lastGCTime && lastGCTime + tunables.highFrequencyThresholdUsec() > currentTime; } }; template struct Callback { F op; void* data; Callback() : op(nullptr), data(nullptr) {} Callback(F op, void* data) : op(op), data(data) {} }; template using CallbackVector = Vector, 4, SystemAllocPolicy>; template class ChainedIter { Iter0 iter0_; Iter1 iter1_; public: ChainedIter(const Iter0& iter0, const Iter1& iter1) : iter0_(iter0), iter1_(iter1) {} bool done() const { return iter0_.done() && iter1_.done(); } void next() { MOZ_ASSERT(!done()); if (!iter0_.done()) { iter0_.next(); } else { MOZ_ASSERT(!iter1_.done()); iter1_.next(); } } T get() const { MOZ_ASSERT(!done()); if (!iter0_.done()) return iter0_.get(); MOZ_ASSERT(!iter1_.done()); return iter1_.get(); } operator T() const { return get(); } T operator->() const { return get(); } }; typedef HashMap, SystemAllocPolicy> RootedValueMap; using AllocKinds = mozilla::EnumSet; class GCRuntime { public: explicit GCRuntime(JSRuntime* rt); MOZ_MUST_USE bool init(uint32_t maxbytes, uint32_t maxNurseryBytes); void finishRoots(); void finish(); inline bool needZealousGC(); MOZ_MUST_USE bool addRoot(Value* vp, const char* name); void removeRoot(Value* vp); void setMarkStackLimit(size_t limit, AutoLockGC& lock); MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); MOZ_MUST_USE bool triggerGC(JS::gcreason::Reason reason); void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock); // The return value indicates if we were able to do the GC. bool triggerZoneGC(Zone* zone, JS::gcreason::Reason reason); void maybeGC(Zone* zone); void minorGC(JS::gcreason::Reason reason, gcstats::Phase phase = gcstats::PHASE_MINOR_GC) JS_HAZ_GC_CALL; void evictNursery(JS::gcreason::Reason reason = JS::gcreason::EVICT_NURSERY) { minorGC(reason, gcstats::PHASE_EVICT_NURSERY); } // The return value indicates whether a major GC was performed. bool gcIfRequested(); void gc(JSGCInvocationKind gckind, JS::gcreason::Reason reason); void startGC(JSGCInvocationKind gckind, JS::gcreason::Reason reason, int64_t millis = 0); void gcSlice(JS::gcreason::Reason reason, int64_t millis = 0); void finishGC(JS::gcreason::Reason reason); void abortGC(); void startDebugGC(JSGCInvocationKind gckind, SliceBudget& budget); void debugGCSlice(SliceBudget& budget); void triggerFullGCForAtoms() { MOZ_ASSERT(fullGCForAtomsRequested_); fullGCForAtomsRequested_ = false; MOZ_RELEASE_ASSERT(triggerGC(JS::gcreason::ALLOC_TRIGGER)); } void runDebugGC(); inline void poke(); enum TraceOrMarkRuntime { TraceRuntime, MarkRuntime }; void traceRuntime(JSTracer* trc, AutoLockForExclusiveAccess& lock); void traceRuntimeForMinorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock); void notifyDidPaint(); void shrinkBuffers(); void onOutOfMallocMemory(); void onOutOfMallocMemory(const AutoLockGC& lock); size_t maxMallocBytesAllocated() { return maxMallocBytes; } uint64_t nextCellUniqueId() { MOZ_ASSERT(nextCellUniqueId_ > 0); uint64_t uid = ++nextCellUniqueId_; return uid; } #ifdef DEBUG bool shutdownCollectedEverything() const { return arenasEmptyAtShutdown; } #endif public: // Internal public interface State state() const { return incrementalState; } bool isHeapCompacting() const { return state() == State::Compact; } bool isForegroundSweeping() const { return state() == State::Sweep; } bool isBackgroundSweeping() { return helperState.isBackgroundSweeping(); } void waitBackgroundSweepEnd() { helperState.waitBackgroundSweepEnd(); } void waitBackgroundSweepOrAllocEnd() { helperState.waitBackgroundSweepEnd(); allocTask.cancel(GCParallelTask::CancelAndWait); } void requestMinorGC(JS::gcreason::Reason reason); #ifdef DEBUG bool onBackgroundThread() { return helperState.onBackgroundThread(); } #endif // DEBUG void lockGC() { lock.lock(); } void unlockGC() { lock.unlock(); } #ifdef DEBUG bool isAllocAllowed() { return noGCOrAllocationCheck == 0; } void disallowAlloc() { ++noGCOrAllocationCheck; } void allowAlloc() { MOZ_ASSERT(!isAllocAllowed()); --noGCOrAllocationCheck; } bool isNurseryAllocAllowed() { return noNurseryAllocationCheck == 0; } void disallowNurseryAlloc() { ++noNurseryAllocationCheck; } void allowNurseryAlloc() { MOZ_ASSERT(!isNurseryAllocAllowed()); --noNurseryAllocationCheck; } bool isStrictProxyCheckingEnabled() { return disableStrictProxyCheckingCount == 0; } void disableStrictProxyChecking() { ++disableStrictProxyCheckingCount; } void enableStrictProxyChecking() { MOZ_ASSERT(disableStrictProxyCheckingCount > 0); --disableStrictProxyCheckingCount; } #endif // DEBUG bool isInsideUnsafeRegion() { return inUnsafeRegion != 0; } void enterUnsafeRegion() { ++inUnsafeRegion; } void leaveUnsafeRegion() { MOZ_ASSERT(inUnsafeRegion > 0); --inUnsafeRegion; } void verifyIsSafeToGC() { MOZ_DIAGNOSTIC_ASSERT(!isInsideUnsafeRegion(), "[AutoAssertNoGC] possible GC in GC-unsafe region"); } void setAlwaysPreserveCode() { alwaysPreserveCode = true; } bool isIncrementalGCAllowed() const { return incrementalAllowed; } void disallowIncrementalGC() { incrementalAllowed = false; } bool isIncrementalGCEnabled() const { return mode == JSGC_MODE_INCREMENTAL && incrementalAllowed; } bool isIncrementalGCInProgress() const { return state() != State::NotActive; } bool isGenerationalGCEnabled() const { return generationalDisabled == 0; } void disableGenerationalGC(); void enableGenerationalGC(); void disableCompactingGC(); void enableCompactingGC(); bool isCompactingGCEnabled() const; void setGrayRootsTracer(JSTraceDataOp traceOp, void* data); MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); void setMaxMallocBytes(size_t value); int32_t getMallocBytes() const { return mallocBytesUntilGC; } void resetMallocBytes(); bool isTooMuchMalloc() const { return mallocBytesUntilGC <= 0; } void updateMallocCounter(JS::Zone* zone, size_t nbytes); void onTooMuchMalloc(); void setGCCallback(JSGCCallback callback, void* data); void callGCCallback(JSGCStatus status) const; void setObjectsTenuredCallback(JSObjectsTenuredCallback callback, void* data); void callObjectsTenuredCallback(); MOZ_MUST_USE bool addFinalizeCallback(JSFinalizeCallback callback, void* data); void removeFinalizeCallback(JSFinalizeCallback func); MOZ_MUST_USE bool addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, void* data); void removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback); MOZ_MUST_USE bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback, void* data); void removeWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback); JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); JS::GCNurseryCollectionCallback setNurseryCollectionCallback( JS::GCNurseryCollectionCallback callback); JS::DoCycleCollectionCallback setDoCycleCollectionCallback(JS::DoCycleCollectionCallback callback); void callDoCycleCollectionCallback(JSContext* cx); void setFullCompartmentChecks(bool enable); bool isManipulatingDeadZones() { return manipulatingDeadZones; } void setManipulatingDeadZones(bool value) { manipulatingDeadZones = value; } unsigned objectsMarkedInDeadZonesCount() { return objectsMarkedInDeadZones; } void incObjectsMarkedInDeadZone() { MOZ_ASSERT(manipulatingDeadZones); ++objectsMarkedInDeadZones; } JS::Zone* getCurrentZoneGroup() { return currentZoneGroup; } void setFoundBlackGrayEdges(TenuredCell& target) { AutoEnterOOMUnsafeRegion oomUnsafe; if (!foundBlackGrayEdges.append(&target)) oomUnsafe.crash("OOM|small: failed to insert into foundBlackGrayEdges"); } uint64_t gcNumber() const { return number; } uint64_t minorGCCount() const { return minorGCNumber; } void incMinorGcNumber() { ++minorGCNumber; ++number; } uint64_t majorGCCount() const { return majorGCNumber; } void incMajorGcNumber() { ++majorGCNumber; ++number; } int64_t defaultSliceBudget() const { return defaultTimeBudget_; } bool isIncrementalGc() const { return isIncremental; } bool isFullGc() const { return isFull; } bool isCompactingGc() const { return isCompacting; } bool minorGCRequested() const { return minorGCTriggerReason != JS::gcreason::NO_REASON; } bool majorGCRequested() const { return majorGCTriggerReason != JS::gcreason::NO_REASON; } bool isGcNeeded() { return minorGCRequested() || majorGCRequested(); } bool fullGCForAtomsRequested() const { return fullGCForAtomsRequested_; } double computeHeapGrowthFactor(size_t lastBytes); size_t computeTriggerBytes(double growthFactor, size_t lastBytes); JSGCMode gcMode() const { return mode; } void setGCMode(JSGCMode m) { mode = m; marker.setGCMode(mode); } inline void updateOnFreeArenaAlloc(const ChunkInfo& info); inline void updateOnArenaFree(const ChunkInfo& info); ChunkPool& fullChunks(const AutoLockGC& lock) { return fullChunks_; } ChunkPool& availableChunks(const AutoLockGC& lock) { return availableChunks_; } ChunkPool& emptyChunks(const AutoLockGC& lock) { return emptyChunks_; } const ChunkPool& fullChunks(const AutoLockGC& lock) const { return fullChunks_; } const ChunkPool& availableChunks(const AutoLockGC& lock) const { return availableChunks_; } const ChunkPool& emptyChunks(const AutoLockGC& lock) const { return emptyChunks_; } typedef ChainedIter NonEmptyChunksIter; NonEmptyChunksIter allNonEmptyChunks() { return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_), ChunkPool::Iter(fullChunks_)); } Chunk* getOrAllocChunk(const AutoLockGC& lock, AutoMaybeStartBackgroundAllocation& maybeStartBGAlloc); void recycleChunk(Chunk* chunk, const AutoLockGC& lock); // Free certain LifoAlloc blocks when it is safe to do so. void freeUnusedLifoBlocksAfterSweeping(LifoAlloc* lifo); void freeAllLifoBlocksAfterSweeping(LifoAlloc* lifo); void freeAllLifoBlocksAfterMinorGC(LifoAlloc* lifo); // Queue a thunk to run after the next minor GC. void callAfterMinorGC(void (*thunk)(void* data), void* data) { nursery.queueSweepAction(thunk, data); } // Public here for ReleaseArenaLists and FinalizeTypedArenas. void releaseArena(Arena* arena, const AutoLockGC& lock); void releaseHeldRelocatedArenas(); void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock); // Allocator template MOZ_MUST_USE bool checkAllocatorState(JSContext* cx, AllocKind kind); template JSObject* tryNewNurseryObject(JSContext* cx, size_t thingSize, size_t nDynamicSlots, const Class* clasp); template static JSObject* tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thingSize, size_t nDynamicSlots); template static T* tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize); static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind); private: enum IncrementalProgress { NotFinished = 0, Finished }; // For ArenaLists::allocateFromArena() friend class ArenaLists; Chunk* pickChunk(const AutoLockGC& lock, AutoMaybeStartBackgroundAllocation& maybeStartBGAlloc); Arena* allocateArena(Chunk* chunk, Zone* zone, AllocKind kind, ShouldCheckThresholds checkThresholds, const AutoLockGC& lock); void arenaAllocatedDuringGC(JS::Zone* zone, Arena* arena); // Allocator internals MOZ_MUST_USE bool gcIfNeededPerAllocation(JSContext* cx); template static void checkIncrementalZoneState(ExclusiveContext* cx, T* t); static TenuredCell* refillFreeListFromAnyThread(ExclusiveContext* cx, AllocKind thingKind, size_t thingSize); static TenuredCell* refillFreeListFromMainThread(JSContext* cx, AllocKind thingKind, size_t thingSize); static TenuredCell* refillFreeListOffMainThread(ExclusiveContext* cx, AllocKind thingKind); /* * Return the list of chunks that can be released outside the GC lock. * Must be called either during the GC or with the GC lock taken. */ friend class BackgroundDecommitTask; ChunkPool expireEmptyChunkPool(const AutoLockGC& lock); void freeEmptyChunks(JSRuntime* rt, const AutoLockGC& lock); void prepareToFreeChunk(ChunkInfo& info); friend class BackgroundAllocTask; friend class AutoMaybeStartBackgroundAllocation; bool wantBackgroundAllocation(const AutoLockGC& lock) const; void startBackgroundAllocTaskIfIdle(); void requestMajorGC(JS::gcreason::Reason reason); SliceBudget defaultBudget(JS::gcreason::Reason reason, int64_t millis); void budgetIncrementalGC(SliceBudget& budget, AutoLockForExclusiveAccess& lock); void resetIncrementalGC(AbortReason reason, AutoLockForExclusiveAccess& lock); // Assert if the system state is such that we should never // receive a request to do GC work. void checkCanCallAPI(); // Check if the system state is such that GC has been supressed // or otherwise delayed. MOZ_MUST_USE bool checkIfGCAllowedInCurrentState(JS::gcreason::Reason reason); gcstats::ZoneGCStats scanZonesBeforeGC(); void collect(bool nonincrementalByAPI, SliceBudget budget, JS::gcreason::Reason reason) JS_HAZ_GC_CALL; MOZ_MUST_USE bool gcCycle(bool nonincrementalByAPI, SliceBudget& budget, JS::gcreason::Reason reason); void incrementalCollectSlice(SliceBudget& budget, JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); void pushZealSelectedObjects(); void purgeRuntime(AutoLockForExclusiveAccess& lock); MOZ_MUST_USE bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); bool shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime, JS::gcreason::Reason reason, bool canAllocateMoreCode); void traceRuntimeForMajorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock); void traceRuntimeAtoms(JSTracer* trc, AutoLockForExclusiveAccess& lock); void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark, AutoLockForExclusiveAccess& lock); void bufferGrayRoots(); void maybeDoCycleCollection(); void markCompartments(); IncrementalProgress drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase); template void markWeakReferences(gcstats::Phase phase); void markWeakReferencesInCurrentGroup(gcstats::Phase phase); template void markGrayReferences(gcstats::Phase phase); void markBufferedGrayRoots(JS::Zone* zone); void markGrayReferencesInCurrentGroup(gcstats::Phase phase); void markAllWeakReferences(gcstats::Phase phase); void markAllGrayReferences(gcstats::Phase phase); void beginSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); void findZoneGroups(AutoLockForExclusiveAccess& lock); MOZ_MUST_USE bool findInterZoneEdges(); void getNextZoneGroup(); void endMarkingZoneGroup(); void beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock); bool shouldReleaseObservedTypes(); void endSweepingZoneGroup(); IncrementalProgress sweepPhase(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); void sweepZones(FreeOp* fop, bool lastGC); void decommitAllWithoutUnlocking(const AutoLockGC& lock); void startDecommit(); void queueZonesForBackgroundSweep(ZoneList& zones); void sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks); void assertBackgroundSweepingFinished(); bool shouldCompact(); void beginCompactPhase(); IncrementalProgress compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); void endCompactPhase(JS::gcreason::Reason reason); void sweepTypesAfterCompacting(Zone* zone); void sweepZoneAfterCompacting(Zone* zone); MOZ_MUST_USE bool relocateArenas(Zone* zone, JS::gcreason::Reason reason, Arena*& relocatedListOut, SliceBudget& sliceBudget); void updateTypeDescrObjects(MovingTracer* trc, Zone* zone); void updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds, size_t bgTaskCount); void updateAllCellPointers(MovingTracer* trc, Zone* zone); void updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess& lock); void protectAndHoldArenas(Arena* arenaList); void unprotectHeldRelocatedArenas(); void releaseRelocatedArenas(Arena* arenaList); void releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, const AutoLockGC& lock); void finishCollection(JS::gcreason::Reason reason); void computeNonIncrementalMarkingForValidation(AutoLockForExclusiveAccess& lock); void validateIncrementalMarking(); void finishMarkingValidation(); #ifdef DEBUG void checkForCompartmentMismatches(); #endif void callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const; void callWeakPointerZoneGroupCallbacks() const; void callWeakPointerCompartmentCallbacks(JSCompartment* comp) const; public: JSRuntime* rt; /* Embedders can use this zone however they wish. */ JS::Zone* systemZone; /* List of compartments and zones (protected by the GC lock). */ ZoneVector zones; Nursery nursery; StoreBuffer storeBuffer; gcstats::Statistics stats; GCMarker marker; /* Track heap usage for this runtime. */ HeapUsage usage; /* GC scheduling state and parameters. */ GCSchedulingTunables tunables; GCSchedulingState schedulingState; MemProfiler mMemProfiler; private: // When empty, chunks reside in the emptyChunks pool and are re-used as // needed or eventually expired if not re-used. The emptyChunks pool gets // refilled from the background allocation task heuristically so that empty // chunks should always available for immediate allocation without syscalls. ChunkPool emptyChunks_; // Chunks which have had some, but not all, of their arenas allocated live // in the available chunk lists. When all available arenas in a chunk have // been allocated, the chunk is removed from the available list and moved // to the fullChunks pool. During a GC, if all arenas are free, the chunk // is moved back to the emptyChunks pool and scheduled for eventual // release. ChunkPool availableChunks_; // When all arenas in a chunk are used, it is moved to the fullChunks pool // so as to reduce the cost of operations on the available lists. ChunkPool fullChunks_; RootedValueMap rootsHash; size_t maxMallocBytes; // An incrementing id used to assign unique ids to cells that require one. mozilla::Atomic nextCellUniqueId_; /* * Number of the committed arenas in all GC chunks including empty chunks. */ mozilla::Atomic numArenasFreeCommitted; VerifyPreTracer* verifyPreData; private: bool chunkAllocationSinceLastGC; int64_t lastGCTime; JSGCMode mode; mozilla::Atomic numActiveZoneIters; /* During shutdown, the GC needs to clean up every possible object. */ bool cleanUpEverything; // Gray marking must be done after all black marking is complete. However, // we do not have write barriers on XPConnect roots. Therefore, XPConnect // roots must be accumulated in the first slice of incremental GC. We // accumulate these roots in each zone's gcGrayRoots vector and then mark // them later, after black marking is complete for each compartment. This // accumulation can fail, but in that case we switch to non-incremental GC. enum class GrayBufferState { Unused, Okay, Failed }; GrayBufferState grayBufferState; bool hasBufferedGrayRoots() const { return grayBufferState == GrayBufferState::Okay; } // Clear each zone's gray buffers, but do not change the current state. void resetBufferedGrayRoots() const; // Reset the gray buffering state to Unused. void clearBufferedGrayRoots() { grayBufferState = GrayBufferState::Unused; resetBufferedGrayRoots(); } mozilla::Atomic majorGCTriggerReason; JS::gcreason::Reason minorGCTriggerReason; /* Perform full GC if rt->keepAtoms() becomes false. */ bool fullGCForAtomsRequested_; /* Incremented at the start of every minor GC. */ uint64_t minorGCNumber; /* Incremented at the start of every major GC. */ uint64_t majorGCNumber; /* The major GC number at which to release observed type information. */ uint64_t jitReleaseNumber; /* Incremented on every GC slice. */ uint64_t number; /* The number at the time of the most recent GC's first slice. */ uint64_t startNumber; /* Whether the currently running GC can finish in multiple slices. */ bool isIncremental; /* Whether all zones are being collected in first GC slice. */ bool isFull; /* Whether the heap will be compacted at the end of GC. */ bool isCompacting; /* The invocation kind of the current GC, taken from the first slice. */ JSGCInvocationKind invocationKind; /* The initial GC reason, taken from the first slice. */ JS::gcreason::Reason initialReason; #ifdef DEBUG /* * If this is 0, all cross-compartment proxies must be registered in the * wrapper map. This checking must be disabled temporarily while creating * new wrappers. When non-zero, this records the recursion depth of wrapper * creation. */ uintptr_t disableStrictProxyCheckingCount; #endif /* * The current incremental GC phase. This is also used internally in * non-incremental GC. */ State incrementalState; /* Indicates that the last incremental slice exhausted the mark stack. */ bool lastMarkSlice; /* Whether any sweeping will take place in the separate GC helper thread. */ bool sweepOnBackgroundThread; /* Whether observed type information is being released in the current GC. */ bool releaseObservedTypes; /* Whether any black->gray edges were found during marking. */ BlackGrayEdgeVector foundBlackGrayEdges; /* Singly linekd list of zones to be swept in the background. */ ZoneList backgroundSweepZones; /* * Free LIFO blocks are transferred to this allocator before being freed on * the background GC thread after sweeping. */ LifoAlloc blocksToFreeAfterSweeping; /* * Free LIFO blocks are transferred to this allocator before being freed * after minor GC. */ LifoAlloc blocksToFreeAfterMinorGC; /* Index of current zone group (for stats). */ unsigned zoneGroupIndex; /* * Incremental sweep state. */ JS::Zone* zoneGroups; JS::Zone* currentZoneGroup; bool sweepingTypes; unsigned finalizePhase; JS::Zone* sweepZone; AllocKind sweepKind; bool abortSweepAfterCurrentGroup; /* * Concurrent sweep infrastructure. */ void startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); void joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); /* * List head of arenas allocated during the sweep phase. */ Arena* arenasAllocatedDuringSweep; /* * Incremental compacting state. */ bool startedCompacting; ZoneList zonesToMaybeCompact; Arena* relocatedArenasToRelease; /* * Indicates that a GC slice has taken place in the middle of an animation * frame, rather than at the beginning. In this case, the next slice will be * delayed so that we don't get back-to-back slices. */ bool interFrameGC; /* Default budget for incremental GC slice. See js/SliceBudget.h. */ int64_t defaultTimeBudget_; /* * We disable incremental GC if we encounter a Class with a trace hook * that does not implement write barriers. */ bool incrementalAllowed; /* * GGC can be enabled from the command line while testing. */ unsigned generationalDisabled; /* * Whether compacting GC can is enabled globally. */ bool compactingEnabled; /* * Some code cannot tolerate compacting GC so it can be disabled temporarily * with AutoDisableCompactingGC which uses this counter. */ unsigned compactingDisabledCount; /* * This is true if we are in the middle of a brain transplant (e.g., * JS_TransplantObject) or some other operation that can manipulate * dead zones. */ bool manipulatingDeadZones; /* * This field is incremented each time we mark an object inside a * zone with no incoming cross-compartment pointers. Typically if * this happens it signals that an incremental GC is marking too much * stuff. At various times we check this counter and, if it has changed, we * run an immediate, non-incremental GC to clean up the dead * zones. This should happen very rarely. */ unsigned objectsMarkedInDeadZones; bool poked; bool fullCompartmentChecks; Callback gcCallback; Callback gcDoCycleCollectionCallback; Callback tenuredCallback; CallbackVector finalizeCallbacks; CallbackVector updateWeakPointerZoneGroupCallbacks; CallbackVector updateWeakPointerCompartmentCallbacks; /* * Malloc counter to measure memory pressure for GC scheduling. It runs * from maxMallocBytes down to zero. */ mozilla::Atomic mallocBytesUntilGC; /* * Whether a GC has been triggered as a result of mallocBytesUntilGC * falling below zero. */ mozilla::Atomic mallocGCTriggered; /* * The trace operations to trace embedding-specific GC roots. One is for * tracing through black roots and the other is for tracing through gray * roots. The black/gray distinction is only relevant to the cycle * collector. */ CallbackVector blackRootTracers; Callback grayRootTracer; /* Always preserve JIT code during GCs, for testing. */ bool alwaysPreserveCode; /* * Some regions of code are hard for the static rooting hazard analysis to * understand. In those cases, we trade the static analysis for a dynamic * analysis. When this is non-zero, we should assert if we trigger, or * might trigger, a GC. */ int inUnsafeRegion; #ifdef DEBUG size_t noGCOrAllocationCheck; size_t noNurseryAllocationCheck; bool arenasEmptyAtShutdown; #endif /* Synchronize GC heap access between main thread and GCHelperState. */ friend class js::AutoLockGC; js::Mutex lock; BackgroundAllocTask allocTask; BackgroundDecommitTask decommitTask; GCHelperState helperState; /* * During incremental sweeping, this field temporarily holds the arenas of * the current AllocKind being swept in order of increasing free space. */ SortedArenaList incrementalSweepList; friend class js::GCHelperState; friend class MarkingValidator; friend class AutoTraceSession; friend class AutoEnterIteration; }; /* Prevent compartments and zones from being collected during iteration. */ class MOZ_RAII AutoEnterIteration { GCRuntime* gc; public: explicit AutoEnterIteration(GCRuntime* gc_) : gc(gc_) { ++gc->numActiveZoneIters; } ~AutoEnterIteration() { MOZ_ASSERT(gc->numActiveZoneIters); --gc->numActiveZoneIters; } }; // After pulling a Chunk out of the empty chunks pool, we want to run the // background allocator to refill it. The code that takes Chunks does so under // the GC lock. We need to start the background allocation under the helper // threads lock. To avoid lock inversion we have to delay the start until after // we are outside the GC lock. This class handles that delay automatically. class MOZ_RAII AutoMaybeStartBackgroundAllocation { GCRuntime* gc; public: AutoMaybeStartBackgroundAllocation() : gc(nullptr) {} void tryToStartBackgroundAllocation(GCRuntime& gc) { this->gc = &gc; } ~AutoMaybeStartBackgroundAllocation() { if (gc) gc->startBackgroundAllocTaskIfIdle(); } }; inline bool GCRuntime::needZealousGC() { return false; } } /* namespace gc */ } /* namespace js */ #endif