diff options
Diffstat (limited to 'js/src/gc/Heap.h')
| -rw-r--r-- | js/src/gc/Heap.h | 1385 |
1 files changed, 1385 insertions, 0 deletions
diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h new file mode 100644 index 000000000..2a9390e91 --- /dev/null +++ b/js/src/gc/Heap.h @@ -0,0 +1,1385 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Heap_h +#define gc_Heap_h + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/EnumeratedRange.h" +#include "mozilla/PodOperations.h" + +#include <stddef.h> +#include <stdint.h> + +#include "jsfriendapi.h" +#include "jspubtd.h" +#include "jstypes.h" +#include "jsutil.h" + +#include "ds/BitArray.h" +#include "gc/Memory.h" +#include "js/GCAPI.h" +#include "js/HeapAPI.h" +#include "js/RootingAPI.h" +#include "js/TracingAPI.h" + +struct JSRuntime; + +namespace JS { +namespace shadow { +struct Runtime; +} // namespace shadow +} // namespace JS + +namespace js { + +class AutoLockGC; +class FreeOp; + +extern bool +RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone); + +#ifdef DEBUG + +// Barriers can't be triggered during backend Ion compilation, which may run on +// a helper thread. +extern bool +CurrentThreadIsIonCompiling(); +#endif + +// The return value indicates if anything was unmarked. +extern bool +UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind); + +extern void +TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); + +namespace gc { + +class Arena; +class ArenaCellSet; +class ArenaList; +class SortedArenaList; +struct Chunk; + +/* + * This flag allows an allocation site to request a specific heap based upon the + * estimated lifetime or lifetime requirements of objects allocated from that + * site. + */ +enum InitialHeap { + DefaultHeap, + TenuredHeap +}; + +/* The GC allocation kinds. */ +// FIXME: uint8_t would make more sense for the underlying type, but causes +// miscompilations in GCC (fixed in 4.8.5 and 4.9.3). See also bug 1143966. +enum class AllocKind { + FIRST, + OBJECT_FIRST = FIRST, + FUNCTION = FIRST, + FUNCTION_EXTENDED, + OBJECT0, + OBJECT0_BACKGROUND, + OBJECT2, + OBJECT2_BACKGROUND, + OBJECT4, + OBJECT4_BACKGROUND, + OBJECT8, + OBJECT8_BACKGROUND, + OBJECT12, + OBJECT12_BACKGROUND, + OBJECT16, + OBJECT16_BACKGROUND, + OBJECT_LIMIT, + OBJECT_LAST = OBJECT_LIMIT - 1, + SCRIPT, + LAZY_SCRIPT, + SHAPE, + ACCESSOR_SHAPE, + BASE_SHAPE, + OBJECT_GROUP, + FAT_INLINE_STRING, + STRING, + EXTERNAL_STRING, + FAT_INLINE_ATOM, + ATOM, + SYMBOL, + JITCODE, + SCOPE, + LIMIT, + LAST = LIMIT - 1 +}; + +// Macro to enumerate the different allocation kinds supplying information about +// the trace kind, C++ type and allocation size. +#define FOR_EACH_OBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType */ \ + D(FUNCTION, Object, JSObject, JSFunction) \ + D(FUNCTION_EXTENDED, Object, JSObject, FunctionExtended) \ + D(OBJECT0, Object, JSObject, JSObject_Slots0) \ + D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0) \ + D(OBJECT2, Object, JSObject, JSObject_Slots2) \ + D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2) \ + D(OBJECT4, Object, JSObject, JSObject_Slots4) \ + D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4) \ + D(OBJECT8, Object, JSObject, JSObject_Slots8) \ + D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8) \ + D(OBJECT12, Object, JSObject, JSObject_Slots12) \ + D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12) \ + D(OBJECT16, Object, JSObject, JSObject_Slots16) \ + D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16) + +#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType */ \ + D(SCRIPT, Script, JSScript, JSScript) \ + D(LAZY_SCRIPT, LazyScript, js::LazyScript, js::LazyScript) \ + D(SHAPE, Shape, js::Shape, js::Shape) \ + D(ACCESSOR_SHAPE, Shape, js::AccessorShape, js::AccessorShape) \ + D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape) \ + D(OBJECT_GROUP, ObjectGroup, js::ObjectGroup, js::ObjectGroup) \ + D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString) \ + D(STRING, String, JSString, JSString) \ + D(EXTERNAL_STRING, String, JSExternalString, JSExternalString) \ + D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom) \ + D(ATOM, String, js::NormalAtom, js::NormalAtom) \ + D(SYMBOL, Symbol, JS::Symbol, JS::Symbol) \ + D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode) \ + D(SCOPE, Scope, js::Scope, js::Scope) + +#define FOR_EACH_ALLOCKIND(D) \ + FOR_EACH_OBJECT_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_ALLOCKIND(D) + +static_assert(int(AllocKind::FIRST) == 0, "Various places depend on AllocKind starting at 0, " + "please audit them carefully!"); +static_assert(int(AllocKind::OBJECT_FIRST) == 0, "Various places depend on AllocKind::OBJECT_FIRST " + "being 0, please audit them carefully!"); + +inline bool +IsAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT; +} + +inline bool +IsValidAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LAST; +} + +inline bool +IsObjectAllocKind(AllocKind kind) +{ + return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST; +} + +inline bool +IsShapeAllocKind(AllocKind kind) +{ + return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE; +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +AllAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all object alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT)) +ObjectAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over alloc kinds from |first| to |limit|, exclusive. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT) +{ + MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!"); + MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!"); + return mozilla::MakeEnumeratedRange(first, limit); +} + +// AllAllocKindArray<ValueType> gives an enumerated array of ValueTypes, +// with each index corresponding to a particular alloc kind. +template<typename ValueType> using AllAllocKindArray = + mozilla::EnumeratedArray<AllocKind, AllocKind::LIMIT, ValueType>; + +// ObjectAllocKindArray<ValueType> gives an enumerated array of ValueTypes, +// with each index corresponding to a particular object alloc kind. +template<typename ValueType> using ObjectAllocKindArray = + mozilla::EnumeratedArray<AllocKind, AllocKind::OBJECT_LIMIT, ValueType>; + +static inline JS::TraceKind +MapAllocToTraceKind(AllocKind kind) +{ + static const JS::TraceKind map[] = { +#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType) \ + JS::TraceKind::traceKind, +FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) +#undef EXPAND_ELEMENT + }; + + static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT), + "AllocKind-to-TraceKind mapping must be in sync"); + return map[size_t(kind)]; +} + +/* + * This must be an upper bound, but we do not need the least upper bound, so + * we just exclude non-background objects. + */ +static const size_t MAX_BACKGROUND_FINALIZE_KINDS = + size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2; + +class TenuredCell; + +// A GC cell is the base class for all GC things. +struct Cell +{ + public: + MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); } + MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; + MOZ_ALWAYS_INLINE TenuredCell& asTenured(); + + inline JSRuntime* runtimeFromMainThread() const; + inline JS::shadow::Runtime* shadowRuntimeFromMainThread() const; + + // Note: Unrestricted access to the runtime of a GC thing from an arbitrary + // thread can easily lead to races. Use this method very carefully. + inline JSRuntime* runtimeFromAnyThread() const; + inline JS::shadow::Runtime* shadowRuntimeFromAnyThread() const; + + // May be overridden by GC thing kinds that have a compartment pointer. + inline JSCompartment* maybeCompartment() const { return nullptr; } + + inline StoreBuffer* storeBuffer() const; + + inline JS::TraceKind getTraceKind() const; + + static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone); + +#ifdef DEBUG + inline bool isAligned() const; + void dump(FILE* fp) const; + void dump() const; +#endif + + protected: + inline uintptr_t address() const; + inline Chunk* chunk() const; +} JS_HAZ_GC_THING; + +// A GC TenuredCell gets behaviors that are valid for things in the Tenured +// heap, such as access to the arena and mark bits. +class TenuredCell : public Cell +{ + public: + // Construct a TenuredCell from a void*, making various sanity assertions. + static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr); + static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr); + + // Mark bit management. + MOZ_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const; + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const; + MOZ_ALWAYS_INLINE void unmark(uint32_t color) const; + MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); + + // Note: this is in TenuredCell because JSObject subclasses are sometimes + // used tagged. + static MOZ_ALWAYS_INLINE bool isNullLike(const Cell* thing) { return !thing; } + + // Access to the arena. + inline Arena* arena() const; + inline AllocKind getAllocKind() const; + inline JS::TraceKind getTraceKind() const; + inline JS::Zone* zone() const; + inline JS::Zone* zoneFromAnyThread() const; + inline bool isInsideZone(JS::Zone* zone) const; + + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const { + return JS::shadow::Zone::asShadowZone(zone()); + } + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const { + return JS::shadow::Zone::asShadowZone(zoneFromAnyThread()); + } + + static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing); + static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing); + + static MOZ_ALWAYS_INLINE void writeBarrierPost(void* cellp, TenuredCell* prior, + TenuredCell* next); + + // Default implementation for kinds that don't require fixup. + void fixupAfterMovingGC() {} + +#ifdef DEBUG + inline bool isAligned() const; +#endif +}; + +/* Cells are aligned to CellShift, so the largest tagged null pointer is: */ +const uintptr_t LargestTaggedNullCellPointer = (1 << CellShift) - 1; + +constexpr size_t +DivideAndRoundUp(size_t numerator, size_t divisor) { + return (numerator + divisor - 1) / divisor; +} + +const size_t ArenaCellCount = ArenaSize / CellSize; +static_assert(ArenaSize % CellSize == 0, "Arena size must be a multiple of cell size"); + +/* + * The mark bitmap has one bit per each GC cell. For multi-cell GC things this + * wastes space but allows to avoid expensive devisions by thing's size when + * accessing the bitmap. In addition this allows to use some bits for colored + * marking during the cycle GC. + */ +const size_t ArenaBitmapBits = ArenaCellCount; +const size_t ArenaBitmapBytes = DivideAndRoundUp(ArenaBitmapBits, 8); +const size_t ArenaBitmapWords = DivideAndRoundUp(ArenaBitmapBits, JS_BITS_PER_WORD); + +/* + * A FreeSpan represents a contiguous sequence of free cells in an Arena. It + * can take two forms. + * + * - In an empty span, |first| and |last| are both zero. + * + * - In a non-empty span, |first| is the address of the first free thing in the + * span, and |last| is the address of the last free thing in the span. + * Furthermore, the memory pointed to by |last| holds a FreeSpan structure + * that points to the next span (which may be empty); this works because + * sizeof(FreeSpan) is less than the smallest thingSize. + */ +class FreeSpan +{ + friend class Arena; + friend class ArenaCellIterImpl; + + uint16_t first; + uint16_t last; + + public: + // This inits just |first| and |last|; if the span is non-empty it doesn't + // do anything with the next span stored at |last|. + void initBounds(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + checkRange(firstArg, lastArg, arena); + first = firstArg; + last = lastArg; + } + + void initAsEmpty() { + first = 0; + last = 0; + } + + // This sets |first| and |last|, and also sets the next span stored at + // |last| as empty. (As a result, |firstArg| and |lastArg| cannot represent + // an empty span.) + void initFinal(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + initBounds(firstArg, lastArg, arena); + FreeSpan* last = nextSpanUnchecked(arena); + last->initAsEmpty(); + checkSpan(arena); + } + + bool isEmpty() const { + return !first; + } + + Arena* getArenaUnchecked() { return reinterpret_cast<Arena*>(this); } + inline Arena* getArena(); + + static size_t offsetOfFirst() { + return offsetof(FreeSpan, first); + } + + static size_t offsetOfLast() { + return offsetof(FreeSpan, last); + } + + // Like nextSpan(), but no checking of the following span is done. + FreeSpan* nextSpanUnchecked(const Arena* arena) const { + MOZ_ASSERT(arena && !isEmpty()); + return reinterpret_cast<FreeSpan*>(uintptr_t(arena) + last); + } + + const FreeSpan* nextSpan(const Arena* arena) const { + checkSpan(arena); + return nextSpanUnchecked(arena); + } + + MOZ_ALWAYS_INLINE TenuredCell* allocate(size_t thingSize) { + // Eschew the usual checks, because this might be the placeholder span. + // If this is somehow an invalid, non-empty span, checkSpan() will catch it. + Arena* arena = getArenaUnchecked(); + checkSpan(arena); + uintptr_t thing = uintptr_t(arena) + first; + if (first < last) { + // We have space for at least two more things, so do a simple bump-allocate. + first += thingSize; + } else if (MOZ_LIKELY(first)) { + // The last space points to the next free span (which may be empty). + const FreeSpan* next = nextSpan(arena); + first = next->first; + last = next->last; + } else { + return nullptr; // The span is empty. + } + checkSpan(arena); + JS_EXTRA_POISON(reinterpret_cast<void*>(thing), JS_ALLOCATED_TENURED_PATTERN, thingSize); + MemProfiler::SampleTenured(reinterpret_cast<void*>(thing), thingSize); + return reinterpret_cast<TenuredCell*>(thing); + } + + inline void checkSpan(const Arena* arena) const; + inline void checkRange(uintptr_t first, uintptr_t last, const Arena* arena) const; +}; + +/* + * Arenas are the allocation units of the tenured heap in the GC. An arena + * is 4kiB in size and 4kiB-aligned. It starts with several header fields + * followed by some bytes of padding. The remainder of the arena is filled + * with GC things of a particular AllocKind. The padding ensures that the + * GC thing array ends exactly at the end of the arena: + * + * <----------------------------------------------> = ArenaSize bytes + * +---------------+---------+----+----+-----+----+ + * | header fields | padding | T0 | T1 | ... | Tn | + * +---------------+---------+----+----+-----+----+ + * <-------------------------> = first thing offset + */ +class Arena +{ + static JS_FRIEND_DATA(const uint32_t) ThingSizes[]; + static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[]; + static JS_FRIEND_DATA(const uint32_t) ThingsPerArena[]; + + /* + * The first span of free things in the arena. Most of these spans are + * stored as offsets in free regions of the data array, and most operations + * on FreeSpans take an Arena pointer for safety. However, the FreeSpans + * used for allocation are stored here, at the start of an Arena, and use + * their own address to grab the next span within the same Arena. + */ + FreeSpan firstFreeSpan; + + public: + /* + * The zone that this Arena is contained within, when allocated. The offset + * of this field must match the ArenaZoneOffset stored in js/HeapAPI.h, + * as is statically asserted below. + */ + JS::Zone* zone; + + /* + * Arena::next has two purposes: when unallocated, it points to the next + * available Arena. When allocated, it points to the next Arena in the same + * zone and with the same alloc kind. + */ + Arena* next; + + private: + /* + * One of the AllocKind constants or AllocKind::LIMIT when the arena does + * not contain any GC things and is on the list of empty arenas in the GC + * chunk. + * + * We use 8 bits for the alloc kind so the compiler can use byte-level + * memory instructions to access it. + */ + size_t allocKind : 8; + + public: + /* + * When collecting we sometimes need to keep an auxillary list of arenas, + * for which we use the following fields. This happens for several reasons: + * + * When recursive marking uses too much stack, the marking is delayed and + * the corresponding arenas are put into a stack. To distinguish the bottom + * of the stack from the arenas not present in the stack we use the + * markOverflow flag to tag arenas on the stack. + * + * Delayed marking is also used for arenas that we allocate into during an + * incremental GC. In this case, we intend to mark all the objects in the + * arena, and it's faster to do this marking in bulk. + * + * When sweeping we keep track of which arenas have been allocated since + * the end of the mark phase. This allows us to tell whether a pointer to + * an unmarked object is yet to be finalized or has already been + * reallocated. We set the allocatedDuringIncremental flag for this and + * clear it at the end of the sweep phase. + * + * To minimize the size of the header fields we record the next linkage as + * address() >> ArenaShift and pack it with the allocKind and the flags. + */ + size_t hasDelayedMarking : 1; + size_t allocatedDuringIncremental : 1; + size_t markOverflow : 1; + size_t auxNextLink : JS_BITS_PER_WORD - 8 - 1 - 1 - 1; + static_assert(ArenaShift >= 8 + 1 + 1 + 1, + "Arena::auxNextLink packing assumes that ArenaShift has " + "enough bits to cover allocKind and hasDelayedMarking."); + + /* + * If non-null, points to an ArenaCellSet that represents the set of cells + * in this arena that are in the nursery's store buffer. + */ + ArenaCellSet* bufferedCells; + + /* + * The size of data should be |ArenaSize - offsetof(data)|, but the offset + * is not yet known to the compiler, so we do it by hand. |firstFreeSpan| + * takes up 8 bytes on 64-bit due to alignment requirements; the rest are + * obvious. This constant is stored in js/HeapAPI.h. + */ + uint8_t data[ArenaSize - ArenaHeaderSize]; + + void init(JS::Zone* zoneArg, AllocKind kind); + + // Sets |firstFreeSpan| to the Arena's entire valid range, and + // also sets the next span stored at |firstFreeSpan.last| as empty. + void setAsFullyUnused() { + AllocKind kind = getAllocKind(); + firstFreeSpan.first = firstThingOffset(kind); + firstFreeSpan.last = lastThingOffset(kind); + FreeSpan* last = firstFreeSpan.nextSpanUnchecked(this); + last->initAsEmpty(); + } + + void setAsNotAllocated() { + firstFreeSpan.initAsEmpty(); + zone = nullptr; + allocKind = size_t(AllocKind::LIMIT); + hasDelayedMarking = 0; + allocatedDuringIncremental = 0; + markOverflow = 0; + auxNextLink = 0; + bufferedCells = nullptr; + } + + uintptr_t address() const { + checkAddress(); + return uintptr_t(this); + } + + inline void checkAddress() const; + + inline Chunk* chunk() const; + + bool allocated() const { + MOZ_ASSERT(IsAllocKind(AllocKind(allocKind))); + return IsValidAllocKind(AllocKind(allocKind)); + } + + AllocKind getAllocKind() const { + MOZ_ASSERT(allocated()); + return AllocKind(allocKind); + } + + FreeSpan* getFirstFreeSpan() { return &firstFreeSpan; } + + static size_t thingSize(AllocKind kind) { return ThingSizes[size_t(kind)]; } + static size_t thingsPerArena(AllocKind kind) { return ThingsPerArena[size_t(kind)]; } + static size_t thingsSpan(AllocKind kind) { return thingsPerArena(kind) * thingSize(kind); } + + static size_t firstThingOffset(AllocKind kind) { return FirstThingOffsets[size_t(kind)]; } + static size_t lastThingOffset(AllocKind kind) { return ArenaSize - thingSize(kind); } + + size_t getThingSize() const { return thingSize(getAllocKind()); } + size_t getThingsPerArena() const { return thingsPerArena(getAllocKind()); } + size_t getThingsSpan() const { return getThingsPerArena() * getThingSize(); } + + uintptr_t thingsStart() const { return address() + firstThingOffset(getAllocKind()); } + uintptr_t thingsEnd() const { return address() + ArenaSize; } + + bool isEmpty() const { + // Arena is empty if its first span covers the whole arena. + firstFreeSpan.checkSpan(this); + AllocKind kind = getAllocKind(); + return firstFreeSpan.first == firstThingOffset(kind) && + firstFreeSpan.last == lastThingOffset(kind); + } + + bool hasFreeThings() const { return !firstFreeSpan.isEmpty(); } + + size_t numFreeThings(size_t thingSize) const { + firstFreeSpan.checkSpan(this); + size_t numFree = 0; + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) + numFree += (span->last - span->first) / thingSize + 1; + return numFree; + } + + size_t countFreeCells() { return numFreeThings(getThingSize()); } + size_t countUsedCells() { return getThingsPerArena() - countFreeCells(); } + + bool inFreeList(uintptr_t thing) { + uintptr_t base = address(); + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) { + /* If the thing comes before the current span, it's not free. */ + if (thing < base + span->first) + return false; + + /* If we find it before the end of the span, it's free. */ + if (thing <= base + span->last) + return true; + } + return false; + } + + static bool isAligned(uintptr_t thing, size_t thingSize) { + /* Things ends at the arena end. */ + uintptr_t tailOffset = ArenaSize - (thing & ArenaMask); + return tailOffset % thingSize == 0; + } + + Arena* getNextDelayedMarking() const { + MOZ_ASSERT(hasDelayedMarking); + return reinterpret_cast<Arena*>(auxNextLink << ArenaShift); + } + + void setNextDelayedMarking(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(!auxNextLink && !hasDelayedMarking); + hasDelayedMarking = 1; + if (arena) + auxNextLink = arena->address() >> ArenaShift; + } + + void unsetDelayedMarking() { + MOZ_ASSERT(hasDelayedMarking); + hasDelayedMarking = 0; + auxNextLink = 0; + } + + Arena* getNextAllocDuringSweep() const { + MOZ_ASSERT(allocatedDuringIncremental); + return reinterpret_cast<Arena*>(auxNextLink << ArenaShift); + } + + void setNextAllocDuringSweep(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(!auxNextLink && !allocatedDuringIncremental); + allocatedDuringIncremental = 1; + if (arena) + auxNextLink = arena->address() >> ArenaShift; + } + + void unsetAllocDuringSweep() { + MOZ_ASSERT(allocatedDuringIncremental); + allocatedDuringIncremental = 0; + auxNextLink = 0; + } + + template <typename T> + size_t finalize(FreeOp* fop, AllocKind thingKind, size_t thingSize); + + static void staticAsserts(); + + void unmarkAll(); + + static size_t offsetOfBufferedCells() { + return offsetof(Arena, bufferedCells); + } +}; + +static_assert(ArenaZoneOffset == offsetof(Arena, zone), + "The hardcoded API zone offset must match the actual offset."); + +static_assert(sizeof(Arena) == ArenaSize, + "ArenaSize must match the actual size of the Arena structure."); + +static_assert(offsetof(Arena, data) == ArenaHeaderSize, + "ArenaHeaderSize must match the actual size of the header fields."); + +inline Arena* +FreeSpan::getArena() +{ + Arena* arena = getArenaUnchecked(); + arena->checkAddress(); + return arena; +} + +inline void +FreeSpan::checkSpan(const Arena* arena) const +{ +#ifdef DEBUG + if (!first) { + MOZ_ASSERT(!first && !last); + return; + } + + arena->checkAddress(); + checkRange(first, last, arena); + + // If there's a following span, it must have a higher address, + // and the gap must be at least 2 * thingSize. + const FreeSpan* next = nextSpanUnchecked(arena); + if (next->first) { + checkRange(next->first, next->last, arena); + size_t thingSize = arena->getThingSize(); + MOZ_ASSERT(last + 2 * thingSize <= next->first); + } +#endif +} + +inline void +FreeSpan::checkRange(uintptr_t first, uintptr_t last, const Arena* arena) const +{ +#ifdef DEBUG + MOZ_ASSERT(arena); + MOZ_ASSERT(first <= last); + AllocKind thingKind = arena->getAllocKind(); + MOZ_ASSERT(first >= Arena::firstThingOffset(thingKind)); + MOZ_ASSERT(last <= Arena::lastThingOffset(thingKind)); + MOZ_ASSERT((last - first) % Arena::thingSize(thingKind) == 0); +#endif +} + +/* + * The tail of the chunk info is shared between all chunks in the system, both + * nursery and tenured. This structure is locatable from any GC pointer by + * aligning to 1MiB. + */ +struct ChunkTrailer +{ + /* Construct a Nursery ChunkTrailer. */ + ChunkTrailer(JSRuntime* rt, StoreBuffer* sb) + : location(ChunkLocation::Nursery), storeBuffer(sb), runtime(rt) + {} + + /* Construct a Tenured heap ChunkTrailer. */ + explicit ChunkTrailer(JSRuntime* rt) + : location(ChunkLocation::TenuredHeap), storeBuffer(nullptr), runtime(rt) + {} + + public: + /* The index the chunk in the nursery, or LocationTenuredHeap. */ + ChunkLocation location; + uint32_t padding; + + /* The store buffer for writes to things in this chunk or nullptr. */ + StoreBuffer* storeBuffer; + + /* This provides quick access to the runtime from absolutely anywhere. */ + JSRuntime* runtime; +}; + +static_assert(sizeof(ChunkTrailer) == ChunkTrailerSize, + "ChunkTrailer size must match the API defined size."); + +/* The chunk header (located at the end of the chunk to preserve arena alignment). */ +struct ChunkInfo +{ + void init() { + next = prev = nullptr; + } + + private: + friend class ChunkPool; + Chunk* next; + Chunk* prev; + + public: + /* Free arenas are linked together with arena.next. */ + Arena* freeArenasHead; + +#if JS_BITS_PER_WORD == 32 + /* + * Calculating sizes and offsets is simpler if sizeof(ChunkInfo) is + * architecture-independent. + */ + char padding[24]; +#endif + + /* + * Decommitted arenas are tracked by a bitmap in the chunk header. We use + * this offset to start our search iteration close to a decommitted arena + * that we can allocate. + */ + uint32_t lastDecommittedArenaOffset; + + /* Number of free arenas, either committed or decommitted. */ + uint32_t numArenasFree; + + /* Number of free, committed arenas. */ + uint32_t numArenasFreeCommitted; +}; + +/* + * Calculating ArenasPerChunk: + * + * In order to figure out how many Arenas will fit in a chunk, we need to know + * how much extra space is available after we allocate the header data. This + * is a problem because the header size depends on the number of arenas in the + * chunk. The two dependent fields are bitmap and decommittedArenas. + * + * For the mark bitmap, we know that each arena will use a fixed number of full + * bytes: ArenaBitmapBytes. The full size of the header data is this number + * multiplied by the eventual number of arenas we have in the header. We, + * conceptually, distribute this header data among the individual arenas and do + * not include it in the header. This way we do not have to worry about its + * variable size: it gets attached to the variable number we are computing. + * + * For the decommitted arena bitmap, we only have 1 bit per arena, so this + * technique will not work. Instead, we observe that we do not have enough + * header info to fill 8 full arenas: it is currently 4 on 64bit, less on + * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas. + * This will not become 63 bytes unless we double the data required in the + * header. Therefore, we just compute the number of bytes required to track + * every possible arena and do not worry about slop bits, since there are too + * few to usefully allocate. + * + * To actually compute the number of arenas we can allocate in a chunk, we + * divide the amount of available space less the header info (not including + * the mark bitmap which is distributed into the arena size) by the size of + * the arena (with the mark bitmap bytes it uses). + */ +const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes; +const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE; +const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkTrailer) - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes; +const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader; + +#ifdef JS_GC_SMALL_CHUNK_SIZE +static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's density."); +#else +static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density."); +#endif + +/* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ +struct ChunkBitmap +{ + volatile uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk]; + + public: + ChunkBitmap() { } + + MOZ_ALWAYS_INLINE void getMarkWordAndMask(const Cell* cell, uint32_t color, + uintptr_t** wordp, uintptr_t* maskp) + { + detail::GetGCThingMarkWordAndMask(uintptr_t(cell), color, wordp, maskp); + } + + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarked(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + return *word & mask; + } + + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, BLACK, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + if (color != BLACK) { + /* + * We use getMarkWordAndMask to recalculate both mask and word as + * doing just mask << color may overflow the mask. + */ + getMarkWordAndMask(cell, color, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + } + return true; + } + + MOZ_ALWAYS_INLINE void unmark(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + *word &= ~mask; + } + + MOZ_ALWAYS_INLINE void copyMarkBit(Cell* dst, const TenuredCell* src, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(dst, color, &word, &mask); + *word = (*word & ~mask) | (src->isMarked(color) ? mask : 0); + } + + void clear() { + memset((void*)bitmap, 0, sizeof(bitmap)); + } + + uintptr_t* arenaBits(Arena* arena) { + static_assert(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD, + "We assume that the part of the bitmap corresponding to the arena " + "has the exact number of words so we do not need to deal with a word " + "that covers bits from two arenas."); + + uintptr_t* word, unused; + getMarkWordAndMask(reinterpret_cast<Cell*>(arena->address()), BLACK, &word, &unused); + return word; + } +}; + +static_assert(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap), + "Ensure our ChunkBitmap actually covers all arenas."); +static_assert(js::gc::ChunkMarkBitmapBits == ArenaBitmapBits * ArenasPerChunk, + "Ensure that the mark bitmap has the right number of bits."); + +typedef BitArray<ArenasPerChunk> PerArenaBitmap; + +const size_t ChunkPadSize = ChunkSize + - (sizeof(Arena) * ArenasPerChunk) + - sizeof(ChunkBitmap) + - sizeof(PerArenaBitmap) + - sizeof(ChunkInfo) + - sizeof(ChunkTrailer); +static_assert(ChunkPadSize < BytesPerArenaWithHeader, + "If the chunk padding is larger than an arena, we should have one more arena."); + +/* + * Chunks contain arenas and associated data structures (mark bitmap, delayed + * marking state). + */ +struct Chunk +{ + Arena arenas[ArenasPerChunk]; + + /* Pad to full size to ensure cache alignment of ChunkInfo. */ + uint8_t padding[ChunkPadSize]; + + ChunkBitmap bitmap; + PerArenaBitmap decommittedArenas; + ChunkInfo info; + ChunkTrailer trailer; + + static Chunk* fromAddress(uintptr_t addr) { + addr &= ~ChunkMask; + return reinterpret_cast<Chunk*>(addr); + } + + static bool withinValidRange(uintptr_t addr) { + uintptr_t offset = addr & ChunkMask; + return Chunk::fromAddress(addr)->isNurseryChunk() + ? offset < ChunkSize - sizeof(ChunkTrailer) + : offset < ArenasPerChunk * ArenaSize; + } + + static size_t arenaIndex(uintptr_t addr) { + MOZ_ASSERT(!Chunk::fromAddress(addr)->isNurseryChunk()); + MOZ_ASSERT(withinValidRange(addr)); + return (addr & ChunkMask) >> ArenaShift; + } + + uintptr_t address() const { + uintptr_t addr = reinterpret_cast<uintptr_t>(this); + MOZ_ASSERT(!(addr & ChunkMask)); + return addr; + } + + bool unused() const { + return info.numArenasFree == ArenasPerChunk; + } + + bool hasAvailableArenas() const { + return info.numArenasFree != 0; + } + + bool isNurseryChunk() const { + return trailer.storeBuffer; + } + + Arena* allocateArena(JSRuntime* rt, JS::Zone* zone, AllocKind kind, const AutoLockGC& lock); + + void releaseArena(JSRuntime* rt, Arena* arena, const AutoLockGC& lock); + void recycleArena(Arena* arena, SortedArenaList& dest, size_t thingsPerArena); + + MOZ_MUST_USE bool decommitOneFreeArena(JSRuntime* rt, AutoLockGC& lock); + void decommitAllArenasWithoutUnlocking(const AutoLockGC& lock); + + static Chunk* allocate(JSRuntime* rt); + void init(JSRuntime* rt); + + private: + void decommitAllArenas(JSRuntime* rt); + + /* Search for a decommitted arena to allocate. */ + unsigned findDecommittedArenaOffset(); + Arena* fetchNextDecommittedArena(); + + void addArenaToFreeList(JSRuntime* rt, Arena* arena); + void addArenaToDecommittedList(JSRuntime* rt, const Arena* arena); + + void updateChunkListAfterAlloc(JSRuntime* rt, const AutoLockGC& lock); + void updateChunkListAfterFree(JSRuntime* rt, const AutoLockGC& lock); + + public: + /* Unlink and return the freeArenasHead. */ + Arena* fetchNextFreeArena(JSRuntime* rt); +}; + +static_assert(sizeof(Chunk) == ChunkSize, + "Ensure the hardcoded chunk size definition actually matches the struct."); +static_assert(js::gc::ChunkMarkBitmapOffset == offsetof(Chunk, bitmap), + "The hardcoded API bitmap offset must match the actual offset."); +static_assert(js::gc::ChunkRuntimeOffset == offsetof(Chunk, trailer) + + offsetof(ChunkTrailer, runtime), + "The hardcoded API runtime offset must match the actual offset."); +static_assert(js::gc::ChunkLocationOffset == offsetof(Chunk, trailer) + + offsetof(ChunkTrailer, location), + "The hardcoded API location offset must match the actual offset."); + +/* + * Tracks the used sizes for owned heap data and automatically maintains the + * memory usage relationship between GCRuntime and Zones. + */ +class HeapUsage +{ + /* + * A heap usage that contains our parent's heap usage, or null if this is + * the top-level usage container. + */ + HeapUsage* parent_; + + /* + * The approximate number of bytes in use on the GC heap, to the nearest + * ArenaSize. This does not include any malloc data. It also does not + * include not-actively-used addresses that are still reserved at the OS + * level for GC usage. It is atomic because it is updated by both the main + * and GC helper threads. + */ + mozilla::Atomic<size_t, mozilla::ReleaseAcquire> gcBytes_; + + public: + explicit HeapUsage(HeapUsage* parent) + : parent_(parent), + gcBytes_(0) + {} + + size_t gcBytes() const { return gcBytes_; } + + void addGCArena() { + gcBytes_ += ArenaSize; + if (parent_) + parent_->addGCArena(); + } + void removeGCArena() { + MOZ_ASSERT(gcBytes_ >= ArenaSize); + gcBytes_ -= ArenaSize; + if (parent_) + parent_->removeGCArena(); + } + + /* Pair to adoptArenas. Adopts the attendant usage statistics. */ + void adopt(HeapUsage& other) { + gcBytes_ += other.gcBytes_; + other.gcBytes_ = 0; + } +}; + +inline void +Arena::checkAddress() const +{ + mozilla::DebugOnly<uintptr_t> addr = uintptr_t(this); + MOZ_ASSERT(addr); + MOZ_ASSERT(!(addr & ArenaMask)); + MOZ_ASSERT(Chunk::withinValidRange(addr)); +} + +inline Chunk* +Arena::chunk() const +{ + return Chunk::fromAddress(address()); +} + +static void +AssertValidColor(const TenuredCell* thing, uint32_t color) +{ +#ifdef DEBUG + Arena* arena = thing->arena(); + MOZ_ASSERT(color < arena->getThingSize() / CellSize); +#endif +} + +MOZ_ALWAYS_INLINE const TenuredCell& +Cell::asTenured() const +{ + MOZ_ASSERT(isTenured()); + return *static_cast<const TenuredCell*>(this); +} + +MOZ_ALWAYS_INLINE TenuredCell& +Cell::asTenured() +{ + MOZ_ASSERT(isTenured()); + return *static_cast<TenuredCell*>(this); +} + +inline JSRuntime* +Cell::runtimeFromMainThread() const +{ + JSRuntime* rt = chunk()->trailer.runtime; + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + return rt; +} + +inline JS::shadow::Runtime* +Cell::shadowRuntimeFromMainThread() const +{ + return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromMainThread()); +} + +inline JSRuntime* +Cell::runtimeFromAnyThread() const +{ + return chunk()->trailer.runtime; +} + +inline JS::shadow::Runtime* +Cell::shadowRuntimeFromAnyThread() const +{ + return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromAnyThread()); +} + +inline uintptr_t +Cell::address() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellSize == 0); + MOZ_ASSERT(Chunk::withinValidRange(addr)); + return addr; +} + +Chunk* +Cell::chunk() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellSize == 0); + addr &= ~ChunkMask; + return reinterpret_cast<Chunk*>(addr); +} + +inline StoreBuffer* +Cell::storeBuffer() const +{ + return chunk()->trailer.storeBuffer; +} + +inline JS::TraceKind +Cell::getTraceKind() const +{ + return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object; +} + +inline bool +InFreeList(Arena* arena, void* thing) +{ + uintptr_t addr = reinterpret_cast<uintptr_t>(thing); + MOZ_ASSERT(Arena::isAligned(addr, arena->getThingSize())); + return arena->inFreeList(addr); +} + +/* static */ MOZ_ALWAYS_INLINE bool +Cell::needWriteBarrierPre(JS::Zone* zone) { + return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier(); +} + +/* static */ MOZ_ALWAYS_INLINE TenuredCell* +TenuredCell::fromPointer(void* ptr) +{ + MOZ_ASSERT(static_cast<TenuredCell*>(ptr)->isTenured()); + return static_cast<TenuredCell*>(ptr); +} + +/* static */ MOZ_ALWAYS_INLINE const TenuredCell* +TenuredCell::fromPointer(const void* ptr) +{ + MOZ_ASSERT(static_cast<const TenuredCell*>(ptr)->isTenured()); + return static_cast<const TenuredCell*>(ptr); +} + +bool +TenuredCell::isMarked(uint32_t color /* = BLACK */) const +{ + MOZ_ASSERT(arena()->allocated()); + AssertValidColor(this, color); + return chunk()->bitmap.isMarked(this, color); +} + +bool +TenuredCell::markIfUnmarked(uint32_t color /* = BLACK */) const +{ + AssertValidColor(this, color); + return chunk()->bitmap.markIfUnmarked(this, color); +} + +void +TenuredCell::unmark(uint32_t color) const +{ + MOZ_ASSERT(color != BLACK); + AssertValidColor(this, color); + chunk()->bitmap.unmark(this, color); +} + +void +TenuredCell::copyMarkBitsFrom(const TenuredCell* src) +{ + ChunkBitmap& bitmap = chunk()->bitmap; + bitmap.copyMarkBit(this, src, BLACK); + bitmap.copyMarkBit(this, src, GRAY); +} + +inline Arena* +TenuredCell::arena() const +{ + MOZ_ASSERT(isTenured()); + uintptr_t addr = address(); + addr &= ~ArenaMask; + return reinterpret_cast<Arena*>(addr); +} + +AllocKind +TenuredCell::getAllocKind() const +{ + return arena()->getAllocKind(); +} + +JS::TraceKind +TenuredCell::getTraceKind() const +{ + return MapAllocToTraceKind(getAllocKind()); +} + +JS::Zone* +TenuredCell::zone() const +{ + JS::Zone* zone = arena()->zone; + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + return zone; +} + +JS::Zone* +TenuredCell::zoneFromAnyThread() const +{ + return arena()->zone; +} + +bool +TenuredCell::isInsideZone(JS::Zone* zone) const +{ + return zone == arena()->zone; +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::readBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(!isNullLike(thing)); + + // It would be good if barriers were never triggered during collection, but + // at the moment this can happen e.g. when rekeying tables containing + // read-barriered GC things after a moving GC. + // + // TODO: Fix this and assert we're not collecting if we're on the main + // thread. + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + // Barriers are only enabled on the main thread and are disabled while collecting. + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier"); + MOZ_ASSERT(tmp == thing); + } + + if (thing->isMarked(GRAY)) { + // There shouldn't be anything marked grey unless we're on the main thread. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); + if (!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)) + UnmarkGrayCellRecursively(thing, thing->getTraceKind()); + } +} + +void +AssertSafeToSkipBarrier(TenuredCell* thing); + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPre(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT_IF(thing, !isNullLike(thing)); + if (!thing) + return; + +#ifdef JS_GC_ZEAL + // When verifying pre barriers we need to switch on all barriers, even + // those on the Atoms Zone. Normally, we never enter a parse task when + // collecting in the atoms zone, so will filter out atoms below. + // Unfortuantely, If we try that when verifying pre-barriers, we'd never be + // able to handle OMT parse tasks at all as we switch on the verifier any + // time we're not doing GC. This would cause us to deadlock, as OMT parsing + // is meant to resume after GC work completes. Instead we filter out any + // OMT barriers that reach us and assert that they would normally not be + // possible. + if (!CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())) { + AssertSafeToSkipBarrier(thing); + return; + } +#endif + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier"); + MOZ_ASSERT(tmp == thing); + } +} + +static MOZ_ALWAYS_INLINE void +AssertValidToSkipBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!IsInsideNursery(thing)); + MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object); +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next) +{ + AssertValidToSkipBarrier(next); +} + +#ifdef DEBUG +bool +Cell::isAligned() const +{ + if (!isTenured()) + return true; + return asTenured().isAligned(); +} + +bool +TenuredCell::isAligned() const +{ + return Arena::isAligned(address(), arena()->getThingSize()); +} +#endif + +static const int32_t ChunkLocationOffsetFromLastByte = + int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask); + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_Heap_h */ |