diff options
Diffstat (limited to 'js/src/jit/Safepoints.cpp')
| -rw-r--r-- | js/src/jit/Safepoints.cpp | 562 |
1 files changed, 562 insertions, 0 deletions
diff --git a/js/src/jit/Safepoints.cpp b/js/src/jit/Safepoints.cpp new file mode 100644 index 000000000..b0bb530f9 --- /dev/null +++ b/js/src/jit/Safepoints.cpp @@ -0,0 +1,562 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Safepoints.h" + +#include "mozilla/MathAlgorithms.h" +#include "mozilla/SizePrintfMacros.h" + +#include "jit/BitSet.h" +#include "jit/JitSpewer.h" +#include "jit/LIR.h" + +using namespace js; +using namespace jit; + +using mozilla::FloorLog2; + +SafepointWriter::SafepointWriter(uint32_t slotCount, uint32_t argumentCount) + : frameSlots_((slotCount / sizeof(intptr_t)) + 1), // Stack slot counts are inclusive. + argumentSlots_(argumentCount / sizeof(intptr_t)) +{ } + +bool +SafepointWriter::init(TempAllocator& alloc) +{ + return frameSlots_.init(alloc) && argumentSlots_.init(alloc); +} + +uint32_t +SafepointWriter::startEntry() +{ + JitSpew(JitSpew_Safepoints, "Encoding safepoint (position %" PRIuSIZE "):", stream_.length()); + return uint32_t(stream_.length()); +} + +void +SafepointWriter::writeOsiCallPointOffset(uint32_t osiCallPointOffset) +{ + stream_.writeUnsigned(osiCallPointOffset); +} + +static void +WriteRegisterMask(CompactBufferWriter& stream, uint32_t bits) +{ + if (sizeof(PackedRegisterMask) == 1) + stream.writeByte(bits); + else + stream.writeUnsigned(bits); +} + +static int32_t +ReadRegisterMask(CompactBufferReader& stream) +{ + if (sizeof(PackedRegisterMask) == 1) + return stream.readByte(); + return stream.readUnsigned(); +} + +static void +WriteFloatRegisterMask(CompactBufferWriter& stream, uint64_t bits) +{ + if (sizeof(FloatRegisters::SetType) == 1) { + stream.writeByte(bits); + } else if (sizeof(FloatRegisters::SetType) == 4) { + stream.writeUnsigned(bits); + } else { + MOZ_ASSERT(sizeof(FloatRegisters::SetType) == 8); + stream.writeUnsigned(bits & 0xffffffff); + stream.writeUnsigned(bits >> 32); + } +} + +static int64_t +ReadFloatRegisterMask(CompactBufferReader& stream) +{ + if (sizeof(FloatRegisters::SetType) == 1) + return stream.readByte(); + if (sizeof(FloatRegisters::SetType) <= 4) + return stream.readUnsigned(); + MOZ_ASSERT(sizeof(FloatRegisters::SetType) == 8); + uint64_t ret = stream.readUnsigned(); + ret |= uint64_t(stream.readUnsigned()) << 32; + return ret; +} + +void +SafepointWriter::writeGcRegs(LSafepoint* safepoint) +{ + LiveGeneralRegisterSet gc(safepoint->gcRegs()); + LiveGeneralRegisterSet spilledGpr(safepoint->liveRegs().gprs()); + LiveFloatRegisterSet spilledFloat(safepoint->liveRegs().fpus()); + LiveGeneralRegisterSet slots(safepoint->slotsOrElementsRegs()); + LiveGeneralRegisterSet valueRegs; + + WriteRegisterMask(stream_, spilledGpr.bits()); + if (!spilledGpr.empty()) { + WriteRegisterMask(stream_, gc.bits()); + WriteRegisterMask(stream_, slots.bits()); + +#ifdef JS_PUNBOX64 + valueRegs = safepoint->valueRegs(); + WriteRegisterMask(stream_, valueRegs.bits()); +#endif + } + + // GC registers are a subset of the spilled registers. + MOZ_ASSERT((valueRegs.bits() & ~spilledGpr.bits()) == 0); + MOZ_ASSERT((gc.bits() & ~spilledGpr.bits()) == 0); + + WriteFloatRegisterMask(stream_, spilledFloat.bits()); + +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_Safepoints)) { + for (GeneralRegisterForwardIterator iter(spilledGpr); iter.more(); ++iter) { + const char* type = gc.has(*iter) + ? "gc" + : slots.has(*iter) + ? "slots" + : valueRegs.has(*iter) + ? "value" + : "any"; + JitSpew(JitSpew_Safepoints, " %s reg: %s", type, (*iter).name()); + } + for (FloatRegisterForwardIterator iter(spilledFloat); iter.more(); ++iter) + JitSpew(JitSpew_Safepoints, " float reg: %s", (*iter).name()); + } +#endif +} + +static void +WriteBitset(const BitSet& set, CompactBufferWriter& stream) +{ + size_t count = set.rawLength(); + const uint32_t* words = set.raw(); + for (size_t i = 0; i < count; i++) + stream.writeUnsigned(words[i]); +} + +static void +MapSlotsToBitset(BitSet& stackSet, BitSet& argumentSet, + CompactBufferWriter& stream, const LSafepoint::SlotList& slots) +{ + stackSet.clear(); + argumentSet.clear(); + + for (uint32_t i = 0; i < slots.length(); i++) { + // Slots are represented at a distance from |fp|. We divide by the + // pointer size, since we only care about pointer-sized/aligned slots + // here. + MOZ_ASSERT(slots[i].slot % sizeof(intptr_t) == 0); + size_t index = slots[i].slot / sizeof(intptr_t); + (slots[i].stack ? stackSet : argumentSet).insert(index); + } + + WriteBitset(stackSet, stream); + WriteBitset(argumentSet, stream); +} + +void +SafepointWriter::writeGcSlots(LSafepoint* safepoint) +{ + LSafepoint::SlotList& slots = safepoint->gcSlots(); + +#ifdef JS_JITSPEW + for (uint32_t i = 0; i < slots.length(); i++) + JitSpew(JitSpew_Safepoints, " gc slot: %u", slots[i].slot); +#endif + + MapSlotsToBitset(frameSlots_, argumentSlots_, stream_, slots); +} + +void +SafepointWriter::writeSlotsOrElementsSlots(LSafepoint* safepoint) +{ + LSafepoint::SlotList& slots = safepoint->slotsOrElementsSlots(); + + stream_.writeUnsigned(slots.length()); + + for (uint32_t i = 0; i < slots.length(); i++) { + if (!slots[i].stack) + MOZ_CRASH(); +#ifdef JS_JITSPEW + JitSpew(JitSpew_Safepoints, " slots/elements slot: %d", slots[i].slot); +#endif + stream_.writeUnsigned(slots[i].slot); + } +} + +void +SafepointWriter::writeValueSlots(LSafepoint* safepoint) +{ + LSafepoint::SlotList& slots = safepoint->valueSlots(); + +#ifdef JS_JITSPEW + for (uint32_t i = 0; i < slots.length(); i++) + JitSpew(JitSpew_Safepoints, " gc value: %u", slots[i].slot); +#endif + + MapSlotsToBitset(frameSlots_, argumentSlots_, stream_, slots); +} + +#if defined(JS_JITSPEW) && defined(JS_NUNBOX32) +static void +DumpNunboxPart(const LAllocation& a) +{ + Fprinter& out = JitSpewPrinter(); + if (a.isStackSlot()) { + out.printf("stack %d", a.toStackSlot()->slot()); + } else if (a.isArgument()) { + out.printf("arg %d", a.toArgument()->index()); + } else { + out.printf("reg %s", a.toGeneralReg()->reg().name()); + } +} +#endif // DEBUG + +// Nunbox part encoding: +// +// Reg = 000 +// Stack = 001 +// Arg = 010 +// +// [vwu] nentries: +// uint16_t: tttp ppXX XXXY YYYY +// +// If ttt = Reg, type is reg XXXXX +// If ppp = Reg, payload is reg YYYYY +// +// If ttt != Reg, type is: +// XXXXX if not 11111, otherwise followed by [vwu] +// If ppp != Reg, payload is: +// YYYYY if not 11111, otherwise followed by [vwu] +// +enum NunboxPartKind { + Part_Reg, + Part_Stack, + Part_Arg +}; + +static const uint32_t PART_KIND_BITS = 3; +static const uint32_t PART_KIND_MASK = (1 << PART_KIND_BITS) - 1; +static const uint32_t PART_INFO_BITS = 5; +static const uint32_t PART_INFO_MASK = (1 << PART_INFO_BITS) - 1; + +static const uint32_t MAX_INFO_VALUE = (1 << PART_INFO_BITS) - 1; +static const uint32_t TYPE_KIND_SHIFT = 16 - PART_KIND_BITS; +static const uint32_t PAYLOAD_KIND_SHIFT = TYPE_KIND_SHIFT - PART_KIND_BITS; +static const uint32_t TYPE_INFO_SHIFT = PAYLOAD_KIND_SHIFT - PART_INFO_BITS; +static const uint32_t PAYLOAD_INFO_SHIFT = TYPE_INFO_SHIFT - PART_INFO_BITS; + +JS_STATIC_ASSERT(PAYLOAD_INFO_SHIFT == 0); + +#ifdef JS_NUNBOX32 +static inline NunboxPartKind +AllocationToPartKind(const LAllocation& a) +{ + if (a.isRegister()) + return Part_Reg; + if (a.isStackSlot()) + return Part_Stack; + MOZ_ASSERT(a.isArgument()); + return Part_Arg; +} + +// gcc 4.5 doesn't actually inline CanEncodeInfoInHeader when only +// using the "inline" keyword, and miscompiles the function as well +// when doing block reordering with branch prediction information. +// See bug 799295 comment 71. +static MOZ_ALWAYS_INLINE bool +CanEncodeInfoInHeader(const LAllocation& a, uint32_t* out) +{ + if (a.isGeneralReg()) { + *out = a.toGeneralReg()->reg().code(); + return true; + } + + if (a.isStackSlot()) + *out = a.toStackSlot()->slot(); + else + *out = a.toArgument()->index(); + + return *out < MAX_INFO_VALUE; +} + +void +SafepointWriter::writeNunboxParts(LSafepoint* safepoint) +{ + LSafepoint::NunboxList& entries = safepoint->nunboxParts(); + +# ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_Safepoints)) { + for (uint32_t i = 0; i < entries.length(); i++) { + SafepointNunboxEntry& entry = entries[i]; + if (entry.type.isUse() || entry.payload.isUse()) + continue; + JitSpewHeader(JitSpew_Safepoints); + Fprinter& out = JitSpewPrinter(); + out.printf(" nunbox (type in "); + DumpNunboxPart(entry.type); + out.printf(", payload in "); + DumpNunboxPart(entry.payload); + out.printf(")\n"); + } + } +# endif + + // Safepoints are permitted to have partially filled in entries for nunboxes, + // provided that only the type is live and not the payload. Omit these from + // the written safepoint. + + size_t pos = stream_.length(); + stream_.writeUnsigned(entries.length()); + + size_t count = 0; + for (size_t i = 0; i < entries.length(); i++) { + SafepointNunboxEntry& entry = entries[i]; + + if (entry.payload.isUse()) { + // No allocation associated with the payload. + continue; + } + + if (entry.type.isUse()) { + // No allocation associated with the type. Look for another + // safepoint entry with an allocation for the type. + entry.type = safepoint->findTypeAllocation(entry.typeVreg); + if (entry.type.isUse()) + continue; + } + + count++; + + uint16_t header = 0; + + header |= (AllocationToPartKind(entry.type) << TYPE_KIND_SHIFT); + header |= (AllocationToPartKind(entry.payload) << PAYLOAD_KIND_SHIFT); + + uint32_t typeVal; + bool typeExtra = !CanEncodeInfoInHeader(entry.type, &typeVal); + if (!typeExtra) + header |= (typeVal << TYPE_INFO_SHIFT); + else + header |= (MAX_INFO_VALUE << TYPE_INFO_SHIFT); + + uint32_t payloadVal; + bool payloadExtra = !CanEncodeInfoInHeader(entry.payload, &payloadVal); + if (!payloadExtra) + header |= (payloadVal << PAYLOAD_INFO_SHIFT); + else + header |= (MAX_INFO_VALUE << PAYLOAD_INFO_SHIFT); + + stream_.writeFixedUint16_t(header); + if (typeExtra) + stream_.writeUnsigned(typeVal); + if (payloadExtra) + stream_.writeUnsigned(payloadVal); + } + + // Update the stream with the actual number of safepoint entries written. + stream_.writeUnsignedAt(pos, count, entries.length()); +} +#endif + +void +SafepointWriter::encode(LSafepoint* safepoint) +{ + uint32_t safepointOffset = startEntry(); + + MOZ_ASSERT(safepoint->osiCallPointOffset()); + + writeOsiCallPointOffset(safepoint->osiCallPointOffset()); + writeGcRegs(safepoint); + writeGcSlots(safepoint); + writeValueSlots(safepoint); + +#ifdef JS_NUNBOX32 + writeNunboxParts(safepoint); +#endif + + writeSlotsOrElementsSlots(safepoint); + + endEntry(); + safepoint->setOffset(safepointOffset); +} + +void +SafepointWriter::endEntry() +{ + JitSpew(JitSpew_Safepoints, " -- entry ended at %d", uint32_t(stream_.length())); +} + +SafepointReader::SafepointReader(IonScript* script, const SafepointIndex* si) + : stream_(script->safepoints() + si->safepointOffset(), + script->safepoints() + script->safepointsSize()), + frameSlots_((script->frameSlots() / sizeof(intptr_t)) + 1), // Stack slot counts are inclusive. + argumentSlots_(script->argumentSlots() / sizeof(intptr_t)) +{ + osiCallPointOffset_ = stream_.readUnsigned(); + + // gcSpills is a subset of allGprSpills. + allGprSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_)); + if (allGprSpills_.empty()) { + gcSpills_ = allGprSpills_; + valueSpills_ = allGprSpills_; + slotsOrElementsSpills_ = allGprSpills_; + } else { + gcSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_)); + slotsOrElementsSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_)); +#ifdef JS_PUNBOX64 + valueSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_)); +#endif + } + allFloatSpills_ = FloatRegisterSet(ReadFloatRegisterMask(stream_)); + + advanceFromGcRegs(); +} + +uint32_t +SafepointReader::osiReturnPointOffset() const +{ + return osiCallPointOffset_ + Assembler::PatchWrite_NearCallSize(); +} + +CodeLocationLabel +SafepointReader::InvalidationPatchPoint(IonScript* script, const SafepointIndex* si) +{ + SafepointReader reader(script, si); + + return CodeLocationLabel(script->method(), CodeOffset(reader.osiCallPointOffset())); +} + +void +SafepointReader::advanceFromGcRegs() +{ + currentSlotChunk_ = 0; + nextSlotChunkNumber_ = 0; + currentSlotsAreStack_ = true; +} + +bool +SafepointReader::getSlotFromBitmap(SafepointSlotEntry* entry) +{ + while (currentSlotChunk_ == 0) { + // Are there any more chunks to read? + if (currentSlotsAreStack_) { + if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(frameSlots_)) { + nextSlotChunkNumber_ = 0; + currentSlotsAreStack_ = false; + continue; + } + } else if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(argumentSlots_)) { + return false; + } + + // Yes, read the next chunk. + currentSlotChunk_ = stream_.readUnsigned(); + nextSlotChunkNumber_++; + } + + // The current chunk still has bits in it, so get the next bit, then mask + // it out of the slot chunk. + uint32_t bit = FloorLog2(currentSlotChunk_); + currentSlotChunk_ &= ~(1 << bit); + + // Return the slot, and re-scale it by the pointer size, reversing the + // transformation in MapSlotsToBitset. + entry->stack = currentSlotsAreStack_; + entry->slot = (((nextSlotChunkNumber_ - 1) * BitSet::BitsPerWord) + bit) * sizeof(intptr_t); + return true; +} + +bool +SafepointReader::getGcSlot(SafepointSlotEntry* entry) +{ + if (getSlotFromBitmap(entry)) + return true; + advanceFromGcSlots(); + return false; +} + +void +SafepointReader::advanceFromGcSlots() +{ + // No, reset the counter. + currentSlotChunk_ = 0; + nextSlotChunkNumber_ = 0; + currentSlotsAreStack_ = true; +} + +bool +SafepointReader::getValueSlot(SafepointSlotEntry* entry) +{ + if (getSlotFromBitmap(entry)) + return true; + advanceFromValueSlots(); + return false; +} + +void +SafepointReader::advanceFromValueSlots() +{ +#ifdef JS_NUNBOX32 + nunboxSlotsRemaining_ = stream_.readUnsigned(); +#else + nunboxSlotsRemaining_ = 0; + advanceFromNunboxSlots(); +#endif +} + +static inline LAllocation +PartFromStream(CompactBufferReader& stream, NunboxPartKind kind, uint32_t info) +{ + if (kind == Part_Reg) + return LGeneralReg(Register::FromCode(info)); + + if (info == MAX_INFO_VALUE) + info = stream.readUnsigned(); + + if (kind == Part_Stack) + return LStackSlot(info); + + MOZ_ASSERT(kind == Part_Arg); + return LArgument(info); +} + +bool +SafepointReader::getNunboxSlot(LAllocation* type, LAllocation* payload) +{ + if (!nunboxSlotsRemaining_--) { + advanceFromNunboxSlots(); + return false; + } + + uint16_t header = stream_.readFixedUint16_t(); + NunboxPartKind typeKind = (NunboxPartKind)((header >> TYPE_KIND_SHIFT) & PART_KIND_MASK); + NunboxPartKind payloadKind = (NunboxPartKind)((header >> PAYLOAD_KIND_SHIFT) & PART_KIND_MASK); + uint32_t typeInfo = (header >> TYPE_INFO_SHIFT) & PART_INFO_MASK; + uint32_t payloadInfo = (header >> PAYLOAD_INFO_SHIFT) & PART_INFO_MASK; + + *type = PartFromStream(stream_, typeKind, typeInfo); + *payload = PartFromStream(stream_, payloadKind, payloadInfo); + return true; +} + +void +SafepointReader::advanceFromNunboxSlots() +{ + slotsOrElementsSlotsRemaining_ = stream_.readUnsigned(); +} + +bool +SafepointReader::getSlotsOrElementsSlot(SafepointSlotEntry* entry) +{ + if (!slotsOrElementsSlotsRemaining_--) + return false; + entry->stack = true; + entry->slot = stream_.readUnsigned(); + return true; +} |