diff options
Diffstat (limited to 'js/src/jit/JitFrames.cpp')
| -rw-r--r-- | js/src/jit/JitFrames.cpp | 3158 |
1 files changed, 3158 insertions, 0 deletions
diff --git a/js/src/jit/JitFrames.cpp b/js/src/jit/JitFrames.cpp new file mode 100644 index 000000000..646442b4c --- /dev/null +++ b/js/src/jit/JitFrames.cpp @@ -0,0 +1,3158 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/JitFrames-inl.h" + +#include "mozilla/SizePrintfMacros.h" + +#include "jsfun.h" +#include "jsobj.h" +#include "jsscript.h" +#include "jsutil.h" + +#include "gc/Marking.h" +#include "jit/BaselineDebugModeOSR.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/Ion.h" +#include "jit/JitcodeMap.h" +#include "jit/JitCompartment.h" +#include "jit/JitSpewer.h" +#include "jit/MacroAssembler.h" +#include "jit/PcScriptCache.h" +#include "jit/Recover.h" +#include "jit/Safepoints.h" +#include "jit/Snapshots.h" +#include "jit/VMFunctions.h" +#include "vm/ArgumentsObject.h" +#include "vm/Debugger.h" +#include "vm/Interpreter.h" +#include "vm/SPSProfiler.h" +#include "vm/TraceLogging.h" +#include "vm/TypeInference.h" + +#include "jsscriptinlines.h" +#include "gc/Nursery-inl.h" +#include "jit/JitFrameIterator-inl.h" +#include "vm/Debugger-inl.h" +#include "vm/Probes-inl.h" +#include "vm/TypeInference-inl.h" + +namespace js { +namespace jit { + +// Given a slot index, returns the offset, in bytes, of that slot from an +// JitFrameLayout. Slot distances are uniform across architectures, however, +// the distance does depend on the size of the frame header. +static inline int32_t +OffsetOfFrameSlot(int32_t slot) +{ + return -slot; +} + +static inline uint8_t* +AddressOfFrameSlot(JitFrameLayout* fp, int32_t slot) +{ + return (uint8_t*) fp + OffsetOfFrameSlot(slot); +} + +static inline uintptr_t +ReadFrameSlot(JitFrameLayout* fp, int32_t slot) +{ + return *(uintptr_t*) AddressOfFrameSlot(fp, slot); +} + +static inline void +WriteFrameSlot(JitFrameLayout* fp, int32_t slot, uintptr_t value) +{ + *(uintptr_t*) AddressOfFrameSlot(fp, slot) = value; +} + +static inline double +ReadFrameDoubleSlot(JitFrameLayout* fp, int32_t slot) +{ + return *(double*) AddressOfFrameSlot(fp, slot); +} + +static inline float +ReadFrameFloat32Slot(JitFrameLayout* fp, int32_t slot) +{ + return *(float*) AddressOfFrameSlot(fp, slot); +} + +static inline int32_t +ReadFrameInt32Slot(JitFrameLayout* fp, int32_t slot) +{ + return *(int32_t*) AddressOfFrameSlot(fp, slot); +} + +static inline bool +ReadFrameBooleanSlot(JitFrameLayout* fp, int32_t slot) +{ + return *(bool*) AddressOfFrameSlot(fp, slot); +} + +JitFrameIterator::JitFrameIterator() + : current_(nullptr), + type_(JitFrame_Exit), + returnAddressToFp_(nullptr), + frameSize_(0), + cachedSafepointIndex_(nullptr), + activation_(nullptr) +{ +} + +JitFrameIterator::JitFrameIterator(JSContext* cx) + : current_(cx->runtime()->jitTop), + type_(JitFrame_Exit), + returnAddressToFp_(nullptr), + frameSize_(0), + cachedSafepointIndex_(nullptr), + activation_(cx->runtime()->activation()->asJit()) +{ + if (activation_->bailoutData()) { + current_ = activation_->bailoutData()->fp(); + frameSize_ = activation_->bailoutData()->topFrameSize(); + type_ = JitFrame_Bailout; + } +} + +JitFrameIterator::JitFrameIterator(const ActivationIterator& activations) + : current_(activations.jitTop()), + type_(JitFrame_Exit), + returnAddressToFp_(nullptr), + frameSize_(0), + cachedSafepointIndex_(nullptr), + activation_(activations->asJit()) +{ + if (activation_->bailoutData()) { + current_ = activation_->bailoutData()->fp(); + frameSize_ = activation_->bailoutData()->topFrameSize(); + type_ = JitFrame_Bailout; + } +} + +bool +JitFrameIterator::checkInvalidation() const +{ + IonScript* dummy; + return checkInvalidation(&dummy); +} + +bool +JitFrameIterator::checkInvalidation(IonScript** ionScriptOut) const +{ + JSScript* script = this->script(); + if (isBailoutJS()) { + *ionScriptOut = activation_->bailoutData()->ionScript(); + return !script->hasIonScript() || script->ionScript() != *ionScriptOut; + } + + uint8_t* returnAddr = returnAddressToFp(); + // N.B. the current IonScript is not the same as the frame's + // IonScript if the frame has since been invalidated. + bool invalidated = !script->hasIonScript() || + !script->ionScript()->containsReturnAddress(returnAddr); + if (!invalidated) + return false; + + int32_t invalidationDataOffset = ((int32_t*) returnAddr)[-1]; + uint8_t* ionScriptDataOffset = returnAddr + invalidationDataOffset; + IonScript* ionScript = (IonScript*) Assembler::GetPointer(ionScriptDataOffset); + MOZ_ASSERT(ionScript->containsReturnAddress(returnAddr)); + *ionScriptOut = ionScript; + return true; +} + +CalleeToken +JitFrameIterator::calleeToken() const +{ + return ((JitFrameLayout*) current_)->calleeToken(); +} + +JSFunction* +JitFrameIterator::callee() const +{ + MOZ_ASSERT(isScripted()); + MOZ_ASSERT(isFunctionFrame()); + return CalleeTokenToFunction(calleeToken()); +} + +JSFunction* +JitFrameIterator::maybeCallee() const +{ + if (isScripted() && (isFunctionFrame())) + return callee(); + return nullptr; +} + +bool +JitFrameIterator::isBareExit() const +{ + if (type_ != JitFrame_Exit) + return false; + return exitFrame()->isBareExit(); +} + +bool +JitFrameIterator::isFunctionFrame() const +{ + return CalleeTokenIsFunction(calleeToken()); +} + +JSScript* +JitFrameIterator::script() const +{ + MOZ_ASSERT(isScripted()); + if (isBaselineJS()) + return baselineFrame()->script(); + JSScript* script = ScriptFromCalleeToken(calleeToken()); + MOZ_ASSERT(script); + return script; +} + +void +JitFrameIterator::baselineScriptAndPc(JSScript** scriptRes, jsbytecode** pcRes) const +{ + MOZ_ASSERT(isBaselineJS()); + JSScript* script = this->script(); + if (scriptRes) + *scriptRes = script; + + MOZ_ASSERT(pcRes); + + // Use the frame's override pc, if we have one. This should only happen + // when we're in FinishBailoutToBaseline, handling an exception or toggling + // debug mode. + if (jsbytecode* overridePc = baselineFrame()->maybeOverridePc()) { + *pcRes = overridePc; + return; + } + + // Else, there must be an ICEntry for the current return address. + uint8_t* retAddr = returnAddressToFp(); + ICEntry& icEntry = script->baselineScript()->icEntryFromReturnAddress(retAddr); + *pcRes = icEntry.pc(script); +} + +Value* +JitFrameIterator::actualArgs() const +{ + return jsFrame()->argv() + 1; +} + +uint8_t* +JitFrameIterator::prevFp() const +{ + return current_ + current()->prevFrameLocalSize() + current()->headerSize(); +} + +JitFrameIterator& +JitFrameIterator::operator++() +{ + MOZ_ASSERT(type_ != JitFrame_Entry); + + frameSize_ = prevFrameLocalSize(); + cachedSafepointIndex_ = nullptr; + + // If the next frame is the entry frame, just exit. Don't update current_, + // since the entry and first frames overlap. + if (current()->prevType() == JitFrame_Entry) { + type_ = JitFrame_Entry; + return *this; + } + + type_ = current()->prevType(); + returnAddressToFp_ = current()->returnAddress(); + current_ = prevFp(); + + return *this; +} + +uintptr_t* +JitFrameIterator::spillBase() const +{ + MOZ_ASSERT(isIonJS()); + + // Get the base address to where safepoint registers are spilled. + // Out-of-line calls do not unwind the extra padding space used to + // aggregate bailout tables, so we use frameSize instead of frameLocals, + // which would only account for local stack slots. + return reinterpret_cast<uintptr_t*>(fp() - ionScript()->frameSize()); +} + +MachineState +JitFrameIterator::machineState() const +{ + MOZ_ASSERT(isIonScripted()); + + // The MachineState is used by GCs for marking call-sites. + if (MOZ_UNLIKELY(isBailoutJS())) + return *activation_->bailoutData()->machineState(); + + SafepointReader reader(ionScript(), safepoint()); + uintptr_t* spill = spillBase(); + MachineState machine; + + for (GeneralRegisterBackwardIterator iter(reader.allGprSpills()); iter.more(); ++iter) + machine.setRegisterLocation(*iter, --spill); + + uint8_t* spillAlign = alignDoubleSpillWithOffset(reinterpret_cast<uint8_t*>(spill), 0); + + char* floatSpill = reinterpret_cast<char*>(spillAlign); + FloatRegisterSet fregs = reader.allFloatSpills().set(); + fregs = fregs.reduceSetForPush(); + for (FloatRegisterBackwardIterator iter(fregs); iter.more(); ++iter) { + floatSpill -= (*iter).size(); + for (uint32_t a = 0; a < (*iter).numAlignedAliased(); a++) { + // Only say that registers that actually start here start here. + // e.g. d0 should not start at s1, only at s0. + FloatRegister ftmp; + (*iter).alignedAliased(a, &ftmp); + machine.setRegisterLocation(ftmp, (double*)floatSpill); + } + } + + return machine; +} + +static uint32_t +NumArgAndLocalSlots(const InlineFrameIterator& frame) +{ + JSScript* script = frame.script(); + return CountArgSlots(script, frame.maybeCalleeTemplate()) + script->nfixed(); +} + +static void +CloseLiveIteratorIon(JSContext* cx, const InlineFrameIterator& frame, uint32_t stackSlot) +{ + SnapshotIterator si = frame.snapshotIterator(); + + // Skip stack slots until we reach the iterator object. + uint32_t skipSlots = NumArgAndLocalSlots(frame) + stackSlot - 1; + + for (unsigned i = 0; i < skipSlots; i++) + si.skip(); + + Value v = si.read(); + RootedObject obj(cx, &v.toObject()); + + if (cx->isExceptionPending()) + UnwindIteratorForException(cx, obj); + else + UnwindIteratorForUncatchableException(cx, obj); +} + +class IonFrameStackDepthOp +{ + uint32_t depth_; + + public: + explicit IonFrameStackDepthOp(const InlineFrameIterator& frame) { + uint32_t base = NumArgAndLocalSlots(frame); + SnapshotIterator si = frame.snapshotIterator(); + MOZ_ASSERT(si.numAllocations() >= base); + depth_ = si.numAllocations() - base; + } + + uint32_t operator()() { return depth_; } +}; + +class TryNoteIterIon : public TryNoteIter<IonFrameStackDepthOp> +{ + public: + TryNoteIterIon(JSContext* cx, const InlineFrameIterator& frame) + : TryNoteIter(cx, frame.script(), frame.pc(), IonFrameStackDepthOp(frame)) + { } +}; + +static void +HandleExceptionIon(JSContext* cx, const InlineFrameIterator& frame, ResumeFromException* rfe, + bool* overrecursed) +{ + if (cx->compartment()->isDebuggee()) { + // We need to bail when there is a catchable exception, and we are the + // debuggee of a Debugger with a live onExceptionUnwind hook, or if a + // Debugger has observed this frame (e.g., for onPop). + bool shouldBail = Debugger::hasLiveHook(cx->global(), Debugger::OnExceptionUnwind); + RematerializedFrame* rematFrame = nullptr; + if (!shouldBail) { + JitActivation* act = cx->runtime()->activation()->asJit(); + rematFrame = act->lookupRematerializedFrame(frame.frame().fp(), frame.frameNo()); + shouldBail = rematFrame && rematFrame->isDebuggee(); + } + + if (shouldBail) { + // If we have an exception from within Ion and the debugger is active, + // we do the following: + // + // 1. Bailout to baseline to reconstruct a baseline frame. + // 2. Resume immediately into the exception tail afterwards, and + // handle the exception again with the top frame now a baseline + // frame. + // + // An empty exception info denotes that we're propagating an Ion + // exception due to debug mode, which BailoutIonToBaseline needs to + // know. This is because we might not be able to fully reconstruct up + // to the stack depth at the snapshot, as we could've thrown in the + // middle of a call. + ExceptionBailoutInfo propagateInfo; + uint32_t retval = ExceptionHandlerBailout(cx, frame, rfe, propagateInfo, overrecursed); + if (retval == BAILOUT_RETURN_OK) + return; + } + + MOZ_ASSERT_IF(rematFrame, !Debugger::inFrameMaps(rematFrame)); + } + + RootedScript script(cx, frame.script()); + if (!script->hasTrynotes()) + return; + + for (TryNoteIterIon tni(cx, frame); !tni.done(); ++tni) { + JSTryNote* tn = *tni; + + switch (tn->kind) { + case JSTRY_FOR_IN: { + MOZ_ASSERT(JSOp(*(script->main() + tn->start + tn->length)) == JSOP_ENDITER); + MOZ_ASSERT(tn->stackDepth > 0); + + uint32_t localSlot = tn->stackDepth; + CloseLiveIteratorIon(cx, frame, localSlot); + break; + } + + case JSTRY_FOR_OF: + case JSTRY_LOOP: + break; + + case JSTRY_CATCH: + if (cx->isExceptionPending()) { + // Ion can compile try-catch, but bailing out to catch + // exceptions is slow. Reset the warm-up counter so that if we + // catch many exceptions we won't Ion-compile the script. + script->resetWarmUpCounter(); + + // Bailout at the start of the catch block. + jsbytecode* catchPC = script->main() + tn->start + tn->length; + ExceptionBailoutInfo excInfo(frame.frameNo(), catchPC, tn->stackDepth); + uint32_t retval = ExceptionHandlerBailout(cx, frame, rfe, excInfo, overrecursed); + if (retval == BAILOUT_RETURN_OK) + return; + + // Error on bailout clears pending exception. + MOZ_ASSERT(!cx->isExceptionPending()); + } + break; + + default: + MOZ_CRASH("Unexpected try note"); + } + } +} + +static void +OnLeaveBaselineFrame(JSContext* cx, const JitFrameIterator& frame, jsbytecode* pc, + ResumeFromException* rfe, bool frameOk) +{ + BaselineFrame* baselineFrame = frame.baselineFrame(); + if (jit::DebugEpilogue(cx, baselineFrame, pc, frameOk)) { + rfe->kind = ResumeFromException::RESUME_FORCED_RETURN; + rfe->framePointer = frame.fp() - BaselineFrame::FramePointerOffset; + rfe->stackPointer = reinterpret_cast<uint8_t*>(baselineFrame); + } +} + +static inline void +ForcedReturn(JSContext* cx, const JitFrameIterator& frame, jsbytecode* pc, + ResumeFromException* rfe) +{ + OnLeaveBaselineFrame(cx, frame, pc, rfe, true); +} + +static inline void +BaselineFrameAndStackPointersFromTryNote(JSTryNote* tn, const JitFrameIterator& frame, + uint8_t** framePointer, uint8_t** stackPointer) +{ + JSScript* script = frame.baselineFrame()->script(); + *framePointer = frame.fp() - BaselineFrame::FramePointerOffset; + *stackPointer = *framePointer - BaselineFrame::Size() - + (script->nfixed() + tn->stackDepth) * sizeof(Value); +} + +static void +SettleOnTryNote(JSContext* cx, JSTryNote* tn, const JitFrameIterator& frame, + EnvironmentIter& ei, ResumeFromException* rfe, jsbytecode** pc) +{ + RootedScript script(cx, frame.baselineFrame()->script()); + + // Unwind environment chain (pop block objects). + if (cx->isExceptionPending()) + UnwindEnvironment(cx, ei, UnwindEnvironmentToTryPc(script, tn)); + + // Compute base pointer and stack pointer. + BaselineFrameAndStackPointersFromTryNote(tn, frame, &rfe->framePointer, &rfe->stackPointer); + + // Compute the pc. + *pc = script->main() + tn->start + tn->length; +} + +struct AutoBaselineHandlingException +{ + BaselineFrame* frame; + AutoBaselineHandlingException(BaselineFrame* frame, jsbytecode* pc) + : frame(frame) + { + frame->setIsHandlingException(); + frame->setOverridePc(pc); + } + ~AutoBaselineHandlingException() { + frame->unsetIsHandlingException(); + frame->clearOverridePc(); + } +}; + +class BaselineFrameStackDepthOp +{ + BaselineFrame* frame_; + public: + explicit BaselineFrameStackDepthOp(BaselineFrame* frame) + : frame_(frame) + { } + uint32_t operator()() { + MOZ_ASSERT(frame_->numValueSlots() >= frame_->script()->nfixed()); + return frame_->numValueSlots() - frame_->script()->nfixed(); + } +}; + +class TryNoteIterBaseline : public TryNoteIter<BaselineFrameStackDepthOp> +{ + public: + TryNoteIterBaseline(JSContext* cx, BaselineFrame* frame, jsbytecode* pc) + : TryNoteIter(cx, frame->script(), pc, BaselineFrameStackDepthOp(frame)) + { } +}; + +// Close all live iterators on a BaselineFrame due to exception unwinding. The +// pc parameter is updated to where the envs have been unwound to. +static void +CloseLiveIteratorsBaselineForUncatchableException(JSContext* cx, const JitFrameIterator& frame, + jsbytecode* pc) +{ + for (TryNoteIterBaseline tni(cx, frame.baselineFrame(), pc); !tni.done(); ++tni) { + JSTryNote* tn = *tni; + + if (tn->kind == JSTRY_FOR_IN) { + uint8_t* framePointer; + uint8_t* stackPointer; + BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer, &stackPointer); + Value iterValue(*(Value*) stackPointer); + RootedObject iterObject(cx, &iterValue.toObject()); + UnwindIteratorForUncatchableException(cx, iterObject); + } + } +} + +static bool +ProcessTryNotesBaseline(JSContext* cx, const JitFrameIterator& frame, EnvironmentIter& ei, + ResumeFromException* rfe, jsbytecode** pc) +{ + RootedScript script(cx, frame.baselineFrame()->script()); + + for (TryNoteIterBaseline tni(cx, frame.baselineFrame(), *pc); !tni.done(); ++tni) { + JSTryNote* tn = *tni; + + MOZ_ASSERT(cx->isExceptionPending()); + switch (tn->kind) { + case JSTRY_CATCH: { + // If we're closing a legacy generator, we have to skip catch + // blocks. + if (cx->isClosingGenerator()) + continue; + + SettleOnTryNote(cx, tn, frame, ei, rfe, pc); + + // Ion can compile try-catch, but bailing out to catch + // exceptions is slow. Reset the warm-up counter so that if we + // catch many exceptions we won't Ion-compile the script. + script->resetWarmUpCounter(); + + // Resume at the start of the catch block. + rfe->kind = ResumeFromException::RESUME_CATCH; + rfe->target = script->baselineScript()->nativeCodeForPC(script, *pc); + return true; + } + + case JSTRY_FINALLY: { + SettleOnTryNote(cx, tn, frame, ei, rfe, pc); + rfe->kind = ResumeFromException::RESUME_FINALLY; + rfe->target = script->baselineScript()->nativeCodeForPC(script, *pc); + // Drop the exception instead of leaking cross compartment data. + if (!cx->getPendingException(MutableHandleValue::fromMarkedLocation(&rfe->exception))) + rfe->exception = UndefinedValue(); + cx->clearPendingException(); + return true; + } + + case JSTRY_FOR_IN: { + uint8_t* framePointer; + uint8_t* stackPointer; + BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer, &stackPointer); + Value iterValue(*(Value*) stackPointer); + RootedObject iterObject(cx, &iterValue.toObject()); + if (!UnwindIteratorForException(cx, iterObject)) { + // See comment in the JSTRY_FOR_IN case in Interpreter.cpp's + // ProcessTryNotes. + SettleOnTryNote(cx, tn, frame, ei, rfe, pc); + MOZ_ASSERT(**pc == JSOP_ENDITER); + return false; + } + break; + } + + case JSTRY_FOR_OF: + case JSTRY_LOOP: + break; + + default: + MOZ_CRASH("Invalid try note"); + } + } + return true; +} + +static void +HandleExceptionBaseline(JSContext* cx, const JitFrameIterator& frame, ResumeFromException* rfe, + jsbytecode* pc) +{ + MOZ_ASSERT(frame.isBaselineJS()); + + bool frameOk = false; + RootedScript script(cx, frame.baselineFrame()->script()); + + if (script->hasScriptCounts()) { + PCCounts* counts = script->getThrowCounts(pc); + // If we failed to allocate, then skip the increment and continue to + // handle the exception. + if (counts) + counts->numExec()++; + } + + // We may be propagating a forced return from the interrupt + // callback, which cannot easily force a return. + if (cx->isPropagatingForcedReturn()) { + cx->clearPropagatingForcedReturn(); + ForcedReturn(cx, frame, pc, rfe); + return; + } + + again: + if (cx->isExceptionPending()) { + if (!cx->isClosingGenerator()) { + switch (Debugger::onExceptionUnwind(cx, frame.baselineFrame())) { + case JSTRAP_ERROR: + // Uncatchable exception. + MOZ_ASSERT(!cx->isExceptionPending()); + goto again; + + case JSTRAP_CONTINUE: + case JSTRAP_THROW: + MOZ_ASSERT(cx->isExceptionPending()); + break; + + case JSTRAP_RETURN: + if (script->hasTrynotes()) + CloseLiveIteratorsBaselineForUncatchableException(cx, frame, pc); + ForcedReturn(cx, frame, pc, rfe); + return; + + default: + MOZ_CRASH("Invalid trap status"); + } + } + + if (script->hasTrynotes()) { + EnvironmentIter ei(cx, frame.baselineFrame(), pc); + if (!ProcessTryNotesBaseline(cx, frame, ei, rfe, &pc)) + goto again; + if (rfe->kind != ResumeFromException::RESUME_ENTRY_FRAME) { + // No need to increment the PCCounts number of execution here, + // as the interpreter increments any PCCounts if present. + MOZ_ASSERT_IF(script->hasScriptCounts(), script->maybeGetPCCounts(pc)); + return; + } + } + + frameOk = HandleClosingGeneratorReturn(cx, frame.baselineFrame(), frameOk); + frameOk = Debugger::onLeaveFrame(cx, frame.baselineFrame(), pc, frameOk); + } else if (script->hasTrynotes()) { + CloseLiveIteratorsBaselineForUncatchableException(cx, frame, pc); + } + + OnLeaveBaselineFrame(cx, frame, pc, rfe, frameOk); +} + +struct AutoDeleteDebugModeOSRInfo +{ + BaselineFrame* frame; + explicit AutoDeleteDebugModeOSRInfo(BaselineFrame* frame) : frame(frame) { MOZ_ASSERT(frame); } + ~AutoDeleteDebugModeOSRInfo() { frame->deleteDebugModeOSRInfo(); } +}; + +struct AutoResetLastProfilerFrameOnReturnFromException +{ + JSContext* cx; + ResumeFromException* rfe; + + AutoResetLastProfilerFrameOnReturnFromException(JSContext* cx, ResumeFromException* rfe) + : cx(cx), rfe(rfe) {} + + ~AutoResetLastProfilerFrameOnReturnFromException() { + if (!cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + return; + + MOZ_ASSERT(cx->runtime()->jitActivation == cx->runtime()->profilingActivation()); + + void* lastProfilingFrame = getLastProfilingFrame(); + cx->runtime()->jitActivation->setLastProfilingFrame(lastProfilingFrame); + } + + void* getLastProfilingFrame() { + switch (rfe->kind) { + case ResumeFromException::RESUME_ENTRY_FRAME: + return nullptr; + + // The following all return into baseline frames. + case ResumeFromException::RESUME_CATCH: + case ResumeFromException::RESUME_FINALLY: + case ResumeFromException::RESUME_FORCED_RETURN: + return rfe->framePointer + BaselineFrame::FramePointerOffset; + + // When resuming into a bailed-out ion frame, use the bailout info to + // find the frame we are resuming into. + case ResumeFromException::RESUME_BAILOUT: + return rfe->bailoutInfo->incomingStack; + } + + MOZ_CRASH("Invalid ResumeFromException type!"); + return nullptr; + } +}; + +void +HandleException(ResumeFromException* rfe) +{ + JSContext* cx = GetJSContextFromMainThread(); + TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime()); + + AutoResetLastProfilerFrameOnReturnFromException profFrameReset(cx, rfe); + + rfe->kind = ResumeFromException::RESUME_ENTRY_FRAME; + + JitSpew(JitSpew_IonInvalidate, "handling exception"); + + // Clear any Ion return override that's been set. + // This may happen if a callVM function causes an invalidation (setting the + // override), and then fails, bypassing the bailout handlers that would + // otherwise clear the return override. + if (cx->runtime()->jitRuntime()->hasIonReturnOverride()) + cx->runtime()->jitRuntime()->takeIonReturnOverride(); + + JitActivation* activation = cx->runtime()->activation()->asJit(); + +#ifdef CHECK_OSIPOINT_REGISTERS + if (JitOptions.checkOsiPointRegisters) + activation->setCheckRegs(false); +#endif + + // The Debugger onExceptionUnwind hook (reachable via + // HandleExceptionBaseline below) may cause on-stack recompilation of + // baseline scripts, which may patch return addresses on the stack. Since + // JitFrameIterators cache the previous frame's return address when + // iterating, we need a variant here that is automatically updated should + // on-stack recompilation occur. + DebugModeOSRVolatileJitFrameIterator iter(cx); + while (!iter.isEntry()) { + bool overrecursed = false; + if (iter.isIonJS()) { + // Search each inlined frame for live iterator objects, and close + // them. + InlineFrameIterator frames(cx, &iter); + + // Invalidation state will be the same for all inlined scripts in the frame. + IonScript* ionScript = nullptr; + bool invalidated = iter.checkInvalidation(&ionScript); + +#ifdef JS_TRACE_LOGGING + if (logger && cx->compartment()->isDebuggee() && logger->enabled()) { + logger->disable(/* force = */ true, + "Forcefully disabled tracelogger, due to " + "throwing an exception with an active Debugger " + "in IonMonkey."); + } +#endif + + for (;;) { + HandleExceptionIon(cx, frames, rfe, &overrecursed); + + if (rfe->kind == ResumeFromException::RESUME_BAILOUT) { + if (invalidated) + ionScript->decrementInvalidationCount(cx->runtime()->defaultFreeOp()); + return; + } + + MOZ_ASSERT(rfe->kind == ResumeFromException::RESUME_ENTRY_FRAME); + + // When profiling, each frame popped needs a notification that + // the function has exited, so invoke the probe that a function + // is exiting. + + JSScript* script = frames.script(); + probes::ExitScript(cx, script, script->functionNonDelazifying(), + /* popSPSFrame = */ false); + if (!frames.more()) { + TraceLogStopEvent(logger, TraceLogger_IonMonkey); + TraceLogStopEvent(logger, TraceLogger_Scripts); + break; + } + ++frames; + } + + activation->removeIonFrameRecovery(iter.jsFrame()); + if (invalidated) + ionScript->decrementInvalidationCount(cx->runtime()->defaultFreeOp()); + + } else if (iter.isBaselineJS()) { + // Set a flag on the frame to signal to DebugModeOSR that we're + // handling an exception. Also ensure the frame has an override + // pc. We clear the frame's override pc when we leave this block, + // this is fine because we're either: + // + // (1) Going to enter a catch or finally block. We don't want to + // keep the old pc when we're executing JIT code. + // (2) Going to pop the frame, either here or a forced return. + // In this case nothing will observe the frame's pc. + // (3) Performing an exception bailout. In this case + // FinishBailoutToBaseline will set the pc to the resume pc + // and clear it before it returns to JIT code. + jsbytecode* pc; + iter.baselineScriptAndPc(nullptr, &pc); + AutoBaselineHandlingException handlingException(iter.baselineFrame(), pc); + + HandleExceptionBaseline(cx, iter, rfe, pc); + + // If we are propagating an exception through a frame with + // on-stack recompile info, we should free the allocated + // RecompileInfo struct before we leave this block, as we will not + // be returning to the recompile handler. + AutoDeleteDebugModeOSRInfo deleteDebugModeOSRInfo(iter.baselineFrame()); + + if (rfe->kind != ResumeFromException::RESUME_ENTRY_FRAME && + rfe->kind != ResumeFromException::RESUME_FORCED_RETURN) + { + return; + } + + TraceLogStopEvent(logger, TraceLogger_Baseline); + TraceLogStopEvent(logger, TraceLogger_Scripts); + + // Unwind profiler pseudo-stack + JSScript* script = iter.script(); + probes::ExitScript(cx, script, script->functionNonDelazifying(), + /* popSPSFrame = */ false); + + if (rfe->kind == ResumeFromException::RESUME_FORCED_RETURN) + return; + } + + JitFrameLayout* current = iter.isScripted() ? iter.jsFrame() : nullptr; + + ++iter; + + if (current) { + // Unwind the frame by updating jitTop. This is necessary so that + // (1) debugger exception unwind and leave frame hooks don't see this + // frame when they use ScriptFrameIter, and (2) ScriptFrameIter does + // not crash when accessing an IonScript that's destroyed by the + // ionScript->decref call. + EnsureBareExitFrame(cx, current); + } + + if (overrecursed) { + // We hit an overrecursion error during bailout. Report it now. + ReportOverRecursed(cx); + } + } + + rfe->stackPointer = iter.fp(); +} + +// Turns a JitFrameLayout into an ExitFrameLayout. Note that it has to be a +// bare exit frame so it's ignored by MarkJitExitFrame. +void +EnsureBareExitFrame(JSContext* cx, JitFrameLayout* frame) +{ + ExitFrameLayout* exitFrame = reinterpret_cast<ExitFrameLayout*>(frame); + + if (cx->runtime()->jitTop == (uint8_t*)frame) { + // If we already called this function for the current frame, do + // nothing. + MOZ_ASSERT(exitFrame->isBareExit()); + return; + } + +#ifdef DEBUG + JitFrameIterator iter(cx); + while (!iter.isScripted()) + ++iter; + MOZ_ASSERT(iter.current() == frame, "|frame| must be the top JS frame"); + + MOZ_ASSERT((uint8_t*)exitFrame->footer() >= cx->runtime()->jitTop, + "Must have space for ExitFooterFrame before jitTop"); +#endif + + cx->runtime()->jitTop = (uint8_t*)frame; + *exitFrame->footer()->addressOfJitCode() = ExitFrameLayout::BareToken(); + MOZ_ASSERT(exitFrame->isBareExit()); +} + +CalleeToken +MarkCalleeToken(JSTracer* trc, CalleeToken token) +{ + switch (CalleeTokenTag tag = GetCalleeTokenTag(token)) { + case CalleeToken_Function: + case CalleeToken_FunctionConstructing: + { + JSFunction* fun = CalleeTokenToFunction(token); + TraceRoot(trc, &fun, "jit-callee"); + return CalleeToToken(fun, tag == CalleeToken_FunctionConstructing); + } + case CalleeToken_Script: + { + JSScript* script = CalleeTokenToScript(token); + TraceRoot(trc, &script, "jit-script"); + return CalleeToToken(script); + } + default: + MOZ_CRASH("unknown callee token type"); + } +} + +uintptr_t* +JitFrameLayout::slotRef(SafepointSlotEntry where) +{ + if (where.stack) + return (uintptr_t*)((uint8_t*)this - where.slot); + return (uintptr_t*)((uint8_t*)argv() + where.slot); +} + +#ifdef JS_NUNBOX32 +static inline uintptr_t +ReadAllocation(const JitFrameIterator& frame, const LAllocation* a) +{ + if (a->isGeneralReg()) { + Register reg = a->toGeneralReg()->reg(); + return frame.machineState().read(reg); + } + return *frame.jsFrame()->slotRef(SafepointSlotEntry(a)); +} +#endif + +static void +MarkThisAndArguments(JSTracer* trc, const JitFrameIterator& frame) +{ + // Mark |this| and any extra actual arguments for an Ion frame. Marking of + // formal arguments is taken care of by the frame's safepoint/snapshot, + // except when the script might have lazy arguments or rest, in which case + // we mark them as well. We also have to mark formals if we have a LazyLink + // frame. + + JitFrameLayout* layout = frame.isExitFrameLayout<LazyLinkExitFrameLayout>() + ? frame.exitFrame()->as<LazyLinkExitFrameLayout>()->jsFrame() + : frame.jsFrame(); + + if (!CalleeTokenIsFunction(layout->calleeToken())) + return; + + size_t nargs = layout->numActualArgs(); + size_t nformals = 0; + + JSFunction* fun = CalleeTokenToFunction(layout->calleeToken()); + if (!frame.isExitFrameLayout<LazyLinkExitFrameLayout>() && + !fun->nonLazyScript()->mayReadFrameArgsDirectly()) + { + nformals = fun->nargs(); + } + + size_t newTargetOffset = Max(nargs, fun->nargs()); + + Value* argv = layout->argv(); + + // Trace |this|. + TraceRoot(trc, argv, "ion-thisv"); + + // Trace actual arguments beyond the formals. Note + 1 for thisv. + for (size_t i = nformals + 1; i < nargs + 1; i++) + TraceRoot(trc, &argv[i], "ion-argv"); + + // Always mark the new.target from the frame. It's not in the snapshots. + // +1 to pass |this| + if (CalleeTokenIsConstructing(layout->calleeToken())) + TraceRoot(trc, &argv[1 + newTargetOffset], "ion-newTarget"); +} + +#ifdef JS_NUNBOX32 +static inline void +WriteAllocation(const JitFrameIterator& frame, const LAllocation* a, uintptr_t value) +{ + if (a->isGeneralReg()) { + Register reg = a->toGeneralReg()->reg(); + frame.machineState().write(reg, value); + } else { + *frame.jsFrame()->slotRef(SafepointSlotEntry(a)) = value; + } +} +#endif + +static void +MarkIonJSFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + JitFrameLayout* layout = (JitFrameLayout*)frame.fp(); + + layout->replaceCalleeToken(MarkCalleeToken(trc, layout->calleeToken())); + + IonScript* ionScript = nullptr; + if (frame.checkInvalidation(&ionScript)) { + // This frame has been invalidated, meaning that its IonScript is no + // longer reachable through the callee token (JSFunction/JSScript->ion + // is now nullptr or recompiled). Manually trace it here. + IonScript::Trace(trc, ionScript); + } else { + ionScript = frame.ionScriptFromCalleeToken(); + } + + MarkThisAndArguments(trc, frame); + + const SafepointIndex* si = ionScript->getSafepointIndex(frame.returnAddressToFp()); + + SafepointReader safepoint(ionScript, si); + + // Scan through slots which contain pointers (or on punboxing systems, + // actual values). + SafepointSlotEntry entry; + + while (safepoint.getGcSlot(&entry)) { + uintptr_t* ref = layout->slotRef(entry); + TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(ref), "ion-gc-slot"); + } + + while (safepoint.getValueSlot(&entry)) { + Value* v = (Value*)layout->slotRef(entry); + TraceRoot(trc, v, "ion-gc-slot"); + } + + uintptr_t* spill = frame.spillBase(); + LiveGeneralRegisterSet gcRegs = safepoint.gcSpills(); + LiveGeneralRegisterSet valueRegs = safepoint.valueSpills(); + for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills()); iter.more(); ++iter) { + --spill; + if (gcRegs.has(*iter)) + TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(spill), "ion-gc-spill"); + else if (valueRegs.has(*iter)) + TraceRoot(trc, reinterpret_cast<Value*>(spill), "ion-value-spill"); + } + +#ifdef JS_NUNBOX32 + LAllocation type, payload; + while (safepoint.getNunboxSlot(&type, &payload)) { + JSValueTag tag = JSValueTag(ReadAllocation(frame, &type)); + uintptr_t rawPayload = ReadAllocation(frame, &payload); + + Value v = Value::fromTagAndPayload(tag, rawPayload); + TraceRoot(trc, &v, "ion-torn-value"); + + if (v != Value::fromTagAndPayload(tag, rawPayload)) { + // GC moved the value, replace the stored payload. + rawPayload = *v.payloadUIntPtr(); + WriteAllocation(frame, &payload, rawPayload); + } + } +#endif +} + +static void +MarkBailoutFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + JitFrameLayout* layout = (JitFrameLayout*)frame.fp(); + + layout->replaceCalleeToken(MarkCalleeToken(trc, layout->calleeToken())); + + // We have to mark the list of actual arguments, as only formal arguments + // are represented in the Snapshot. + MarkThisAndArguments(trc, frame); + + // Under a bailout, do not have a Safepoint to only iterate over GC-things. + // Thus we use a SnapshotIterator to trace all the locations which would be + // used to reconstruct the Baseline frame. + // + // Note that at the time where this function is called, we have not yet + // started to reconstruct baseline frames. + + // The vector of recover instructions is already traced as part of the + // JitActivation. + SnapshotIterator snapIter(frame, frame.activation()->bailoutData()->machineState()); + + // For each instruction, we read the allocations without evaluating the + // recover instruction, nor reconstructing the frame. We are only looking at + // tracing readable allocations. + while (true) { + while (snapIter.moreAllocations()) + snapIter.traceAllocation(trc); + + if (!snapIter.moreInstructions()) + break; + snapIter.nextInstruction(); + } + +} + +void +UpdateIonJSFrameForMinorGC(JSTracer* trc, const JitFrameIterator& frame) +{ + // Minor GCs may move slots/elements allocated in the nursery. Update + // any slots/elements pointers stored in this frame. + + JitFrameLayout* layout = (JitFrameLayout*)frame.fp(); + + IonScript* ionScript = nullptr; + if (frame.checkInvalidation(&ionScript)) { + // This frame has been invalidated, meaning that its IonScript is no + // longer reachable through the callee token (JSFunction/JSScript->ion + // is now nullptr or recompiled). + } else { + ionScript = frame.ionScriptFromCalleeToken(); + } + + Nursery& nursery = trc->runtime()->gc.nursery; + + const SafepointIndex* si = ionScript->getSafepointIndex(frame.returnAddressToFp()); + SafepointReader safepoint(ionScript, si); + + LiveGeneralRegisterSet slotsRegs = safepoint.slotsOrElementsSpills(); + uintptr_t* spill = frame.spillBase(); + for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills()); iter.more(); ++iter) { + --spill; + if (slotsRegs.has(*iter)) + nursery.forwardBufferPointer(reinterpret_cast<HeapSlot**>(spill)); + } + + // Skip to the right place in the safepoint + SafepointSlotEntry entry; + while (safepoint.getGcSlot(&entry)); + while (safepoint.getValueSlot(&entry)); +#ifdef JS_NUNBOX32 + LAllocation type, payload; + while (safepoint.getNunboxSlot(&type, &payload)); +#endif + + while (safepoint.getSlotsOrElementsSlot(&entry)) { + HeapSlot** slots = reinterpret_cast<HeapSlot**>(layout->slotRef(entry)); + nursery.forwardBufferPointer(slots); + } +} + +static void +MarkJitStubFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + // Mark the ICStub pointer stored in the stub frame. This is necessary + // so that we don't destroy the stub code after unlinking the stub. + + MOZ_ASSERT(frame.type() == JitFrame_IonStub || frame.type() == JitFrame_BaselineStub); + JitStubFrameLayout* layout = (JitStubFrameLayout*)frame.fp(); + + if (ICStub* stub = layout->maybeStubPtr()) { + MOZ_ASSERT(ICStub::CanMakeCalls(stub->kind())); + stub->trace(trc); + } +} + +static void +MarkIonAccessorICFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + MOZ_ASSERT(frame.type() == JitFrame_IonAccessorIC); + IonAccessorICFrameLayout* layout = (IonAccessorICFrameLayout*)frame.fp(); + TraceRoot(trc, layout->stubCode(), "ion-ic-accessor-code"); +} + +#ifdef JS_CODEGEN_MIPS32 +uint8_t* +alignDoubleSpillWithOffset(uint8_t* pointer, int32_t offset) +{ + uint32_t address = reinterpret_cast<uint32_t>(pointer); + address = (address - offset) & ~(ABIStackAlignment - 1); + return reinterpret_cast<uint8_t*>(address); +} + +static void +MarkJitExitFrameCopiedArguments(JSTracer* trc, const VMFunction* f, ExitFooterFrame* footer) +{ + uint8_t* doubleArgs = reinterpret_cast<uint8_t*>(footer); + doubleArgs = alignDoubleSpillWithOffset(doubleArgs, sizeof(intptr_t)); + if (f->outParam == Type_Handle) + doubleArgs -= sizeof(Value); + doubleArgs -= f->doubleByRefArgs() * sizeof(double); + + for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) { + if (f->argProperties(explicitArg) == VMFunction::DoubleByRef) { + // Arguments with double size can only have RootValue type. + if (f->argRootType(explicitArg) == VMFunction::RootValue) + TraceRoot(trc, reinterpret_cast<Value*>(doubleArgs), "ion-vm-args"); + else + MOZ_ASSERT(f->argRootType(explicitArg) == VMFunction::RootNone); + doubleArgs += sizeof(double); + } + } +} +#else +static void +MarkJitExitFrameCopiedArguments(JSTracer* trc, const VMFunction* f, ExitFooterFrame* footer) +{ + // This is NO-OP on other platforms. +} +#endif + +static void +MarkJitExitFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + ExitFooterFrame* footer = frame.exitFrame()->footer(); + + // Mark the code of the code handling the exit path. This is needed because + // invalidated script are no longer marked because data are erased by the + // invalidation and relocation data are no longer reliable. So the VM + // wrapper or the invalidation code may be GC if no JitCode keep reference + // on them. + MOZ_ASSERT(uintptr_t(footer->jitCode()) != uintptr_t(-1)); + + // This corresponds to the case where we have build a fake exit frame which + // handles the case of a native function call. We need to mark the argument + // vector of the function call, and also new.target if it was a constructing + // call. + if (frame.isExitFrameLayout<NativeExitFrameLayout>()) { + NativeExitFrameLayout* native = frame.exitFrame()->as<NativeExitFrameLayout>(); + size_t len = native->argc() + 2; + Value* vp = native->vp(); + TraceRootRange(trc, len, vp, "ion-native-args"); + if (frame.isExitFrameLayout<ConstructNativeExitFrameLayout>()) + TraceRoot(trc, vp + len, "ion-native-new-target"); + return; + } + + if (frame.isExitFrameLayout<IonOOLNativeExitFrameLayout>()) { + IonOOLNativeExitFrameLayout* oolnative = + frame.exitFrame()->as<IonOOLNativeExitFrameLayout>(); + TraceRoot(trc, oolnative->stubCode(), "ion-ool-native-code"); + TraceRoot(trc, oolnative->vp(), "iol-ool-native-vp"); + size_t len = oolnative->argc() + 1; + TraceRootRange(trc, len, oolnative->thisp(), "ion-ool-native-thisargs"); + return; + } + + if (frame.isExitFrameLayout<IonOOLPropertyOpExitFrameLayout>() || + frame.isExitFrameLayout<IonOOLSetterOpExitFrameLayout>()) + { + // A SetterOp frame is a different size, but that's the only relevant + // difference between the two. The fields that need marking are all in + // the common base class. + IonOOLPropertyOpExitFrameLayout* oolgetter = + frame.isExitFrameLayout<IonOOLPropertyOpExitFrameLayout>() + ? frame.exitFrame()->as<IonOOLPropertyOpExitFrameLayout>() + : frame.exitFrame()->as<IonOOLSetterOpExitFrameLayout>(); + TraceRoot(trc, oolgetter->stubCode(), "ion-ool-property-op-code"); + TraceRoot(trc, oolgetter->vp(), "ion-ool-property-op-vp"); + TraceRoot(trc, oolgetter->id(), "ion-ool-property-op-id"); + TraceRoot(trc, oolgetter->obj(), "ion-ool-property-op-obj"); + return; + } + + if (frame.isExitFrameLayout<IonOOLProxyExitFrameLayout>()) { + IonOOLProxyExitFrameLayout* oolproxy = frame.exitFrame()->as<IonOOLProxyExitFrameLayout>(); + TraceRoot(trc, oolproxy->stubCode(), "ion-ool-proxy-code"); + TraceRoot(trc, oolproxy->vp(), "ion-ool-proxy-vp"); + TraceRoot(trc, oolproxy->id(), "ion-ool-proxy-id"); + TraceRoot(trc, oolproxy->proxy(), "ion-ool-proxy-proxy"); + return; + } + + if (frame.isExitFrameLayout<IonDOMExitFrameLayout>()) { + IonDOMExitFrameLayout* dom = frame.exitFrame()->as<IonDOMExitFrameLayout>(); + TraceRoot(trc, dom->thisObjAddress(), "ion-dom-args"); + if (dom->isMethodFrame()) { + IonDOMMethodExitFrameLayout* method = + reinterpret_cast<IonDOMMethodExitFrameLayout*>(dom); + size_t len = method->argc() + 2; + Value* vp = method->vp(); + TraceRootRange(trc, len, vp, "ion-dom-args"); + } else { + TraceRoot(trc, dom->vp(), "ion-dom-args"); + } + return; + } + + if (frame.isExitFrameLayout<LazyLinkExitFrameLayout>()) { + LazyLinkExitFrameLayout* ll = frame.exitFrame()->as<LazyLinkExitFrameLayout>(); + JitFrameLayout* layout = ll->jsFrame(); + + TraceRoot(trc, ll->stubCode(), "lazy-link-code"); + layout->replaceCalleeToken(MarkCalleeToken(trc, layout->calleeToken())); + MarkThisAndArguments(trc, frame); + return; + } + + if (frame.isBareExit()) { + // Nothing to mark. Fake exit frame pushed for VM functions with + // nothing to mark on the stack. + return; + } + + TraceRoot(trc, footer->addressOfJitCode(), "ion-exit-code"); + + const VMFunction* f = footer->function(); + if (f == nullptr) + return; + + // Mark arguments of the VM wrapper. + uint8_t* argBase = frame.exitFrame()->argBase(); + for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) { + switch (f->argRootType(explicitArg)) { + case VMFunction::RootNone: + break; + case VMFunction::RootObject: { + // Sometimes we can bake in HandleObjects to nullptr. + JSObject** pobj = reinterpret_cast<JSObject**>(argBase); + if (*pobj) + TraceRoot(trc, pobj, "ion-vm-args"); + break; + } + case VMFunction::RootString: + case VMFunction::RootPropertyName: + TraceRoot(trc, reinterpret_cast<JSString**>(argBase), "ion-vm-args"); + break; + case VMFunction::RootFunction: + TraceRoot(trc, reinterpret_cast<JSFunction**>(argBase), "ion-vm-args"); + break; + case VMFunction::RootValue: + TraceRoot(trc, reinterpret_cast<Value*>(argBase), "ion-vm-args"); + break; + case VMFunction::RootCell: + TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(argBase), "ion-vm-args"); + break; + } + + switch (f->argProperties(explicitArg)) { + case VMFunction::WordByValue: + case VMFunction::WordByRef: + argBase += sizeof(void*); + break; + case VMFunction::DoubleByValue: + case VMFunction::DoubleByRef: + argBase += 2 * sizeof(void*); + break; + } + } + + if (f->outParam == Type_Handle) { + switch (f->outParamRootType) { + case VMFunction::RootNone: + MOZ_CRASH("Handle outparam must have root type"); + case VMFunction::RootObject: + TraceRoot(trc, footer->outParam<JSObject*>(), "ion-vm-out"); + break; + case VMFunction::RootString: + case VMFunction::RootPropertyName: + TraceRoot(trc, footer->outParam<JSString*>(), "ion-vm-out"); + break; + case VMFunction::RootFunction: + TraceRoot(trc, footer->outParam<JSFunction*>(), "ion-vm-out"); + break; + case VMFunction::RootValue: + TraceRoot(trc, footer->outParam<Value>(), "ion-vm-outvp"); + break; + case VMFunction::RootCell: + TraceGenericPointerRoot(trc, footer->outParam<gc::Cell*>(), "ion-vm-out"); + break; + } + } + + MarkJitExitFrameCopiedArguments(trc, f, footer); +} + +static void +MarkRectifierFrame(JSTracer* trc, const JitFrameIterator& frame) +{ + // Mark thisv. + // + // Baseline JIT code generated as part of the ICCall_Fallback stub may use + // it if we're calling a constructor that returns a primitive value. + RectifierFrameLayout* layout = (RectifierFrameLayout*)frame.fp(); + TraceRoot(trc, &layout->argv()[0], "ion-thisv"); +} + +static void +MarkJitActivation(JSTracer* trc, const JitActivationIterator& activations) +{ + JitActivation* activation = activations->asJit(); + +#ifdef CHECK_OSIPOINT_REGISTERS + if (JitOptions.checkOsiPointRegisters) { + // GC can modify spilled registers, breaking our register checks. + // To handle this, we disable these checks for the current VM call + // when a GC happens. + activation->setCheckRegs(false); + } +#endif + + activation->markRematerializedFrames(trc); + activation->markIonRecovery(trc); + + for (JitFrameIterator frames(activations); !frames.done(); ++frames) { + switch (frames.type()) { + case JitFrame_Exit: + MarkJitExitFrame(trc, frames); + break; + case JitFrame_BaselineJS: + frames.baselineFrame()->trace(trc, frames); + break; + case JitFrame_IonJS: + MarkIonJSFrame(trc, frames); + break; + case JitFrame_BaselineStub: + case JitFrame_IonStub: + MarkJitStubFrame(trc, frames); + break; + case JitFrame_Bailout: + MarkBailoutFrame(trc, frames); + break; + case JitFrame_Rectifier: + MarkRectifierFrame(trc, frames); + break; + case JitFrame_IonAccessorIC: + MarkIonAccessorICFrame(trc, frames); + break; + default: + MOZ_CRASH("unexpected frame type"); + } + } +} + +void +MarkJitActivations(JSRuntime* rt, JSTracer* trc) +{ + for (JitActivationIterator activations(rt); !activations.done(); ++activations) + MarkJitActivation(trc, activations); +} + +JSCompartment* +TopmostIonActivationCompartment(JSRuntime* rt) +{ + for (JitActivationIterator activations(rt); !activations.done(); ++activations) { + for (JitFrameIterator frames(activations); !frames.done(); ++frames) { + if (frames.type() == JitFrame_IonJS) + return activations.activation()->compartment(); + } + } + return nullptr; +} + +void UpdateJitActivationsForMinorGC(JSRuntime* rt, JSTracer* trc) +{ + MOZ_ASSERT(trc->runtime()->isHeapMinorCollecting()); + for (JitActivationIterator activations(rt); !activations.done(); ++activations) { + for (JitFrameIterator frames(activations); !frames.done(); ++frames) { + if (frames.type() == JitFrame_IonJS) + UpdateIonJSFrameForMinorGC(trc, frames); + } + } +} + +void +GetPcScript(JSContext* cx, JSScript** scriptRes, jsbytecode** pcRes) +{ + JitSpew(JitSpew_IonSnapshots, "Recover PC & Script from the last frame."); + + // Recover the return address so that we can look it up in the + // PcScriptCache, as script/pc computation is expensive. + JSRuntime* rt = cx->runtime(); + JitActivationIterator iter(rt); + JitFrameIterator it(iter); + uint8_t* retAddr; + if (it.isExitFrame()) { + ++it; + + // Skip rectifier frames. + if (it.isRectifier()) { + ++it; + MOZ_ASSERT(it.isBaselineStub() || it.isBaselineJS() || it.isIonJS()); + } + + // Skip Baseline/Ion stub and accessor IC frames. + if (it.isBaselineStub()) { + ++it; + MOZ_ASSERT(it.isBaselineJS()); + } else if (it.isIonStub() || it.isIonAccessorIC()) { + ++it; + MOZ_ASSERT(it.isIonJS()); + } + + MOZ_ASSERT(it.isBaselineJS() || it.isIonJS()); + + // Don't use the return address if the BaselineFrame has an override pc. + // The override pc is cheap to get, so we won't benefit from the cache, + // and the override pc could change without the return address changing. + // Moreover, sometimes when an override pc is present during exception + // handling, the return address is set to nullptr as a sanity check, + // since we do not return to the frame that threw the exception. + if (!it.isBaselineJS() || !it.baselineFrame()->hasOverridePc()) { + retAddr = it.returnAddressToFp(); + MOZ_ASSERT(retAddr); + } else { + retAddr = nullptr; + } + } else { + MOZ_ASSERT(it.isBailoutJS()); + retAddr = it.returnAddress(); + } + + uint32_t hash; + if (retAddr) { + hash = PcScriptCache::Hash(retAddr); + + // Lazily initialize the cache. The allocation may safely fail and will not GC. + if (MOZ_UNLIKELY(rt->ionPcScriptCache == nullptr)) { + rt->ionPcScriptCache = (PcScriptCache*)js_malloc(sizeof(struct PcScriptCache)); + if (rt->ionPcScriptCache) + rt->ionPcScriptCache->clear(rt->gc.gcNumber()); + } + + if (rt->ionPcScriptCache && rt->ionPcScriptCache->get(rt, hash, retAddr, scriptRes, pcRes)) + return; + } + + // Lookup failed: undertake expensive process to recover the innermost inlined frame. + jsbytecode* pc = nullptr; + if (it.isIonJS() || it.isBailoutJS()) { + InlineFrameIterator ifi(cx, &it); + *scriptRes = ifi.script(); + pc = ifi.pc(); + } else { + MOZ_ASSERT(it.isBaselineJS()); + it.baselineScriptAndPc(scriptRes, &pc); + } + + if (pcRes) + *pcRes = pc; + + // Add entry to cache. + if (retAddr && rt->ionPcScriptCache) + rt->ionPcScriptCache->add(hash, retAddr, pc, *scriptRes); +} + +uint32_t +OsiIndex::returnPointDisplacement() const +{ + // In general, pointer arithmetic on code is bad, but in this case, + // getting the return address from a call instruction, stepping over pools + // would be wrong. + return callPointDisplacement_ + Assembler::PatchWrite_NearCallSize(); +} + +RInstructionResults::RInstructionResults(JitFrameLayout* fp) + : results_(nullptr), + fp_(fp), + initialized_(false) +{ +} + +RInstructionResults::RInstructionResults(RInstructionResults&& src) + : results_(mozilla::Move(src.results_)), + fp_(src.fp_), + initialized_(src.initialized_) +{ + src.initialized_ = false; +} + +RInstructionResults& +RInstructionResults::operator=(RInstructionResults&& rhs) +{ + MOZ_ASSERT(&rhs != this, "self-moves are prohibited"); + this->~RInstructionResults(); + new(this) RInstructionResults(mozilla::Move(rhs)); + return *this; +} + +RInstructionResults::~RInstructionResults() +{ + // results_ is freed by the UniquePtr. +} + +bool +RInstructionResults::init(JSContext* cx, uint32_t numResults) +{ + if (numResults) { + results_ = cx->make_unique<Values>(); + if (!results_ || !results_->growBy(numResults)) + return false; + + Value guard = MagicValue(JS_ION_BAILOUT); + for (size_t i = 0; i < numResults; i++) + (*results_)[i].init(guard); + } + + initialized_ = true; + return true; +} + +bool +RInstructionResults::isInitialized() const +{ + return initialized_; +} + +#ifdef DEBUG +size_t +RInstructionResults::length() const +{ + return results_->length(); +} +#endif + +JitFrameLayout* +RInstructionResults::frame() const +{ + MOZ_ASSERT(fp_); + return fp_; +} + +HeapPtr<Value>& +RInstructionResults::operator [](size_t index) +{ + return (*results_)[index]; +} + +void +RInstructionResults::trace(JSTracer* trc) +{ + // Note: The vector necessary exists, otherwise this object would not have + // been stored on the activation from where the trace function is called. + TraceRange(trc, results_->length(), results_->begin(), "ion-recover-results"); +} + + +SnapshotIterator::SnapshotIterator(const JitFrameIterator& iter, const MachineState* machineState) + : snapshot_(iter.ionScript()->snapshots(), + iter.snapshotOffset(), + iter.ionScript()->snapshotsRVATableSize(), + iter.ionScript()->snapshotsListSize()), + recover_(snapshot_, + iter.ionScript()->recovers(), + iter.ionScript()->recoversSize()), + fp_(iter.jsFrame()), + machine_(machineState), + ionScript_(iter.ionScript()), + instructionResults_(nullptr) +{ +} + +SnapshotIterator::SnapshotIterator() + : snapshot_(nullptr, 0, 0, 0), + recover_(snapshot_, nullptr, 0), + fp_(nullptr), + ionScript_(nullptr), + instructionResults_(nullptr) +{ +} + +int32_t +SnapshotIterator::readOuterNumActualArgs() const +{ + return fp_->numActualArgs(); +} + +uintptr_t +SnapshotIterator::fromStack(int32_t offset) const +{ + return ReadFrameSlot(fp_, offset); +} + +static Value +FromObjectPayload(uintptr_t payload) +{ + // Note: Both MIRType::Object and MIRType::ObjectOrNull are encoded in + // snapshots using JSVAL_TYPE_OBJECT. + return ObjectOrNullValue(reinterpret_cast<JSObject*>(payload)); +} + +static Value +FromStringPayload(uintptr_t payload) +{ + return StringValue(reinterpret_cast<JSString*>(payload)); +} + +static Value +FromSymbolPayload(uintptr_t payload) +{ + return SymbolValue(reinterpret_cast<JS::Symbol*>(payload)); +} + +static Value +FromTypedPayload(JSValueType type, uintptr_t payload) +{ + switch (type) { + case JSVAL_TYPE_INT32: + return Int32Value(payload); + case JSVAL_TYPE_BOOLEAN: + return BooleanValue(!!payload); + case JSVAL_TYPE_STRING: + return FromStringPayload(payload); + case JSVAL_TYPE_SYMBOL: + return FromSymbolPayload(payload); + case JSVAL_TYPE_OBJECT: + return FromObjectPayload(payload); + default: + MOZ_CRASH("unexpected type - needs payload"); + } +} + +bool +SnapshotIterator::allocationReadable(const RValueAllocation& alloc, ReadMethod rm) +{ + // If we have to recover stores, and if we are not interested in the + // default value of the instruction, then we have to check if the recover + // instruction results are available. + if (alloc.needSideEffect() && !(rm & RM_AlwaysDefault)) { + if (!hasInstructionResults()) + return false; + } + + switch (alloc.mode()) { + case RValueAllocation::DOUBLE_REG: + return hasRegister(alloc.fpuReg()); + + case RValueAllocation::TYPED_REG: + return hasRegister(alloc.reg2()); + +#if defined(JS_NUNBOX32) + case RValueAllocation::UNTYPED_REG_REG: + return hasRegister(alloc.reg()) && hasRegister(alloc.reg2()); + case RValueAllocation::UNTYPED_REG_STACK: + return hasRegister(alloc.reg()) && hasStack(alloc.stackOffset2()); + case RValueAllocation::UNTYPED_STACK_REG: + return hasStack(alloc.stackOffset()) && hasRegister(alloc.reg2()); + case RValueAllocation::UNTYPED_STACK_STACK: + return hasStack(alloc.stackOffset()) && hasStack(alloc.stackOffset2()); +#elif defined(JS_PUNBOX64) + case RValueAllocation::UNTYPED_REG: + return hasRegister(alloc.reg()); + case RValueAllocation::UNTYPED_STACK: + return hasStack(alloc.stackOffset()); +#endif + + case RValueAllocation::RECOVER_INSTRUCTION: + return hasInstructionResult(alloc.index()); + case RValueAllocation::RI_WITH_DEFAULT_CST: + return rm & RM_AlwaysDefault || hasInstructionResult(alloc.index()); + + default: + return true; + } +} + +Value +SnapshotIterator::allocationValue(const RValueAllocation& alloc, ReadMethod rm) +{ + switch (alloc.mode()) { + case RValueAllocation::CONSTANT: + return ionScript_->getConstant(alloc.index()); + + case RValueAllocation::CST_UNDEFINED: + return UndefinedValue(); + + case RValueAllocation::CST_NULL: + return NullValue(); + + case RValueAllocation::DOUBLE_REG: + return DoubleValue(fromRegister(alloc.fpuReg())); + + case RValueAllocation::ANY_FLOAT_REG: + { + union { + double d; + float f; + } pun; + MOZ_ASSERT(alloc.fpuReg().isSingle()); + pun.d = fromRegister(alloc.fpuReg()); + // The register contains the encoding of a float32. We just read + // the bits without making any conversion. + return Float32Value(pun.f); + } + + case RValueAllocation::ANY_FLOAT_STACK: + return Float32Value(ReadFrameFloat32Slot(fp_, alloc.stackOffset())); + + case RValueAllocation::TYPED_REG: + return FromTypedPayload(alloc.knownType(), fromRegister(alloc.reg2())); + + case RValueAllocation::TYPED_STACK: + { + switch (alloc.knownType()) { + case JSVAL_TYPE_DOUBLE: + return DoubleValue(ReadFrameDoubleSlot(fp_, alloc.stackOffset2())); + case JSVAL_TYPE_INT32: + return Int32Value(ReadFrameInt32Slot(fp_, alloc.stackOffset2())); + case JSVAL_TYPE_BOOLEAN: + return BooleanValue(ReadFrameBooleanSlot(fp_, alloc.stackOffset2())); + case JSVAL_TYPE_STRING: + return FromStringPayload(fromStack(alloc.stackOffset2())); + case JSVAL_TYPE_SYMBOL: + return FromSymbolPayload(fromStack(alloc.stackOffset2())); + case JSVAL_TYPE_OBJECT: + return FromObjectPayload(fromStack(alloc.stackOffset2())); + default: + MOZ_CRASH("Unexpected type"); + } + } + +#if defined(JS_NUNBOX32) + case RValueAllocation::UNTYPED_REG_REG: + { + return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())), + fromRegister(alloc.reg2())); + } + + case RValueAllocation::UNTYPED_REG_STACK: + { + return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())), + fromStack(alloc.stackOffset2())); + } + + case RValueAllocation::UNTYPED_STACK_REG: + { + return Value::fromTagAndPayload(JSValueTag(fromStack(alloc.stackOffset())), + fromRegister(alloc.reg2())); + } + + case RValueAllocation::UNTYPED_STACK_STACK: + { + return Value::fromTagAndPayload(JSValueTag(fromStack(alloc.stackOffset())), + fromStack(alloc.stackOffset2())); + } +#elif defined(JS_PUNBOX64) + case RValueAllocation::UNTYPED_REG: + { + return Value::fromRawBits(fromRegister(alloc.reg())); + } + + case RValueAllocation::UNTYPED_STACK: + { + return Value::fromRawBits(fromStack(alloc.stackOffset())); + } +#endif + + case RValueAllocation::RECOVER_INSTRUCTION: + return fromInstructionResult(alloc.index()); + + case RValueAllocation::RI_WITH_DEFAULT_CST: + if (rm & RM_Normal && hasInstructionResult(alloc.index())) + return fromInstructionResult(alloc.index()); + MOZ_ASSERT(rm & RM_AlwaysDefault); + return ionScript_->getConstant(alloc.index2()); + + default: + MOZ_CRASH("huh?"); + } +} + +const FloatRegisters::RegisterContent* +SnapshotIterator::floatAllocationPointer(const RValueAllocation& alloc) const +{ + switch (alloc.mode()) { + case RValueAllocation::ANY_FLOAT_REG: + return machine_->address(alloc.fpuReg()); + + case RValueAllocation::ANY_FLOAT_STACK: + return (FloatRegisters::RegisterContent*) AddressOfFrameSlot(fp_, alloc.stackOffset()); + + default: + MOZ_CRASH("Not a float allocation."); + } +} + +Value +SnapshotIterator::maybeRead(const RValueAllocation& a, MaybeReadFallback& fallback) +{ + if (allocationReadable(a)) + return allocationValue(a); + + if (fallback.canRecoverResults()) { + // Code paths which are calling maybeRead are not always capable of + // returning an error code, as these code paths used to be infallible. + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!initInstructionResults(fallback)) + oomUnsafe.crash("js::jit::SnapshotIterator::maybeRead"); + + if (allocationReadable(a)) + return allocationValue(a); + + MOZ_ASSERT_UNREACHABLE("All allocations should be readable."); + } + + return fallback.unreadablePlaceholder(); +} + +void +SnapshotIterator::writeAllocationValuePayload(const RValueAllocation& alloc, const Value& v) +{ + uintptr_t payload = *v.payloadUIntPtr(); +#if defined(JS_PUNBOX64) + // Do not write back the tag, as this will trigger an assertion when we will + // reconstruct the JS Value while marking again or when bailing out. + payload &= JSVAL_PAYLOAD_MASK; +#endif + + switch (alloc.mode()) { + case RValueAllocation::CONSTANT: + ionScript_->getConstant(alloc.index()) = v; + break; + + case RValueAllocation::CST_UNDEFINED: + case RValueAllocation::CST_NULL: + case RValueAllocation::DOUBLE_REG: + case RValueAllocation::ANY_FLOAT_REG: + case RValueAllocation::ANY_FLOAT_STACK: + MOZ_CRASH("Not a GC thing: Unexpected write"); + break; + + case RValueAllocation::TYPED_REG: + machine_->write(alloc.reg2(), payload); + break; + + case RValueAllocation::TYPED_STACK: + switch (alloc.knownType()) { + default: + MOZ_CRASH("Not a GC thing: Unexpected write"); + break; + case JSVAL_TYPE_STRING: + case JSVAL_TYPE_SYMBOL: + case JSVAL_TYPE_OBJECT: + WriteFrameSlot(fp_, alloc.stackOffset2(), payload); + break; + } + break; + +#if defined(JS_NUNBOX32) + case RValueAllocation::UNTYPED_REG_REG: + case RValueAllocation::UNTYPED_STACK_REG: + machine_->write(alloc.reg2(), payload); + break; + + case RValueAllocation::UNTYPED_REG_STACK: + case RValueAllocation::UNTYPED_STACK_STACK: + WriteFrameSlot(fp_, alloc.stackOffset2(), payload); + break; +#elif defined(JS_PUNBOX64) + case RValueAllocation::UNTYPED_REG: + machine_->write(alloc.reg(), v.asRawBits()); + break; + + case RValueAllocation::UNTYPED_STACK: + WriteFrameSlot(fp_, alloc.stackOffset(), v.asRawBits()); + break; +#endif + + case RValueAllocation::RECOVER_INSTRUCTION: + MOZ_CRASH("Recover instructions are handled by the JitActivation."); + break; + + case RValueAllocation::RI_WITH_DEFAULT_CST: + // Assume that we are always going to be writing on the default value + // while tracing. + ionScript_->getConstant(alloc.index2()) = v; + break; + + default: + MOZ_CRASH("huh?"); + } +} + +void +SnapshotIterator::traceAllocation(JSTracer* trc) +{ + RValueAllocation alloc = readAllocation(); + if (!allocationReadable(alloc, RM_AlwaysDefault)) + return; + + Value v = allocationValue(alloc, RM_AlwaysDefault); + if (!v.isMarkable()) + return; + + Value copy = v; + TraceRoot(trc, &v, "ion-typed-reg"); + if (v != copy) { + MOZ_ASSERT(SameType(v, copy)); + writeAllocationValuePayload(alloc, v); + } +} + +const RResumePoint* +SnapshotIterator::resumePoint() const +{ + return instruction()->toResumePoint(); +} + +uint32_t +SnapshotIterator::numAllocations() const +{ + return instruction()->numOperands(); +} + +uint32_t +SnapshotIterator::pcOffset() const +{ + return resumePoint()->pcOffset(); +} + +void +SnapshotIterator::skipInstruction() +{ + MOZ_ASSERT(snapshot_.numAllocationsRead() == 0); + size_t numOperands = instruction()->numOperands(); + for (size_t i = 0; i < numOperands; i++) + skip(); + nextInstruction(); +} + +bool +SnapshotIterator::initInstructionResults(MaybeReadFallback& fallback) +{ + MOZ_ASSERT(fallback.canRecoverResults()); + JSContext* cx = fallback.maybeCx; + + // If there is only one resume point in the list of instructions, then there + // is no instruction to recover, and thus no need to register any results. + if (recover_.numInstructions() == 1) + return true; + + JitFrameLayout* fp = fallback.frame->jsFrame(); + RInstructionResults* results = fallback.activation->maybeIonFrameRecovery(fp); + if (!results) { + AutoCompartment ac(cx, fallback.frame->script()->compartment()); + + // We do not have the result yet, which means that an observable stack + // slot is requested. As we do not want to bailout every time for the + // same reason, we need to recompile without optimizing away the + // observable stack slots. The script would later be recompiled to have + // support for Argument objects. + if (fallback.consequence == MaybeReadFallback::Fallback_Invalidate) + ionScript_->invalidate(cx, /* resetUses = */ false, "Observe recovered instruction."); + + // Register the list of result on the activation. We need to do that + // before we initialize the list such as if any recover instruction + // cause a GC, we can ensure that the results are properly traced by the + // activation. + RInstructionResults tmp(fallback.frame->jsFrame()); + if (!fallback.activation->registerIonFrameRecovery(mozilla::Move(tmp))) + return false; + + results = fallback.activation->maybeIonFrameRecovery(fp); + + // Start a new snapshot at the beginning of the JitFrameIterator. This + // SnapshotIterator is used for evaluating the content of all recover + // instructions. The result is then saved on the JitActivation. + MachineState machine = fallback.frame->machineState(); + SnapshotIterator s(*fallback.frame, &machine); + if (!s.computeInstructionResults(cx, results)) { + + // If the evaluation failed because of OOMs, then we discard the + // current set of result that we collected so far. + fallback.activation->removeIonFrameRecovery(fp); + return false; + } + } + + MOZ_ASSERT(results->isInitialized()); + MOZ_ASSERT(results->length() == recover_.numInstructions() - 1); + instructionResults_ = results; + return true; +} + +bool +SnapshotIterator::computeInstructionResults(JSContext* cx, RInstructionResults* results) const +{ + MOZ_ASSERT(!results->isInitialized()); + MOZ_ASSERT(recover_.numInstructionsRead() == 1); + + // The last instruction will always be a resume point. + size_t numResults = recover_.numInstructions() - 1; + if (!results->isInitialized()) { + if (!results->init(cx, numResults)) + return false; + + // No need to iterate over the only resume point. + if (!numResults) { + MOZ_ASSERT(results->isInitialized()); + return true; + } + + // Use AutoEnterAnalysis to avoid invoking the object metadata callback, + // which could try to walk the stack while bailing out. + AutoEnterAnalysis enter(cx); + + // Fill with the results of recover instructions. + SnapshotIterator s(*this); + s.instructionResults_ = results; + while (s.moreInstructions()) { + // Skip resume point and only interpret recover instructions. + if (s.instruction()->isResumePoint()) { + s.skipInstruction(); + continue; + } + + if (!s.instruction()->recover(cx, s)) + return false; + s.nextInstruction(); + } + } + + MOZ_ASSERT(results->isInitialized()); + return true; +} + +void +SnapshotIterator::storeInstructionResult(const Value& v) +{ + uint32_t currIns = recover_.numInstructionsRead() - 1; + MOZ_ASSERT((*instructionResults_)[currIns].isMagic(JS_ION_BAILOUT)); + (*instructionResults_)[currIns] = v; +} + +Value +SnapshotIterator::fromInstructionResult(uint32_t index) const +{ + MOZ_ASSERT(!(*instructionResults_)[index].isMagic(JS_ION_BAILOUT)); + return (*instructionResults_)[index]; +} + +void +SnapshotIterator::settleOnFrame() +{ + // Check that the current instruction can still be use. + MOZ_ASSERT(snapshot_.numAllocationsRead() == 0); + while (!instruction()->isResumePoint()) + skipInstruction(); +} + +void +SnapshotIterator::nextFrame() +{ + nextInstruction(); + settleOnFrame(); +} + +Value +SnapshotIterator::maybeReadAllocByIndex(size_t index) +{ + while (index--) { + MOZ_ASSERT(moreAllocations()); + skip(); + } + + Value s; + { + // This MaybeReadFallback method cannot GC. + JS::AutoSuppressGCAnalysis nogc; + MaybeReadFallback fallback(UndefinedValue()); + s = maybeRead(fallback); + } + + while (moreAllocations()) + skip(); + + return s; +} + +JitFrameLayout* +JitFrameIterator::jsFrame() const +{ + MOZ_ASSERT(isScripted()); + if (isBailoutJS()) + return (JitFrameLayout*) activation_->bailoutData()->fp(); + + return (JitFrameLayout*) fp(); +} + +IonScript* +JitFrameIterator::ionScript() const +{ + MOZ_ASSERT(isIonScripted()); + if (isBailoutJS()) + return activation_->bailoutData()->ionScript(); + + IonScript* ionScript = nullptr; + if (checkInvalidation(&ionScript)) + return ionScript; + return ionScriptFromCalleeToken(); +} + +IonScript* +JitFrameIterator::ionScriptFromCalleeToken() const +{ + MOZ_ASSERT(isIonJS()); + MOZ_ASSERT(!checkInvalidation()); + return script()->ionScript(); +} + +const SafepointIndex* +JitFrameIterator::safepoint() const +{ + MOZ_ASSERT(isIonJS()); + if (!cachedSafepointIndex_) + cachedSafepointIndex_ = ionScript()->getSafepointIndex(returnAddressToFp()); + return cachedSafepointIndex_; +} + +SnapshotOffset +JitFrameIterator::snapshotOffset() const +{ + MOZ_ASSERT(isIonScripted()); + if (isBailoutJS()) + return activation_->bailoutData()->snapshotOffset(); + return osiIndex()->snapshotOffset(); +} + +const OsiIndex* +JitFrameIterator::osiIndex() const +{ + MOZ_ASSERT(isIonJS()); + SafepointReader reader(ionScript(), safepoint()); + return ionScript()->getOsiIndex(reader.osiReturnPointOffset()); +} + +InlineFrameIterator::InlineFrameIterator(JSContext* cx, const JitFrameIterator* iter) + : calleeTemplate_(cx), + calleeRVA_(), + script_(cx) +{ + resetOn(iter); +} + +InlineFrameIterator::InlineFrameIterator(JSRuntime* rt, const JitFrameIterator* iter) + : calleeTemplate_(rt->contextFromMainThread()), + calleeRVA_(), + script_(rt->contextFromMainThread()) +{ + resetOn(iter); +} + +InlineFrameIterator::InlineFrameIterator(JSContext* cx, const InlineFrameIterator* iter) + : frame_(iter ? iter->frame_ : nullptr), + framesRead_(0), + frameCount_(iter ? iter->frameCount_ : UINT32_MAX), + calleeTemplate_(cx), + calleeRVA_(), + script_(cx) +{ + if (frame_) { + machine_ = iter->machine_; + start_ = SnapshotIterator(*frame_, &machine_); + + // findNextFrame will iterate to the next frame and init. everything. + // Therefore to settle on the same frame, we report one frame less readed. + framesRead_ = iter->framesRead_ - 1; + findNextFrame(); + } +} + +void +InlineFrameIterator::resetOn(const JitFrameIterator* iter) +{ + frame_ = iter; + framesRead_ = 0; + frameCount_ = UINT32_MAX; + + if (iter) { + machine_ = iter->machineState(); + start_ = SnapshotIterator(*iter, &machine_); + findNextFrame(); + } +} + +void +InlineFrameIterator::findNextFrame() +{ + MOZ_ASSERT(more()); + + si_ = start_; + + // Read the initial frame out of the C stack. + calleeTemplate_ = frame_->maybeCallee(); + calleeRVA_ = RValueAllocation(); + script_ = frame_->script(); + MOZ_ASSERT(script_->hasBaselineScript()); + + // Settle on the outermost frame without evaluating any instructions before + // looking for a pc. + si_.settleOnFrame(); + + pc_ = script_->offsetToPC(si_.pcOffset()); + numActualArgs_ = 0xbadbad; + + // This unfortunately is O(n*m), because we must skip over outer frames + // before reading inner ones. + + // The first time (frameCount_ == UINT32_MAX) we do not know the number of + // frames that we are going to inspect. So we are iterating until there is + // no more frames, to settle on the inner most frame and to count the number + // of frames. + size_t remaining = (frameCount_ != UINT32_MAX) ? frameNo() - 1 : SIZE_MAX; + + size_t i = 1; + for (; i <= remaining && si_.moreFrames(); i++) { + MOZ_ASSERT(IsIonInlinablePC(pc_)); + + // Recover the number of actual arguments from the script. + if (JSOp(*pc_) != JSOP_FUNAPPLY) + numActualArgs_ = GET_ARGC(pc_); + if (JSOp(*pc_) == JSOP_FUNCALL) { + MOZ_ASSERT(GET_ARGC(pc_) > 0); + numActualArgs_ = GET_ARGC(pc_) - 1; + } else if (IsGetPropPC(pc_)) { + numActualArgs_ = 0; + } else if (IsSetPropPC(pc_)) { + numActualArgs_ = 1; + } + + if (numActualArgs_ == 0xbadbad) + MOZ_CRASH("Couldn't deduce the number of arguments of an ionmonkey frame"); + + // Skip over non-argument slots, as well as |this|. + bool skipNewTarget = JSOp(*pc_) == JSOP_NEW; + unsigned skipCount = (si_.numAllocations() - 1) - numActualArgs_ - 1 - skipNewTarget; + for (unsigned j = 0; j < skipCount; j++) + si_.skip(); + + // This value should correspond to the function which is being inlined. + // The value must be readable to iterate over the inline frame. Most of + // the time, these functions are stored as JSFunction constants, + // register which are holding the JSFunction pointer, or recover + // instruction with Default value. + Value funval = si_.readWithDefault(&calleeRVA_); + + // Skip extra value allocations. + while (si_.moreAllocations()) + si_.skip(); + + si_.nextFrame(); + + calleeTemplate_ = &funval.toObject().as<JSFunction>(); + + // Inlined functions may be clones that still point to the lazy script + // for the executed script, if they are clones. The actual script + // exists though, just make sure the function points to it. + script_ = calleeTemplate_->existingScript(); + MOZ_ASSERT(script_->hasBaselineScript()); + + pc_ = script_->offsetToPC(si_.pcOffset()); + } + + // The first time we do not know the number of frames, we only settle on the + // last frame, and update the number of frames based on the number of + // iteration that we have done. + if (frameCount_ == UINT32_MAX) { + MOZ_ASSERT(!si_.moreFrames()); + frameCount_ = i; + } + + framesRead_++; +} + +JSFunction* +InlineFrameIterator::callee(MaybeReadFallback& fallback) const +{ + MOZ_ASSERT(isFunctionFrame()); + if (calleeRVA_.mode() == RValueAllocation::INVALID || !fallback.canRecoverResults()) + return calleeTemplate_; + + SnapshotIterator s(si_); + // :TODO: Handle allocation failures from recover instruction. + Value funval = s.maybeRead(calleeRVA_, fallback); + return &funval.toObject().as<JSFunction>(); +} + +JSObject* +InlineFrameIterator::computeEnvironmentChain(const Value& envChainValue, + MaybeReadFallback& fallback, + bool* hasInitialEnv) const +{ + if (envChainValue.isObject()) { + if (hasInitialEnv) { + if (fallback.canRecoverResults()) { + RootedObject obj(fallback.maybeCx, &envChainValue.toObject()); + *hasInitialEnv = isFunctionFrame() && + callee(fallback)->needsFunctionEnvironmentObjects(); + return obj; + } else { + JS::AutoSuppressGCAnalysis nogc; // If we cannot recover then we cannot GC. + *hasInitialEnv = isFunctionFrame() && + callee(fallback)->needsFunctionEnvironmentObjects(); + } + } + + return &envChainValue.toObject(); + } + + // Note we can hit this case even for functions with a CallObject, in case + // we are walking the frame during the function prologue, before the env + // chain has been initialized. + if (isFunctionFrame()) + return callee(fallback)->environment(); + + // Ion does not handle non-function scripts that have anything other than + // the global on their env chain. + MOZ_ASSERT(!script()->isForEval()); + MOZ_ASSERT(!script()->hasNonSyntacticScope()); + return &script()->global().lexicalEnvironment(); +} + +bool +InlineFrameIterator::isFunctionFrame() const +{ + return !!calleeTemplate_; +} + +MachineState +MachineState::FromBailout(RegisterDump::GPRArray& regs, RegisterDump::FPUArray& fpregs) +{ + MachineState machine; + + for (unsigned i = 0; i < Registers::Total; i++) + machine.setRegisterLocation(Register::FromCode(i), ®s[i].r); +#ifdef JS_CODEGEN_ARM + float* fbase = (float*)&fpregs[0]; + for (unsigned i = 0; i < FloatRegisters::TotalDouble; i++) + machine.setRegisterLocation(FloatRegister(i, FloatRegister::Double), &fpregs[i].d); + for (unsigned i = 0; i < FloatRegisters::TotalSingle; i++) + machine.setRegisterLocation(FloatRegister(i, FloatRegister::Single), (double*)&fbase[i]); +#elif defined(JS_CODEGEN_MIPS32) + float* fbase = (float*)&fpregs[0]; + for (unsigned i = 0; i < FloatRegisters::TotalDouble; i++) { + machine.setRegisterLocation(FloatRegister::FromIndex(i, FloatRegister::Double), + &fpregs[i].d); + } + for (unsigned i = 0; i < FloatRegisters::TotalSingle; i++) { + machine.setRegisterLocation(FloatRegister::FromIndex(i, FloatRegister::Single), + (double*)&fbase[i]); + } +#elif defined(JS_CODEGEN_MIPS64) + for (unsigned i = 0; i < FloatRegisters::TotalPhys; i++) { + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Double), &fpregs[i]); + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Single), &fpregs[i]); + } +#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + for (unsigned i = 0; i < FloatRegisters::TotalPhys; i++) { + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Single), &fpregs[i]); + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Double), &fpregs[i]); + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Simd128), &fpregs[i]); + } +#elif defined(JS_CODEGEN_ARM64) + for (unsigned i = 0; i < FloatRegisters::TotalPhys; i++) { + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Single), &fpregs[i]); + machine.setRegisterLocation(FloatRegister(i, FloatRegisters::Double), &fpregs[i]); + } + +#elif defined(JS_CODEGEN_NONE) + MOZ_CRASH(); +#else +# error "Unknown architecture!" +#endif + return machine; +} + +bool +InlineFrameIterator::isConstructing() const +{ + // Skip the current frame and look at the caller's. + if (more()) { + InlineFrameIterator parent(GetJSContextFromMainThread(), this); + ++parent; + + // Inlined Getters and Setters are never constructing. + if (IsGetPropPC(parent.pc()) || IsSetPropPC(parent.pc())) + return false; + + // In the case of a JS frame, look up the pc from the snapshot. + MOZ_ASSERT(IsCallPC(parent.pc())); + + return (JSOp)*parent.pc() == JSOP_NEW; + } + + return frame_->isConstructing(); +} + +bool +JitFrameIterator::isConstructing() const +{ + return CalleeTokenIsConstructing(calleeToken()); +} + +unsigned +JitFrameIterator::numActualArgs() const +{ + if (isScripted()) + return jsFrame()->numActualArgs(); + + MOZ_ASSERT(isExitFrameLayout<NativeExitFrameLayout>()); + return exitFrame()->as<NativeExitFrameLayout>()->argc(); +} + +void +SnapshotIterator::warnUnreadableAllocation() +{ + fprintf(stderr, "Warning! Tried to access unreadable value allocation (possible f.arguments).\n"); +} + +struct DumpOp { + explicit DumpOp(unsigned int i) : i_(i) {} + + unsigned int i_; + void operator()(const Value& v) { + fprintf(stderr, " actual (arg %d): ", i_); +#ifdef DEBUG + DumpValue(v); +#else + fprintf(stderr, "?\n"); +#endif + i_++; + } +}; + +void +JitFrameIterator::dumpBaseline() const +{ + MOZ_ASSERT(isBaselineJS()); + + fprintf(stderr, " JS Baseline frame\n"); + if (isFunctionFrame()) { + fprintf(stderr, " callee fun: "); +#ifdef DEBUG + DumpObject(callee()); +#else + fprintf(stderr, "?\n"); +#endif + } else { + fprintf(stderr, " global frame, no callee\n"); + } + + fprintf(stderr, " file %s line %" PRIuSIZE "\n", + script()->filename(), script()->lineno()); + + JSContext* cx = GetJSContextFromMainThread(); + RootedScript script(cx); + jsbytecode* pc; + baselineScriptAndPc(script.address(), &pc); + + fprintf(stderr, " script = %p, pc = %p (offset %u)\n", (void*)script, pc, uint32_t(script->pcToOffset(pc))); + fprintf(stderr, " current op: %s\n", CodeName[*pc]); + + fprintf(stderr, " actual args: %d\n", numActualArgs()); + + BaselineFrame* frame = baselineFrame(); + + for (unsigned i = 0; i < frame->numValueSlots(); i++) { + fprintf(stderr, " slot %u: ", i); +#ifdef DEBUG + Value* v = frame->valueSlot(i); + DumpValue(*v); +#else + fprintf(stderr, "?\n"); +#endif + } +} + +void +InlineFrameIterator::dump() const +{ + MaybeReadFallback fallback(UndefinedValue()); + + if (more()) + fprintf(stderr, " JS frame (inlined)\n"); + else + fprintf(stderr, " JS frame\n"); + + bool isFunction = false; + if (isFunctionFrame()) { + isFunction = true; + fprintf(stderr, " callee fun: "); +#ifdef DEBUG + DumpObject(callee(fallback)); +#else + fprintf(stderr, "?\n"); +#endif + } else { + fprintf(stderr, " global frame, no callee\n"); + } + + fprintf(stderr, " file %s line %" PRIuSIZE "\n", + script()->filename(), script()->lineno()); + + fprintf(stderr, " script = %p, pc = %p\n", (void*) script(), pc()); + fprintf(stderr, " current op: %s\n", CodeName[*pc()]); + + if (!more()) { + numActualArgs(); + } + + SnapshotIterator si = snapshotIterator(); + fprintf(stderr, " slots: %u\n", si.numAllocations() - 1); + for (unsigned i = 0; i < si.numAllocations() - 1; i++) { + if (isFunction) { + if (i == 0) + fprintf(stderr, " env chain: "); + else if (i == 1) + fprintf(stderr, " this: "); + else if (i - 2 < calleeTemplate()->nargs()) + fprintf(stderr, " formal (arg %d): ", i - 2); + else { + if (i - 2 == calleeTemplate()->nargs() && numActualArgs() > calleeTemplate()->nargs()) { + DumpOp d(calleeTemplate()->nargs()); + unaliasedForEachActual(GetJSContextFromMainThread(), d, ReadFrame_Overflown, fallback); + } + + fprintf(stderr, " slot %d: ", int(i - 2 - calleeTemplate()->nargs())); + } + } else + fprintf(stderr, " slot %u: ", i); +#ifdef DEBUG + DumpValue(si.maybeRead(fallback)); +#else + fprintf(stderr, "?\n"); +#endif + } + + fputc('\n', stderr); +} + +void +JitFrameIterator::dump() const +{ + switch (type_) { + case JitFrame_Entry: + fprintf(stderr, " Entry frame\n"); + fprintf(stderr, " Frame size: %u\n", unsigned(current()->prevFrameLocalSize())); + break; + case JitFrame_BaselineJS: + dumpBaseline(); + break; + case JitFrame_BaselineStub: + fprintf(stderr, " Baseline stub frame\n"); + fprintf(stderr, " Frame size: %u\n", unsigned(current()->prevFrameLocalSize())); + break; + case JitFrame_Bailout: + case JitFrame_IonJS: + { + InlineFrameIterator frames(GetJSContextFromMainThread(), this); + for (;;) { + frames.dump(); + if (!frames.more()) + break; + ++frames; + } + break; + } + case JitFrame_IonStub: + fprintf(stderr, " Ion stub frame\n"); + fprintf(stderr, " Frame size: %u\n", unsigned(current()->prevFrameLocalSize())); + break; + case JitFrame_Rectifier: + fprintf(stderr, " Rectifier frame\n"); + fprintf(stderr, " Frame size: %u\n", unsigned(current()->prevFrameLocalSize())); + break; + case JitFrame_IonAccessorIC: + fprintf(stderr, " Ion scripted accessor IC\n"); + fprintf(stderr, " Frame size: %u\n", unsigned(current()->prevFrameLocalSize())); + break; + case JitFrame_Exit: + fprintf(stderr, " Exit frame\n"); + break; + }; + fputc('\n', stderr); +} + +#ifdef DEBUG +bool +JitFrameIterator::verifyReturnAddressUsingNativeToBytecodeMap() +{ + MOZ_ASSERT(returnAddressToFp_ != nullptr); + + // Only handle Ion frames for now. + if (type_ != JitFrame_IonJS && type_ != JitFrame_BaselineJS) + return true; + + JSRuntime* rt = js::TlsPerThreadData.get()->runtimeIfOnOwnerThread(); + + // Don't verify on non-main-thread. + if (!rt) + return true; + + // Don't verify if sampling is being suppressed. + if (!rt->isProfilerSamplingEnabled()) + return true; + + if (rt->isHeapMinorCollecting()) + return true; + + JitRuntime* jitrt = rt->jitRuntime(); + + // Look up and print bytecode info for the native address. + const JitcodeGlobalEntry* entry = jitrt->getJitcodeGlobalTable()->lookup(returnAddressToFp_); + if (!entry) + return true; + + JitSpew(JitSpew_Profiling, "Found nativeToBytecode entry for %p: %p - %p", + returnAddressToFp_, entry->nativeStartAddr(), entry->nativeEndAddr()); + + JitcodeGlobalEntry::BytecodeLocationVector location; + uint32_t depth = UINT32_MAX; + if (!entry->callStackAtAddr(rt, returnAddressToFp_, location, &depth)) + return false; + MOZ_ASSERT(depth > 0 && depth != UINT32_MAX); + MOZ_ASSERT(location.length() == depth); + + JitSpew(JitSpew_Profiling, "Found bytecode location of depth %d:", depth); + for (size_t i = 0; i < location.length(); i++) { + JitSpew(JitSpew_Profiling, " %s:%" PRIuSIZE " - %" PRIuSIZE, + location[i].script->filename(), location[i].script->lineno(), + size_t(location[i].pc - location[i].script->code())); + } + + if (type_ == JitFrame_IonJS) { + // Create an InlineFrameIterator here and verify the mapped info against the iterator info. + InlineFrameIterator inlineFrames(GetJSContextFromMainThread(), this); + for (size_t idx = 0; idx < location.length(); idx++) { + MOZ_ASSERT(idx < location.length()); + MOZ_ASSERT_IF(idx < location.length() - 1, inlineFrames.more()); + + JitSpew(JitSpew_Profiling, + "Match %d: ION %s:%" PRIuSIZE "(%" PRIuSIZE ") vs N2B %s:%" PRIuSIZE "(%" PRIuSIZE ")", + (int)idx, + inlineFrames.script()->filename(), + inlineFrames.script()->lineno(), + size_t(inlineFrames.pc() - inlineFrames.script()->code()), + location[idx].script->filename(), + location[idx].script->lineno(), + size_t(location[idx].pc - location[idx].script->code())); + + MOZ_ASSERT(inlineFrames.script() == location[idx].script); + + if (inlineFrames.more()) + ++inlineFrames; + } + } + + return true; +} +#endif // DEBUG + +JitProfilingFrameIterator::JitProfilingFrameIterator( + JSRuntime* rt, const JS::ProfilingFrameIterator::RegisterState& state) +{ + // If no profilingActivation is live, initialize directly to + // end-of-iteration state. + if (!rt->profilingActivation()) { + type_ = JitFrame_Entry; + fp_ = nullptr; + returnAddressToFp_ = nullptr; + return; + } + + MOZ_ASSERT(rt->profilingActivation()->isJit()); + + JitActivation* act = rt->profilingActivation()->asJit(); + + // If the top JitActivation has a null lastProfilingFrame, assume that + // it's a trivially empty activation, and initialize directly + // to end-of-iteration state. + if (!act->lastProfilingFrame()) { + type_ = JitFrame_Entry; + fp_ = nullptr; + returnAddressToFp_ = nullptr; + return; + } + + // Get the fp from the current profilingActivation + fp_ = (uint8_t*) act->lastProfilingFrame(); + void* lastCallSite = act->lastProfilingCallSite(); + + JitcodeGlobalTable* table = rt->jitRuntime()->getJitcodeGlobalTable(); + + // Profiler sampling must NOT be suppressed if we are here. + MOZ_ASSERT(rt->isProfilerSamplingEnabled()); + + // Try initializing with sampler pc + if (tryInitWithPC(state.pc)) + return; + + // Try initializing with sampler pc using native=>bytecode table. + if (tryInitWithTable(table, state.pc, rt, /* forLastCallSite = */ false)) + return; + + // Try initializing with lastProfilingCallSite pc + if (lastCallSite) { + if (tryInitWithPC(lastCallSite)) + return; + + // Try initializing with lastProfilingCallSite pc using native=>bytecode table. + if (tryInitWithTable(table, lastCallSite, rt, /* forLastCallSite = */ true)) + return; + } + + MOZ_ASSERT(frameScript()->hasBaselineScript()); + + // If nothing matches, for now just assume we are at the start of the last frame's + // baseline jit code. + type_ = JitFrame_BaselineJS; + returnAddressToFp_ = frameScript()->baselineScript()->method()->raw(); +} + +template <typename ReturnType = CommonFrameLayout*> +inline ReturnType +GetPreviousRawFrame(CommonFrameLayout* frame) +{ + size_t prevSize = frame->prevFrameLocalSize() + frame->headerSize(); + return ReturnType((uint8_t*)frame + prevSize); +} + +JitProfilingFrameIterator::JitProfilingFrameIterator(void* exitFrame) +{ + // Skip the exit frame. + ExitFrameLayout* frame = (ExitFrameLayout*) exitFrame; + moveToNextFrame(frame); +} + +bool +JitProfilingFrameIterator::tryInitWithPC(void* pc) +{ + JSScript* callee = frameScript(); + + // Check for Ion first, since it's more likely for hot code. + if (callee->hasIonScript() && callee->ionScript()->method()->containsNativePC(pc)) { + type_ = JitFrame_IonJS; + returnAddressToFp_ = pc; + return true; + } + + // Check for containment in Baseline jitcode second. + if (callee->hasBaselineScript() && callee->baselineScript()->method()->containsNativePC(pc)) { + type_ = JitFrame_BaselineJS; + returnAddressToFp_ = pc; + return true; + } + + return false; +} + +bool +JitProfilingFrameIterator::tryInitWithTable(JitcodeGlobalTable* table, void* pc, JSRuntime* rt, + bool forLastCallSite) +{ + if (!pc) + return false; + + const JitcodeGlobalEntry* entry = table->lookup(pc); + if (!entry) + return false; + + JSScript* callee = frameScript(); + + MOZ_ASSERT(entry->isIon() || entry->isBaseline() || entry->isIonCache() || entry->isDummy()); + + // Treat dummy lookups as an empty frame sequence. + if (entry->isDummy()) { + type_ = JitFrame_Entry; + fp_ = nullptr; + returnAddressToFp_ = nullptr; + return true; + } + + if (entry->isIon()) { + // If looked-up callee doesn't match frame callee, don't accept lastProfilingCallSite + if (entry->ionEntry().getScript(0) != callee) + return false; + + type_ = JitFrame_IonJS; + returnAddressToFp_ = pc; + return true; + } + + if (entry->isBaseline()) { + // If looked-up callee doesn't match frame callee, don't accept lastProfilingCallSite + if (forLastCallSite && entry->baselineEntry().script() != callee) + return false; + + type_ = JitFrame_BaselineJS; + returnAddressToFp_ = pc; + return true; + } + + if (entry->isIonCache()) { + void* ptr = entry->ionCacheEntry().rejoinAddr(); + const JitcodeGlobalEntry& ionEntry = table->lookupInfallible(ptr); + MOZ_ASSERT(ionEntry.isIon()); + + if (ionEntry.ionEntry().getScript(0) != callee) + return false; + + type_ = JitFrame_IonJS; + returnAddressToFp_ = pc; + return true; + } + + return false; +} + +void +JitProfilingFrameIterator::fixBaselineReturnAddress() +{ + MOZ_ASSERT(type_ == JitFrame_BaselineJS); + BaselineFrame* bl = (BaselineFrame*)(fp_ - BaselineFrame::FramePointerOffset - + BaselineFrame::Size()); + + // Debug mode OSR for Baseline uses a "continuation fixer" and stashes the + // actual return address in an auxiliary structure. + if (BaselineDebugModeOSRInfo* info = bl->getDebugModeOSRInfo()) { + returnAddressToFp_ = info->resumeAddr; + return; + } + + // Resuming a generator via .throw() pushes a bogus return address onto + // the stack. We have the actual jsbytecode* stashed on the frame itself; + // translate that into the Baseline code address. + if (jsbytecode* override = bl->maybeOverridePc()) { + JSScript* script = bl->script(); + returnAddressToFp_ = script->baselineScript()->nativeCodeForPC(script, override); + return; + } +} + +void +JitProfilingFrameIterator::operator++() +{ + JitFrameLayout* frame = framePtr(); + moveToNextFrame(frame); +} + +void +JitProfilingFrameIterator::moveToNextFrame(CommonFrameLayout* frame) +{ + /* + * fp_ points to a Baseline or Ion frame. The possible call-stacks + * patterns occurring between this frame and a previous Ion or Baseline + * frame are as follows: + * + * <Baseline-Or-Ion> + * ^ + * | + * ^--- Ion + * | + * ^--- Baseline Stub <---- Baseline + * | + * ^--- Argument Rectifier + * | ^ + * | | + * | ^--- Ion + * | | + * | ^--- Baseline Stub <---- Baseline + * | + * ^--- Entry Frame (From C++) + * Exit Frame (From previous JitActivation) + * ^ + * | + * ^--- Ion + * | + * ^--- Baseline + * | + * ^--- Baseline Stub <---- Baseline + */ + FrameType prevType = frame->prevType(); + + if (prevType == JitFrame_IonJS) { + returnAddressToFp_ = frame->returnAddress(); + fp_ = GetPreviousRawFrame<uint8_t*>(frame); + type_ = JitFrame_IonJS; + return; + } + + if (prevType == JitFrame_BaselineJS) { + returnAddressToFp_ = frame->returnAddress(); + fp_ = GetPreviousRawFrame<uint8_t*>(frame); + type_ = JitFrame_BaselineJS; + fixBaselineReturnAddress(); + return; + } + + if (prevType == JitFrame_BaselineStub) { + BaselineStubFrameLayout* stubFrame = GetPreviousRawFrame<BaselineStubFrameLayout*>(frame); + MOZ_ASSERT(stubFrame->prevType() == JitFrame_BaselineJS); + + returnAddressToFp_ = stubFrame->returnAddress(); + fp_ = ((uint8_t*) stubFrame->reverseSavedFramePtr()) + + jit::BaselineFrame::FramePointerOffset; + type_ = JitFrame_BaselineJS; + return; + } + + if (prevType == JitFrame_Rectifier) { + RectifierFrameLayout* rectFrame = GetPreviousRawFrame<RectifierFrameLayout*>(frame); + FrameType rectPrevType = rectFrame->prevType(); + + if (rectPrevType == JitFrame_IonJS) { + returnAddressToFp_ = rectFrame->returnAddress(); + fp_ = GetPreviousRawFrame<uint8_t*>(rectFrame); + type_ = JitFrame_IonJS; + return; + } + + if (rectPrevType == JitFrame_BaselineStub) { + BaselineStubFrameLayout* stubFrame = + GetPreviousRawFrame<BaselineStubFrameLayout*>(rectFrame); + returnAddressToFp_ = stubFrame->returnAddress(); + fp_ = ((uint8_t*) stubFrame->reverseSavedFramePtr()) + + jit::BaselineFrame::FramePointerOffset; + type_ = JitFrame_BaselineJS; + return; + } + + MOZ_CRASH("Bad frame type prior to rectifier frame."); + } + + if (prevType == JitFrame_IonAccessorIC) { + IonAccessorICFrameLayout* accessorFrame = + GetPreviousRawFrame<IonAccessorICFrameLayout*>(frame); + + MOZ_ASSERT(accessorFrame->prevType() == JitFrame_IonJS); + + returnAddressToFp_ = accessorFrame->returnAddress(); + fp_ = GetPreviousRawFrame<uint8_t*>(accessorFrame); + type_ = JitFrame_IonJS; + return; + } + + if (prevType == JitFrame_Entry) { + // No previous frame, set to null to indicate that JitFrameIterator is done() + returnAddressToFp_ = nullptr; + fp_ = nullptr; + type_ = JitFrame_Entry; + return; + } + + MOZ_CRASH("Bad frame type."); +} + +JitFrameLayout* +InvalidationBailoutStack::fp() const +{ + return (JitFrameLayout*) (sp() + ionScript_->frameSize()); +} + +void +InvalidationBailoutStack::checkInvariants() const +{ +#ifdef DEBUG + JitFrameLayout* frame = fp(); + CalleeToken token = frame->calleeToken(); + MOZ_ASSERT(token); + + uint8_t* rawBase = ionScript()->method()->raw(); + uint8_t* rawLimit = rawBase + ionScript()->method()->instructionsSize(); + uint8_t* osiPoint = osiPointReturnAddress(); + MOZ_ASSERT(rawBase <= osiPoint && osiPoint <= rawLimit); +#endif +} + +void +AssertJitStackInvariants(JSContext* cx) +{ + for (JitActivationIterator activations(cx->runtime()); !activations.done(); ++activations) { + JitFrameIterator frames(activations); + size_t prevFrameSize = 0; + size_t frameSize = 0; + bool isScriptedCallee = false; + for (; !frames.done(); ++frames) { + size_t calleeFp = reinterpret_cast<size_t>(frames.fp()); + size_t callerFp = reinterpret_cast<size_t>(frames.prevFp()); + MOZ_ASSERT(callerFp >= calleeFp); + prevFrameSize = frameSize; + frameSize = callerFp - calleeFp; + + if (frames.prevType() == JitFrame_Rectifier) { + MOZ_RELEASE_ASSERT(frameSize % JitStackAlignment == 0, + "The rectifier frame should keep the alignment"); + + size_t expectedFrameSize = 0 +#if defined(JS_CODEGEN_X86) + + sizeof(void*) /* frame pointer */ +#endif + + sizeof(Value) * (frames.callee()->nargs() + + 1 /* |this| argument */ + + frames.isConstructing() /* new.target */) + + sizeof(JitFrameLayout); + MOZ_RELEASE_ASSERT(frameSize >= expectedFrameSize, + "The frame is large enough to hold all arguments"); + MOZ_RELEASE_ASSERT(expectedFrameSize + JitStackAlignment > frameSize, + "The frame size is optimal"); + } + + if (frames.isExitFrame()) { + // For the moment, we do not keep the JitStackAlignment + // alignment for exit frames. + frameSize -= ExitFrameLayout::Size(); + } + + if (frames.isIonJS()) { + // Ideally, we should not have such requirement, but keep the + // alignment-delta as part of the Safepoint such that we can pad + // accordingly when making out-of-line calls. In the mean time, + // let us have check-points where we can garantee that + // everything can properly be aligned before adding complexity. + MOZ_RELEASE_ASSERT(frames.ionScript()->frameSize() % JitStackAlignment == 0, + "Ensure that if the Ion frame is aligned, then the spill base is also aligned"); + + if (isScriptedCallee) { + MOZ_RELEASE_ASSERT(prevFrameSize % JitStackAlignment == 0, + "The ion frame should keep the alignment"); + } + } + + // The stack is dynamically aligned by baseline stubs before calling + // any jitted code. + if (frames.prevType() == JitFrame_BaselineStub && isScriptedCallee) { + MOZ_RELEASE_ASSERT(calleeFp % JitStackAlignment == 0, + "The baseline stub restores the stack alignment"); + } + + isScriptedCallee = false + || frames.isScripted() + || frames.type() == JitFrame_Rectifier; + } + + MOZ_RELEASE_ASSERT(frames.type() == JitFrame_Entry, + "The first frame of a Jit activation should be an entry frame"); + MOZ_RELEASE_ASSERT(reinterpret_cast<size_t>(frames.fp()) % JitStackAlignment == 0, + "The entry frame should be properly aligned"); + } +} + +} // namespace jit +} // namespace js |