diff options
Diffstat (limited to 'js/src/jit/BaselineDebugModeOSR.cpp')
| -rw-r--r-- | js/src/jit/BaselineDebugModeOSR.cpp | 1184 |
1 files changed, 1184 insertions, 0 deletions
diff --git a/js/src/jit/BaselineDebugModeOSR.cpp b/js/src/jit/BaselineDebugModeOSR.cpp new file mode 100644 index 000000000..8b937ee24 --- /dev/null +++ b/js/src/jit/BaselineDebugModeOSR.cpp @@ -0,0 +1,1184 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/BaselineDebugModeOSR.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/SizePrintfMacros.h" + +#include "jit/BaselineIC.h" +#include "jit/JitcodeMap.h" +#include "jit/Linker.h" +#include "jit/PerfSpewer.h" + +#include "jit/JitFrames-inl.h" +#include "jit/MacroAssembler-inl.h" +#include "vm/Stack-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::DebugOnly; + +struct DebugModeOSREntry +{ + JSScript* script; + BaselineScript* oldBaselineScript; + ICStub* oldStub; + ICStub* newStub; + BaselineDebugModeOSRInfo* recompInfo; + uint32_t pcOffset; + ICEntry::Kind frameKind; + + explicit DebugModeOSREntry(JSScript* script) + : script(script), + oldBaselineScript(script->baselineScript()), + oldStub(nullptr), + newStub(nullptr), + recompInfo(nullptr), + pcOffset(uint32_t(-1)), + frameKind(ICEntry::Kind_Invalid) + { } + + DebugModeOSREntry(JSScript* script, uint32_t pcOffset) + : script(script), + oldBaselineScript(script->baselineScript()), + oldStub(nullptr), + newStub(nullptr), + recompInfo(nullptr), + pcOffset(pcOffset), + frameKind(ICEntry::Kind_Invalid) + { } + + DebugModeOSREntry(JSScript* script, const ICEntry& icEntry) + : script(script), + oldBaselineScript(script->baselineScript()), + oldStub(nullptr), + newStub(nullptr), + recompInfo(nullptr), + pcOffset(icEntry.pcOffset()), + frameKind(icEntry.kind()) + { +#ifdef DEBUG + MOZ_ASSERT(pcOffset == icEntry.pcOffset()); + MOZ_ASSERT(frameKind == icEntry.kind()); +#endif + } + + DebugModeOSREntry(JSScript* script, BaselineDebugModeOSRInfo* info) + : script(script), + oldBaselineScript(script->baselineScript()), + oldStub(nullptr), + newStub(nullptr), + recompInfo(nullptr), + pcOffset(script->pcToOffset(info->pc)), + frameKind(info->frameKind) + { +#ifdef DEBUG + MOZ_ASSERT(pcOffset == script->pcToOffset(info->pc)); + MOZ_ASSERT(frameKind == info->frameKind); +#endif + } + + DebugModeOSREntry(DebugModeOSREntry&& other) + : script(other.script), + oldBaselineScript(other.oldBaselineScript), + oldStub(other.oldStub), + newStub(other.newStub), + recompInfo(other.recompInfo ? other.takeRecompInfo() : nullptr), + pcOffset(other.pcOffset), + frameKind(other.frameKind) + { } + + ~DebugModeOSREntry() { + // Note that this is nulled out when the recompInfo is taken by the + // frame. The frame then has the responsibility of freeing the + // recompInfo. + js_delete(recompInfo); + } + + bool needsRecompileInfo() const { + return frameKind == ICEntry::Kind_CallVM || + frameKind == ICEntry::Kind_WarmupCounter || + frameKind == ICEntry::Kind_StackCheck || + frameKind == ICEntry::Kind_EarlyStackCheck || + frameKind == ICEntry::Kind_DebugTrap || + frameKind == ICEntry::Kind_DebugPrologue || + frameKind == ICEntry::Kind_DebugEpilogue; + } + + bool recompiled() const { + return oldBaselineScript != script->baselineScript(); + } + + BaselineDebugModeOSRInfo* takeRecompInfo() { + MOZ_ASSERT(needsRecompileInfo() && recompInfo); + BaselineDebugModeOSRInfo* tmp = recompInfo; + recompInfo = nullptr; + return tmp; + } + + bool allocateRecompileInfo(JSContext* cx) { + MOZ_ASSERT(script); + MOZ_ASSERT(needsRecompileInfo()); + + // If we are returning to a frame which needs a continuation fixer, + // allocate the recompile info up front so that the patching function + // is infallible. + jsbytecode* pc = script->offsetToPC(pcOffset); + + // XXX: Work around compiler error disallowing using bitfields + // with the template magic of new_. + ICEntry::Kind kind = frameKind; + recompInfo = cx->new_<BaselineDebugModeOSRInfo>(pc, kind); + return !!recompInfo; + } + + ICFallbackStub* fallbackStub() const { + MOZ_ASSERT(script); + MOZ_ASSERT(oldStub); + return script->baselineScript()->icEntryFromPCOffset(pcOffset).fallbackStub(); + } +}; + +typedef Vector<DebugModeOSREntry> DebugModeOSREntryVector; + +class UniqueScriptOSREntryIter +{ + const DebugModeOSREntryVector& entries_; + size_t index_; + + public: + explicit UniqueScriptOSREntryIter(const DebugModeOSREntryVector& entries) + : entries_(entries), + index_(0) + { } + + bool done() { + return index_ == entries_.length(); + } + + const DebugModeOSREntry& entry() { + MOZ_ASSERT(!done()); + return entries_[index_]; + } + + UniqueScriptOSREntryIter& operator++() { + MOZ_ASSERT(!done()); + while (++index_ < entries_.length()) { + bool unique = true; + for (size_t i = 0; i < index_; i++) { + if (entries_[i].script == entries_[index_].script) { + unique = false; + break; + } + } + if (unique) + break; + } + return *this; + } +}; + +static bool +CollectJitStackScripts(JSContext* cx, const Debugger::ExecutionObservableSet& obs, + const ActivationIterator& activation, DebugModeOSREntryVector& entries) +{ + ICStub* prevFrameStubPtr = nullptr; + bool needsRecompileHandler = false; + for (JitFrameIterator iter(activation); !iter.done(); ++iter) { + switch (iter.type()) { + case JitFrame_BaselineJS: { + JSScript* script = iter.script(); + + if (!obs.shouldRecompileOrInvalidate(script)) { + prevFrameStubPtr = nullptr; + break; + } + + BaselineFrame* frame = iter.baselineFrame(); + + if (BaselineDebugModeOSRInfo* info = frame->getDebugModeOSRInfo()) { + // If patching a previously patched yet unpopped frame, we can + // use the BaselineDebugModeOSRInfo on the frame directly to + // patch. Indeed, we cannot use iter.returnAddressToFp(), as + // it points into the debug mode OSR handler and cannot be + // used to look up a corresponding ICEntry. + // + // See cases F and G in PatchBaselineFramesForDebugMode. + if (!entries.append(DebugModeOSREntry(script, info))) + return false; + } else if (frame->isHandlingException()) { + // We are in the middle of handling an exception and the frame + // must have an override pc. + uint32_t offset = script->pcToOffset(frame->overridePc()); + if (!entries.append(DebugModeOSREntry(script, offset))) + return false; + } else { + // The frame must be settled on a pc with an ICEntry. + uint8_t* retAddr = iter.returnAddressToFp(); + BaselineICEntry& icEntry = script->baselineScript()->icEntryFromReturnAddress(retAddr); + if (!entries.append(DebugModeOSREntry(script, icEntry))) + return false; + } + + if (entries.back().needsRecompileInfo()) { + if (!entries.back().allocateRecompileInfo(cx)) + return false; + + needsRecompileHandler |= true; + } + entries.back().oldStub = prevFrameStubPtr; + prevFrameStubPtr = nullptr; + break; + } + + case JitFrame_BaselineStub: + prevFrameStubPtr = + reinterpret_cast<BaselineStubFrameLayout*>(iter.fp())->maybeStubPtr(); + break; + + case JitFrame_IonJS: { + InlineFrameIterator inlineIter(cx, &iter); + while (true) { + if (obs.shouldRecompileOrInvalidate(inlineIter.script())) { + if (!entries.append(DebugModeOSREntry(inlineIter.script()))) + return false; + } + if (!inlineIter.more()) + break; + ++inlineIter; + } + break; + } + + default:; + } + } + + // Initialize the on-stack recompile handler, which may fail, so that + // patching the stack is infallible. + if (needsRecompileHandler) { + JitRuntime* rt = cx->runtime()->jitRuntime(); + if (!rt->getBaselineDebugModeOSRHandlerAddress(cx, true)) + return false; + } + + return true; +} + +static bool +CollectInterpreterStackScripts(JSContext* cx, const Debugger::ExecutionObservableSet& obs, + const ActivationIterator& activation, + DebugModeOSREntryVector& entries) +{ + // Collect interpreter frame stacks with IonScript or BaselineScript as + // well. These do not need to be patched, but do need to be invalidated + // and recompiled. + InterpreterActivation* act = activation.activation()->asInterpreter(); + for (InterpreterFrameIterator iter(act); !iter.done(); ++iter) { + JSScript* script = iter.frame()->script(); + if (obs.shouldRecompileOrInvalidate(script)) { + if (!entries.append(DebugModeOSREntry(iter.frame()->script()))) + return false; + } + } + return true; +} + +#ifdef JS_JITSPEW +static const char* +ICEntryKindToString(ICEntry::Kind kind) +{ + switch (kind) { + case ICEntry::Kind_Op: + return "IC"; + case ICEntry::Kind_NonOp: + return "non-op IC"; + case ICEntry::Kind_CallVM: + return "callVM"; + case ICEntry::Kind_WarmupCounter: + return "warmup counter"; + case ICEntry::Kind_StackCheck: + return "stack check"; + case ICEntry::Kind_EarlyStackCheck: + return "early stack check"; + case ICEntry::Kind_DebugTrap: + return "debug trap"; + case ICEntry::Kind_DebugPrologue: + return "debug prologue"; + case ICEntry::Kind_DebugEpilogue: + return "debug epilogue"; + default: + MOZ_CRASH("bad ICEntry kind"); + } +} +#endif // JS_JITSPEW + +static void +SpewPatchBaselineFrame(uint8_t* oldReturnAddress, uint8_t* newReturnAddress, + JSScript* script, ICEntry::Kind frameKind, jsbytecode* pc) +{ + JitSpew(JitSpew_BaselineDebugModeOSR, + "Patch return %p -> %p on BaselineJS frame (%s:%" PRIuSIZE ") from %s at %s", + oldReturnAddress, newReturnAddress, script->filename(), script->lineno(), + ICEntryKindToString(frameKind), CodeName[(JSOp)*pc]); +} + +static void +SpewPatchBaselineFrameFromExceptionHandler(uint8_t* oldReturnAddress, uint8_t* newReturnAddress, + JSScript* script, jsbytecode* pc) +{ + JitSpew(JitSpew_BaselineDebugModeOSR, + "Patch return %p -> %p on BaselineJS frame (%s:%" PRIuSIZE ") from exception handler at %s", + oldReturnAddress, newReturnAddress, script->filename(), script->lineno(), + CodeName[(JSOp)*pc]); +} + +static void +SpewPatchStubFrame(ICStub* oldStub, ICStub* newStub) +{ + JitSpew(JitSpew_BaselineDebugModeOSR, + "Patch stub %p -> %p on BaselineStub frame (%s)", + oldStub, newStub, newStub ? ICStub::KindString(newStub->kind()) : "exception handler"); +} + +static void +PatchBaselineFramesForDebugMode(JSContext* cx, const Debugger::ExecutionObservableSet& obs, + const ActivationIterator& activation, + DebugModeOSREntryVector& entries, size_t* start) +{ + // + // Recompile Patching Overview + // + // When toggling debug mode with live baseline scripts on the stack, we + // could have entered the VM via the following ways from the baseline + // script. + // + // Off to On: + // A. From a "can call" stub. + // B. From a VM call. + // H. From inside HandleExceptionBaseline. + // I. From inside the interrupt handler via the prologue stack check. + // J. From the warmup counter in the prologue. + // + // On to Off: + // - All the ways above. + // C. From the debug trap handler. + // D. From the debug prologue. + // E. From the debug epilogue. + // + // Cycles (On to Off to On)+ or (Off to On to Off)+: + // F. Undo cases B, C, D, E, I or J above on previously patched yet unpopped + // frames. + // + // In general, we patch the return address from the VM call to return to a + // "continuation fixer" to fix up machine state (registers and stack + // state). Specifics on what needs to be done are documented below. + // + + CommonFrameLayout* prev = nullptr; + size_t entryIndex = *start; + + for (JitFrameIterator iter(activation); !iter.done(); ++iter) { + switch (iter.type()) { + case JitFrame_BaselineJS: { + // If the script wasn't recompiled or is not observed, there's + // nothing to patch. + if (!obs.shouldRecompileOrInvalidate(iter.script())) + break; + + DebugModeOSREntry& entry = entries[entryIndex]; + + if (!entry.recompiled()) { + entryIndex++; + break; + } + + JSScript* script = entry.script; + uint32_t pcOffset = entry.pcOffset; + jsbytecode* pc = script->offsetToPC(pcOffset); + + MOZ_ASSERT(script == iter.script()); + MOZ_ASSERT(pcOffset < script->length()); + + BaselineScript* bl = script->baselineScript(); + ICEntry::Kind kind = entry.frameKind; + + if (kind == ICEntry::Kind_Op) { + // Case A above. + // + // Patching these cases needs to patch both the stub frame and + // the baseline frame. The stub frame is patched below. For + // the baseline frame here, we resume right after the IC + // returns. + // + // Since we're using the same IC stub code, we can resume + // directly to the IC resume address. + uint8_t* retAddr = bl->returnAddressForIC(bl->icEntryFromPCOffset(pcOffset)); + SpewPatchBaselineFrame(prev->returnAddress(), retAddr, script, kind, pc); + DebugModeOSRVolatileJitFrameIterator::forwardLiveIterators( + cx, prev->returnAddress(), retAddr); + prev->setReturnAddress(retAddr); + entryIndex++; + break; + } + + if (kind == ICEntry::Kind_Invalid) { + // Case H above. + // + // We are recompiling on-stack scripts from inside the + // exception handler, by way of an onExceptionUnwind + // invocation, on a pc without an ICEntry. This means the + // frame must have an override pc. + // + // If profiling is off, patch the resume address to nullptr, + // to ensure the old address is not used anywhere. + // + // If profiling is on, JitProfilingFrameIterator requires a + // valid return address. + MOZ_ASSERT(iter.baselineFrame()->isHandlingException()); + MOZ_ASSERT(iter.baselineFrame()->overridePc() == pc); + uint8_t* retAddr; + if (cx->runtime()->spsProfiler.enabled()) + retAddr = bl->nativeCodeForPC(script, pc); + else + retAddr = nullptr; + SpewPatchBaselineFrameFromExceptionHandler(prev->returnAddress(), retAddr, + script, pc); + DebugModeOSRVolatileJitFrameIterator::forwardLiveIterators( + cx, prev->returnAddress(), retAddr); + prev->setReturnAddress(retAddr); + entryIndex++; + break; + } + + // Case F above. + // + // We undo a previous recompile by handling cases B, C, D, E, I or J + // like normal, except that we retrieve the pc information via + // the previous OSR debug info stashed on the frame. + BaselineDebugModeOSRInfo* info = iter.baselineFrame()->getDebugModeOSRInfo(); + if (info) { + MOZ_ASSERT(info->pc == pc); + MOZ_ASSERT(info->frameKind == kind); + MOZ_ASSERT(kind == ICEntry::Kind_CallVM || + kind == ICEntry::Kind_WarmupCounter || + kind == ICEntry::Kind_StackCheck || + kind == ICEntry::Kind_EarlyStackCheck || + kind == ICEntry::Kind_DebugTrap || + kind == ICEntry::Kind_DebugPrologue || + kind == ICEntry::Kind_DebugEpilogue); + + // We will have allocated a new recompile info, so delete the + // existing one. + iter.baselineFrame()->deleteDebugModeOSRInfo(); + } + + // The RecompileInfo must already be allocated so that this + // function may be infallible. + BaselineDebugModeOSRInfo* recompInfo = entry.takeRecompInfo(); + + bool popFrameReg; + switch (kind) { + case ICEntry::Kind_CallVM: { + // Case B above. + // + // Patching returns from a VM call. After fixing up the the + // continuation for unsynced values (the frame register is + // popped by the callVM trampoline), we resume at the + // return-from-callVM address. The assumption here is that all + // callVMs which can trigger debug mode OSR are the *only* + // callVMs generated for their respective pc locations in the + // baseline JIT code. + BaselineICEntry& callVMEntry = bl->callVMEntryFromPCOffset(pcOffset); + recompInfo->resumeAddr = bl->returnAddressForIC(callVMEntry); + popFrameReg = false; + break; + } + + case ICEntry::Kind_WarmupCounter: { + // Case J above. + // + // Patching mechanism is identical to a CallVM. This is + // handled especially only because the warmup counter VM call is + // part of the prologue, and not tied an opcode. + BaselineICEntry& warmupCountEntry = bl->warmupCountICEntry(); + recompInfo->resumeAddr = bl->returnAddressForIC(warmupCountEntry); + popFrameReg = false; + break; + } + + case ICEntry::Kind_StackCheck: + case ICEntry::Kind_EarlyStackCheck: { + // Case I above. + // + // Patching mechanism is identical to a CallVM. This is + // handled especially only because the stack check VM call is + // part of the prologue, and not tied an opcode. + bool earlyCheck = kind == ICEntry::Kind_EarlyStackCheck; + BaselineICEntry& stackCheckEntry = bl->stackCheckICEntry(earlyCheck); + recompInfo->resumeAddr = bl->returnAddressForIC(stackCheckEntry); + popFrameReg = false; + break; + } + + case ICEntry::Kind_DebugTrap: + // Case C above. + // + // Debug traps are emitted before each op, so we resume at the + // same op. Calling debug trap handlers is done via a toggled + // call to a thunk (DebugTrapHandler) that takes care tearing + // down its own stub frame so we don't need to worry about + // popping the frame reg. + recompInfo->resumeAddr = bl->nativeCodeForPC(script, pc, &recompInfo->slotInfo); + popFrameReg = false; + break; + + case ICEntry::Kind_DebugPrologue: + // Case D above. + // + // We patch a jump directly to the right place in the prologue + // after popping the frame reg and checking for forced return. + recompInfo->resumeAddr = bl->postDebugPrologueAddr(); + popFrameReg = true; + break; + + default: + // Case E above. + // + // We patch a jump directly to the epilogue after popping the + // frame reg and checking for forced return. + MOZ_ASSERT(kind == ICEntry::Kind_DebugEpilogue); + recompInfo->resumeAddr = bl->epilogueEntryAddr(); + popFrameReg = true; + break; + } + + SpewPatchBaselineFrame(prev->returnAddress(), recompInfo->resumeAddr, + script, kind, recompInfo->pc); + + // The recompile handler must already be created so that this + // function may be infallible. + JitRuntime* rt = cx->runtime()->jitRuntime(); + void* handlerAddr = rt->getBaselineDebugModeOSRHandlerAddress(cx, popFrameReg); + MOZ_ASSERT(handlerAddr); + + prev->setReturnAddress(reinterpret_cast<uint8_t*>(handlerAddr)); + iter.baselineFrame()->setDebugModeOSRInfo(recompInfo); + iter.baselineFrame()->setOverridePc(recompInfo->pc); + + entryIndex++; + break; + } + + case JitFrame_BaselineStub: { + JitFrameIterator prev(iter); + ++prev; + BaselineFrame* prevFrame = prev.baselineFrame(); + if (!obs.shouldRecompileOrInvalidate(prevFrame->script())) + break; + + DebugModeOSREntry& entry = entries[entryIndex]; + + // If the script wasn't recompiled, there's nothing to patch. + if (!entry.recompiled()) + break; + + BaselineStubFrameLayout* layout = + reinterpret_cast<BaselineStubFrameLayout*>(iter.fp()); + MOZ_ASSERT(layout->maybeStubPtr() == entry.oldStub); + + // Patch baseline stub frames for case A above. + // + // We need to patch the stub frame to point to an ICStub belonging + // to the recompiled baseline script. These stubs are allocated up + // front in CloneOldBaselineStub. They share the same JitCode as + // the old baseline script's stubs, so we don't need to patch the + // exit frame's return address. + // + // Subtlety here: the debug trap handler of case C above pushes a + // stub frame with a null stub pointer. This handler will exist + // across recompiling the script, so we don't patch anything for + // such stub frames. We will return to that handler, which takes + // care of cleaning up the stub frame. + // + // Note that for stub pointers that are already on the C stack + // (i.e. fallback calls), we need to check for recompilation using + // DebugModeOSRVolatileStub. + if (layout->maybeStubPtr()) { + MOZ_ASSERT(entry.newStub || prevFrame->isHandlingException()); + SpewPatchStubFrame(entry.oldStub, entry.newStub); + layout->setStubPtr(entry.newStub); + } + + break; + } + + case JitFrame_IonJS: { + // Nothing to patch. + InlineFrameIterator inlineIter(cx, &iter); + while (true) { + if (obs.shouldRecompileOrInvalidate(inlineIter.script())) + entryIndex++; + if (!inlineIter.more()) + break; + ++inlineIter; + } + break; + } + + default:; + } + + prev = iter.current(); + } + + *start = entryIndex; +} + +static void +SkipInterpreterFrameEntries(const Debugger::ExecutionObservableSet& obs, + const ActivationIterator& activation, + DebugModeOSREntryVector& entries, size_t* start) +{ + size_t entryIndex = *start; + + // Skip interpreter frames, which do not need patching. + InterpreterActivation* act = activation.activation()->asInterpreter(); + for (InterpreterFrameIterator iter(act); !iter.done(); ++iter) { + if (obs.shouldRecompileOrInvalidate(iter.frame()->script())) + entryIndex++; + } + + *start = entryIndex; +} + +static bool +RecompileBaselineScriptForDebugMode(JSContext* cx, JSScript* script, + Debugger::IsObserving observing) +{ + BaselineScript* oldBaselineScript = script->baselineScript(); + + // If a script is on the stack multiple times, it may have already + // been recompiled. + if (oldBaselineScript->hasDebugInstrumentation() == observing) + return true; + + JitSpew(JitSpew_BaselineDebugModeOSR, "Recompiling (%s:%" PRIuSIZE ") for %s", + script->filename(), script->lineno(), observing ? "DEBUGGING" : "NORMAL EXECUTION"); + + script->setBaselineScript(cx->runtime(), nullptr); + + MethodStatus status = BaselineCompile(cx, script, /* forceDebugMode = */ observing); + if (status != Method_Compiled) { + // We will only fail to recompile for debug mode due to OOM. Restore + // the old baseline script in case something doesn't properly + // propagate OOM. + MOZ_ASSERT(status == Method_Error); + script->setBaselineScript(cx->runtime(), oldBaselineScript); + return false; + } + + // Don't destroy the old baseline script yet, since if we fail any of the + // recompiles we need to rollback all the old baseline scripts. + MOZ_ASSERT(script->baselineScript()->hasDebugInstrumentation() == observing); + return true; +} + +#define PATCHABLE_ICSTUB_KIND_LIST(_) \ + _(Call_Scripted) \ + _(Call_AnyScripted) \ + _(Call_Native) \ + _(Call_ClassHook) \ + _(Call_ScriptedApplyArray) \ + _(Call_ScriptedApplyArguments) \ + _(Call_ScriptedFunCall) \ + _(GetElem_NativePrototypeCallNativeName) \ + _(GetElem_NativePrototypeCallNativeSymbol) \ + _(GetElem_NativePrototypeCallScriptedName) \ + _(GetElem_NativePrototypeCallScriptedSymbol) \ + _(GetProp_CallScripted) \ + _(GetProp_CallNative) \ + _(GetProp_CallNativeGlobal) \ + _(GetProp_CallDOMProxyNative) \ + _(GetProp_CallDOMProxyWithGenerationNative) \ + _(GetProp_DOMProxyShadowed) \ + _(GetProp_Generic) \ + _(SetProp_CallScripted) \ + _(SetProp_CallNative) + +static bool +CloneOldBaselineStub(JSContext* cx, DebugModeOSREntryVector& entries, size_t entryIndex) +{ + DebugModeOSREntry& entry = entries[entryIndex]; + if (!entry.oldStub) + return true; + + ICStub* oldStub = entry.oldStub; + MOZ_ASSERT(ICStub::CanMakeCalls(oldStub->kind())); + + if (entry.frameKind == ICEntry::Kind_Invalid) { + // The exception handler can modify the frame's override pc while + // unwinding scopes. This is fine, but if we have a stub frame, the code + // code below will get confused: the entry's pcOffset doesn't match the + // stub that's still on the stack. To prevent that, we just set the new + // stub to nullptr as we will never return to this stub frame anyway. + entry.newStub = nullptr; + return true; + } + + // Get the new fallback stub from the recompiled baseline script. + ICFallbackStub* fallbackStub = entry.fallbackStub(); + + // We don't need to clone fallback stubs, as they are guaranteed to + // exist. Furthermore, their JitCode is cached and should be the same even + // across the recompile. + if (oldStub->isFallback()) { + MOZ_ASSERT(oldStub->jitCode() == fallbackStub->jitCode()); + entry.newStub = fallbackStub; + return true; + } + + // Check if we have already cloned the stub on a younger frame. Ignore + // frames that entered the exception handler (entries[i].newStub is nullptr + // in that case, see above). + for (size_t i = 0; i < entryIndex; i++) { + if (oldStub == entries[i].oldStub && entries[i].frameKind != ICEntry::Kind_Invalid) { + MOZ_ASSERT(entries[i].newStub); + entry.newStub = entries[i].newStub; + return true; + } + } + + // Some stubs are monitored, get the first stub in the monitor chain from + // the new fallback stub if so. + ICStub* firstMonitorStub; + if (fallbackStub->isMonitoredFallback()) { + ICMonitoredFallbackStub* monitored = fallbackStub->toMonitoredFallbackStub(); + firstMonitorStub = monitored->fallbackMonitorStub()->firstMonitorStub(); + } else { + firstMonitorStub = nullptr; + } + ICStubSpace* stubSpace = ICStubCompiler::StubSpaceForKind(oldStub->kind(), entry.script, + ICStubCompiler::Engine::Baseline); + + // Clone the existing stub into the recompiled IC. + // + // Note that since JitCode is a GC thing, cloning an ICStub with the same + // JitCode ensures it won't be collected. + switch (oldStub->kind()) { +#define CASE_KIND(kindName) \ + case ICStub::kindName: \ + entry.newStub = IC##kindName::Clone(cx, stubSpace, firstMonitorStub, \ + *oldStub->to##kindName()); \ + break; + PATCHABLE_ICSTUB_KIND_LIST(CASE_KIND) +#undef CASE_KIND + + default: + MOZ_CRASH("Bad stub kind"); + } + + if (!entry.newStub) + return false; + + fallbackStub->addNewStub(entry.newStub); + return true; +} + +static bool +InvalidateScriptsInZone(JSContext* cx, Zone* zone, const Vector<DebugModeOSREntry>& entries) +{ + RecompileInfoVector invalid; + for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) { + JSScript* script = iter.entry().script; + if (script->compartment()->zone() != zone) + continue; + + if (script->hasIonScript()) { + if (!invalid.append(script->ionScript()->recompileInfo())) { + ReportOutOfMemory(cx); + return false; + } + } + + // Cancel off-thread Ion compile for anything that has a + // BaselineScript. If we relied on the call to Invalidate below to + // cancel off-thread Ion compiles, only those with existing IonScripts + // would be cancelled. + if (script->hasBaselineScript()) + CancelOffThreadIonCompile(script); + } + + // No need to cancel off-thread Ion compiles again, we already did it + // above. + Invalidate(zone->types, cx->runtime()->defaultFreeOp(), invalid, + /* resetUses = */ true, /* cancelOffThread = */ false); + return true; +} + +static void +UndoRecompileBaselineScriptsForDebugMode(JSContext* cx, + const DebugModeOSREntryVector& entries) +{ + // In case of failure, roll back the entire set of active scripts so that + // we don't have to patch return addresses on the stack. + for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) { + const DebugModeOSREntry& entry = iter.entry(); + JSScript* script = entry.script; + BaselineScript* baselineScript = script->baselineScript(); + if (entry.recompiled()) { + script->setBaselineScript(cx->runtime(), entry.oldBaselineScript); + BaselineScript::Destroy(cx->runtime()->defaultFreeOp(), baselineScript); + } + } +} + +bool +jit::RecompileOnStackBaselineScriptsForDebugMode(JSContext* cx, + const Debugger::ExecutionObservableSet& obs, + Debugger::IsObserving observing) +{ + // First recompile the active scripts on the stack and patch the live + // frames. + Vector<DebugModeOSREntry> entries(cx); + + for (ActivationIterator iter(cx->runtime()); !iter.done(); ++iter) { + if (iter->isJit()) { + if (!CollectJitStackScripts(cx, obs, iter, entries)) + return false; + } else if (iter->isInterpreter()) { + if (!CollectInterpreterStackScripts(cx, obs, iter, entries)) + return false; + } + } + + if (entries.empty()) + return true; + + // When the profiler is enabled, we need to have suppressed sampling, + // since the basline jit scripts are in a state of flux. + MOZ_ASSERT(!cx->runtime()->isProfilerSamplingEnabled()); + + // Invalidate all scripts we are recompiling. + if (Zone* zone = obs.singleZone()) { + if (!InvalidateScriptsInZone(cx, zone, entries)) + return false; + } else { + typedef Debugger::ExecutionObservableSet::ZoneRange ZoneRange; + for (ZoneRange r = obs.zones()->all(); !r.empty(); r.popFront()) { + if (!InvalidateScriptsInZone(cx, r.front(), entries)) + return false; + } + } + + // Try to recompile all the scripts. If we encounter an error, we need to + // roll back as if none of the compilations happened, so that we don't + // crash. + for (size_t i = 0; i < entries.length(); i++) { + JSScript* script = entries[i].script; + AutoCompartment ac(cx, script->compartment()); + if (!RecompileBaselineScriptForDebugMode(cx, script, observing) || + !CloneOldBaselineStub(cx, entries, i)) + { + UndoRecompileBaselineScriptsForDebugMode(cx, entries); + return false; + } + } + + // If all recompiles succeeded, destroy the old baseline scripts and patch + // the live frames. + // + // After this point the function must be infallible. + + for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) { + const DebugModeOSREntry& entry = iter.entry(); + if (entry.recompiled()) + BaselineScript::Destroy(cx->runtime()->defaultFreeOp(), entry.oldBaselineScript); + } + + size_t processed = 0; + for (ActivationIterator iter(cx->runtime()); !iter.done(); ++iter) { + if (iter->isJit()) + PatchBaselineFramesForDebugMode(cx, obs, iter, entries, &processed); + else if (iter->isInterpreter()) + SkipInterpreterFrameEntries(obs, iter, entries, &processed); + } + MOZ_ASSERT(processed == entries.length()); + + return true; +} + +void +BaselineDebugModeOSRInfo::popValueInto(PCMappingSlotInfo::SlotLocation loc, Value* vp) +{ + switch (loc) { + case PCMappingSlotInfo::SlotInR0: + valueR0 = vp[stackAdjust]; + break; + case PCMappingSlotInfo::SlotInR1: + valueR1 = vp[stackAdjust]; + break; + case PCMappingSlotInfo::SlotIgnore: + break; + default: + MOZ_CRASH("Bad slot location"); + } + + stackAdjust++; +} + +static inline bool +HasForcedReturn(BaselineDebugModeOSRInfo* info, bool rv) +{ + ICEntry::Kind kind = info->frameKind; + + // The debug epilogue always checks its resumption value, so we don't need + // to check rv. + if (kind == ICEntry::Kind_DebugEpilogue) + return true; + + // |rv| is the value in ReturnReg. If true, in the case of the prologue, + // it means a forced return. + if (kind == ICEntry::Kind_DebugPrologue) + return rv; + + // N.B. The debug trap handler handles its own forced return, so no + // need to deal with it here. + return false; +} + +static inline bool +IsReturningFromCallVM(BaselineDebugModeOSRInfo* info) +{ + // Keep this in sync with EmitBranchIsReturningFromCallVM. + // + // The stack check entries are returns from a callVM, but have a special + // kind because they do not exist in a 1-1 relationship with a pc offset. + return info->frameKind == ICEntry::Kind_CallVM || + info->frameKind == ICEntry::Kind_WarmupCounter || + info->frameKind == ICEntry::Kind_StackCheck || + info->frameKind == ICEntry::Kind_EarlyStackCheck; +} + +static void +EmitBranchICEntryKind(MacroAssembler& masm, Register entry, ICEntry::Kind kind, Label* label) +{ + masm.branch32(MacroAssembler::Equal, + Address(entry, offsetof(BaselineDebugModeOSRInfo, frameKind)), + Imm32(kind), label); +} + +static void +EmitBranchIsReturningFromCallVM(MacroAssembler& masm, Register entry, Label* label) +{ + // Keep this in sync with IsReturningFromCallVM. + EmitBranchICEntryKind(masm, entry, ICEntry::Kind_CallVM, label); + EmitBranchICEntryKind(masm, entry, ICEntry::Kind_WarmupCounter, label); + EmitBranchICEntryKind(masm, entry, ICEntry::Kind_StackCheck, label); + EmitBranchICEntryKind(masm, entry, ICEntry::Kind_EarlyStackCheck, label); +} + +static void +SyncBaselineDebugModeOSRInfo(BaselineFrame* frame, Value* vp, bool rv) +{ + BaselineDebugModeOSRInfo* info = frame->debugModeOSRInfo(); + MOZ_ASSERT(info); + MOZ_ASSERT(frame->script()->baselineScript()->containsCodeAddress(info->resumeAddr)); + + if (HasForcedReturn(info, rv)) { + // Load the frame's rval and overwrite the resume address to go to the + // epilogue. + MOZ_ASSERT(R0 == JSReturnOperand); + info->valueR0 = frame->returnValue(); + info->resumeAddr = frame->script()->baselineScript()->epilogueEntryAddr(); + return; + } + + // Read stack values and make sure R0 and R1 have the right values if we + // aren't returning from a callVM. + // + // In the case of returning from a callVM, we don't need to restore R0 and + // R1 ourself since we'll return into code that does it if needed. + if (!IsReturningFromCallVM(info)) { + unsigned numUnsynced = info->slotInfo.numUnsynced(); + MOZ_ASSERT(numUnsynced <= 2); + if (numUnsynced > 0) + info->popValueInto(info->slotInfo.topSlotLocation(), vp); + if (numUnsynced > 1) + info->popValueInto(info->slotInfo.nextSlotLocation(), vp); + } + + // Scale stackAdjust. + info->stackAdjust *= sizeof(Value); +} + +static void +FinishBaselineDebugModeOSR(BaselineFrame* frame) +{ + frame->deleteDebugModeOSRInfo(); + + // We will return to JIT code now so we have to clear the override pc. + frame->clearOverridePc(); +} + +void +BaselineFrame::deleteDebugModeOSRInfo() +{ + js_delete(getDebugModeOSRInfo()); + flags_ &= ~HAS_DEBUG_MODE_OSR_INFO; +} + +JitCode* +JitRuntime::getBaselineDebugModeOSRHandler(JSContext* cx) +{ + if (!baselineDebugModeOSRHandler_) { + AutoLockForExclusiveAccess lock(cx); + AutoCompartment ac(cx, cx->runtime()->atomsCompartment(lock), &lock); + uint32_t offset; + if (JitCode* code = generateBaselineDebugModeOSRHandler(cx, &offset)) { + baselineDebugModeOSRHandler_ = code; + baselineDebugModeOSRHandlerNoFrameRegPopAddr_ = code->raw() + offset; + } + } + + return baselineDebugModeOSRHandler_; +} + +void* +JitRuntime::getBaselineDebugModeOSRHandlerAddress(JSContext* cx, bool popFrameReg) +{ + if (!getBaselineDebugModeOSRHandler(cx)) + return nullptr; + return popFrameReg + ? baselineDebugModeOSRHandler_->raw() + : baselineDebugModeOSRHandlerNoFrameRegPopAddr_; +} + +static void +EmitBaselineDebugModeOSRHandlerTail(MacroAssembler& masm, Register temp, bool returnFromCallVM) +{ + // Save real return address on the stack temporarily. + // + // If we're returning from a callVM, we don't need to worry about R0 and + // R1 but do need to propagate the original ReturnReg value. Otherwise we + // need to worry about R0 and R1 but can clobber ReturnReg. Indeed, on + // x86, R1 contains ReturnReg. + if (returnFromCallVM) { + masm.push(ReturnReg); + } else { + masm.pushValue(Address(temp, offsetof(BaselineDebugModeOSRInfo, valueR0))); + masm.pushValue(Address(temp, offsetof(BaselineDebugModeOSRInfo, valueR1))); + } + masm.push(BaselineFrameReg); + masm.push(Address(temp, offsetof(BaselineDebugModeOSRInfo, resumeAddr))); + + // Call a stub to free the allocated info. + masm.setupUnalignedABICall(temp); + masm.loadBaselineFramePtr(BaselineFrameReg, temp); + masm.passABIArg(temp); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, FinishBaselineDebugModeOSR)); + + // Restore saved values. + AllocatableGeneralRegisterSet jumpRegs(GeneralRegisterSet::All()); + if (returnFromCallVM) { + jumpRegs.take(ReturnReg); + } else { + jumpRegs.take(R0); + jumpRegs.take(R1); + } + jumpRegs.take(BaselineFrameReg); + Register target = jumpRegs.takeAny(); + + masm.pop(target); + masm.pop(BaselineFrameReg); + if (returnFromCallVM) { + masm.pop(ReturnReg); + } else { + masm.popValue(R1); + masm.popValue(R0); + } + + masm.jump(target); +} + +JitCode* +JitRuntime::generateBaselineDebugModeOSRHandler(JSContext* cx, uint32_t* noFrameRegPopOffsetOut) +{ + MacroAssembler masm(cx); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + regs.take(BaselineFrameReg); + regs.take(ReturnReg); + Register temp = regs.takeAny(); + Register syncedStackStart = regs.takeAny(); + + // Pop the frame reg. + masm.pop(BaselineFrameReg); + + // Not all patched baseline frames are returning from a situation where + // the frame reg is already fixed up. + CodeOffset noFrameRegPopOffset(masm.currentOffset()); + + // Record the stack pointer for syncing. + masm.moveStackPtrTo(syncedStackStart); + masm.push(ReturnReg); + masm.push(BaselineFrameReg); + + // Call a stub to fully initialize the info. + masm.setupUnalignedABICall(temp); + masm.loadBaselineFramePtr(BaselineFrameReg, temp); + masm.passABIArg(temp); + masm.passABIArg(syncedStackStart); + masm.passABIArg(ReturnReg); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, SyncBaselineDebugModeOSRInfo)); + + // Discard stack values depending on how many were unsynced, as we always + // have a fully synced stack in the recompile handler. We arrive here via + // a callVM, and prepareCallVM in BaselineCompiler always fully syncs the + // stack. + masm.pop(BaselineFrameReg); + masm.pop(ReturnReg); + masm.loadPtr(Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfScratchValue()), temp); + masm.addToStackPtr(Address(temp, offsetof(BaselineDebugModeOSRInfo, stackAdjust))); + + // Emit two tails for the case of returning from a callVM and all other + // cases, as the state we need to restore differs depending on the case. + Label returnFromCallVM, end; + EmitBranchIsReturningFromCallVM(masm, temp, &returnFromCallVM); + + EmitBaselineDebugModeOSRHandlerTail(masm, temp, /* returnFromCallVM = */ false); + masm.jump(&end); + masm.bind(&returnFromCallVM); + EmitBaselineDebugModeOSRHandlerTail(masm, temp, /* returnFromCallVM = */ true); + masm.bind(&end); + + Linker linker(masm); + AutoFlushICache afc("BaselineDebugModeOSRHandler"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + if (!code) + return nullptr; + + *noFrameRegPopOffsetOut = noFrameRegPopOffset.offset(); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "BaselineDebugModeOSRHandler"); +#endif + + return code; +} + +/* static */ void +DebugModeOSRVolatileJitFrameIterator::forwardLiveIterators(JSContext* cx, + uint8_t* oldAddr, uint8_t* newAddr) +{ + DebugModeOSRVolatileJitFrameIterator* iter; + for (iter = cx->liveVolatileJitFrameIterators_; iter; iter = iter->prev) { + if (iter->returnAddressToFp_ == oldAddr) + iter->returnAddressToFp_ = newAddr; + } +} |