diff options
Diffstat (limited to 'js/src/jit/Ion.cpp')
| -rw-r--r-- | js/src/jit/Ion.cpp | 3560 |
1 files changed, 3560 insertions, 0 deletions
diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp new file mode 100644 index 000000000..2a158ed7e --- /dev/null +++ b/js/src/jit/Ion.cpp @@ -0,0 +1,3560 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Ion.h" + +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/SizePrintfMacros.h" +#include "mozilla/ThreadLocal.h" + +#include "jscompartment.h" +#include "jsgc.h" +#include "jsprf.h" + +#include "gc/Marking.h" +#include "jit/AliasAnalysis.h" +#include "jit/AlignmentMaskAnalysis.h" +#include "jit/BacktrackingAllocator.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineInspector.h" +#include "jit/BaselineJIT.h" +#include "jit/CodeGenerator.h" +#include "jit/EagerSimdUnbox.h" +#include "jit/EdgeCaseAnalysis.h" +#include "jit/EffectiveAddressAnalysis.h" +#include "jit/FlowAliasAnalysis.h" +#include "jit/FoldLinearArithConstants.h" +#include "jit/InstructionReordering.h" +#include "jit/IonAnalysis.h" +#include "jit/IonBuilder.h" +#include "jit/IonOptimizationLevels.h" +#include "jit/JitcodeMap.h" +#include "jit/JitCommon.h" +#include "jit/JitCompartment.h" +#include "jit/JitSpewer.h" +#include "jit/LICM.h" +#include "jit/LIR.h" +#include "jit/LoopUnroller.h" +#include "jit/Lowering.h" +#include "jit/PerfSpewer.h" +#include "jit/RangeAnalysis.h" +#include "jit/ScalarReplacement.h" +#include "jit/Sink.h" +#include "jit/StupidAllocator.h" +#include "jit/ValueNumbering.h" +#include "jit/WasmBCE.h" +#include "vm/Debugger.h" +#include "vm/HelperThreads.h" +#include "vm/TraceLogging.h" + +#include "jscompartmentinlines.h" +#include "jsobjinlines.h" +#include "jsscriptinlines.h" + +#include "jit/JitFrames-inl.h" +#include "jit/shared/Lowering-shared-inl.h" +#include "vm/Debugger-inl.h" +#include "vm/EnvironmentObject-inl.h" +#include "vm/Stack-inl.h" + +using namespace js; +using namespace js::jit; + +// Assert that JitCode is gc::Cell aligned. +JS_STATIC_ASSERT(sizeof(JitCode) % gc::CellSize == 0); + +static MOZ_THREAD_LOCAL(JitContext*) TlsJitContext; + +static JitContext* +CurrentJitContext() +{ + if (!TlsJitContext.init()) + return nullptr; + return TlsJitContext.get(); +} + +void +jit::SetJitContext(JitContext* ctx) +{ + TlsJitContext.set(ctx); +} + +JitContext* +jit::GetJitContext() +{ + MOZ_ASSERT(CurrentJitContext()); + return CurrentJitContext(); +} + +JitContext* +jit::MaybeGetJitContext() +{ + return CurrentJitContext(); +} + +JitContext::JitContext(JSContext* cx, TempAllocator* temp) + : cx(cx), + temp(temp), + runtime(CompileRuntime::get(cx->runtime())), + compartment(CompileCompartment::get(cx->compartment())), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext(ExclusiveContext* cx, TempAllocator* temp) + : cx(nullptr), + temp(temp), + runtime(CompileRuntime::get(cx->runtime_)), + compartment(nullptr), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext(CompileRuntime* rt, CompileCompartment* comp, TempAllocator* temp) + : cx(nullptr), + temp(temp), + runtime(rt), + compartment(comp), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext(CompileRuntime* rt) + : cx(nullptr), + temp(nullptr), + runtime(rt), + compartment(nullptr), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext(TempAllocator* temp) + : cx(nullptr), + temp(temp), + runtime(nullptr), + compartment(nullptr), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext(CompileRuntime* rt, TempAllocator* temp) + : cx(nullptr), + temp(temp), + runtime(rt), + compartment(nullptr), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::JitContext() + : cx(nullptr), + temp(nullptr), + runtime(nullptr), + compartment(nullptr), + prev_(CurrentJitContext()), + assemblerCount_(0) +{ + SetJitContext(this); +} + +JitContext::~JitContext() +{ + SetJitContext(prev_); +} + +bool +jit::InitializeIon() +{ + if (!TlsJitContext.init()) + return false; + CheckLogging(); +#if defined(JS_CODEGEN_ARM) + InitARMFlags(); +#endif + CheckPerf(); + return true; +} + +JitRuntime::JitRuntime(JSRuntime* rt) + : execAlloc_(rt), + backedgeExecAlloc_(rt), + exceptionTail_(nullptr), + bailoutTail_(nullptr), + profilerExitFrameTail_(nullptr), + enterJIT_(nullptr), + bailoutHandler_(nullptr), + argumentsRectifier_(nullptr), + argumentsRectifierReturnAddr_(nullptr), + invalidator_(nullptr), + debugTrapHandler_(nullptr), + baselineDebugModeOSRHandler_(nullptr), + functionWrappers_(nullptr), + osrTempData_(nullptr), + preventBackedgePatching_(false), + backedgeTarget_(BackedgeLoopHeader), + ionReturnOverride_(MagicValue(JS_ARG_POISON)), + jitcodeGlobalTable_(nullptr) +{ +} + +JitRuntime::~JitRuntime() +{ + js_delete(functionWrappers_); + freeOsrTempData(); + + // By this point, the jitcode global table should be empty. + MOZ_ASSERT_IF(jitcodeGlobalTable_, jitcodeGlobalTable_->empty()); + js_delete(jitcodeGlobalTable_); +} + +bool +JitRuntime::initialize(JSContext* cx, AutoLockForExclusiveAccess& lock) +{ + AutoCompartment ac(cx, cx->atomsCompartment(lock), &lock); + + JitContext jctx(cx, nullptr); + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return false; + + functionWrappers_ = cx->new_<VMWrapperMap>(cx); + if (!functionWrappers_ || !functionWrappers_->init()) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting profiler exit frame tail stub"); + profilerExitFrameTail_ = generateProfilerExitFrameTailStub(cx); + if (!profilerExitFrameTail_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting exception tail stub"); + + void* handler = JS_FUNC_TO_DATA_PTR(void*, jit::HandleException); + + exceptionTail_ = generateExceptionTailStub(cx, handler); + if (!exceptionTail_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting bailout tail stub"); + bailoutTail_ = generateBailoutTailStub(cx); + if (!bailoutTail_) + return false; + + if (cx->runtime()->jitSupportsFloatingPoint) { + JitSpew(JitSpew_Codegen, "# Emitting bailout tables"); + + // Initialize some Ion-only stubs that require floating-point support. + if (!bailoutTables_.reserve(FrameSizeClass::ClassLimit().classId())) + return false; + + for (uint32_t id = 0;; id++) { + FrameSizeClass class_ = FrameSizeClass::FromClass(id); + if (class_ == FrameSizeClass::ClassLimit()) + break; + bailoutTables_.infallibleAppend((JitCode*)nullptr); + JitSpew(JitSpew_Codegen, "# Bailout table"); + bailoutTables_[id] = generateBailoutTable(cx, id); + if (!bailoutTables_[id]) + return false; + } + + JitSpew(JitSpew_Codegen, "# Emitting bailout handler"); + bailoutHandler_ = generateBailoutHandler(cx); + if (!bailoutHandler_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting invalidator"); + invalidator_ = generateInvalidator(cx); + if (!invalidator_) + return false; + } + + JitSpew(JitSpew_Codegen, "# Emitting sequential arguments rectifier"); + argumentsRectifier_ = generateArgumentsRectifier(cx, &argumentsRectifierReturnAddr_); + if (!argumentsRectifier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting EnterJIT sequence"); + enterJIT_ = generateEnterJIT(cx, EnterJitOptimized); + if (!enterJIT_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting EnterBaselineJIT sequence"); + enterBaselineJIT_ = generateEnterJIT(cx, EnterJitBaseline); + if (!enterBaselineJIT_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Value"); + valuePreBarrier_ = generatePreBarrier(cx, MIRType::Value); + if (!valuePreBarrier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for String"); + stringPreBarrier_ = generatePreBarrier(cx, MIRType::String); + if (!stringPreBarrier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Object"); + objectPreBarrier_ = generatePreBarrier(cx, MIRType::Object); + if (!objectPreBarrier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Shape"); + shapePreBarrier_ = generatePreBarrier(cx, MIRType::Shape); + if (!shapePreBarrier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for ObjectGroup"); + objectGroupPreBarrier_ = generatePreBarrier(cx, MIRType::ObjectGroup); + if (!objectGroupPreBarrier_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting malloc stub"); + mallocStub_ = generateMallocStub(cx); + if (!mallocStub_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting free stub"); + freeStub_ = generateFreeStub(cx); + if (!freeStub_) + return false; + + JitSpew(JitSpew_Codegen, "# Emitting VM function wrappers"); + for (VMFunction* fun = VMFunction::functions; fun; fun = fun->next) { + JitSpew(JitSpew_Codegen, "# VM function wrapper"); + if (!generateVMWrapper(cx, *fun)) + return false; + } + + JitSpew(JitSpew_Codegen, "# Emitting lazy link stub"); + lazyLinkStub_ = generateLazyLinkStub(cx); + if (!lazyLinkStub_) + return false; + + jitcodeGlobalTable_ = cx->new_<JitcodeGlobalTable>(); + if (!jitcodeGlobalTable_) + return false; + + return true; +} + +JitCode* +JitRuntime::debugTrapHandler(JSContext* cx) +{ + if (!debugTrapHandler_) { + // JitRuntime code stubs are shared across compartments and have to + // be allocated in the atoms compartment. + AutoLockForExclusiveAccess lock(cx); + AutoCompartment ac(cx, cx->runtime()->atomsCompartment(lock), &lock); + debugTrapHandler_ = generateDebugTrapHandler(cx); + } + return debugTrapHandler_; +} + +uint8_t* +JitRuntime::allocateOsrTempData(size_t size) +{ + osrTempData_ = (uint8_t*)js_realloc(osrTempData_, size); + return osrTempData_; +} + +void +JitRuntime::freeOsrTempData() +{ + js_free(osrTempData_); + osrTempData_ = nullptr; +} + +void +JitRuntime::patchIonBackedges(JSRuntime* rt, BackedgeTarget target) +{ + if (target == BackedgeLoopHeader) { + // We must be on the main thread. The caller must use + // AutoPreventBackedgePatching to ensure we don't reenter. + MOZ_ASSERT(preventBackedgePatching_); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + } else { + // We must be called from InterruptRunningJitCode, or a signal handler + // triggered there. rt->handlingJitInterrupt() ensures we can't reenter + // this code. + MOZ_ASSERT(!preventBackedgePatching_); + MOZ_ASSERT(rt->handlingJitInterrupt()); + } + + // Do nothing if we know all backedges are already jumping to `target`. + if (backedgeTarget_ == target) + return; + + backedgeTarget_ = target; + + backedgeExecAlloc_.makeAllWritable(); + + // Patch all loop backedges in Ion code so that they either jump to the + // normal loop header or to an interrupt handler each time they run. + for (InlineListIterator<PatchableBackedge> iter(backedgeList_.begin()); + iter != backedgeList_.end(); + iter++) + { + PatchableBackedge* patchableBackedge = *iter; + if (target == BackedgeLoopHeader) + PatchBackedge(patchableBackedge->backedge, patchableBackedge->loopHeader, target); + else + PatchBackedge(patchableBackedge->backedge, patchableBackedge->interruptCheck, target); + } + + backedgeExecAlloc_.makeAllExecutable(); +} + +JitCompartment::JitCompartment() + : stubCodes_(nullptr), + cacheIRStubCodes_(nullptr), + baselineGetPropReturnAddr_(nullptr), + baselineSetPropReturnAddr_(nullptr), + stringConcatStub_(nullptr), + regExpMatcherStub_(nullptr), + regExpSearcherStub_(nullptr), + regExpTesterStub_(nullptr) +{ + baselineCallReturnAddrs_[0] = baselineCallReturnAddrs_[1] = nullptr; +} + +JitCompartment::~JitCompartment() +{ + js_delete(stubCodes_); + js_delete(cacheIRStubCodes_); +} + +bool +JitCompartment::initialize(JSContext* cx) +{ + stubCodes_ = cx->new_<ICStubCodeMap>(cx->runtime()); + if (!stubCodes_) + return false; + + if (!stubCodes_->init()) { + ReportOutOfMemory(cx); + return false; + } + + cacheIRStubCodes_ = cx->new_<CacheIRStubCodeMap>(cx->runtime()); + if (!cacheIRStubCodes_) + return false; + + if (!cacheIRStubCodes_->init()) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool +JitCompartment::ensureIonStubsExist(JSContext* cx) +{ + if (!stringConcatStub_) { + stringConcatStub_ = generateStringConcatStub(cx); + if (!stringConcatStub_) + return false; + } + + return true; +} + +void +jit::FinishOffThreadBuilder(JSRuntime* runtime, IonBuilder* builder, + const AutoLockHelperThreadState& locked) +{ + // Clean the references to the pending IonBuilder, if we just finished it. + if (builder->script()->baselineScript()->hasPendingIonBuilder() && + builder->script()->baselineScript()->pendingIonBuilder() == builder) + { + builder->script()->baselineScript()->removePendingIonBuilder(builder->script()); + } + + // If the builder is still in one of the helper thread list, then remove it. + if (builder->isInList()) { + MOZ_ASSERT(runtime); + runtime->ionLazyLinkListRemove(builder); + } + + // Clear the recompiling flag of the old ionScript, since we continue to + // use the old ionScript if recompiling fails. + if (builder->script()->hasIonScript()) + builder->script()->ionScript()->clearRecompiling(); + + // Clean up if compilation did not succeed. + if (builder->script()->isIonCompilingOffThread()) { + IonScript* ion = + builder->abortReason() == AbortReason_Disable ? ION_DISABLED_SCRIPT : nullptr; + builder->script()->setIonScript(runtime, ion); + } + + // The builder is allocated into its LifoAlloc, so destroying that will + // destroy the builder and all other data accumulated during compilation, + // except any final codegen (which includes an assembler and needs to be + // explicitly destroyed). + js_delete(builder->backgroundCodegen()); + js_delete(builder->alloc().lifoAlloc()); +} + +static bool +LinkCodeGen(JSContext* cx, IonBuilder* builder, CodeGenerator *codegen) +{ + RootedScript script(cx, builder->script()); + TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime()); + TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script); + AutoTraceLog logScript(logger, event); + AutoTraceLog logLink(logger, TraceLogger_IonLinking); + + if (!codegen->link(cx, builder->constraints())) + return false; + + return true; +} + +static bool +LinkBackgroundCodeGen(JSContext* cx, IonBuilder* builder) +{ + CodeGenerator* codegen = builder->backgroundCodegen(); + if (!codegen) + return false; + + JitContext jctx(cx, &builder->alloc()); + + // Root the assembler until the builder is finished below. As it was + // constructed off thread, the assembler has not been rooted previously, + // though any GC activity would discard the builder. + MacroAssembler::AutoRooter masm(cx, &codegen->masm); + + return LinkCodeGen(cx, builder, codegen); +} + +void +jit::LinkIonScript(JSContext* cx, HandleScript calleeScript) +{ + IonBuilder* builder; + + { + AutoLockHelperThreadState lock; + + // Get the pending builder from the Ion frame. + MOZ_ASSERT(calleeScript->hasBaselineScript()); + builder = calleeScript->baselineScript()->pendingIonBuilder(); + calleeScript->baselineScript()->removePendingIonBuilder(calleeScript); + + // Remove from pending. + cx->runtime()->ionLazyLinkListRemove(builder); + } + + { + AutoEnterAnalysis enterTypes(cx); + if (!LinkBackgroundCodeGen(cx, builder)) { + // Silently ignore OOM during code generation. The assembly code + // doesn't has code to handle it after linking happened. So it's + // not OK to throw a catchable exception from there. + cx->clearPendingException(); + + // Reset the TypeZone's compiler output for this script, if any. + InvalidateCompilerOutputsForScript(cx, calleeScript); + } + } + + { + AutoLockHelperThreadState lock; + FinishOffThreadBuilder(cx->runtime(), builder, lock); + } +} + +uint8_t* +jit::LazyLinkTopActivation(JSContext* cx) +{ + // First frame should be an exit frame. + JitFrameIterator it(cx); + LazyLinkExitFrameLayout* ll = it.exitFrame()->as<LazyLinkExitFrameLayout>(); + RootedScript calleeScript(cx, ScriptFromCalleeToken(ll->jsFrame()->calleeToken())); + + LinkIonScript(cx, calleeScript); + + MOZ_ASSERT(calleeScript->hasBaselineScript()); + MOZ_ASSERT(calleeScript->baselineOrIonRawPointer()); + + return calleeScript->baselineOrIonRawPointer(); +} + +/* static */ void +JitRuntime::Mark(JSTracer* trc, AutoLockForExclusiveAccess& lock) +{ + MOZ_ASSERT(!trc->runtime()->isHeapMinorCollecting()); + + // Shared stubs are allocated in the atoms compartment, so do not iterate + // them after the atoms heap after it has been "finished." + if (trc->runtime()->atomsAreFinished()) + return; + + Zone* zone = trc->runtime()->atomsCompartment(lock)->zone(); + for (auto i = zone->cellIter<JitCode>(); !i.done(); i.next()) { + JitCode* code = i; + TraceRoot(trc, &code, "wrapper"); + } +} + +/* static */ void +JitRuntime::MarkJitcodeGlobalTableUnconditionally(JSTracer* trc) +{ + if (trc->runtime()->spsProfiler.enabled() && + trc->runtime()->hasJitRuntime() && + trc->runtime()->jitRuntime()->hasJitcodeGlobalTable()) + { + trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markUnconditionally(trc); + } +} + +/* static */ bool +JitRuntime::MarkJitcodeGlobalTableIteratively(JSTracer* trc) +{ + if (trc->runtime()->hasJitRuntime() && + trc->runtime()->jitRuntime()->hasJitcodeGlobalTable()) + { + return trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markIteratively(trc); + } + return false; +} + +/* static */ void +JitRuntime::SweepJitcodeGlobalTable(JSRuntime* rt) +{ + if (rt->hasJitRuntime() && rt->jitRuntime()->hasJitcodeGlobalTable()) + rt->jitRuntime()->getJitcodeGlobalTable()->sweep(rt); +} + +void +JitCompartment::mark(JSTracer* trc, JSCompartment* compartment) +{ + // Free temporary OSR buffer. + trc->runtime()->jitRuntime()->freeOsrTempData(); +} + +void +JitCompartment::sweep(FreeOp* fop, JSCompartment* compartment) +{ + // Any outstanding compilations should have been cancelled by the GC. + MOZ_ASSERT(!HasOffThreadIonCompile(compartment)); + + stubCodes_->sweep(); + cacheIRStubCodes_->sweep(); + + // If the sweep removed the ICCall_Fallback stub, nullptr the baselineCallReturnAddr_ field. + if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CALL_KEY)) + baselineCallReturnAddrs_[0] = nullptr; + if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CONSTRUCT_KEY)) + baselineCallReturnAddrs_[1] = nullptr; + + // Similarly for the ICGetProp_Fallback stub. + if (!stubCodes_->lookup(ICGetProp_Fallback::Compiler::BASELINE_KEY)) + baselineGetPropReturnAddr_ = nullptr; + if (!stubCodes_->lookup(ICSetProp_Fallback::Compiler::BASELINE_KEY)) + baselineSetPropReturnAddr_ = nullptr; + + JSRuntime* rt = fop->runtime(); + if (stringConcatStub_ && !IsMarkedUnbarriered(rt, &stringConcatStub_)) + stringConcatStub_ = nullptr; + + if (regExpMatcherStub_ && !IsMarkedUnbarriered(rt, ®ExpMatcherStub_)) + regExpMatcherStub_ = nullptr; + + if (regExpSearcherStub_ && !IsMarkedUnbarriered(rt, ®ExpSearcherStub_)) + regExpSearcherStub_ = nullptr; + + if (regExpTesterStub_ && !IsMarkedUnbarriered(rt, ®ExpTesterStub_)) + regExpTesterStub_ = nullptr; + + for (ReadBarrieredObject& obj : simdTemplateObjects_) { + if (obj && IsAboutToBeFinalized(&obj)) + obj.set(nullptr); + } +} + +void +JitCompartment::toggleBarriers(bool enabled) +{ + // Toggle barriers in compartment wide stubs that have patchable pre barriers. + if (regExpMatcherStub_) + regExpMatcherStub_->togglePreBarriers(enabled, Reprotect); + if (regExpSearcherStub_) + regExpSearcherStub_->togglePreBarriers(enabled, Reprotect); + if (regExpTesterStub_) + regExpTesterStub_->togglePreBarriers(enabled, Reprotect); + + // Toggle barriers in baseline IC stubs. + for (ICStubCodeMap::Enum e(*stubCodes_); !e.empty(); e.popFront()) { + JitCode* code = *e.front().value().unsafeGet(); + code->togglePreBarriers(enabled, Reprotect); + } + for (CacheIRStubCodeMap::Enum e(*cacheIRStubCodes_); !e.empty(); e.popFront()) { + JitCode* code = *e.front().value().unsafeGet(); + code->togglePreBarriers(enabled, Reprotect); + } +} + +size_t +JitCompartment::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const +{ + size_t n = mallocSizeOf(this); + if (stubCodes_) + n += stubCodes_->sizeOfIncludingThis(mallocSizeOf); + if (cacheIRStubCodes_) + n += cacheIRStubCodes_->sizeOfIncludingThis(mallocSizeOf); + return n; +} + +JitCode* +JitRuntime::getBailoutTable(const FrameSizeClass& frameClass) const +{ + MOZ_ASSERT(frameClass != FrameSizeClass::None()); + return bailoutTables_[frameClass.classId()]; +} + +JitCode* +JitRuntime::getVMWrapper(const VMFunction& f) const +{ + MOZ_ASSERT(functionWrappers_); + MOZ_ASSERT(functionWrappers_->initialized()); + JitRuntime::VMWrapperMap::Ptr p = functionWrappers_->readonlyThreadsafeLookup(&f); + MOZ_ASSERT(p); + + return p->value(); +} + +template <AllowGC allowGC> +JitCode* +JitCode::New(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize, + ExecutablePool* pool, CodeKind kind) +{ + JitCode* codeObj = Allocate<JitCode, allowGC>(cx); + if (!codeObj) { + pool->release(headerSize + bufferSize, kind); + return nullptr; + } + + new (codeObj) JitCode(code, bufferSize, headerSize, pool, kind); + return codeObj; +} + +template +JitCode* +JitCode::New<CanGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize, + ExecutablePool* pool, CodeKind kind); + +template +JitCode* +JitCode::New<NoGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize, + ExecutablePool* pool, CodeKind kind); + +void +JitCode::copyFrom(MacroAssembler& masm) +{ + // Store the JitCode pointer right before the code buffer, so we can + // recover the gcthing from relocation tables. + *(JitCode**)(code_ - sizeof(JitCode*)) = this; + insnSize_ = masm.instructionsSize(); + masm.executableCopy(code_); + + jumpRelocTableBytes_ = masm.jumpRelocationTableBytes(); + masm.copyJumpRelocationTable(code_ + jumpRelocTableOffset()); + + dataRelocTableBytes_ = masm.dataRelocationTableBytes(); + masm.copyDataRelocationTable(code_ + dataRelocTableOffset()); + + preBarrierTableBytes_ = masm.preBarrierTableBytes(); + masm.copyPreBarrierTable(code_ + preBarrierTableOffset()); + + masm.processCodeLabels(code_); +} + +void +JitCode::traceChildren(JSTracer* trc) +{ + // Note that we cannot mark invalidated scripts, since we've basically + // corrupted the code stream by injecting bailouts. + if (invalidated()) + return; + + if (jumpRelocTableBytes_) { + uint8_t* start = code_ + jumpRelocTableOffset(); + CompactBufferReader reader(start, start + jumpRelocTableBytes_); + MacroAssembler::TraceJumpRelocations(trc, this, reader); + } + if (dataRelocTableBytes_) { + // If we're moving objects, we need writable JIT code. + bool movingObjects = trc->runtime()->isHeapMinorCollecting() || zone()->isGCCompacting(); + MaybeAutoWritableJitCode awjc(this, movingObjects ? Reprotect : DontReprotect); + + uint8_t* start = code_ + dataRelocTableOffset(); + CompactBufferReader reader(start, start + dataRelocTableBytes_); + MacroAssembler::TraceDataRelocations(trc, this, reader); + } +} + +void +JitCode::finalize(FreeOp* fop) +{ + // If this jitcode had a bytecode map, it must have already been removed. +#ifdef DEBUG + JSRuntime* rt = fop->runtime(); + if (hasBytecodeMap_) { + MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable()); + MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw())); + } +#endif + + MOZ_ASSERT(pool_); + + // With W^X JIT code, reprotecting memory for each JitCode instance is + // slow, so we record the ranges and poison them later all at once. It's + // safe to ignore OOM here, it just means we won't poison the code. + if (fop->appendJitPoisonRange(JitPoisonRange(pool_, code_ - headerSize_, + headerSize_ + bufferSize_))) + { + pool_->addRef(); + } + code_ = nullptr; + + // Code buffers are stored inside ExecutablePools. Pools are refcounted. + // Releasing the pool may free it. Horrible hack: if we are using perf + // integration, we don't want to reuse code addresses, so we just leak the + // memory instead. + if (!PerfEnabled()) + pool_->release(headerSize_ + bufferSize_, CodeKind(kind_)); + pool_ = nullptr; +} + +void +JitCode::togglePreBarriers(bool enabled, ReprotectCode reprotect) +{ + uint8_t* start = code_ + preBarrierTableOffset(); + CompactBufferReader reader(start, start + preBarrierTableBytes_); + + if (!reader.more()) + return; + + MaybeAutoWritableJitCode awjc(this, reprotect); + do { + size_t offset = reader.readUnsigned(); + CodeLocationLabel loc(this, CodeOffset(offset)); + if (enabled) + Assembler::ToggleToCmp(loc); + else + Assembler::ToggleToJmp(loc); + } while (reader.more()); +} + +IonScript::IonScript() + : method_(nullptr), + deoptTable_(nullptr), + osrPc_(nullptr), + osrEntryOffset_(0), + skipArgCheckEntryOffset_(0), + invalidateEpilogueOffset_(0), + invalidateEpilogueDataOffset_(0), + numBailouts_(0), + hasProfilingInstrumentation_(false), + recompiling_(false), + runtimeData_(0), + runtimeSize_(0), + cacheIndex_(0), + cacheEntries_(0), + safepointIndexOffset_(0), + safepointIndexEntries_(0), + safepointsStart_(0), + safepointsSize_(0), + frameSlots_(0), + frameSize_(0), + bailoutTable_(0), + bailoutEntries_(0), + osiIndexOffset_(0), + osiIndexEntries_(0), + snapshots_(0), + snapshotsListSize_(0), + snapshotsRVATableSize_(0), + constantTable_(0), + constantEntries_(0), + backedgeList_(0), + backedgeEntries_(0), + invalidationCount_(0), + recompileInfo_(), + osrPcMismatchCounter_(0), + fallbackStubSpace_() +{ +} + +IonScript* +IonScript::New(JSContext* cx, RecompileInfo recompileInfo, + uint32_t frameSlots, uint32_t argumentSlots, uint32_t frameSize, + size_t snapshotsListSize, size_t snapshotsRVATableSize, + size_t recoversSize, size_t bailoutEntries, + size_t constants, size_t safepointIndices, + size_t osiIndices, size_t cacheEntries, + size_t runtimeSize, size_t safepointsSize, + size_t backedgeEntries, size_t sharedStubEntries, + OptimizationLevel optimizationLevel) +{ + constexpr size_t DataAlignment = sizeof(void*); + + if (snapshotsListSize >= MAX_BUFFER_SIZE || + (bailoutEntries >= MAX_BUFFER_SIZE / sizeof(uint32_t))) + { + ReportOutOfMemory(cx); + return nullptr; + } + + // This should not overflow on x86, because the memory is already allocated + // *somewhere* and if their total overflowed there would be no memory left + // at all. + size_t paddedSnapshotsSize = AlignBytes(snapshotsListSize + snapshotsRVATableSize, DataAlignment); + size_t paddedRecoversSize = AlignBytes(recoversSize, DataAlignment); + size_t paddedBailoutSize = AlignBytes(bailoutEntries * sizeof(uint32_t), DataAlignment); + size_t paddedConstantsSize = AlignBytes(constants * sizeof(Value), DataAlignment); + size_t paddedSafepointIndicesSize = AlignBytes(safepointIndices * sizeof(SafepointIndex), DataAlignment); + size_t paddedOsiIndicesSize = AlignBytes(osiIndices * sizeof(OsiIndex), DataAlignment); + size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(uint32_t), DataAlignment); + size_t paddedRuntimeSize = AlignBytes(runtimeSize, DataAlignment); + size_t paddedSafepointSize = AlignBytes(safepointsSize, DataAlignment); + size_t paddedBackedgeSize = AlignBytes(backedgeEntries * sizeof(PatchableBackedge), DataAlignment); + size_t paddedSharedStubSize = AlignBytes(sharedStubEntries * sizeof(IonICEntry), DataAlignment); + + size_t bytes = paddedSnapshotsSize + + paddedRecoversSize + + paddedBailoutSize + + paddedConstantsSize + + paddedSafepointIndicesSize + + paddedOsiIndicesSize + + paddedCacheEntriesSize + + paddedRuntimeSize + + paddedSafepointSize + + paddedBackedgeSize + + paddedSharedStubSize; + IonScript* script = cx->zone()->pod_malloc_with_extra<IonScript, uint8_t>(bytes); + if (!script) + return nullptr; + new (script) IonScript(); + + uint32_t offsetCursor = sizeof(IonScript); + + script->runtimeData_ = offsetCursor; + script->runtimeSize_ = runtimeSize; + offsetCursor += paddedRuntimeSize; + + script->cacheIndex_ = offsetCursor; + script->cacheEntries_ = cacheEntries; + offsetCursor += paddedCacheEntriesSize; + + script->safepointIndexOffset_ = offsetCursor; + script->safepointIndexEntries_ = safepointIndices; + offsetCursor += paddedSafepointIndicesSize; + + script->safepointsStart_ = offsetCursor; + script->safepointsSize_ = safepointsSize; + offsetCursor += paddedSafepointSize; + + script->bailoutTable_ = offsetCursor; + script->bailoutEntries_ = bailoutEntries; + offsetCursor += paddedBailoutSize; + + script->osiIndexOffset_ = offsetCursor; + script->osiIndexEntries_ = osiIndices; + offsetCursor += paddedOsiIndicesSize; + + script->snapshots_ = offsetCursor; + script->snapshotsListSize_ = snapshotsListSize; + script->snapshotsRVATableSize_ = snapshotsRVATableSize; + offsetCursor += paddedSnapshotsSize; + + script->recovers_ = offsetCursor; + script->recoversSize_ = recoversSize; + offsetCursor += paddedRecoversSize; + + script->constantTable_ = offsetCursor; + script->constantEntries_ = constants; + offsetCursor += paddedConstantsSize; + + script->backedgeList_ = offsetCursor; + script->backedgeEntries_ = backedgeEntries; + offsetCursor += paddedBackedgeSize; + + script->sharedStubList_ = offsetCursor; + script->sharedStubEntries_ = sharedStubEntries; + offsetCursor += paddedSharedStubSize; + + script->frameSlots_ = frameSlots; + script->argumentSlots_ = argumentSlots; + + script->frameSize_ = frameSize; + + script->recompileInfo_ = recompileInfo; + script->optimizationLevel_ = optimizationLevel; + + return script; +} + +void +IonScript::adoptFallbackStubs(FallbackICStubSpace* stubSpace) + +{ + fallbackStubSpace()->adoptFrom(stubSpace); +} + +void +IonScript::trace(JSTracer* trc) +{ + if (method_) + TraceEdge(trc, &method_, "method"); + + if (deoptTable_) + TraceEdge(trc, &deoptTable_, "deoptimizationTable"); + + for (size_t i = 0; i < numConstants(); i++) + TraceEdge(trc, &getConstant(i), "constant"); + + // Mark all IC stub codes hanging off the IC stub entries. + for (size_t i = 0; i < numSharedStubs(); i++) { + IonICEntry& ent = sharedStubList()[i]; + ent.trace(trc); + } + + // Trace caches so that the JSScript pointer can be updated if moved. + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).trace(trc); +} + +/* static */ void +IonScript::writeBarrierPre(Zone* zone, IonScript* ionScript) +{ + if (zone->needsIncrementalBarrier()) + ionScript->trace(zone->barrierTracer()); +} + +void +IonScript::copySnapshots(const SnapshotWriter* writer) +{ + MOZ_ASSERT(writer->listSize() == snapshotsListSize_); + memcpy((uint8_t*)this + snapshots_, + writer->listBuffer(), snapshotsListSize_); + + MOZ_ASSERT(snapshotsRVATableSize_); + MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_); + memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_, + writer->RVATableBuffer(), snapshotsRVATableSize_); +} + +void +IonScript::copyRecovers(const RecoverWriter* writer) +{ + MOZ_ASSERT(writer->size() == recoversSize_); + memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_); +} + +void +IonScript::copySafepoints(const SafepointWriter* writer) +{ + MOZ_ASSERT(writer->size() == safepointsSize_); + memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_); +} + +void +IonScript::copyBailoutTable(const SnapshotOffset* table) +{ + memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t)); +} + +void +IonScript::copyConstants(const Value* vp) +{ + for (size_t i = 0; i < constantEntries_; i++) + constants()[i].init(vp[i]); +} + +void +IonScript::copyPatchableBackedges(JSContext* cx, JitCode* code, + PatchableBackedgeInfo* backedges, + MacroAssembler& masm) +{ + JitRuntime* jrt = cx->runtime()->jitRuntime(); + JitRuntime::AutoPreventBackedgePatching apbp(cx->runtime()); + + for (size_t i = 0; i < backedgeEntries_; i++) { + PatchableBackedgeInfo& info = backedges[i]; + PatchableBackedge* patchableBackedge = &backedgeList()[i]; + + info.backedge.fixup(&masm); + CodeLocationJump backedge(code, info.backedge); + CodeLocationLabel loopHeader(code, CodeOffset(info.loopHeader->offset())); + CodeLocationLabel interruptCheck(code, CodeOffset(info.interruptCheck->offset())); + new(patchableBackedge) PatchableBackedge(backedge, loopHeader, interruptCheck); + + // Point the backedge to either of its possible targets, matching the + // other backedges in the runtime. + if (jrt->backedgeTarget() == JitRuntime::BackedgeInterruptCheck) + PatchBackedge(backedge, interruptCheck, JitRuntime::BackedgeInterruptCheck); + else + PatchBackedge(backedge, loopHeader, JitRuntime::BackedgeLoopHeader); + + jrt->addPatchableBackedge(patchableBackedge); + } +} + +void +IonScript::copySafepointIndices(const SafepointIndex* si, MacroAssembler& masm) +{ + // Jumps in the caches reflect the offset of those jumps in the compiled + // code, not the absolute positions of the jumps. Update according to the + // final code address now. + SafepointIndex* table = safepointIndices(); + memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex)); +} + +void +IonScript::copyOsiIndices(const OsiIndex* oi, MacroAssembler& masm) +{ + memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex)); +} + +void +IonScript::copyRuntimeData(const uint8_t* data) +{ + memcpy(runtimeData(), data, runtimeSize()); +} + +void +IonScript::copyCacheEntries(const uint32_t* caches, MacroAssembler& masm) +{ + memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t)); + + // Jumps in the caches reflect the offset of those jumps in the compiled + // code, not the absolute positions of the jumps. Update according to the + // final code address now. + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).updateBaseAddress(method_, masm); +} + +const SafepointIndex* +IonScript::getSafepointIndex(uint32_t disp) const +{ + MOZ_ASSERT(safepointIndexEntries_ > 0); + + const SafepointIndex* table = safepointIndices(); + if (safepointIndexEntries_ == 1) { + MOZ_ASSERT(disp == table[0].displacement()); + return &table[0]; + } + + size_t minEntry = 0; + size_t maxEntry = safepointIndexEntries_ - 1; + uint32_t min = table[minEntry].displacement(); + uint32_t max = table[maxEntry].displacement(); + + // Raise if the element is not in the list. + MOZ_ASSERT(min <= disp && disp <= max); + + // Approximate the location of the FrameInfo. + size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry; + uint32_t guessDisp = table[guess].displacement(); + + if (table[guess].displacement() == disp) + return &table[guess]; + + // Doing a linear scan from the guess should be more efficient in case of + // small group which are equally distributed on the code. + // + // such as: <... ... ... ... . ... ...> + if (guessDisp > disp) { + while (--guess >= minEntry) { + guessDisp = table[guess].displacement(); + MOZ_ASSERT(guessDisp >= disp); + if (guessDisp == disp) + return &table[guess]; + } + } else { + while (++guess <= maxEntry) { + guessDisp = table[guess].displacement(); + MOZ_ASSERT(guessDisp <= disp); + if (guessDisp == disp) + return &table[guess]; + } + } + + MOZ_CRASH("displacement not found."); +} + +const OsiIndex* +IonScript::getOsiIndex(uint32_t disp) const +{ + const OsiIndex* end = osiIndices() + osiIndexEntries_; + for (const OsiIndex* it = osiIndices(); it != end; ++it) { + if (it->returnPointDisplacement() == disp) + return it; + } + + MOZ_CRASH("Failed to find OSI point return address"); +} + +const OsiIndex* +IonScript::getOsiIndex(uint8_t* retAddr) const +{ + JitSpew(JitSpew_IonInvalidate, "IonScript %p has method %p raw %p", (void*) this, (void*) + method(), method()->raw()); + + MOZ_ASSERT(containsCodeAddress(retAddr)); + uint32_t disp = retAddr - method()->raw(); + return getOsiIndex(disp); +} + +void +IonScript::Trace(JSTracer* trc, IonScript* script) +{ + if (script != ION_DISABLED_SCRIPT) + script->trace(trc); +} + +void +IonScript::Destroy(FreeOp* fop, IonScript* script) +{ + script->unlinkFromRuntime(fop); + + /* + * When the script contains pointers to nursery things, the store buffer can + * contain entries that point into the fallback stub space. Since we can + * destroy scripts outside the context of a GC, this situation could result + * in us trying to mark invalid store buffer entries. + * + * Defer freeing any allocated blocks until after the next minor GC. + */ + script->fallbackStubSpace_.freeAllAfterMinorGC(fop->runtime()); + + fop->delete_(script); +} + +void +JS::DeletePolicy<js::jit::IonScript>::operator()(const js::jit::IonScript* script) +{ + IonScript::Destroy(rt_->defaultFreeOp(), const_cast<IonScript*>(script)); +} + +void +IonScript::toggleBarriers(bool enabled, ReprotectCode reprotect) +{ + method()->togglePreBarriers(enabled, reprotect); +} + +void +IonScript::purgeOptimizedStubs(Zone* zone) +{ + for (size_t i = 0; i < numSharedStubs(); i++) { + IonICEntry& entry = sharedStubList()[i]; + if (!entry.hasStub()) + continue; + + ICStub* lastStub = entry.firstStub(); + while (lastStub->next()) + lastStub = lastStub->next(); + + if (lastStub->isFallback()) { + // Unlink all stubs allocated in the optimized space. + ICStub* stub = entry.firstStub(); + ICStub* prev = nullptr; + + while (stub->next()) { + if (!stub->allocatedInFallbackSpace()) { + lastStub->toFallbackStub()->unlinkStub(zone, prev, stub); + stub = stub->next(); + continue; + } + + prev = stub; + stub = stub->next(); + } + + lastStub->toFallbackStub()->setInvalid(); + + if (lastStub->isMonitoredFallback()) { + // Monitor stubs can't make calls, so are always in the + // optimized stub space. + ICTypeMonitor_Fallback* lastMonStub = + lastStub->toMonitoredFallbackStub()->fallbackMonitorStub(); + lastMonStub->resetMonitorStubChain(zone); + lastMonStub->setInvalid(); + } + } else if (lastStub->isTypeMonitor_Fallback()) { + lastStub->toTypeMonitor_Fallback()->resetMonitorStubChain(zone); + lastStub->toTypeMonitor_Fallback()->setInvalid(); + } else { + MOZ_ASSERT(lastStub->isTableSwitch()); + } + } + +#ifdef DEBUG + // All remaining stubs must be allocated in the fallback space. + for (size_t i = 0; i < numSharedStubs(); i++) { + IonICEntry& entry = sharedStubList()[i]; + if (!entry.hasStub()) + continue; + + ICStub* stub = entry.firstStub(); + while (stub->next()) { + MOZ_ASSERT(stub->allocatedInFallbackSpace()); + stub = stub->next(); + } + } +#endif +} + +void +IonScript::purgeCaches() +{ + // Don't reset any ICs if we're invalidated, otherwise, repointing the + // inline jump could overwrite an invalidation marker. These ICs can + // no longer run, however, the IC slow paths may be active on the stack. + // ICs therefore are required to check for invalidation before patching, + // to ensure the same invariant. + if (invalidated()) + return; + + if (numCaches() == 0) + return; + + AutoWritableJitCode awjc(method()); + for (size_t i = 0; i < numCaches(); i++) + getCacheFromIndex(i).reset(DontReprotect); +} + +void +IonScript::unlinkFromRuntime(FreeOp* fop) +{ + // The writes to the executable buffer below may clobber backedge jumps, so + // make sure that those backedges are unlinked from the runtime and not + // reclobbered with garbage if an interrupt is requested. + JitRuntime* jrt = fop->runtime()->jitRuntime(); + JitRuntime::AutoPreventBackedgePatching apbp(fop->runtime()); + for (size_t i = 0; i < backedgeEntries_; i++) + jrt->removePatchableBackedge(&backedgeList()[i]); + + // Clear the list of backedges, so that this method is idempotent. It is + // called during destruction, and may be additionally called when the + // script is invalidated. + backedgeEntries_ = 0; +} + +void +jit::ToggleBarriers(JS::Zone* zone, bool needs) +{ + JSRuntime* rt = zone->runtimeFromMainThread(); + if (!rt->hasJitRuntime()) + return; + + for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next()) { + if (script->hasIonScript()) + script->ionScript()->toggleBarriers(needs); + if (script->hasBaselineScript()) + script->baselineScript()->toggleBarriers(needs); + } + + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + if (comp->jitCompartment()) + comp->jitCompartment()->toggleBarriers(needs); + } +} + +namespace js { +namespace jit { + +static void +OptimizeSinCos(MIRGenerator *mir, MIRGraph &graph) +{ + // Now, we are looking for: + // var y = sin(x); + // var z = cos(x); + // Graph before: + // - 1 op + // - 6 mathfunction op1 Sin + // - 7 mathfunction op1 Cos + // Graph will look like: + // - 1 op + // - 5 sincos op1 + // - 6 mathfunction sincos5 Sin + // - 7 mathfunction sincos5 Cos + for (MBasicBlockIterator block(graph.begin()); block != graph.end(); block++) { + for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ) { + MInstruction *ins = *iter++; + if (!ins->isMathFunction() || ins->isRecoveredOnBailout()) + continue; + + MMathFunction *insFunc = ins->toMathFunction(); + if (insFunc->function() != MMathFunction::Sin && insFunc->function() != MMathFunction::Cos) + continue; + + // Check if sin/cos is already optimized. + if (insFunc->getOperand(0)->type() == MIRType::SinCosDouble) + continue; + + // insFunc is either a |sin(x)| or |cos(x)| instruction. The + // following loop iterates over the uses of |x| to check if both + // |sin(x)| and |cos(x)| instructions exist. + bool hasSin = false; + bool hasCos = false; + for (MUseDefIterator uses(insFunc->input()); uses; uses++) + { + if (!uses.def()->isInstruction()) + continue; + + // We should replacing the argument of the sin/cos just when it + // is dominated by the |block|. + if (!block->dominates(uses.def()->block())) + continue; + + MInstruction *insUse = uses.def()->toInstruction(); + if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout()) + continue; + + MMathFunction *mathIns = insUse->toMathFunction(); + if (!hasSin && mathIns->function() == MMathFunction::Sin) { + hasSin = true; + JitSpew(JitSpew_Sincos, "Found sin in block %d.", mathIns->block()->id()); + } + else if (!hasCos && mathIns->function() == MMathFunction::Cos) { + hasCos = true; + JitSpew(JitSpew_Sincos, "Found cos in block %d.", mathIns->block()->id()); + } + + if (hasCos && hasSin) + break; + } + + if (!hasCos || !hasSin) { + JitSpew(JitSpew_Sincos, "No sin/cos pair found."); + continue; + } + + JitSpew(JitSpew_Sincos, "Found, at least, a pair sin/cos. Adding sincos in block %d", + block->id()); + // Adding the MSinCos and replacing the parameters of the + // sin(x)/cos(x) to sin(sincos(x))/cos(sincos(x)). + MSinCos *insSinCos = MSinCos::New(graph.alloc(), + insFunc->input(), + insFunc->toMathFunction()->cache()); + insSinCos->setImplicitlyUsedUnchecked(); + block->insertBefore(insFunc, insSinCos); + for (MUseDefIterator uses(insFunc->input()); uses; ) + { + MDefinition* def = uses.def(); + uses++; + if (!def->isInstruction()) + continue; + + // We should replacing the argument of the sin/cos just when it + // is dominated by the |block|. + if (!block->dominates(def->block())) + continue; + + MInstruction *insUse = def->toInstruction(); + if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout()) + continue; + + MMathFunction *mathIns = insUse->toMathFunction(); + if (mathIns->function() != MMathFunction::Sin && mathIns->function() != MMathFunction::Cos) + continue; + + mathIns->replaceOperand(0, insSinCos); + JitSpew(JitSpew_Sincos, "Replacing %s by sincos in block %d", + mathIns->function() == MMathFunction::Sin ? "sin" : "cos", + mathIns->block()->id()); + } + } + } +} + +bool +OptimizeMIR(MIRGenerator* mir) +{ + MIRGraph& graph = mir->graph(); + GraphSpewer& gs = mir->graphSpewer(); + TraceLoggerThread* logger; + if (GetJitContext()->onMainThread()) + logger = TraceLoggerForMainThread(GetJitContext()->runtime); + else + logger = TraceLoggerForCurrentThread(); + + if (mir->shouldCancel("Start")) + return false; + + if (!mir->compilingWasm()) { + if (!MakeMRegExpHoistable(mir, graph)) + return false; + + if (mir->shouldCancel("Make MRegExp Hoistable")) + return false; + } + + gs.spewPass("BuildSSA"); + AssertBasicGraphCoherency(graph); + + if (!JitOptions.disablePgo && !mir->compilingWasm()) { + AutoTraceLog log(logger, TraceLogger_PruneUnusedBranches); + if (!PruneUnusedBranches(mir, graph)) + return false; + gs.spewPass("Prune Unused Branches"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Prune Unused Branches")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_FoldTests); + if (!FoldTests(graph)) + return false; + gs.spewPass("Fold Tests"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Fold Tests")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_SplitCriticalEdges); + if (!SplitCriticalEdges(graph)) + return false; + gs.spewPass("Split Critical Edges"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Split Critical Edges")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_RenumberBlocks); + RenumberBlocks(graph); + gs.spewPass("Renumber Blocks"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Renumber Blocks")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_DominatorTree); + if (!BuildDominatorTree(graph)) + return false; + // No spew: graph not changed. + + if (mir->shouldCancel("Dominator Tree")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_PhiAnalysis); + // Aggressive phi elimination must occur before any code elimination. If the + // script contains a try-statement, we only compiled the try block and not + // the catch or finally blocks, so in this case it's also invalid to use + // aggressive phi elimination. + Observability observability = graph.hasTryBlock() + ? ConservativeObservability + : AggressiveObservability; + if (!EliminatePhis(mir, graph, observability)) + return false; + gs.spewPass("Eliminate phis"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Eliminate phis")) + return false; + + if (!BuildPhiReverseMapping(graph)) + return false; + AssertExtendedGraphCoherency(graph); + // No spew: graph not changed. + + if (mir->shouldCancel("Phi reverse mapping")) + return false; + } + + if (!JitOptions.disableRecoverIns && mir->optimizationInfo().scalarReplacementEnabled()) { + AutoTraceLog log(logger, TraceLogger_ScalarReplacement); + if (!ScalarReplacement(mir, graph)) + return false; + gs.spewPass("Scalar Replacement"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Scalar Replacement")) + return false; + } + + if (!mir->compilingWasm()) { + AutoTraceLog log(logger, TraceLogger_ApplyTypes); + if (!ApplyTypeInformation(mir, graph)) + return false; + gs.spewPass("Apply types"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Apply types")) + return false; + } + + if (!JitOptions.disableRecoverIns && mir->optimizationInfo().eagerSimdUnboxEnabled()) { + AutoTraceLog log(logger, TraceLogger_EagerSimdUnbox); + if (!EagerSimdUnbox(mir, graph)) + return false; + gs.spewPass("Eager Simd Unbox"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Eager Simd Unbox")) + return false; + } + + if (mir->optimizationInfo().amaEnabled()) { + AutoTraceLog log(logger, TraceLogger_AlignmentMaskAnalysis); + AlignmentMaskAnalysis ama(graph); + if (!ama.analyze()) + return false; + gs.spewPass("Alignment Mask Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Alignment Mask Analysis")) + return false; + } + + ValueNumberer gvn(mir, graph); + if (!gvn.init()) + return false; + + // Alias analysis is required for LICM and GVN so that we don't move + // loads across stores. + if (mir->optimizationInfo().licmEnabled() || + mir->optimizationInfo().gvnEnabled()) + { + { + AutoTraceLog log(logger, TraceLogger_AliasAnalysis); + if (JitOptions.disableFlowAA) { + AliasAnalysis analysis(mir, graph); + if (!analysis.analyze()) + return false; + } else { + FlowAliasAnalysis analysis(mir, graph); + if (!analysis.analyze()) + return false; + } + + gs.spewPass("Alias analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Alias analysis")) + return false; + } + + if (!mir->compilingWasm()) { + // Eliminating dead resume point operands requires basic block + // instructions to be numbered. Reuse the numbering computed during + // alias analysis. + if (!EliminateDeadResumePointOperands(mir, graph)) + return false; + + if (mir->shouldCancel("Eliminate dead resume point operands")) + return false; + } + } + + if (mir->optimizationInfo().gvnEnabled()) { + AutoTraceLog log(logger, TraceLogger_GVN); + if (!gvn.run(ValueNumberer::UpdateAliasAnalysis)) + return false; + gs.spewPass("GVN"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("GVN")) + return false; + } + + if (mir->optimizationInfo().licmEnabled()) { + AutoTraceLog log(logger, TraceLogger_LICM); + // LICM can hoist instructions from conditional branches and trigger + // repeated bailouts. Disable it if this script is known to bailout + // frequently. + JSScript* script = mir->info().script(); + if (!script || !script->hadFrequentBailouts()) { + if (!LICM(mir, graph)) + return false; + gs.spewPass("LICM"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("LICM")) + return false; + } + } + + RangeAnalysis r(mir, graph); + if (mir->optimizationInfo().rangeAnalysisEnabled()) { + AutoTraceLog log(logger, TraceLogger_RangeAnalysis); + if (!r.addBetaNodes()) + return false; + gs.spewPass("Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA Beta")) + return false; + + if (!r.analyze() || !r.addRangeAssertions()) + return false; + gs.spewPass("Range Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Range Analysis")) + return false; + + if (!r.removeBetaNodes()) + return false; + gs.spewPass("De-Beta"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA De-Beta")) + return false; + + if (mir->optimizationInfo().gvnEnabled()) { + bool shouldRunUCE = false; + if (!r.prepareForUCE(&shouldRunUCE)) + return false; + gs.spewPass("RA check UCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("RA check UCE")) + return false; + + if (shouldRunUCE) { + if (!gvn.run(ValueNumberer::DontUpdateAliasAnalysis)) + return false; + gs.spewPass("UCE After RA"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("UCE After RA")) + return false; + } + } + + if (mir->optimizationInfo().autoTruncateEnabled()) { + if (!r.truncate()) + return false; + gs.spewPass("Truncate Doubles"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Truncate Doubles")) + return false; + } + + if (mir->optimizationInfo().loopUnrollingEnabled()) { + AutoTraceLog log(logger, TraceLogger_LoopUnrolling); + + if (!UnrollLoops(graph, r.loopIterationBounds)) + return false; + + gs.spewPass("Unroll Loops"); + AssertExtendedGraphCoherency(graph); + } + } + + if (!JitOptions.disableRecoverIns) { + AutoTraceLog log(logger, TraceLogger_Sink); + if (!Sink(mir, graph)) + return false; + gs.spewPass("Sink"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Sink")) + return false; + } + + if (!JitOptions.disableRecoverIns && mir->optimizationInfo().rangeAnalysisEnabled()) { + AutoTraceLog log(logger, TraceLogger_RemoveUnnecessaryBitops); + if (!r.removeUnnecessaryBitops()) + return false; + gs.spewPass("Remove Unnecessary Bitops"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Remove Unnecessary Bitops")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_FoldLinearArithConstants); + if (!FoldLinearArithConstants(mir, graph)) + return false; + gs.spewPass("Fold Linear Arithmetic Constants"); + AssertBasicGraphCoherency(graph); + + if (mir->shouldCancel("Fold Linear Arithmetic Constants")) + return false; + } + + if (mir->optimizationInfo().eaaEnabled()) { + AutoTraceLog log(logger, TraceLogger_EffectiveAddressAnalysis); + EffectiveAddressAnalysis eaa(mir, graph); + if (!eaa.analyze()) + return false; + gs.spewPass("Effective Address Analysis"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Effective Address Analysis")) + return false; + } + + if (mir->optimizationInfo().sincosEnabled()) { + AutoTraceLog log(logger, TraceLogger_Sincos); + OptimizeSinCos(mir, graph); + gs.spewPass("Sincos optimization"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Sincos optimization")) + return false; + } + + { + AutoTraceLog log(logger, TraceLogger_EliminateDeadCode); + if (!EliminateDeadCode(mir, graph)) + return false; + gs.spewPass("DCE"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("DCE")) + return false; + } + + if (mir->optimizationInfo().instructionReorderingEnabled()) { + AutoTraceLog log(logger, TraceLogger_ReorderInstructions); + if (!ReorderInstructions(mir, graph)) + return false; + gs.spewPass("Reordering"); + + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Reordering")) + return false; + } + + // Make loops contiguous. We do this after GVN/UCE and range analysis, + // which can remove CFG edges, exposing more blocks that can be moved. + { + AutoTraceLog log(logger, TraceLogger_MakeLoopsContiguous); + if (!MakeLoopsContiguous(graph)) + return false; + gs.spewPass("Make loops contiguous"); + AssertExtendedGraphCoherency(graph); + + if (mir->shouldCancel("Make loops contiguous")) + return false; + } + + // Passes after this point must not move instructions; these analyses + // depend on knowing the final order in which instructions will execute. + + if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) { + AutoTraceLog log(logger, TraceLogger_EdgeCaseAnalysis); + EdgeCaseAnalysis edgeCaseAnalysis(mir, graph); + if (!edgeCaseAnalysis.analyzeLate()) + return false; + gs.spewPass("Edge Case Analysis (Late)"); + AssertGraphCoherency(graph); + + if (mir->shouldCancel("Edge Case Analysis (Late)")) + return false; + } + + if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) { + AutoTraceLog log(logger, TraceLogger_EliminateRedundantChecks); + // Note: check elimination has to run after all other passes that move + // instructions. Since check uses are replaced with the actual index, + // code motion after this pass could incorrectly move a load or store + // before its bounds check. + if (!EliminateRedundantChecks(graph)) + return false; + gs.spewPass("Bounds Check Elimination"); + AssertGraphCoherency(graph); + } + + if (!mir->compilingWasm()) { + AutoTraceLog log(logger, TraceLogger_AddKeepAliveInstructions); + if (!AddKeepAliveInstructions(graph)) + return false; + gs.spewPass("Add KeepAlive Instructions"); + AssertGraphCoherency(graph); + } + + if (mir->compilingWasm()) { + if (!EliminateBoundsChecks(mir, graph)) + return false; + gs.spewPass("Redundant Bounds Check Elimination"); + AssertGraphCoherency(graph); + } + + DumpMIRExpressions(graph); + + return true; +} + +LIRGraph* +GenerateLIR(MIRGenerator* mir) +{ + MIRGraph& graph = mir->graph(); + GraphSpewer& gs = mir->graphSpewer(); + + TraceLoggerThread* logger; + if (GetJitContext()->onMainThread()) + logger = TraceLoggerForMainThread(GetJitContext()->runtime); + else + logger = TraceLoggerForCurrentThread(); + + LIRGraph* lir = mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph); + if (!lir || !lir->init()) + return nullptr; + + LIRGenerator lirgen(mir, graph, *lir); + { + AutoTraceLog log(logger, TraceLogger_GenerateLIR); + if (!lirgen.generate()) + return nullptr; + gs.spewPass("Generate LIR"); + + if (mir->shouldCancel("Generate LIR")) + return nullptr; + } + + AllocationIntegrityState integrity(*lir); + + { + AutoTraceLog log(logger, TraceLogger_RegisterAllocation); + + IonRegisterAllocator allocator = mir->optimizationInfo().registerAllocator(); + + switch (allocator) { + case RegisterAllocator_Backtracking: + case RegisterAllocator_Testbed: { +#ifdef DEBUG + if (!integrity.record()) + return nullptr; +#endif + + BacktrackingAllocator regalloc(mir, &lirgen, *lir, + allocator == RegisterAllocator_Testbed); + if (!regalloc.go()) + return nullptr; + +#ifdef DEBUG + if (!integrity.check(false)) + return nullptr; +#endif + + gs.spewPass("Allocate Registers [Backtracking]"); + break; + } + + case RegisterAllocator_Stupid: { + // Use the integrity checker to populate safepoint information, so + // run it in all builds. + if (!integrity.record()) + return nullptr; + + StupidAllocator regalloc(mir, &lirgen, *lir); + if (!regalloc.go()) + return nullptr; + if (!integrity.check(true)) + return nullptr; + gs.spewPass("Allocate Registers [Stupid]"); + break; + } + + default: + MOZ_CRASH("Bad regalloc"); + } + + if (mir->shouldCancel("Allocate Registers")) + return nullptr; + } + + return lir; +} + +CodeGenerator* +GenerateCode(MIRGenerator* mir, LIRGraph* lir) +{ + TraceLoggerThread* logger; + if (GetJitContext()->onMainThread()) + logger = TraceLoggerForMainThread(GetJitContext()->runtime); + else + logger = TraceLoggerForCurrentThread(); + AutoTraceLog log(logger, TraceLogger_GenerateCode); + + CodeGenerator* codegen = js_new<CodeGenerator>(mir, lir); + if (!codegen) + return nullptr; + + if (!codegen->generate()) { + js_delete(codegen); + return nullptr; + } + + return codegen; +} + +CodeGenerator* +CompileBackEnd(MIRGenerator* mir) +{ + // Everything in CompileBackEnd can potentially run on a helper thread. + AutoEnterIonCompilation enter(mir->safeForMinorGC()); + AutoSpewEndFunction spewEndFunction(mir); + + if (!OptimizeMIR(mir)) + return nullptr; + + LIRGraph* lir = GenerateLIR(mir); + if (!lir) + return nullptr; + + return GenerateCode(mir, lir); +} + +// Find a finished builder for the compartment. +static IonBuilder* +GetFinishedBuilder(JSContext* cx, GlobalHelperThreadState::IonBuilderVector& finished) +{ + for (size_t i = 0; i < finished.length(); i++) { + IonBuilder* testBuilder = finished[i]; + if (testBuilder->compartment == CompileCompartment::get(cx->compartment())) { + HelperThreadState().remove(finished, &i); + return testBuilder; + } + } + + return nullptr; +} + +void +AttachFinishedCompilations(JSContext* cx) +{ + JitCompartment* ion = cx->compartment()->jitCompartment(); + if (!ion) + return; + + { + AutoLockHelperThreadState lock; + + GlobalHelperThreadState::IonBuilderVector& finished = HelperThreadState().ionFinishedList(lock); + + // Incorporate any off thread compilations for the compartment which have + // finished, failed or have been cancelled. + while (true) { + // Find a finished builder for the compartment. + IonBuilder* builder = GetFinishedBuilder(cx, finished); + if (!builder) + break; + + JSScript* script = builder->script(); + MOZ_ASSERT(script->hasBaselineScript()); + script->baselineScript()->setPendingIonBuilder(cx->runtime(), script, builder); + cx->runtime()->ionLazyLinkListAdd(builder); + + // Don't keep more than 100 lazy link builders. + // Link the oldest ones immediately. + while (cx->runtime()->ionLazyLinkListSize() > 100) { + jit::IonBuilder* builder = cx->runtime()->ionLazyLinkList().getLast(); + RootedScript script(cx, builder->script()); + + AutoUnlockHelperThreadState unlock(lock); + AutoCompartment ac(cx, script->compartment()); + jit::LinkIonScript(cx, script); + } + + continue; + } + } +} + +static void +TrackAllProperties(JSContext* cx, JSObject* obj) +{ + MOZ_ASSERT(obj->isSingleton()); + + for (Shape::Range<NoGC> range(obj->as<NativeObject>().lastProperty()); !range.empty(); range.popFront()) + EnsureTrackPropertyTypes(cx, obj, range.front().propid()); +} + +static void +TrackPropertiesForSingletonScopes(JSContext* cx, JSScript* script, BaselineFrame* baselineFrame) +{ + // Ensure that all properties of singleton call objects which the script + // could access are tracked. These are generally accessed through + // ALIASEDVAR operations in baseline and will not be tracked even if they + // have been accessed in baseline code. + JSObject* environment = script->functionNonDelazifying() + ? script->functionNonDelazifying()->environment() + : nullptr; + + while (environment && !environment->is<GlobalObject>()) { + if (environment->is<CallObject>() && environment->isSingleton()) + TrackAllProperties(cx, environment); + environment = environment->enclosingEnvironment(); + } + + if (baselineFrame) { + JSObject* scope = baselineFrame->environmentChain(); + if (scope->is<CallObject>() && scope->isSingleton()) + TrackAllProperties(cx, scope); + } +} + +static void +TrackIonAbort(JSContext* cx, JSScript* script, jsbytecode* pc, const char* message) +{ + if (!cx->runtime()->jitRuntime()->isOptimizationTrackingEnabled(cx->runtime())) + return; + + // Only bother tracking aborts of functions we're attempting to + // Ion-compile after successfully running in Baseline. + if (!script->hasBaselineScript()) + return; + + JitcodeGlobalTable* table = cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); + void* ptr = script->baselineScript()->method()->raw(); + JitcodeGlobalEntry& entry = table->lookupInfallible(ptr); + entry.baselineEntry().trackIonAbort(pc, message); +} + +static void +TrackAndSpewIonAbort(JSContext* cx, JSScript* script, const char* message) +{ + JitSpew(JitSpew_IonAbort, "%s", message); + TrackIonAbort(cx, script, script->code(), message); +} + +static AbortReason +IonCompile(JSContext* cx, JSScript* script, + BaselineFrame* baselineFrame, jsbytecode* osrPc, + bool recompile, OptimizationLevel optimizationLevel) +{ + TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime()); + TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script); + AutoTraceLog logScript(logger, event); + AutoTraceLog logCompile(logger, TraceLogger_IonCompilation); + + // Make sure the script's canonical function isn't lazy. We can't de-lazify + // it in a helper thread. + script->ensureNonLazyCanonicalFunction(cx); + + TrackPropertiesForSingletonScopes(cx, script, baselineFrame); + + LifoAlloc* alloc = cx->new_<LifoAlloc>(TempAllocator::PreferredLifoChunkSize); + if (!alloc) + return AbortReason_Alloc; + + ScopedJSDeletePtr<LifoAlloc> autoDelete(alloc); + + TempAllocator* temp = alloc->new_<TempAllocator>(alloc); + if (!temp) + return AbortReason_Alloc; + + JitContext jctx(cx, temp); + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return AbortReason_Alloc; + + if (!cx->compartment()->jitCompartment()->ensureIonStubsExist(cx)) + return AbortReason_Alloc; + + MIRGraph* graph = alloc->new_<MIRGraph>(temp); + if (!graph) + return AbortReason_Alloc; + + InlineScriptTree* inlineScriptTree = InlineScriptTree::New(temp, nullptr, nullptr, script); + if (!inlineScriptTree) + return AbortReason_Alloc; + + CompileInfo* info = alloc->new_<CompileInfo>(script, script->functionNonDelazifying(), osrPc, + Analysis_None, + script->needsArgsObj(), inlineScriptTree); + if (!info) + return AbortReason_Alloc; + + BaselineInspector* inspector = alloc->new_<BaselineInspector>(script); + if (!inspector) + return AbortReason_Alloc; + + BaselineFrameInspector* baselineFrameInspector = nullptr; + if (baselineFrame) { + baselineFrameInspector = NewBaselineFrameInspector(temp, baselineFrame, info); + if (!baselineFrameInspector) + return AbortReason_Alloc; + } + + CompilerConstraintList* constraints = NewCompilerConstraintList(*temp); + if (!constraints) + return AbortReason_Alloc; + + const OptimizationInfo* optimizationInfo = IonOptimizations.get(optimizationLevel); + const JitCompileOptions options(cx); + + IonBuilder* builder = alloc->new_<IonBuilder>((JSContext*) nullptr, + CompileCompartment::get(cx->compartment()), + options, temp, graph, constraints, + inspector, info, optimizationInfo, + baselineFrameInspector); + if (!builder) + return AbortReason_Alloc; + + if (cx->runtime()->gc.storeBuffer.cancelIonCompilations()) + builder->setNotSafeForMinorGC(); + + MOZ_ASSERT(recompile == builder->script()->hasIonScript()); + MOZ_ASSERT(builder->script()->canIonCompile()); + + RootedScript builderScript(cx, builder->script()); + + if (recompile) + builderScript->ionScript()->setRecompiling(); + + SpewBeginFunction(builder, builderScript); + + bool succeeded; + { + AutoEnterAnalysis enter(cx); + succeeded = builder->build(); + builder->clearForBackEnd(); + } + + if (!succeeded) { + AbortReason reason = builder->abortReason(); + builder->graphSpewer().endFunction(); + if (reason == AbortReason_PreliminaryObjects) { + // Some group was accessed which has associated preliminary objects + // to analyze. Do this now and we will try to build again shortly. + const MIRGenerator::ObjectGroupVector& groups = builder->abortedPreliminaryGroups(); + for (size_t i = 0; i < groups.length(); i++) { + ObjectGroup* group = groups[i]; + if (group->newScript()) { + if (!group->newScript()->maybeAnalyze(cx, group, nullptr, /* force = */ true)) + return AbortReason_Alloc; + } else if (group->maybePreliminaryObjects()) { + group->maybePreliminaryObjects()->maybeAnalyze(cx, group, /* force = */ true); + } else { + MOZ_CRASH("Unexpected aborted preliminary group"); + } + } + } + + if (builder->hadActionableAbort()) { + JSScript* abortScript; + jsbytecode* abortPc; + const char* abortMessage; + builder->actionableAbortLocationAndMessage(&abortScript, &abortPc, &abortMessage); + TrackIonAbort(cx, abortScript, abortPc, abortMessage); + } + + if (cx->isThrowingOverRecursed()) { + // Non-analysis compilations should never fail with stack overflow. + MOZ_CRASH("Stack overflow during compilation"); + } + + return reason; + } + + // If possible, compile the script off thread. + if (options.offThreadCompilationAvailable()) { + JitSpew(JitSpew_IonSyncLogs, "Can't log script %s:%" PRIuSIZE + ". (Compiled on background thread.)", + builderScript->filename(), builderScript->lineno()); + + if (!CreateMIRRootList(*builder)) + return AbortReason_Alloc; + + if (!StartOffThreadIonCompile(cx, builder)) { + JitSpew(JitSpew_IonAbort, "Unable to start off-thread ion compilation."); + builder->graphSpewer().endFunction(); + return AbortReason_Alloc; + } + + if (!recompile) + builderScript->setIonScript(cx->runtime(), ION_COMPILING_SCRIPT); + + // The allocator and associated data will be destroyed after being + // processed in the finishedOffThreadCompilations list. + autoDelete.forget(); + + return AbortReason_NoAbort; + } + + { + ScopedJSDeletePtr<CodeGenerator> codegen; + AutoEnterAnalysis enter(cx); + codegen = CompileBackEnd(builder); + if (!codegen) { + JitSpew(JitSpew_IonAbort, "Failed during back-end compilation."); + if (cx->isExceptionPending()) + return AbortReason_Error; + return AbortReason_Disable; + } + + succeeded = LinkCodeGen(cx, builder, codegen); + } + + if (succeeded) + return AbortReason_NoAbort; + if (cx->isExceptionPending()) + return AbortReason_Error; + return AbortReason_Disable; +} + +static bool +CheckFrame(JSContext* cx, BaselineFrame* frame) +{ + MOZ_ASSERT(!frame->script()->isGenerator()); + MOZ_ASSERT(!frame->isDebuggerEvalFrame()); + MOZ_ASSERT(!frame->isEvalFrame()); + + // This check is to not overrun the stack. + if (frame->isFunctionFrame()) { + if (TooManyActualArguments(frame->numActualArgs())) { + TrackAndSpewIonAbort(cx, frame->script(), "too many actual arguments"); + return false; + } + + if (TooManyFormalArguments(frame->numFormalArgs())) { + TrackAndSpewIonAbort(cx, frame->script(), "too many arguments"); + return false; + } + } + + return true; +} + +static bool +CheckScript(JSContext* cx, JSScript* script, bool osr) +{ + if (script->isForEval()) { + // Eval frames are not yet supported. Supporting this will require new + // logic in pushBailoutFrame to deal with linking prev. + // Additionally, JSOP_DEFVAR support will require baking in isEvalFrame(). + TrackAndSpewIonAbort(cx, script, "eval script"); + return false; + } + + if (script->isGenerator()) { + TrackAndSpewIonAbort(cx, script, "generator script"); + return false; + } + + if (script->hasNonSyntacticScope() && !script->functionNonDelazifying()) { + // Support functions with a non-syntactic global scope but not other + // scripts. For global scripts, IonBuilder currently uses the global + // object as scope chain, this is not valid when the script has a + // non-syntactic global scope. + TrackAndSpewIonAbort(cx, script, "has non-syntactic global scope"); + return false; + } + + if (script->functionHasExtraBodyVarScope() && + script->functionExtraBodyVarScope()->hasEnvironment()) + { + // This restriction will be lifted when intra-function scope chains + // are compilable by Ion. See bug 1273858. + TrackAndSpewIonAbort(cx, script, "has extra var environment"); + return false; + } + + if (script->nTypeSets() >= UINT16_MAX) { + // In this case multiple bytecode ops can share a single observed + // TypeSet (see bug 1303710). + TrackAndSpewIonAbort(cx, script, "too many typesets"); + return false; + } + + return true; +} + +static MethodStatus +CheckScriptSize(JSContext* cx, JSScript* script) +{ + if (!JitOptions.limitScriptSize) + return Method_Compiled; + + uint32_t numLocalsAndArgs = NumLocalsAndArgs(script); + + if (script->length() > MAX_MAIN_THREAD_SCRIPT_SIZE || + numLocalsAndArgs > MAX_MAIN_THREAD_LOCALS_AND_ARGS) + { + if (!OffThreadCompilationAvailable(cx)) { + JitSpew(JitSpew_IonAbort, "Script too large (%" PRIuSIZE " bytes) (%u locals/args)", + script->length(), numLocalsAndArgs); + TrackIonAbort(cx, script, script->code(), "too large"); + return Method_CantCompile; + } + } + + return Method_Compiled; +} + +bool +CanIonCompileScript(JSContext* cx, JSScript* script, bool osr) +{ + if (!script->canIonCompile() || !CheckScript(cx, script, osr)) + return false; + + return CheckScriptSize(cx, script) == Method_Compiled; +} + +static OptimizationLevel +GetOptimizationLevel(HandleScript script, jsbytecode* pc) +{ + return IonOptimizations.levelForScript(script, pc); +} + +static MethodStatus +Compile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc, + bool forceRecompile = false) +{ + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(jit::IsBaselineEnabled(cx)); + MOZ_ASSERT_IF(osrPc != nullptr, LoopEntryCanIonOsr(osrPc)); + + if (!script->hasBaselineScript()) + return Method_Skipped; + + if (script->isDebuggee() || (osrFrame && osrFrame->isDebuggee())) { + TrackAndSpewIonAbort(cx, script, "debugging"); + return Method_Skipped; + } + + if (!CheckScript(cx, script, bool(osrPc))) { + JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno()); + return Method_CantCompile; + } + + MethodStatus status = CheckScriptSize(cx, script); + if (status != Method_Compiled) { + JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno()); + return status; + } + + bool recompile = false; + OptimizationLevel optimizationLevel = GetOptimizationLevel(script, osrPc); + if (optimizationLevel == OptimizationLevel::DontCompile) + return Method_Skipped; + + if (!CanLikelyAllocateMoreExecutableMemory()) { + script->resetWarmUpCounter(); + return Method_Skipped; + } + + if (script->hasIonScript()) { + IonScript* scriptIon = script->ionScript(); + if (!scriptIon->method()) + return Method_CantCompile; + + // Don't recompile/overwrite higher optimized code, + // with a lower optimization level. + if (optimizationLevel <= scriptIon->optimizationLevel() && !forceRecompile) + return Method_Compiled; + + // Don't start compiling if already compiling + if (scriptIon->isRecompiling()) + return Method_Compiled; + + if (osrPc) + scriptIon->resetOsrPcMismatchCounter(); + + recompile = true; + } + + if (script->baselineScript()->hasPendingIonBuilder()) { + IonBuilder* buildIon = script->baselineScript()->pendingIonBuilder(); + if (optimizationLevel <= buildIon->optimizationInfo().level() && !forceRecompile) + return Method_Compiled; + + recompile = true; + } + + AbortReason reason = IonCompile(cx, script, osrFrame, osrPc, recompile, optimizationLevel); + if (reason == AbortReason_Error) + return Method_Error; + + if (reason == AbortReason_Disable) + return Method_CantCompile; + + if (reason == AbortReason_Alloc) { + ReportOutOfMemory(cx); + return Method_Error; + } + + // Compilation succeeded or we invalidated right away or an inlining/alloc abort + if (script->hasIonScript()) + return Method_Compiled; + return Method_Skipped; +} + +} // namespace jit +} // namespace js + +bool +jit::OffThreadCompilationAvailable(JSContext* cx) +{ + // Even if off thread compilation is enabled, compilation must still occur + // on the main thread in some cases. + // + // Require cpuCount > 1 so that Ion compilation jobs and main-thread + // execution are not competing for the same resources. + return cx->runtime()->canUseOffthreadIonCompilation() + && HelperThreadState().cpuCount > 1 + && CanUseExtraThreads(); +} + +MethodStatus +jit::CanEnter(JSContext* cx, RunState& state) +{ + MOZ_ASSERT(jit::IsIonEnabled(cx)); + + JSScript* script = state.script(); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) + return Method_Skipped; + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) + return Method_Skipped; + + // Skip if the code is expected to result in a bailout. + if (script->hasIonScript() && script->ionScript()->bailoutExpected()) + return Method_Skipped; + + RootedScript rscript(cx, script); + + // If constructing, allocate a new |this| object before building Ion. + // Creating |this| is done before building Ion because it may change the + // type information and invalidate compilation results. + if (state.isInvoke()) { + InvokeState& invoke = *state.asInvoke(); + + if (TooManyActualArguments(invoke.args().length())) { + TrackAndSpewIonAbort(cx, script, "too many actual args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (TooManyFormalArguments(invoke.args().callee().as<JSFunction>().nargs())) { + TrackAndSpewIonAbort(cx, script, "too many args"); + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + if (!state.maybeCreateThisForConstructor(cx)) { + if (cx->isThrowingOutOfMemory()) { + cx->recoverFromOutOfMemory(); + return Method_Skipped; + } + return Method_Error; + } + } + + // If --ion-eager is used, compile with Baseline first, so that we + // can directly enter IonMonkey. + if (JitOptions.eagerCompilation && !rscript->hasBaselineScript()) { + MethodStatus status = CanEnterBaselineMethod(cx, state); + if (status != Method_Compiled) + return status; + } + + // Skip if the script is being compiled off thread or can't be + // Ion-compiled (again). MaybeCreateThisForConstructor could have + // started an Ion compilation or marked the script as uncompilable. + if (rscript->isIonCompilingOffThread() || !rscript->canIonCompile()) + return Method_Skipped; + + // Attempt compilation. Returns Method_Compiled if already compiled. + MethodStatus status = Compile(cx, rscript, nullptr, nullptr); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, rscript); + return status; + } + + if (state.script()->baselineScript()->hasPendingIonBuilder()) { + LinkIonScript(cx, state.script()); + if (!state.script()->hasIonScript()) + return jit::Method_Skipped; + } + + return Method_Compiled; +} + +static MethodStatus +BaselineCanEnterAtEntry(JSContext* cx, HandleScript script, BaselineFrame* frame) +{ + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(frame->callee()->nonLazyScript()->canIonCompile()); + MOZ_ASSERT(!frame->callee()->nonLazyScript()->isIonCompilingOffThread()); + MOZ_ASSERT(!frame->callee()->nonLazyScript()->hasIonScript()); + MOZ_ASSERT(frame->isFunctionFrame()); + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(cx, frame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // Attempt compilation. Returns Method_Compiled if already compiled. + MethodStatus status = Compile(cx, script, frame, nullptr); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + return Method_Compiled; +} + +// Decide if a transition from baseline execution to Ion code should occur. +// May compile or recompile the target JSScript. +static MethodStatus +BaselineCanEnterAtBranch(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* pc) +{ + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT((JSOp)*pc == JSOP_LOOPENTRY); + MOZ_ASSERT(LoopEntryCanIonOsr(pc)); + + // Skip if the script has been disabled. + if (!script->canIonCompile()) + return Method_Skipped; + + // Skip if the script is being compiled off thread. + if (script->isIonCompilingOffThread()) + return Method_Skipped; + + // Skip if the code is expected to result in a bailout. + if (script->hasIonScript() && script->ionScript()->bailoutExpected()) + return Method_Skipped; + + // Optionally ignore on user request. + if (!JitOptions.osr) + return Method_Skipped; + + // Mark as forbidden if frame can't be handled. + if (!CheckFrame(cx, osrFrame)) { + ForbidCompilation(cx, script); + return Method_CantCompile; + } + + // Check if the jitcode still needs to get linked and do this + // to have a valid IonScript. + if (script->baselineScript()->hasPendingIonBuilder()) + LinkIonScript(cx, script); + + // By default a recompilation doesn't happen on osr mismatch. + // Decide if we want to force a recompilation if this happens too much. + bool force = false; + if (script->hasIonScript() && pc != script->ionScript()->osrPc()) { + uint32_t count = script->ionScript()->incrOsrPcMismatchCounter(); + if (count <= JitOptions.osrPcMismatchesBeforeRecompile) + return Method_Skipped; + force = true; + } + + // Attempt compilation. + // - Returns Method_Compiled if the right ionscript is present + // (Meaning it was present or a sequantial compile finished) + // - Returns Method_Skipped if pc doesn't match + // (This means a background thread compilation with that pc could have started or not.) + RootedScript rscript(cx, script); + MethodStatus status = Compile(cx, rscript, osrFrame, pc, force); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + // Return the compilation was skipped when the osr pc wasn't adjusted. + // This can happen when there was still an IonScript available and a + // background compilation started, but hasn't finished yet. + // Or when we didn't force a recompile. + if (script->hasIonScript() && pc != script->ionScript()->osrPc()) + return Method_Skipped; + + return Method_Compiled; +} + +bool +jit::IonCompileScriptForBaseline(JSContext* cx, BaselineFrame* frame, jsbytecode* pc) +{ + // A TI OOM will disable TI and Ion. + if (!jit::IsIonEnabled(cx)) + return true; + + RootedScript script(cx, frame->script()); + bool isLoopEntry = JSOp(*pc) == JSOP_LOOPENTRY; + + MOZ_ASSERT(!isLoopEntry || LoopEntryCanIonOsr(pc)); + + if (!script->canIonCompile()) { + // TODO: ASSERT that ion-compilation-disabled checker stub doesn't exist. + // TODO: Clear all optimized stubs. + // TODO: Add a ion-compilation-disabled checker IC stub + script->resetWarmUpCounter(); + return true; + } + + MOZ_ASSERT(!script->isIonCompilingOffThread()); + + // If Ion script exists, but PC is not at a loop entry, then Ion will be entered for + // this script at an appropriate LOOPENTRY or the next time this function is called. + if (script->hasIonScript() && !isLoopEntry) { + JitSpew(JitSpew_BaselineOSR, "IonScript exists, but not at loop entry!"); + // TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist. + // TODO: Clear all optimized stubs. + // TODO: Add a ion-script-already-exists checker stub. + return true; + } + + // Ensure that Ion-compiled code is available. + JitSpew(JitSpew_BaselineOSR, + "WarmUpCounter for %s:%" PRIuSIZE " reached %d at pc %p, trying to switch to Ion!", + script->filename(), script->lineno(), (int) script->getWarmUpCount(), (void*) pc); + + MethodStatus stat; + if (isLoopEntry) { + MOZ_ASSERT(LoopEntryCanIonOsr(pc)); + JitSpew(JitSpew_BaselineOSR, " Compile at loop entry!"); + stat = BaselineCanEnterAtBranch(cx, script, frame, pc); + } else if (frame->isFunctionFrame()) { + JitSpew(JitSpew_BaselineOSR, " Compile function from top for later entry!"); + stat = BaselineCanEnterAtEntry(cx, script, frame); + } else { + return true; + } + + if (stat == Method_Error) { + JitSpew(JitSpew_BaselineOSR, " Compile with Ion errored!"); + return false; + } + + if (stat == Method_CantCompile) + JitSpew(JitSpew_BaselineOSR, " Can't compile with Ion!"); + else if (stat == Method_Skipped) + JitSpew(JitSpew_BaselineOSR, " Skipped compile with Ion!"); + else if (stat == Method_Compiled) + JitSpew(JitSpew_BaselineOSR, " Compiled with Ion!"); + else + MOZ_CRASH("Invalid MethodStatus!"); + + // Failed to compile. Reset warm-up counter and return. + if (stat != Method_Compiled) { + // TODO: If stat == Method_CantCompile, insert stub that just skips the + // warm-up counter entirely, instead of resetting it. + bool bailoutExpected = script->hasIonScript() && script->ionScript()->bailoutExpected(); + if (stat == Method_CantCompile || bailoutExpected) { + JitSpew(JitSpew_BaselineOSR, " Reset WarmUpCounter cantCompile=%s bailoutExpected=%s!", + stat == Method_CantCompile ? "yes" : "no", + bailoutExpected ? "yes" : "no"); + script->resetWarmUpCounter(); + } + return true; + } + + return true; +} + + +MethodStatus +jit::Recompile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc, + bool force) +{ + MOZ_ASSERT(script->hasIonScript()); + if (script->ionScript()->isRecompiling()) + return Method_Compiled; + + MethodStatus status = Compile(cx, script, osrFrame, osrPc, force); + if (status != Method_Compiled) { + if (status == Method_CantCompile) + ForbidCompilation(cx, script); + return status; + } + + return Method_Compiled; +} + +MethodStatus +jit::CanEnterUsingFastInvoke(JSContext* cx, HandleScript script, uint32_t numActualArgs) +{ + MOZ_ASSERT(jit::IsIonEnabled(cx)); + + // Skip if the code is expected to result in a bailout. + if (!script->hasIonScript() || script->ionScript()->bailoutExpected()) + return Method_Skipped; + + // Don't handle arguments underflow, to make this work we would have to pad + // missing arguments with |undefined|. + if (numActualArgs < script->functionNonDelazifying()->nargs()) + return Method_Skipped; + + if (!cx->compartment()->ensureJitCompartmentExists(cx)) + return Method_Error; + + // This can GC, so afterward, script->ion is not guaranteed to be valid. + if (!cx->runtime()->jitRuntime()->enterIon()) + return Method_Error; + + if (!script->hasIonScript()) + return Method_Skipped; + + return Method_Compiled; +} + +static JitExecStatus +EnterIon(JSContext* cx, EnterJitData& data) +{ + JS_CHECK_RECURSION(cx, return JitExec_Aborted); + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(!data.osrFrame); + +#ifdef DEBUG + // See comment in EnterBaseline. + mozilla::Maybe<JS::AutoAssertNoGC> nogc; + nogc.emplace(cx); +#endif + + EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon(); + + // Caller must construct |this| before invoking the Ion function. + MOZ_ASSERT_IF(data.constructing, + data.maxArgv[0].isObject() || data.maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL)); + + data.result.setInt32(data.numActualArgs); + { + AssertCompartmentUnchanged pcc(cx); + ActivationEntryMonitor entryMonitor(cx, data.calleeToken); + JitActivation activation(cx); + +#ifdef DEBUG + nogc.reset(); +#endif + CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv, /* osrFrame = */nullptr, data.calleeToken, + /* envChain = */ nullptr, 0, data.result.address()); + } + + MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride()); + + // Jit callers wrap primitive constructor return, except for derived class constructors. + if (!data.result.isMagic() && data.constructing && + data.result.isPrimitive()) + { + MOZ_ASSERT(data.maxArgv[0].isObject()); + data.result = data.maxArgv[0]; + } + + // Release temporary buffer used for OSR into Ion. + cx->runtime()->getJitRuntime(cx)->freeOsrTempData(); + + MOZ_ASSERT_IF(data.result.isMagic(), data.result.isMagic(JS_ION_ERROR)); + return data.result.isMagic() ? JitExec_Error : JitExec_Ok; +} + +bool +jit::SetEnterJitData(JSContext* cx, EnterJitData& data, RunState& state, + MutableHandle<GCVector<Value>> vals) +{ + data.osrFrame = nullptr; + + if (state.isInvoke()) { + const CallArgs& args = state.asInvoke()->args(); + unsigned numFormals = state.script()->functionNonDelazifying()->nargs(); + data.constructing = state.asInvoke()->constructing(); + data.numActualArgs = args.length(); + data.maxArgc = Max(args.length(), numFormals) + 1; + data.envChain = nullptr; + data.calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), data.constructing); + + if (data.numActualArgs >= numFormals) { + data.maxArgv = args.base() + 1; + } else { + MOZ_ASSERT(vals.empty()); + unsigned numPushedArgs = Max(args.length(), numFormals); + if (!vals.reserve(numPushedArgs + 1 + data.constructing)) + return false; + + // Append |this| and any provided arguments. + for (size_t i = 1; i < args.length() + 2; ++i) + vals.infallibleAppend(args.base()[i]); + + // Pad missing arguments with |undefined|. + while (vals.length() < numFormals + 1) + vals.infallibleAppend(UndefinedValue()); + + if (data.constructing) + vals.infallibleAppend(args.newTarget()); + + MOZ_ASSERT(vals.length() >= numFormals + 1 + data.constructing); + data.maxArgv = vals.begin(); + } + } else { + data.constructing = false; + data.numActualArgs = 0; + data.maxArgc = 0; + data.maxArgv = nullptr; + data.envChain = state.asExecute()->environmentChain(); + + data.calleeToken = CalleeToToken(state.script()); + + if (state.script()->isForEval() && state.script()->isDirectEvalInFunction()) { + // Push newTarget onto the stack. + if (!vals.reserve(1)) + return false; + + data.maxArgc = 1; + data.maxArgv = vals.begin(); + if (state.asExecute()->newTarget().isNull()) { + ScriptFrameIter iter(cx); + vals.infallibleAppend(iter.newTarget()); + } else { + vals.infallibleAppend(state.asExecute()->newTarget()); + } + } + } + + return true; +} + +JitExecStatus +jit::IonCannon(JSContext* cx, RunState& state) +{ + IonScript* ion = state.script()->ionScript(); + + EnterJitData data(cx); + data.jitcode = ion->method()->raw(); + + Rooted<GCVector<Value>> vals(cx, GCVector<Value>(cx)); + if (!SetEnterJitData(cx, data, state, &vals)) + return JitExec_Error; + + JitExecStatus status = EnterIon(cx, data); + + if (status == JitExec_Ok) + state.setReturnValue(data.result); + + return status; +} + +JitExecStatus +jit::FastInvoke(JSContext* cx, HandleFunction fun, CallArgs& args) +{ + JS_CHECK_RECURSION(cx, return JitExec_Error); + + RootedScript script(cx, fun->nonLazyScript()); + + if (!Debugger::checkNoExecute(cx, script)) + return JitExec_Error; + +#ifdef DEBUG + // See comment in EnterBaseline. + mozilla::Maybe<JS::AutoAssertNoGC> nogc; + nogc.emplace(cx); +#endif + + IonScript* ion = script->ionScript(); + JitCode* code = ion->method(); + void* jitcode = code->raw(); + + MOZ_ASSERT(jit::IsIonEnabled(cx)); + MOZ_ASSERT(!ion->bailoutExpected()); + + ActivationEntryMonitor entryMonitor(cx, CalleeToToken(script)); + JitActivation activation(cx); + + EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon(); + void* calleeToken = CalleeToToken(fun, /* constructing = */ false); + + RootedValue result(cx, Int32Value(args.length())); + MOZ_ASSERT(args.length() >= fun->nargs()); + +#ifdef DEBUG + nogc.reset(); +#endif + CALL_GENERATED_CODE(enter, jitcode, args.length() + 1, args.array() - 1, /* osrFrame = */nullptr, + calleeToken, /* envChain = */ nullptr, 0, result.address()); + + MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride()); + + args.rval().set(result); + + MOZ_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR)); + return result.isMagic() ? JitExec_Error : JitExec_Ok; +} + +static void +InvalidateActivation(FreeOp* fop, const JitActivationIterator& activations, bool invalidateAll) +{ + JitSpew(JitSpew_IonInvalidate, "BEGIN invalidating activation"); + +#ifdef CHECK_OSIPOINT_REGISTERS + if (JitOptions.checkOsiPointRegisters) + activations->asJit()->setCheckRegs(false); +#endif + + size_t frameno = 1; + + for (JitFrameIterator it(activations); !it.done(); ++it, ++frameno) { + MOZ_ASSERT_IF(frameno == 1, it.isExitFrame() || it.type() == JitFrame_Bailout); + +#ifdef JS_JITSPEW + switch (it.type()) { + case JitFrame_Exit: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " exit frame @ %p", frameno, it.fp()); + break; + case JitFrame_BaselineJS: + case JitFrame_IonJS: + case JitFrame_Bailout: + { + MOZ_ASSERT(it.isScripted()); + const char* type = "Unknown"; + if (it.isIonJS()) + type = "Optimized"; + else if (it.isBaselineJS()) + type = "Baseline"; + else if (it.isBailoutJS()) + type = "Bailing"; + JitSpew(JitSpew_IonInvalidate, + "#%" PRIuSIZE " %s JS frame @ %p, %s:%" PRIuSIZE " (fun: %p, script: %p, pc %p)", + frameno, type, it.fp(), it.script()->maybeForwardedFilename(), + it.script()->lineno(), it.maybeCallee(), (JSScript*)it.script(), + it.returnAddressToFp()); + break; + } + case JitFrame_IonStub: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion stub frame @ %p", frameno, it.fp()); + break; + case JitFrame_BaselineStub: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " baseline stub frame @ %p", frameno, it.fp()); + break; + case JitFrame_Rectifier: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " rectifier frame @ %p", frameno, it.fp()); + break; + case JitFrame_IonAccessorIC: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion IC getter/setter frame @ %p", frameno, it.fp()); + break; + case JitFrame_Entry: + JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " entry frame @ %p", frameno, it.fp()); + break; + } +#endif // JS_JITSPEW + + if (!it.isIonScripted()) + continue; + + bool calledFromLinkStub = false; + JitCode* lazyLinkStub = fop->runtime()->jitRuntime()->lazyLinkStub(); + if (it.returnAddressToFp() >= lazyLinkStub->raw() && + it.returnAddressToFp() < lazyLinkStub->rawEnd()) + { + calledFromLinkStub = true; + } + + // See if the frame has already been invalidated. + if (!calledFromLinkStub && it.checkInvalidation()) + continue; + + JSScript* script = it.script(); + if (!script->hasIonScript()) + continue; + + if (!invalidateAll && !script->ionScript()->invalidated()) + continue; + + IonScript* ionScript = script->ionScript(); + + // Purge ICs before we mark this script as invalidated. This will + // prevent lastJump_ from appearing to be a bogus pointer, just + // in case anyone tries to read it. + ionScript->purgeCaches(); + ionScript->purgeOptimizedStubs(script->zone()); + + // Clean up any pointers from elsewhere in the runtime to this IonScript + // which is about to become disconnected from its JSScript. + ionScript->unlinkFromRuntime(fop); + + // This frame needs to be invalidated. We do the following: + // + // 1. Increment the reference counter to keep the ionScript alive + // for the invalidation bailout or for the exception handler. + // 2. Determine safepoint that corresponds to the current call. + // 3. From safepoint, get distance to the OSI-patchable offset. + // 4. From the IonScript, determine the distance between the + // call-patchable offset and the invalidation epilogue. + // 5. Patch the OSI point with a call-relative to the + // invalidation epilogue. + // + // The code generator ensures that there's enough space for us + // to patch in a call-relative operation at each invalidation + // point. + // + // Note: you can't simplify this mechanism to "just patch the + // instruction immediately after the call" because things may + // need to move into a well-defined register state (using move + // instructions after the call) in to capture an appropriate + // snapshot after the call occurs. + + ionScript->incrementInvalidationCount(); + + JitCode* ionCode = ionScript->method(); + + JS::Zone* zone = script->zone(); + if (zone->needsIncrementalBarrier()) { + // We're about to remove edges from the JSScript to gcthings + // embedded in the JitCode. Perform one final trace of the + // JitCode for the incremental GC, as it must know about + // those edges. + ionCode->traceChildren(zone->barrierTracer()); + } + ionCode->setInvalidated(); + + // Don't adjust OSI points in the linkStub (which don't exist), or in a + // bailout path. + if (calledFromLinkStub || it.isBailoutJS()) + continue; + + // Write the delta (from the return address offset to the + // IonScript pointer embedded into the invalidation epilogue) + // where the safepointed call instruction used to be. We rely on + // the call sequence causing the safepoint being >= the size of + // a uint32, which is checked during safepoint index + // construction. + AutoWritableJitCode awjc(ionCode); + const SafepointIndex* si = ionScript->getSafepointIndex(it.returnAddressToFp()); + CodeLocationLabel dataLabelToMunge(it.returnAddressToFp()); + ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() - + (it.returnAddressToFp() - ionCode->raw()); + Assembler::PatchWrite_Imm32(dataLabelToMunge, Imm32(delta)); + + CodeLocationLabel osiPatchPoint = SafepointReader::InvalidationPatchPoint(ionScript, si); + CodeLocationLabel invalidateEpilogue(ionCode, CodeOffset(ionScript->invalidateEpilogueOffset())); + + JitSpew(JitSpew_IonInvalidate, " ! Invalidate ionScript %p (inv count %" PRIuSIZE ") -> patching osipoint %p", + ionScript, ionScript->invalidationCount(), (void*) osiPatchPoint.raw()); + Assembler::PatchWrite_NearCall(osiPatchPoint, invalidateEpilogue); + } + + JitSpew(JitSpew_IonInvalidate, "END invalidating activation"); +} + +void +jit::InvalidateAll(FreeOp* fop, Zone* zone) +{ + // The caller should previously have cancelled off thread compilation. +#ifdef DEBUG + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) + MOZ_ASSERT(!HasOffThreadIonCompile(comp)); +#endif + + for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) { + if (iter->compartment()->zone() == zone) { + JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC"); + InvalidateActivation(fop, iter, true); + } + } +} + + +void +jit::Invalidate(TypeZone& types, FreeOp* fop, + const RecompileInfoVector& invalid, bool resetUses, + bool cancelOffThread) +{ + JitSpew(JitSpew_IonInvalidate, "Start invalidation."); + + // Add an invalidation reference to all invalidated IonScripts to indicate + // to the traversal which frames have been invalidated. + size_t numInvalidations = 0; + for (size_t i = 0; i < invalid.length(); i++) { + const CompilerOutput* co = invalid[i].compilerOutput(types); + if (!co) + continue; + MOZ_ASSERT(co->isValid()); + + if (cancelOffThread) + CancelOffThreadIonCompile(co->script()); + + if (!co->ion()) + continue; + + JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%" PRIuSIZE ", IonScript %p", + co->script()->filename(), co->script()->lineno(), co->ion()); + + // Keep the ion script alive during the invalidation and flag this + // ionScript as being invalidated. This increment is removed by the + // loop after the calls to InvalidateActivation. + co->ion()->incrementInvalidationCount(); + numInvalidations++; + } + + if (!numInvalidations) { + JitSpew(JitSpew_IonInvalidate, " No IonScript invalidation."); + return; + } + + for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) + InvalidateActivation(fop, iter, false); + + // Drop the references added above. If a script was never active, its + // IonScript will be immediately destroyed. Otherwise, it will be held live + // until its last invalidated frame is destroyed. + for (size_t i = 0; i < invalid.length(); i++) { + CompilerOutput* co = invalid[i].compilerOutput(types); + if (!co) + continue; + MOZ_ASSERT(co->isValid()); + + JSScript* script = co->script(); + IonScript* ionScript = co->ion(); + if (!ionScript) + continue; + + script->setIonScript(nullptr, nullptr); + ionScript->decrementInvalidationCount(fop); + co->invalidate(); + numInvalidations--; + + // Wait for the scripts to get warm again before doing another + // compile, unless we are recompiling *because* a script got hot + // (resetUses is false). + if (resetUses) + script->resetWarmUpCounter(); + } + + // Make sure we didn't leak references by invalidating the same IonScript + // multiple times in the above loop. + MOZ_ASSERT(!numInvalidations); +} + +void +jit::Invalidate(JSContext* cx, const RecompileInfoVector& invalid, bool resetUses, + bool cancelOffThread) +{ + jit::Invalidate(cx->zone()->types, cx->runtime()->defaultFreeOp(), invalid, resetUses, + cancelOffThread); +} + +void +jit::IonScript::invalidate(JSContext* cx, bool resetUses, const char* reason) +{ + JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason); + + // RecompileInfoVector has inline space for at least one element. + RecompileInfoVector list; + MOZ_RELEASE_ASSERT(list.reserve(1)); + list.infallibleAppend(recompileInfo()); + + Invalidate(cx, list, resetUses, true); +} + +void +jit::Invalidate(JSContext* cx, JSScript* script, bool resetUses, bool cancelOffThread) +{ + MOZ_ASSERT(script->hasIonScript()); + + if (cx->runtime()->spsProfiler.enabled()) { + // Register invalidation with profiler. + // Format of event payload string: + // "<filename>:<lineno>" + + // Get the script filename, if any, and its length. + const char* filename = script->filename(); + if (filename == nullptr) + filename = "<unknown>"; + + // Construct the descriptive string. + char* buf = JS_smprintf("Invalidate %s:%" PRIuSIZE, filename, script->lineno()); + + // Ignore the event on allocation failure. + if (buf) { + cx->runtime()->spsProfiler.markEvent(buf); + JS_smprintf_free(buf); + } + } + + // RecompileInfoVector has inline space for at least one element. + RecompileInfoVector scripts; + MOZ_ASSERT(script->hasIonScript()); + MOZ_RELEASE_ASSERT(scripts.reserve(1)); + scripts.infallibleAppend(script->ionScript()->recompileInfo()); + + Invalidate(cx, scripts, resetUses, cancelOffThread); +} + +static void +FinishInvalidationOf(FreeOp* fop, JSScript* script, IonScript* ionScript) +{ + TypeZone& types = script->zone()->types; + + // Note: If the script is about to be swept, the compiler output may have + // already been destroyed. + if (CompilerOutput* output = ionScript->recompileInfo().compilerOutput(types)) + output->invalidate(); + + // If this script has Ion code on the stack, invalidated() will return + // true. In this case we have to wait until destroying it. + if (!ionScript->invalidated()) + jit::IonScript::Destroy(fop, ionScript); +} + +void +jit::FinishInvalidation(FreeOp* fop, JSScript* script) +{ + // In all cases, nullptr out script->ion to avoid re-entry. + if (script->hasIonScript()) { + IonScript* ion = script->ionScript(); + script->setIonScript(nullptr, nullptr); + FinishInvalidationOf(fop, script, ion); + } +} + +void +jit::ForbidCompilation(JSContext* cx, JSScript* script) +{ + JitSpew(JitSpew_IonAbort, "Disabling Ion compilation of script %s:%" PRIuSIZE, + script->filename(), script->lineno()); + + CancelOffThreadIonCompile(script); + + if (script->hasIonScript()) + Invalidate(cx, script, false); + + script->setIonScript(cx->runtime(), ION_DISABLED_SCRIPT); +} + +AutoFlushICache* +PerThreadData::autoFlushICache() const +{ + return autoFlushICache_; +} + +void +PerThreadData::setAutoFlushICache(AutoFlushICache* afc) +{ + autoFlushICache_ = afc; +} + +// Set the range for the merging of flushes. The flushing is deferred until the end of +// the AutoFlushICache context. Subsequent flushing within this range will is also +// deferred. This is only expected to be defined once for each AutoFlushICache +// context. It assumes the range will be flushed is required to be within an +// AutoFlushICache context. +void +AutoFlushICache::setRange(uintptr_t start, size_t len) +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache(); + MOZ_ASSERT(afc); + MOZ_ASSERT(!afc->start_); + JitSpewCont(JitSpew_CacheFlush, "(%" PRIxPTR " %" PRIxSIZE "):", start, len); + + uintptr_t stop = start + len; + afc->start_ = start; + afc->stop_ = stop; +#endif +} + +// Flush the instruction cache. +// +// If called within a dynamic AutoFlushICache context and if the range is already pending +// flushing for this AutoFlushICache context then the request is ignored with the +// understanding that it will be flushed on exit from the AutoFlushICache context. +// Otherwise the range is flushed immediately. +// +// Updates outside the current code object are typically the exception so they are flushed +// immediately rather than attempting to merge them. +// +// For efficiency it is expected that all large ranges will be flushed within an +// AutoFlushICache, so check. If this assertion is hit then it does not necessarily +// indicate a program fault but it might indicate a lost opportunity to merge cache +// flushing. It can be corrected by wrapping the call in an AutoFlushICache to context. +// +// Note this can be called without TLS PerThreadData defined so this case needs +// to be guarded against. E.g. when patching instructions from the exception +// handler on MacOS running the ARM simulator. +void +AutoFlushICache::flush(uintptr_t start, size_t len) +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + PerThreadData* pt = TlsPerThreadData.get(); + AutoFlushICache* afc = pt ? pt->PerThreadData::autoFlushICache() : nullptr; + if (!afc) { + JitSpewCont(JitSpew_CacheFlush, "#"); + ExecutableAllocator::cacheFlush((void*)start, len); + MOZ_ASSERT(len <= 32); + return; + } + + uintptr_t stop = start + len; + if (start >= afc->start_ && stop <= afc->stop_) { + // Update is within the pending flush range, so defer to the end of the context. + JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "-" : "="); + return; + } + + JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "x" : "*"); + ExecutableAllocator::cacheFlush((void*)start, len); +#endif +} + +// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths +// where the changes are being abandoned. +void +AutoFlushICache::setInhibit() +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache(); + MOZ_ASSERT(afc); + MOZ_ASSERT(afc->start_); + JitSpewCont(JitSpew_CacheFlush, "I"); + afc->inhibit_ = true; +#endif +} + +// The common use case is merging cache flushes when preparing a code object. In this +// case the entire range of the code object is being flushed and as the code is patched +// smaller redundant flushes could occur. The design allows an AutoFlushICache dynamic +// thread local context to be declared in which the range of the code object can be set +// which defers flushing until the end of this dynamic context. The redundant flushing +// within this code range is also deferred avoiding redundant flushing. Flushing outside +// this code range is not affected and proceeds immediately. +// +// In some cases flushing is not necessary, such as when compiling an wasm module which +// is flushed again when dynamically linked, and also in error paths that abandon the +// code. Flushing within the set code range can be inhibited within the AutoFlushICache +// dynamic context by setting an inhibit flag. +// +// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and +// it is assumed that code being assembled or patched is not executed before the exit of +// the respective AutoFlushICache dynamic context. +// +AutoFlushICache::AutoFlushICache(const char* nonce, bool inhibit) +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + : start_(0), + stop_(0), + name_(nonce), + inhibit_(inhibit) +#endif +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + PerThreadData* pt = TlsPerThreadData.get(); + AutoFlushICache* afc = pt->PerThreadData::autoFlushICache(); + if (afc) + JitSpew(JitSpew_CacheFlush, "<%s,%s%s ", nonce, afc->name_, inhibit ? " I" : ""); + else + JitSpewCont(JitSpew_CacheFlush, "<%s%s ", nonce, inhibit ? " I" : ""); + + prev_ = afc; + pt->PerThreadData::setAutoFlushICache(this); +#endif +} + +AutoFlushICache::~AutoFlushICache() +{ +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + PerThreadData* pt = TlsPerThreadData.get(); + MOZ_ASSERT(pt->PerThreadData::autoFlushICache() == this); + + if (!inhibit_ && start_) + ExecutableAllocator::cacheFlush((void*)start_, size_t(stop_ - start_)); + + JitSpewCont(JitSpew_CacheFlush, "%s%s>", name_, start_ ? "" : " U"); + JitSpewFin(JitSpew_CacheFlush); + pt->PerThreadData::setAutoFlushICache(prev_); +#endif +} + +void +jit::PurgeCaches(JSScript* script) +{ + if (script->hasIonScript()) + script->ionScript()->purgeCaches(); +} + +size_t +jit::SizeOfIonData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf) +{ + size_t result = 0; + + if (script->hasIonScript()) + result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf); + + return result; +} + +void +jit::DestroyJitScripts(FreeOp* fop, JSScript* script) +{ + if (script->hasIonScript()) + jit::IonScript::Destroy(fop, script->ionScript()); + + if (script->hasBaselineScript()) + jit::BaselineScript::Destroy(fop, script->baselineScript()); +} + +void +jit::TraceJitScripts(JSTracer* trc, JSScript* script) +{ + if (script->hasIonScript()) + jit::IonScript::Trace(trc, script->ionScript()); + + if (script->hasBaselineScript()) + jit::BaselineScript::Trace(trc, script->baselineScript()); +} + +bool +jit::JitSupportsFloatingPoint() +{ + return js::jit::MacroAssembler::SupportsFloatingPoint(); +} + +bool +jit::JitSupportsUnalignedAccesses() +{ + return js::jit::MacroAssembler::SupportsUnalignedAccesses(); +} + +bool +jit::JitSupportsSimd() +{ + return js::jit::MacroAssembler::SupportsSimd(); +} + +bool +jit::JitSupportsAtomics() +{ +#if defined(JS_CODEGEN_ARM) + // Bug 1146902, bug 1077318: Enable Ion inlining of Atomics + // operations on ARM only when the CPU has byte, halfword, and + // doubleword load-exclusive and store-exclusive instructions, + // until we can add support for systems that don't have those. + return js::jit::HasLDSTREXBHD(); +#else + return true; +#endif +} + +// If you change these, please also change the comment in TempAllocator. +/* static */ const size_t TempAllocator::BallastSize = 16 * 1024; +/* static */ const size_t TempAllocator::PreferredLifoChunkSize = 32 * 1024; |