summaryrefslogtreecommitdiffstats
path: root/js/src/jit/Ion.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/Ion.cpp')
-rw-r--r--js/src/jit/Ion.cpp3560
1 files changed, 3560 insertions, 0 deletions
diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp
new file mode 100644
index 000000000..2a158ed7e
--- /dev/null
+++ b/js/src/jit/Ion.cpp
@@ -0,0 +1,3560 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Ion.h"
+
+#include "mozilla/IntegerPrintfMacros.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/SizePrintfMacros.h"
+#include "mozilla/ThreadLocal.h"
+
+#include "jscompartment.h"
+#include "jsgc.h"
+#include "jsprf.h"
+
+#include "gc/Marking.h"
+#include "jit/AliasAnalysis.h"
+#include "jit/AlignmentMaskAnalysis.h"
+#include "jit/BacktrackingAllocator.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineInspector.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CodeGenerator.h"
+#include "jit/EagerSimdUnbox.h"
+#include "jit/EdgeCaseAnalysis.h"
+#include "jit/EffectiveAddressAnalysis.h"
+#include "jit/FlowAliasAnalysis.h"
+#include "jit/FoldLinearArithConstants.h"
+#include "jit/InstructionReordering.h"
+#include "jit/IonAnalysis.h"
+#include "jit/IonBuilder.h"
+#include "jit/IonOptimizationLevels.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitCommon.h"
+#include "jit/JitCompartment.h"
+#include "jit/JitSpewer.h"
+#include "jit/LICM.h"
+#include "jit/LIR.h"
+#include "jit/LoopUnroller.h"
+#include "jit/Lowering.h"
+#include "jit/PerfSpewer.h"
+#include "jit/RangeAnalysis.h"
+#include "jit/ScalarReplacement.h"
+#include "jit/Sink.h"
+#include "jit/StupidAllocator.h"
+#include "jit/ValueNumbering.h"
+#include "jit/WasmBCE.h"
+#include "vm/Debugger.h"
+#include "vm/HelperThreads.h"
+#include "vm/TraceLogging.h"
+
+#include "jscompartmentinlines.h"
+#include "jsobjinlines.h"
+#include "jsscriptinlines.h"
+
+#include "jit/JitFrames-inl.h"
+#include "jit/shared/Lowering-shared-inl.h"
+#include "vm/Debugger-inl.h"
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/Stack-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// Assert that JitCode is gc::Cell aligned.
+JS_STATIC_ASSERT(sizeof(JitCode) % gc::CellSize == 0);
+
+static MOZ_THREAD_LOCAL(JitContext*) TlsJitContext;
+
+static JitContext*
+CurrentJitContext()
+{
+ if (!TlsJitContext.init())
+ return nullptr;
+ return TlsJitContext.get();
+}
+
+void
+jit::SetJitContext(JitContext* ctx)
+{
+ TlsJitContext.set(ctx);
+}
+
+JitContext*
+jit::GetJitContext()
+{
+ MOZ_ASSERT(CurrentJitContext());
+ return CurrentJitContext();
+}
+
+JitContext*
+jit::MaybeGetJitContext()
+{
+ return CurrentJitContext();
+}
+
+JitContext::JitContext(JSContext* cx, TempAllocator* temp)
+ : cx(cx),
+ temp(temp),
+ runtime(CompileRuntime::get(cx->runtime())),
+ compartment(CompileCompartment::get(cx->compartment())),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext(ExclusiveContext* cx, TempAllocator* temp)
+ : cx(nullptr),
+ temp(temp),
+ runtime(CompileRuntime::get(cx->runtime_)),
+ compartment(nullptr),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext(CompileRuntime* rt, CompileCompartment* comp, TempAllocator* temp)
+ : cx(nullptr),
+ temp(temp),
+ runtime(rt),
+ compartment(comp),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext(CompileRuntime* rt)
+ : cx(nullptr),
+ temp(nullptr),
+ runtime(rt),
+ compartment(nullptr),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext(TempAllocator* temp)
+ : cx(nullptr),
+ temp(temp),
+ runtime(nullptr),
+ compartment(nullptr),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext(CompileRuntime* rt, TempAllocator* temp)
+ : cx(nullptr),
+ temp(temp),
+ runtime(rt),
+ compartment(nullptr),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::JitContext()
+ : cx(nullptr),
+ temp(nullptr),
+ runtime(nullptr),
+ compartment(nullptr),
+ prev_(CurrentJitContext()),
+ assemblerCount_(0)
+{
+ SetJitContext(this);
+}
+
+JitContext::~JitContext()
+{
+ SetJitContext(prev_);
+}
+
+bool
+jit::InitializeIon()
+{
+ if (!TlsJitContext.init())
+ return false;
+ CheckLogging();
+#if defined(JS_CODEGEN_ARM)
+ InitARMFlags();
+#endif
+ CheckPerf();
+ return true;
+}
+
+JitRuntime::JitRuntime(JSRuntime* rt)
+ : execAlloc_(rt),
+ backedgeExecAlloc_(rt),
+ exceptionTail_(nullptr),
+ bailoutTail_(nullptr),
+ profilerExitFrameTail_(nullptr),
+ enterJIT_(nullptr),
+ bailoutHandler_(nullptr),
+ argumentsRectifier_(nullptr),
+ argumentsRectifierReturnAddr_(nullptr),
+ invalidator_(nullptr),
+ debugTrapHandler_(nullptr),
+ baselineDebugModeOSRHandler_(nullptr),
+ functionWrappers_(nullptr),
+ osrTempData_(nullptr),
+ preventBackedgePatching_(false),
+ backedgeTarget_(BackedgeLoopHeader),
+ ionReturnOverride_(MagicValue(JS_ARG_POISON)),
+ jitcodeGlobalTable_(nullptr)
+{
+}
+
+JitRuntime::~JitRuntime()
+{
+ js_delete(functionWrappers_);
+ freeOsrTempData();
+
+ // By this point, the jitcode global table should be empty.
+ MOZ_ASSERT_IF(jitcodeGlobalTable_, jitcodeGlobalTable_->empty());
+ js_delete(jitcodeGlobalTable_);
+}
+
+bool
+JitRuntime::initialize(JSContext* cx, AutoLockForExclusiveAccess& lock)
+{
+ AutoCompartment ac(cx, cx->atomsCompartment(lock), &lock);
+
+ JitContext jctx(cx, nullptr);
+
+ if (!cx->compartment()->ensureJitCompartmentExists(cx))
+ return false;
+
+ functionWrappers_ = cx->new_<VMWrapperMap>(cx);
+ if (!functionWrappers_ || !functionWrappers_->init())
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting profiler exit frame tail stub");
+ profilerExitFrameTail_ = generateProfilerExitFrameTailStub(cx);
+ if (!profilerExitFrameTail_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting exception tail stub");
+
+ void* handler = JS_FUNC_TO_DATA_PTR(void*, jit::HandleException);
+
+ exceptionTail_ = generateExceptionTailStub(cx, handler);
+ if (!exceptionTail_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting bailout tail stub");
+ bailoutTail_ = generateBailoutTailStub(cx);
+ if (!bailoutTail_)
+ return false;
+
+ if (cx->runtime()->jitSupportsFloatingPoint) {
+ JitSpew(JitSpew_Codegen, "# Emitting bailout tables");
+
+ // Initialize some Ion-only stubs that require floating-point support.
+ if (!bailoutTables_.reserve(FrameSizeClass::ClassLimit().classId()))
+ return false;
+
+ for (uint32_t id = 0;; id++) {
+ FrameSizeClass class_ = FrameSizeClass::FromClass(id);
+ if (class_ == FrameSizeClass::ClassLimit())
+ break;
+ bailoutTables_.infallibleAppend((JitCode*)nullptr);
+ JitSpew(JitSpew_Codegen, "# Bailout table");
+ bailoutTables_[id] = generateBailoutTable(cx, id);
+ if (!bailoutTables_[id])
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting bailout handler");
+ bailoutHandler_ = generateBailoutHandler(cx);
+ if (!bailoutHandler_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting invalidator");
+ invalidator_ = generateInvalidator(cx);
+ if (!invalidator_)
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting sequential arguments rectifier");
+ argumentsRectifier_ = generateArgumentsRectifier(cx, &argumentsRectifierReturnAddr_);
+ if (!argumentsRectifier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting EnterJIT sequence");
+ enterJIT_ = generateEnterJIT(cx, EnterJitOptimized);
+ if (!enterJIT_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting EnterBaselineJIT sequence");
+ enterBaselineJIT_ = generateEnterJIT(cx, EnterJitBaseline);
+ if (!enterBaselineJIT_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Value");
+ valuePreBarrier_ = generatePreBarrier(cx, MIRType::Value);
+ if (!valuePreBarrier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for String");
+ stringPreBarrier_ = generatePreBarrier(cx, MIRType::String);
+ if (!stringPreBarrier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Object");
+ objectPreBarrier_ = generatePreBarrier(cx, MIRType::Object);
+ if (!objectPreBarrier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Shape");
+ shapePreBarrier_ = generatePreBarrier(cx, MIRType::Shape);
+ if (!shapePreBarrier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for ObjectGroup");
+ objectGroupPreBarrier_ = generatePreBarrier(cx, MIRType::ObjectGroup);
+ if (!objectGroupPreBarrier_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting malloc stub");
+ mallocStub_ = generateMallocStub(cx);
+ if (!mallocStub_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting free stub");
+ freeStub_ = generateFreeStub(cx);
+ if (!freeStub_)
+ return false;
+
+ JitSpew(JitSpew_Codegen, "# Emitting VM function wrappers");
+ for (VMFunction* fun = VMFunction::functions; fun; fun = fun->next) {
+ JitSpew(JitSpew_Codegen, "# VM function wrapper");
+ if (!generateVMWrapper(cx, *fun))
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting lazy link stub");
+ lazyLinkStub_ = generateLazyLinkStub(cx);
+ if (!lazyLinkStub_)
+ return false;
+
+ jitcodeGlobalTable_ = cx->new_<JitcodeGlobalTable>();
+ if (!jitcodeGlobalTable_)
+ return false;
+
+ return true;
+}
+
+JitCode*
+JitRuntime::debugTrapHandler(JSContext* cx)
+{
+ if (!debugTrapHandler_) {
+ // JitRuntime code stubs are shared across compartments and have to
+ // be allocated in the atoms compartment.
+ AutoLockForExclusiveAccess lock(cx);
+ AutoCompartment ac(cx, cx->runtime()->atomsCompartment(lock), &lock);
+ debugTrapHandler_ = generateDebugTrapHandler(cx);
+ }
+ return debugTrapHandler_;
+}
+
+uint8_t*
+JitRuntime::allocateOsrTempData(size_t size)
+{
+ osrTempData_ = (uint8_t*)js_realloc(osrTempData_, size);
+ return osrTempData_;
+}
+
+void
+JitRuntime::freeOsrTempData()
+{
+ js_free(osrTempData_);
+ osrTempData_ = nullptr;
+}
+
+void
+JitRuntime::patchIonBackedges(JSRuntime* rt, BackedgeTarget target)
+{
+ if (target == BackedgeLoopHeader) {
+ // We must be on the main thread. The caller must use
+ // AutoPreventBackedgePatching to ensure we don't reenter.
+ MOZ_ASSERT(preventBackedgePatching_);
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+ } else {
+ // We must be called from InterruptRunningJitCode, or a signal handler
+ // triggered there. rt->handlingJitInterrupt() ensures we can't reenter
+ // this code.
+ MOZ_ASSERT(!preventBackedgePatching_);
+ MOZ_ASSERT(rt->handlingJitInterrupt());
+ }
+
+ // Do nothing if we know all backedges are already jumping to `target`.
+ if (backedgeTarget_ == target)
+ return;
+
+ backedgeTarget_ = target;
+
+ backedgeExecAlloc_.makeAllWritable();
+
+ // Patch all loop backedges in Ion code so that they either jump to the
+ // normal loop header or to an interrupt handler each time they run.
+ for (InlineListIterator<PatchableBackedge> iter(backedgeList_.begin());
+ iter != backedgeList_.end();
+ iter++)
+ {
+ PatchableBackedge* patchableBackedge = *iter;
+ if (target == BackedgeLoopHeader)
+ PatchBackedge(patchableBackedge->backedge, patchableBackedge->loopHeader, target);
+ else
+ PatchBackedge(patchableBackedge->backedge, patchableBackedge->interruptCheck, target);
+ }
+
+ backedgeExecAlloc_.makeAllExecutable();
+}
+
+JitCompartment::JitCompartment()
+ : stubCodes_(nullptr),
+ cacheIRStubCodes_(nullptr),
+ baselineGetPropReturnAddr_(nullptr),
+ baselineSetPropReturnAddr_(nullptr),
+ stringConcatStub_(nullptr),
+ regExpMatcherStub_(nullptr),
+ regExpSearcherStub_(nullptr),
+ regExpTesterStub_(nullptr)
+{
+ baselineCallReturnAddrs_[0] = baselineCallReturnAddrs_[1] = nullptr;
+}
+
+JitCompartment::~JitCompartment()
+{
+ js_delete(stubCodes_);
+ js_delete(cacheIRStubCodes_);
+}
+
+bool
+JitCompartment::initialize(JSContext* cx)
+{
+ stubCodes_ = cx->new_<ICStubCodeMap>(cx->runtime());
+ if (!stubCodes_)
+ return false;
+
+ if (!stubCodes_->init()) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ cacheIRStubCodes_ = cx->new_<CacheIRStubCodeMap>(cx->runtime());
+ if (!cacheIRStubCodes_)
+ return false;
+
+ if (!cacheIRStubCodes_->init()) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+bool
+JitCompartment::ensureIonStubsExist(JSContext* cx)
+{
+ if (!stringConcatStub_) {
+ stringConcatStub_ = generateStringConcatStub(cx);
+ if (!stringConcatStub_)
+ return false;
+ }
+
+ return true;
+}
+
+void
+jit::FinishOffThreadBuilder(JSRuntime* runtime, IonBuilder* builder,
+ const AutoLockHelperThreadState& locked)
+{
+ // Clean the references to the pending IonBuilder, if we just finished it.
+ if (builder->script()->baselineScript()->hasPendingIonBuilder() &&
+ builder->script()->baselineScript()->pendingIonBuilder() == builder)
+ {
+ builder->script()->baselineScript()->removePendingIonBuilder(builder->script());
+ }
+
+ // If the builder is still in one of the helper thread list, then remove it.
+ if (builder->isInList()) {
+ MOZ_ASSERT(runtime);
+ runtime->ionLazyLinkListRemove(builder);
+ }
+
+ // Clear the recompiling flag of the old ionScript, since we continue to
+ // use the old ionScript if recompiling fails.
+ if (builder->script()->hasIonScript())
+ builder->script()->ionScript()->clearRecompiling();
+
+ // Clean up if compilation did not succeed.
+ if (builder->script()->isIonCompilingOffThread()) {
+ IonScript* ion =
+ builder->abortReason() == AbortReason_Disable ? ION_DISABLED_SCRIPT : nullptr;
+ builder->script()->setIonScript(runtime, ion);
+ }
+
+ // The builder is allocated into its LifoAlloc, so destroying that will
+ // destroy the builder and all other data accumulated during compilation,
+ // except any final codegen (which includes an assembler and needs to be
+ // explicitly destroyed).
+ js_delete(builder->backgroundCodegen());
+ js_delete(builder->alloc().lifoAlloc());
+}
+
+static bool
+LinkCodeGen(JSContext* cx, IonBuilder* builder, CodeGenerator *codegen)
+{
+ RootedScript script(cx, builder->script());
+ TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
+ TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script);
+ AutoTraceLog logScript(logger, event);
+ AutoTraceLog logLink(logger, TraceLogger_IonLinking);
+
+ if (!codegen->link(cx, builder->constraints()))
+ return false;
+
+ return true;
+}
+
+static bool
+LinkBackgroundCodeGen(JSContext* cx, IonBuilder* builder)
+{
+ CodeGenerator* codegen = builder->backgroundCodegen();
+ if (!codegen)
+ return false;
+
+ JitContext jctx(cx, &builder->alloc());
+
+ // Root the assembler until the builder is finished below. As it was
+ // constructed off thread, the assembler has not been rooted previously,
+ // though any GC activity would discard the builder.
+ MacroAssembler::AutoRooter masm(cx, &codegen->masm);
+
+ return LinkCodeGen(cx, builder, codegen);
+}
+
+void
+jit::LinkIonScript(JSContext* cx, HandleScript calleeScript)
+{
+ IonBuilder* builder;
+
+ {
+ AutoLockHelperThreadState lock;
+
+ // Get the pending builder from the Ion frame.
+ MOZ_ASSERT(calleeScript->hasBaselineScript());
+ builder = calleeScript->baselineScript()->pendingIonBuilder();
+ calleeScript->baselineScript()->removePendingIonBuilder(calleeScript);
+
+ // Remove from pending.
+ cx->runtime()->ionLazyLinkListRemove(builder);
+ }
+
+ {
+ AutoEnterAnalysis enterTypes(cx);
+ if (!LinkBackgroundCodeGen(cx, builder)) {
+ // Silently ignore OOM during code generation. The assembly code
+ // doesn't has code to handle it after linking happened. So it's
+ // not OK to throw a catchable exception from there.
+ cx->clearPendingException();
+
+ // Reset the TypeZone's compiler output for this script, if any.
+ InvalidateCompilerOutputsForScript(cx, calleeScript);
+ }
+ }
+
+ {
+ AutoLockHelperThreadState lock;
+ FinishOffThreadBuilder(cx->runtime(), builder, lock);
+ }
+}
+
+uint8_t*
+jit::LazyLinkTopActivation(JSContext* cx)
+{
+ // First frame should be an exit frame.
+ JitFrameIterator it(cx);
+ LazyLinkExitFrameLayout* ll = it.exitFrame()->as<LazyLinkExitFrameLayout>();
+ RootedScript calleeScript(cx, ScriptFromCalleeToken(ll->jsFrame()->calleeToken()));
+
+ LinkIonScript(cx, calleeScript);
+
+ MOZ_ASSERT(calleeScript->hasBaselineScript());
+ MOZ_ASSERT(calleeScript->baselineOrIonRawPointer());
+
+ return calleeScript->baselineOrIonRawPointer();
+}
+
+/* static */ void
+JitRuntime::Mark(JSTracer* trc, AutoLockForExclusiveAccess& lock)
+{
+ MOZ_ASSERT(!trc->runtime()->isHeapMinorCollecting());
+
+ // Shared stubs are allocated in the atoms compartment, so do not iterate
+ // them after the atoms heap after it has been "finished."
+ if (trc->runtime()->atomsAreFinished())
+ return;
+
+ Zone* zone = trc->runtime()->atomsCompartment(lock)->zone();
+ for (auto i = zone->cellIter<JitCode>(); !i.done(); i.next()) {
+ JitCode* code = i;
+ TraceRoot(trc, &code, "wrapper");
+ }
+}
+
+/* static */ void
+JitRuntime::MarkJitcodeGlobalTableUnconditionally(JSTracer* trc)
+{
+ if (trc->runtime()->spsProfiler.enabled() &&
+ trc->runtime()->hasJitRuntime() &&
+ trc->runtime()->jitRuntime()->hasJitcodeGlobalTable())
+ {
+ trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markUnconditionally(trc);
+ }
+}
+
+/* static */ bool
+JitRuntime::MarkJitcodeGlobalTableIteratively(JSTracer* trc)
+{
+ if (trc->runtime()->hasJitRuntime() &&
+ trc->runtime()->jitRuntime()->hasJitcodeGlobalTable())
+ {
+ return trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markIteratively(trc);
+ }
+ return false;
+}
+
+/* static */ void
+JitRuntime::SweepJitcodeGlobalTable(JSRuntime* rt)
+{
+ if (rt->hasJitRuntime() && rt->jitRuntime()->hasJitcodeGlobalTable())
+ rt->jitRuntime()->getJitcodeGlobalTable()->sweep(rt);
+}
+
+void
+JitCompartment::mark(JSTracer* trc, JSCompartment* compartment)
+{
+ // Free temporary OSR buffer.
+ trc->runtime()->jitRuntime()->freeOsrTempData();
+}
+
+void
+JitCompartment::sweep(FreeOp* fop, JSCompartment* compartment)
+{
+ // Any outstanding compilations should have been cancelled by the GC.
+ MOZ_ASSERT(!HasOffThreadIonCompile(compartment));
+
+ stubCodes_->sweep();
+ cacheIRStubCodes_->sweep();
+
+ // If the sweep removed the ICCall_Fallback stub, nullptr the baselineCallReturnAddr_ field.
+ if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CALL_KEY))
+ baselineCallReturnAddrs_[0] = nullptr;
+ if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CONSTRUCT_KEY))
+ baselineCallReturnAddrs_[1] = nullptr;
+
+ // Similarly for the ICGetProp_Fallback stub.
+ if (!stubCodes_->lookup(ICGetProp_Fallback::Compiler::BASELINE_KEY))
+ baselineGetPropReturnAddr_ = nullptr;
+ if (!stubCodes_->lookup(ICSetProp_Fallback::Compiler::BASELINE_KEY))
+ baselineSetPropReturnAddr_ = nullptr;
+
+ JSRuntime* rt = fop->runtime();
+ if (stringConcatStub_ && !IsMarkedUnbarriered(rt, &stringConcatStub_))
+ stringConcatStub_ = nullptr;
+
+ if (regExpMatcherStub_ && !IsMarkedUnbarriered(rt, &regExpMatcherStub_))
+ regExpMatcherStub_ = nullptr;
+
+ if (regExpSearcherStub_ && !IsMarkedUnbarriered(rt, &regExpSearcherStub_))
+ regExpSearcherStub_ = nullptr;
+
+ if (regExpTesterStub_ && !IsMarkedUnbarriered(rt, &regExpTesterStub_))
+ regExpTesterStub_ = nullptr;
+
+ for (ReadBarrieredObject& obj : simdTemplateObjects_) {
+ if (obj && IsAboutToBeFinalized(&obj))
+ obj.set(nullptr);
+ }
+}
+
+void
+JitCompartment::toggleBarriers(bool enabled)
+{
+ // Toggle barriers in compartment wide stubs that have patchable pre barriers.
+ if (regExpMatcherStub_)
+ regExpMatcherStub_->togglePreBarriers(enabled, Reprotect);
+ if (regExpSearcherStub_)
+ regExpSearcherStub_->togglePreBarriers(enabled, Reprotect);
+ if (regExpTesterStub_)
+ regExpTesterStub_->togglePreBarriers(enabled, Reprotect);
+
+ // Toggle barriers in baseline IC stubs.
+ for (ICStubCodeMap::Enum e(*stubCodes_); !e.empty(); e.popFront()) {
+ JitCode* code = *e.front().value().unsafeGet();
+ code->togglePreBarriers(enabled, Reprotect);
+ }
+ for (CacheIRStubCodeMap::Enum e(*cacheIRStubCodes_); !e.empty(); e.popFront()) {
+ JitCode* code = *e.front().value().unsafeGet();
+ code->togglePreBarriers(enabled, Reprotect);
+ }
+}
+
+size_t
+JitCompartment::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
+{
+ size_t n = mallocSizeOf(this);
+ if (stubCodes_)
+ n += stubCodes_->sizeOfIncludingThis(mallocSizeOf);
+ if (cacheIRStubCodes_)
+ n += cacheIRStubCodes_->sizeOfIncludingThis(mallocSizeOf);
+ return n;
+}
+
+JitCode*
+JitRuntime::getBailoutTable(const FrameSizeClass& frameClass) const
+{
+ MOZ_ASSERT(frameClass != FrameSizeClass::None());
+ return bailoutTables_[frameClass.classId()];
+}
+
+JitCode*
+JitRuntime::getVMWrapper(const VMFunction& f) const
+{
+ MOZ_ASSERT(functionWrappers_);
+ MOZ_ASSERT(functionWrappers_->initialized());
+ JitRuntime::VMWrapperMap::Ptr p = functionWrappers_->readonlyThreadsafeLookup(&f);
+ MOZ_ASSERT(p);
+
+ return p->value();
+}
+
+template <AllowGC allowGC>
+JitCode*
+JitCode::New(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind)
+{
+ JitCode* codeObj = Allocate<JitCode, allowGC>(cx);
+ if (!codeObj) {
+ pool->release(headerSize + bufferSize, kind);
+ return nullptr;
+ }
+
+ new (codeObj) JitCode(code, bufferSize, headerSize, pool, kind);
+ return codeObj;
+}
+
+template
+JitCode*
+JitCode::New<CanGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind);
+
+template
+JitCode*
+JitCode::New<NoGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind);
+
+void
+JitCode::copyFrom(MacroAssembler& masm)
+{
+ // Store the JitCode pointer right before the code buffer, so we can
+ // recover the gcthing from relocation tables.
+ *(JitCode**)(code_ - sizeof(JitCode*)) = this;
+ insnSize_ = masm.instructionsSize();
+ masm.executableCopy(code_);
+
+ jumpRelocTableBytes_ = masm.jumpRelocationTableBytes();
+ masm.copyJumpRelocationTable(code_ + jumpRelocTableOffset());
+
+ dataRelocTableBytes_ = masm.dataRelocationTableBytes();
+ masm.copyDataRelocationTable(code_ + dataRelocTableOffset());
+
+ preBarrierTableBytes_ = masm.preBarrierTableBytes();
+ masm.copyPreBarrierTable(code_ + preBarrierTableOffset());
+
+ masm.processCodeLabels(code_);
+}
+
+void
+JitCode::traceChildren(JSTracer* trc)
+{
+ // Note that we cannot mark invalidated scripts, since we've basically
+ // corrupted the code stream by injecting bailouts.
+ if (invalidated())
+ return;
+
+ if (jumpRelocTableBytes_) {
+ uint8_t* start = code_ + jumpRelocTableOffset();
+ CompactBufferReader reader(start, start + jumpRelocTableBytes_);
+ MacroAssembler::TraceJumpRelocations(trc, this, reader);
+ }
+ if (dataRelocTableBytes_) {
+ // If we're moving objects, we need writable JIT code.
+ bool movingObjects = trc->runtime()->isHeapMinorCollecting() || zone()->isGCCompacting();
+ MaybeAutoWritableJitCode awjc(this, movingObjects ? Reprotect : DontReprotect);
+
+ uint8_t* start = code_ + dataRelocTableOffset();
+ CompactBufferReader reader(start, start + dataRelocTableBytes_);
+ MacroAssembler::TraceDataRelocations(trc, this, reader);
+ }
+}
+
+void
+JitCode::finalize(FreeOp* fop)
+{
+ // If this jitcode had a bytecode map, it must have already been removed.
+#ifdef DEBUG
+ JSRuntime* rt = fop->runtime();
+ if (hasBytecodeMap_) {
+ MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable());
+ MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw()));
+ }
+#endif
+
+ MOZ_ASSERT(pool_);
+
+ // With W^X JIT code, reprotecting memory for each JitCode instance is
+ // slow, so we record the ranges and poison them later all at once. It's
+ // safe to ignore OOM here, it just means we won't poison the code.
+ if (fop->appendJitPoisonRange(JitPoisonRange(pool_, code_ - headerSize_,
+ headerSize_ + bufferSize_)))
+ {
+ pool_->addRef();
+ }
+ code_ = nullptr;
+
+ // Code buffers are stored inside ExecutablePools. Pools are refcounted.
+ // Releasing the pool may free it. Horrible hack: if we are using perf
+ // integration, we don't want to reuse code addresses, so we just leak the
+ // memory instead.
+ if (!PerfEnabled())
+ pool_->release(headerSize_ + bufferSize_, CodeKind(kind_));
+ pool_ = nullptr;
+}
+
+void
+JitCode::togglePreBarriers(bool enabled, ReprotectCode reprotect)
+{
+ uint8_t* start = code_ + preBarrierTableOffset();
+ CompactBufferReader reader(start, start + preBarrierTableBytes_);
+
+ if (!reader.more())
+ return;
+
+ MaybeAutoWritableJitCode awjc(this, reprotect);
+ do {
+ size_t offset = reader.readUnsigned();
+ CodeLocationLabel loc(this, CodeOffset(offset));
+ if (enabled)
+ Assembler::ToggleToCmp(loc);
+ else
+ Assembler::ToggleToJmp(loc);
+ } while (reader.more());
+}
+
+IonScript::IonScript()
+ : method_(nullptr),
+ deoptTable_(nullptr),
+ osrPc_(nullptr),
+ osrEntryOffset_(0),
+ skipArgCheckEntryOffset_(0),
+ invalidateEpilogueOffset_(0),
+ invalidateEpilogueDataOffset_(0),
+ numBailouts_(0),
+ hasProfilingInstrumentation_(false),
+ recompiling_(false),
+ runtimeData_(0),
+ runtimeSize_(0),
+ cacheIndex_(0),
+ cacheEntries_(0),
+ safepointIndexOffset_(0),
+ safepointIndexEntries_(0),
+ safepointsStart_(0),
+ safepointsSize_(0),
+ frameSlots_(0),
+ frameSize_(0),
+ bailoutTable_(0),
+ bailoutEntries_(0),
+ osiIndexOffset_(0),
+ osiIndexEntries_(0),
+ snapshots_(0),
+ snapshotsListSize_(0),
+ snapshotsRVATableSize_(0),
+ constantTable_(0),
+ constantEntries_(0),
+ backedgeList_(0),
+ backedgeEntries_(0),
+ invalidationCount_(0),
+ recompileInfo_(),
+ osrPcMismatchCounter_(0),
+ fallbackStubSpace_()
+{
+}
+
+IonScript*
+IonScript::New(JSContext* cx, RecompileInfo recompileInfo,
+ uint32_t frameSlots, uint32_t argumentSlots, uint32_t frameSize,
+ size_t snapshotsListSize, size_t snapshotsRVATableSize,
+ size_t recoversSize, size_t bailoutEntries,
+ size_t constants, size_t safepointIndices,
+ size_t osiIndices, size_t cacheEntries,
+ size_t runtimeSize, size_t safepointsSize,
+ size_t backedgeEntries, size_t sharedStubEntries,
+ OptimizationLevel optimizationLevel)
+{
+ constexpr size_t DataAlignment = sizeof(void*);
+
+ if (snapshotsListSize >= MAX_BUFFER_SIZE ||
+ (bailoutEntries >= MAX_BUFFER_SIZE / sizeof(uint32_t)))
+ {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ // This should not overflow on x86, because the memory is already allocated
+ // *somewhere* and if their total overflowed there would be no memory left
+ // at all.
+ size_t paddedSnapshotsSize = AlignBytes(snapshotsListSize + snapshotsRVATableSize, DataAlignment);
+ size_t paddedRecoversSize = AlignBytes(recoversSize, DataAlignment);
+ size_t paddedBailoutSize = AlignBytes(bailoutEntries * sizeof(uint32_t), DataAlignment);
+ size_t paddedConstantsSize = AlignBytes(constants * sizeof(Value), DataAlignment);
+ size_t paddedSafepointIndicesSize = AlignBytes(safepointIndices * sizeof(SafepointIndex), DataAlignment);
+ size_t paddedOsiIndicesSize = AlignBytes(osiIndices * sizeof(OsiIndex), DataAlignment);
+ size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(uint32_t), DataAlignment);
+ size_t paddedRuntimeSize = AlignBytes(runtimeSize, DataAlignment);
+ size_t paddedSafepointSize = AlignBytes(safepointsSize, DataAlignment);
+ size_t paddedBackedgeSize = AlignBytes(backedgeEntries * sizeof(PatchableBackedge), DataAlignment);
+ size_t paddedSharedStubSize = AlignBytes(sharedStubEntries * sizeof(IonICEntry), DataAlignment);
+
+ size_t bytes = paddedSnapshotsSize +
+ paddedRecoversSize +
+ paddedBailoutSize +
+ paddedConstantsSize +
+ paddedSafepointIndicesSize +
+ paddedOsiIndicesSize +
+ paddedCacheEntriesSize +
+ paddedRuntimeSize +
+ paddedSafepointSize +
+ paddedBackedgeSize +
+ paddedSharedStubSize;
+ IonScript* script = cx->zone()->pod_malloc_with_extra<IonScript, uint8_t>(bytes);
+ if (!script)
+ return nullptr;
+ new (script) IonScript();
+
+ uint32_t offsetCursor = sizeof(IonScript);
+
+ script->runtimeData_ = offsetCursor;
+ script->runtimeSize_ = runtimeSize;
+ offsetCursor += paddedRuntimeSize;
+
+ script->cacheIndex_ = offsetCursor;
+ script->cacheEntries_ = cacheEntries;
+ offsetCursor += paddedCacheEntriesSize;
+
+ script->safepointIndexOffset_ = offsetCursor;
+ script->safepointIndexEntries_ = safepointIndices;
+ offsetCursor += paddedSafepointIndicesSize;
+
+ script->safepointsStart_ = offsetCursor;
+ script->safepointsSize_ = safepointsSize;
+ offsetCursor += paddedSafepointSize;
+
+ script->bailoutTable_ = offsetCursor;
+ script->bailoutEntries_ = bailoutEntries;
+ offsetCursor += paddedBailoutSize;
+
+ script->osiIndexOffset_ = offsetCursor;
+ script->osiIndexEntries_ = osiIndices;
+ offsetCursor += paddedOsiIndicesSize;
+
+ script->snapshots_ = offsetCursor;
+ script->snapshotsListSize_ = snapshotsListSize;
+ script->snapshotsRVATableSize_ = snapshotsRVATableSize;
+ offsetCursor += paddedSnapshotsSize;
+
+ script->recovers_ = offsetCursor;
+ script->recoversSize_ = recoversSize;
+ offsetCursor += paddedRecoversSize;
+
+ script->constantTable_ = offsetCursor;
+ script->constantEntries_ = constants;
+ offsetCursor += paddedConstantsSize;
+
+ script->backedgeList_ = offsetCursor;
+ script->backedgeEntries_ = backedgeEntries;
+ offsetCursor += paddedBackedgeSize;
+
+ script->sharedStubList_ = offsetCursor;
+ script->sharedStubEntries_ = sharedStubEntries;
+ offsetCursor += paddedSharedStubSize;
+
+ script->frameSlots_ = frameSlots;
+ script->argumentSlots_ = argumentSlots;
+
+ script->frameSize_ = frameSize;
+
+ script->recompileInfo_ = recompileInfo;
+ script->optimizationLevel_ = optimizationLevel;
+
+ return script;
+}
+
+void
+IonScript::adoptFallbackStubs(FallbackICStubSpace* stubSpace)
+
+{
+ fallbackStubSpace()->adoptFrom(stubSpace);
+}
+
+void
+IonScript::trace(JSTracer* trc)
+{
+ if (method_)
+ TraceEdge(trc, &method_, "method");
+
+ if (deoptTable_)
+ TraceEdge(trc, &deoptTable_, "deoptimizationTable");
+
+ for (size_t i = 0; i < numConstants(); i++)
+ TraceEdge(trc, &getConstant(i), "constant");
+
+ // Mark all IC stub codes hanging off the IC stub entries.
+ for (size_t i = 0; i < numSharedStubs(); i++) {
+ IonICEntry& ent = sharedStubList()[i];
+ ent.trace(trc);
+ }
+
+ // Trace caches so that the JSScript pointer can be updated if moved.
+ for (size_t i = 0; i < numCaches(); i++)
+ getCacheFromIndex(i).trace(trc);
+}
+
+/* static */ void
+IonScript::writeBarrierPre(Zone* zone, IonScript* ionScript)
+{
+ if (zone->needsIncrementalBarrier())
+ ionScript->trace(zone->barrierTracer());
+}
+
+void
+IonScript::copySnapshots(const SnapshotWriter* writer)
+{
+ MOZ_ASSERT(writer->listSize() == snapshotsListSize_);
+ memcpy((uint8_t*)this + snapshots_,
+ writer->listBuffer(), snapshotsListSize_);
+
+ MOZ_ASSERT(snapshotsRVATableSize_);
+ MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_);
+ memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_,
+ writer->RVATableBuffer(), snapshotsRVATableSize_);
+}
+
+void
+IonScript::copyRecovers(const RecoverWriter* writer)
+{
+ MOZ_ASSERT(writer->size() == recoversSize_);
+ memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_);
+}
+
+void
+IonScript::copySafepoints(const SafepointWriter* writer)
+{
+ MOZ_ASSERT(writer->size() == safepointsSize_);
+ memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_);
+}
+
+void
+IonScript::copyBailoutTable(const SnapshotOffset* table)
+{
+ memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t));
+}
+
+void
+IonScript::copyConstants(const Value* vp)
+{
+ for (size_t i = 0; i < constantEntries_; i++)
+ constants()[i].init(vp[i]);
+}
+
+void
+IonScript::copyPatchableBackedges(JSContext* cx, JitCode* code,
+ PatchableBackedgeInfo* backedges,
+ MacroAssembler& masm)
+{
+ JitRuntime* jrt = cx->runtime()->jitRuntime();
+ JitRuntime::AutoPreventBackedgePatching apbp(cx->runtime());
+
+ for (size_t i = 0; i < backedgeEntries_; i++) {
+ PatchableBackedgeInfo& info = backedges[i];
+ PatchableBackedge* patchableBackedge = &backedgeList()[i];
+
+ info.backedge.fixup(&masm);
+ CodeLocationJump backedge(code, info.backedge);
+ CodeLocationLabel loopHeader(code, CodeOffset(info.loopHeader->offset()));
+ CodeLocationLabel interruptCheck(code, CodeOffset(info.interruptCheck->offset()));
+ new(patchableBackedge) PatchableBackedge(backedge, loopHeader, interruptCheck);
+
+ // Point the backedge to either of its possible targets, matching the
+ // other backedges in the runtime.
+ if (jrt->backedgeTarget() == JitRuntime::BackedgeInterruptCheck)
+ PatchBackedge(backedge, interruptCheck, JitRuntime::BackedgeInterruptCheck);
+ else
+ PatchBackedge(backedge, loopHeader, JitRuntime::BackedgeLoopHeader);
+
+ jrt->addPatchableBackedge(patchableBackedge);
+ }
+}
+
+void
+IonScript::copySafepointIndices(const SafepointIndex* si, MacroAssembler& masm)
+{
+ // Jumps in the caches reflect the offset of those jumps in the compiled
+ // code, not the absolute positions of the jumps. Update according to the
+ // final code address now.
+ SafepointIndex* table = safepointIndices();
+ memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex));
+}
+
+void
+IonScript::copyOsiIndices(const OsiIndex* oi, MacroAssembler& masm)
+{
+ memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex));
+}
+
+void
+IonScript::copyRuntimeData(const uint8_t* data)
+{
+ memcpy(runtimeData(), data, runtimeSize());
+}
+
+void
+IonScript::copyCacheEntries(const uint32_t* caches, MacroAssembler& masm)
+{
+ memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t));
+
+ // Jumps in the caches reflect the offset of those jumps in the compiled
+ // code, not the absolute positions of the jumps. Update according to the
+ // final code address now.
+ for (size_t i = 0; i < numCaches(); i++)
+ getCacheFromIndex(i).updateBaseAddress(method_, masm);
+}
+
+const SafepointIndex*
+IonScript::getSafepointIndex(uint32_t disp) const
+{
+ MOZ_ASSERT(safepointIndexEntries_ > 0);
+
+ const SafepointIndex* table = safepointIndices();
+ if (safepointIndexEntries_ == 1) {
+ MOZ_ASSERT(disp == table[0].displacement());
+ return &table[0];
+ }
+
+ size_t minEntry = 0;
+ size_t maxEntry = safepointIndexEntries_ - 1;
+ uint32_t min = table[minEntry].displacement();
+ uint32_t max = table[maxEntry].displacement();
+
+ // Raise if the element is not in the list.
+ MOZ_ASSERT(min <= disp && disp <= max);
+
+ // Approximate the location of the FrameInfo.
+ size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry;
+ uint32_t guessDisp = table[guess].displacement();
+
+ if (table[guess].displacement() == disp)
+ return &table[guess];
+
+ // Doing a linear scan from the guess should be more efficient in case of
+ // small group which are equally distributed on the code.
+ //
+ // such as: <... ... ... ... . ... ...>
+ if (guessDisp > disp) {
+ while (--guess >= minEntry) {
+ guessDisp = table[guess].displacement();
+ MOZ_ASSERT(guessDisp >= disp);
+ if (guessDisp == disp)
+ return &table[guess];
+ }
+ } else {
+ while (++guess <= maxEntry) {
+ guessDisp = table[guess].displacement();
+ MOZ_ASSERT(guessDisp <= disp);
+ if (guessDisp == disp)
+ return &table[guess];
+ }
+ }
+
+ MOZ_CRASH("displacement not found.");
+}
+
+const OsiIndex*
+IonScript::getOsiIndex(uint32_t disp) const
+{
+ const OsiIndex* end = osiIndices() + osiIndexEntries_;
+ for (const OsiIndex* it = osiIndices(); it != end; ++it) {
+ if (it->returnPointDisplacement() == disp)
+ return it;
+ }
+
+ MOZ_CRASH("Failed to find OSI point return address");
+}
+
+const OsiIndex*
+IonScript::getOsiIndex(uint8_t* retAddr) const
+{
+ JitSpew(JitSpew_IonInvalidate, "IonScript %p has method %p raw %p", (void*) this, (void*)
+ method(), method()->raw());
+
+ MOZ_ASSERT(containsCodeAddress(retAddr));
+ uint32_t disp = retAddr - method()->raw();
+ return getOsiIndex(disp);
+}
+
+void
+IonScript::Trace(JSTracer* trc, IonScript* script)
+{
+ if (script != ION_DISABLED_SCRIPT)
+ script->trace(trc);
+}
+
+void
+IonScript::Destroy(FreeOp* fop, IonScript* script)
+{
+ script->unlinkFromRuntime(fop);
+
+ /*
+ * When the script contains pointers to nursery things, the store buffer can
+ * contain entries that point into the fallback stub space. Since we can
+ * destroy scripts outside the context of a GC, this situation could result
+ * in us trying to mark invalid store buffer entries.
+ *
+ * Defer freeing any allocated blocks until after the next minor GC.
+ */
+ script->fallbackStubSpace_.freeAllAfterMinorGC(fop->runtime());
+
+ fop->delete_(script);
+}
+
+void
+JS::DeletePolicy<js::jit::IonScript>::operator()(const js::jit::IonScript* script)
+{
+ IonScript::Destroy(rt_->defaultFreeOp(), const_cast<IonScript*>(script));
+}
+
+void
+IonScript::toggleBarriers(bool enabled, ReprotectCode reprotect)
+{
+ method()->togglePreBarriers(enabled, reprotect);
+}
+
+void
+IonScript::purgeOptimizedStubs(Zone* zone)
+{
+ for (size_t i = 0; i < numSharedStubs(); i++) {
+ IonICEntry& entry = sharedStubList()[i];
+ if (!entry.hasStub())
+ continue;
+
+ ICStub* lastStub = entry.firstStub();
+ while (lastStub->next())
+ lastStub = lastStub->next();
+
+ if (lastStub->isFallback()) {
+ // Unlink all stubs allocated in the optimized space.
+ ICStub* stub = entry.firstStub();
+ ICStub* prev = nullptr;
+
+ while (stub->next()) {
+ if (!stub->allocatedInFallbackSpace()) {
+ lastStub->toFallbackStub()->unlinkStub(zone, prev, stub);
+ stub = stub->next();
+ continue;
+ }
+
+ prev = stub;
+ stub = stub->next();
+ }
+
+ lastStub->toFallbackStub()->setInvalid();
+
+ if (lastStub->isMonitoredFallback()) {
+ // Monitor stubs can't make calls, so are always in the
+ // optimized stub space.
+ ICTypeMonitor_Fallback* lastMonStub =
+ lastStub->toMonitoredFallbackStub()->fallbackMonitorStub();
+ lastMonStub->resetMonitorStubChain(zone);
+ lastMonStub->setInvalid();
+ }
+ } else if (lastStub->isTypeMonitor_Fallback()) {
+ lastStub->toTypeMonitor_Fallback()->resetMonitorStubChain(zone);
+ lastStub->toTypeMonitor_Fallback()->setInvalid();
+ } else {
+ MOZ_ASSERT(lastStub->isTableSwitch());
+ }
+ }
+
+#ifdef DEBUG
+ // All remaining stubs must be allocated in the fallback space.
+ for (size_t i = 0; i < numSharedStubs(); i++) {
+ IonICEntry& entry = sharedStubList()[i];
+ if (!entry.hasStub())
+ continue;
+
+ ICStub* stub = entry.firstStub();
+ while (stub->next()) {
+ MOZ_ASSERT(stub->allocatedInFallbackSpace());
+ stub = stub->next();
+ }
+ }
+#endif
+}
+
+void
+IonScript::purgeCaches()
+{
+ // Don't reset any ICs if we're invalidated, otherwise, repointing the
+ // inline jump could overwrite an invalidation marker. These ICs can
+ // no longer run, however, the IC slow paths may be active on the stack.
+ // ICs therefore are required to check for invalidation before patching,
+ // to ensure the same invariant.
+ if (invalidated())
+ return;
+
+ if (numCaches() == 0)
+ return;
+
+ AutoWritableJitCode awjc(method());
+ for (size_t i = 0; i < numCaches(); i++)
+ getCacheFromIndex(i).reset(DontReprotect);
+}
+
+void
+IonScript::unlinkFromRuntime(FreeOp* fop)
+{
+ // The writes to the executable buffer below may clobber backedge jumps, so
+ // make sure that those backedges are unlinked from the runtime and not
+ // reclobbered with garbage if an interrupt is requested.
+ JitRuntime* jrt = fop->runtime()->jitRuntime();
+ JitRuntime::AutoPreventBackedgePatching apbp(fop->runtime());
+ for (size_t i = 0; i < backedgeEntries_; i++)
+ jrt->removePatchableBackedge(&backedgeList()[i]);
+
+ // Clear the list of backedges, so that this method is idempotent. It is
+ // called during destruction, and may be additionally called when the
+ // script is invalidated.
+ backedgeEntries_ = 0;
+}
+
+void
+jit::ToggleBarriers(JS::Zone* zone, bool needs)
+{
+ JSRuntime* rt = zone->runtimeFromMainThread();
+ if (!rt->hasJitRuntime())
+ return;
+
+ for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next()) {
+ if (script->hasIonScript())
+ script->ionScript()->toggleBarriers(needs);
+ if (script->hasBaselineScript())
+ script->baselineScript()->toggleBarriers(needs);
+ }
+
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
+ if (comp->jitCompartment())
+ comp->jitCompartment()->toggleBarriers(needs);
+ }
+}
+
+namespace js {
+namespace jit {
+
+static void
+OptimizeSinCos(MIRGenerator *mir, MIRGraph &graph)
+{
+ // Now, we are looking for:
+ // var y = sin(x);
+ // var z = cos(x);
+ // Graph before:
+ // - 1 op
+ // - 6 mathfunction op1 Sin
+ // - 7 mathfunction op1 Cos
+ // Graph will look like:
+ // - 1 op
+ // - 5 sincos op1
+ // - 6 mathfunction sincos5 Sin
+ // - 7 mathfunction sincos5 Cos
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end(); block++) {
+ for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ) {
+ MInstruction *ins = *iter++;
+ if (!ins->isMathFunction() || ins->isRecoveredOnBailout())
+ continue;
+
+ MMathFunction *insFunc = ins->toMathFunction();
+ if (insFunc->function() != MMathFunction::Sin && insFunc->function() != MMathFunction::Cos)
+ continue;
+
+ // Check if sin/cos is already optimized.
+ if (insFunc->getOperand(0)->type() == MIRType::SinCosDouble)
+ continue;
+
+ // insFunc is either a |sin(x)| or |cos(x)| instruction. The
+ // following loop iterates over the uses of |x| to check if both
+ // |sin(x)| and |cos(x)| instructions exist.
+ bool hasSin = false;
+ bool hasCos = false;
+ for (MUseDefIterator uses(insFunc->input()); uses; uses++)
+ {
+ if (!uses.def()->isInstruction())
+ continue;
+
+ // We should replacing the argument of the sin/cos just when it
+ // is dominated by the |block|.
+ if (!block->dominates(uses.def()->block()))
+ continue;
+
+ MInstruction *insUse = uses.def()->toInstruction();
+ if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout())
+ continue;
+
+ MMathFunction *mathIns = insUse->toMathFunction();
+ if (!hasSin && mathIns->function() == MMathFunction::Sin) {
+ hasSin = true;
+ JitSpew(JitSpew_Sincos, "Found sin in block %d.", mathIns->block()->id());
+ }
+ else if (!hasCos && mathIns->function() == MMathFunction::Cos) {
+ hasCos = true;
+ JitSpew(JitSpew_Sincos, "Found cos in block %d.", mathIns->block()->id());
+ }
+
+ if (hasCos && hasSin)
+ break;
+ }
+
+ if (!hasCos || !hasSin) {
+ JitSpew(JitSpew_Sincos, "No sin/cos pair found.");
+ continue;
+ }
+
+ JitSpew(JitSpew_Sincos, "Found, at least, a pair sin/cos. Adding sincos in block %d",
+ block->id());
+ // Adding the MSinCos and replacing the parameters of the
+ // sin(x)/cos(x) to sin(sincos(x))/cos(sincos(x)).
+ MSinCos *insSinCos = MSinCos::New(graph.alloc(),
+ insFunc->input(),
+ insFunc->toMathFunction()->cache());
+ insSinCos->setImplicitlyUsedUnchecked();
+ block->insertBefore(insFunc, insSinCos);
+ for (MUseDefIterator uses(insFunc->input()); uses; )
+ {
+ MDefinition* def = uses.def();
+ uses++;
+ if (!def->isInstruction())
+ continue;
+
+ // We should replacing the argument of the sin/cos just when it
+ // is dominated by the |block|.
+ if (!block->dominates(def->block()))
+ continue;
+
+ MInstruction *insUse = def->toInstruction();
+ if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout())
+ continue;
+
+ MMathFunction *mathIns = insUse->toMathFunction();
+ if (mathIns->function() != MMathFunction::Sin && mathIns->function() != MMathFunction::Cos)
+ continue;
+
+ mathIns->replaceOperand(0, insSinCos);
+ JitSpew(JitSpew_Sincos, "Replacing %s by sincos in block %d",
+ mathIns->function() == MMathFunction::Sin ? "sin" : "cos",
+ mathIns->block()->id());
+ }
+ }
+ }
+}
+
+bool
+OptimizeMIR(MIRGenerator* mir)
+{
+ MIRGraph& graph = mir->graph();
+ GraphSpewer& gs = mir->graphSpewer();
+ TraceLoggerThread* logger;
+ if (GetJitContext()->onMainThread())
+ logger = TraceLoggerForMainThread(GetJitContext()->runtime);
+ else
+ logger = TraceLoggerForCurrentThread();
+
+ if (mir->shouldCancel("Start"))
+ return false;
+
+ if (!mir->compilingWasm()) {
+ if (!MakeMRegExpHoistable(mir, graph))
+ return false;
+
+ if (mir->shouldCancel("Make MRegExp Hoistable"))
+ return false;
+ }
+
+ gs.spewPass("BuildSSA");
+ AssertBasicGraphCoherency(graph);
+
+ if (!JitOptions.disablePgo && !mir->compilingWasm()) {
+ AutoTraceLog log(logger, TraceLogger_PruneUnusedBranches);
+ if (!PruneUnusedBranches(mir, graph))
+ return false;
+ gs.spewPass("Prune Unused Branches");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Prune Unused Branches"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_FoldTests);
+ if (!FoldTests(graph))
+ return false;
+ gs.spewPass("Fold Tests");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Fold Tests"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_SplitCriticalEdges);
+ if (!SplitCriticalEdges(graph))
+ return false;
+ gs.spewPass("Split Critical Edges");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Split Critical Edges"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_RenumberBlocks);
+ RenumberBlocks(graph);
+ gs.spewPass("Renumber Blocks");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Renumber Blocks"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_DominatorTree);
+ if (!BuildDominatorTree(graph))
+ return false;
+ // No spew: graph not changed.
+
+ if (mir->shouldCancel("Dominator Tree"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_PhiAnalysis);
+ // Aggressive phi elimination must occur before any code elimination. If the
+ // script contains a try-statement, we only compiled the try block and not
+ // the catch or finally blocks, so in this case it's also invalid to use
+ // aggressive phi elimination.
+ Observability observability = graph.hasTryBlock()
+ ? ConservativeObservability
+ : AggressiveObservability;
+ if (!EliminatePhis(mir, graph, observability))
+ return false;
+ gs.spewPass("Eliminate phis");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Eliminate phis"))
+ return false;
+
+ if (!BuildPhiReverseMapping(graph))
+ return false;
+ AssertExtendedGraphCoherency(graph);
+ // No spew: graph not changed.
+
+ if (mir->shouldCancel("Phi reverse mapping"))
+ return false;
+ }
+
+ if (!JitOptions.disableRecoverIns && mir->optimizationInfo().scalarReplacementEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_ScalarReplacement);
+ if (!ScalarReplacement(mir, graph))
+ return false;
+ gs.spewPass("Scalar Replacement");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Scalar Replacement"))
+ return false;
+ }
+
+ if (!mir->compilingWasm()) {
+ AutoTraceLog log(logger, TraceLogger_ApplyTypes);
+ if (!ApplyTypeInformation(mir, graph))
+ return false;
+ gs.spewPass("Apply types");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Apply types"))
+ return false;
+ }
+
+ if (!JitOptions.disableRecoverIns && mir->optimizationInfo().eagerSimdUnboxEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_EagerSimdUnbox);
+ if (!EagerSimdUnbox(mir, graph))
+ return false;
+ gs.spewPass("Eager Simd Unbox");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Eager Simd Unbox"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().amaEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_AlignmentMaskAnalysis);
+ AlignmentMaskAnalysis ama(graph);
+ if (!ama.analyze())
+ return false;
+ gs.spewPass("Alignment Mask Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Alignment Mask Analysis"))
+ return false;
+ }
+
+ ValueNumberer gvn(mir, graph);
+ if (!gvn.init())
+ return false;
+
+ // Alias analysis is required for LICM and GVN so that we don't move
+ // loads across stores.
+ if (mir->optimizationInfo().licmEnabled() ||
+ mir->optimizationInfo().gvnEnabled())
+ {
+ {
+ AutoTraceLog log(logger, TraceLogger_AliasAnalysis);
+ if (JitOptions.disableFlowAA) {
+ AliasAnalysis analysis(mir, graph);
+ if (!analysis.analyze())
+ return false;
+ } else {
+ FlowAliasAnalysis analysis(mir, graph);
+ if (!analysis.analyze())
+ return false;
+ }
+
+ gs.spewPass("Alias analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Alias analysis"))
+ return false;
+ }
+
+ if (!mir->compilingWasm()) {
+ // Eliminating dead resume point operands requires basic block
+ // instructions to be numbered. Reuse the numbering computed during
+ // alias analysis.
+ if (!EliminateDeadResumePointOperands(mir, graph))
+ return false;
+
+ if (mir->shouldCancel("Eliminate dead resume point operands"))
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().gvnEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_GVN);
+ if (!gvn.run(ValueNumberer::UpdateAliasAnalysis))
+ return false;
+ gs.spewPass("GVN");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("GVN"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().licmEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_LICM);
+ // LICM can hoist instructions from conditional branches and trigger
+ // repeated bailouts. Disable it if this script is known to bailout
+ // frequently.
+ JSScript* script = mir->info().script();
+ if (!script || !script->hadFrequentBailouts()) {
+ if (!LICM(mir, graph))
+ return false;
+ gs.spewPass("LICM");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("LICM"))
+ return false;
+ }
+ }
+
+ RangeAnalysis r(mir, graph);
+ if (mir->optimizationInfo().rangeAnalysisEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_RangeAnalysis);
+ if (!r.addBetaNodes())
+ return false;
+ gs.spewPass("Beta");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA Beta"))
+ return false;
+
+ if (!r.analyze() || !r.addRangeAssertions())
+ return false;
+ gs.spewPass("Range Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Range Analysis"))
+ return false;
+
+ if (!r.removeBetaNodes())
+ return false;
+ gs.spewPass("De-Beta");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA De-Beta"))
+ return false;
+
+ if (mir->optimizationInfo().gvnEnabled()) {
+ bool shouldRunUCE = false;
+ if (!r.prepareForUCE(&shouldRunUCE))
+ return false;
+ gs.spewPass("RA check UCE");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA check UCE"))
+ return false;
+
+ if (shouldRunUCE) {
+ if (!gvn.run(ValueNumberer::DontUpdateAliasAnalysis))
+ return false;
+ gs.spewPass("UCE After RA");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("UCE After RA"))
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().autoTruncateEnabled()) {
+ if (!r.truncate())
+ return false;
+ gs.spewPass("Truncate Doubles");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Truncate Doubles"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().loopUnrollingEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_LoopUnrolling);
+
+ if (!UnrollLoops(graph, r.loopIterationBounds))
+ return false;
+
+ gs.spewPass("Unroll Loops");
+ AssertExtendedGraphCoherency(graph);
+ }
+ }
+
+ if (!JitOptions.disableRecoverIns) {
+ AutoTraceLog log(logger, TraceLogger_Sink);
+ if (!Sink(mir, graph))
+ return false;
+ gs.spewPass("Sink");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Sink"))
+ return false;
+ }
+
+ if (!JitOptions.disableRecoverIns && mir->optimizationInfo().rangeAnalysisEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_RemoveUnnecessaryBitops);
+ if (!r.removeUnnecessaryBitops())
+ return false;
+ gs.spewPass("Remove Unnecessary Bitops");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Remove Unnecessary Bitops"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_FoldLinearArithConstants);
+ if (!FoldLinearArithConstants(mir, graph))
+ return false;
+ gs.spewPass("Fold Linear Arithmetic Constants");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Fold Linear Arithmetic Constants"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().eaaEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_EffectiveAddressAnalysis);
+ EffectiveAddressAnalysis eaa(mir, graph);
+ if (!eaa.analyze())
+ return false;
+ gs.spewPass("Effective Address Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Effective Address Analysis"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().sincosEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_Sincos);
+ OptimizeSinCos(mir, graph);
+ gs.spewPass("Sincos optimization");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Sincos optimization"))
+ return false;
+ }
+
+ {
+ AutoTraceLog log(logger, TraceLogger_EliminateDeadCode);
+ if (!EliminateDeadCode(mir, graph))
+ return false;
+ gs.spewPass("DCE");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("DCE"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().instructionReorderingEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_ReorderInstructions);
+ if (!ReorderInstructions(mir, graph))
+ return false;
+ gs.spewPass("Reordering");
+
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Reordering"))
+ return false;
+ }
+
+ // Make loops contiguous. We do this after GVN/UCE and range analysis,
+ // which can remove CFG edges, exposing more blocks that can be moved.
+ {
+ AutoTraceLog log(logger, TraceLogger_MakeLoopsContiguous);
+ if (!MakeLoopsContiguous(graph))
+ return false;
+ gs.spewPass("Make loops contiguous");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Make loops contiguous"))
+ return false;
+ }
+
+ // Passes after this point must not move instructions; these analyses
+ // depend on knowing the final order in which instructions will execute.
+
+ if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_EdgeCaseAnalysis);
+ EdgeCaseAnalysis edgeCaseAnalysis(mir, graph);
+ if (!edgeCaseAnalysis.analyzeLate())
+ return false;
+ gs.spewPass("Edge Case Analysis (Late)");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Edge Case Analysis (Late)"))
+ return false;
+ }
+
+ if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) {
+ AutoTraceLog log(logger, TraceLogger_EliminateRedundantChecks);
+ // Note: check elimination has to run after all other passes that move
+ // instructions. Since check uses are replaced with the actual index,
+ // code motion after this pass could incorrectly move a load or store
+ // before its bounds check.
+ if (!EliminateRedundantChecks(graph))
+ return false;
+ gs.spewPass("Bounds Check Elimination");
+ AssertGraphCoherency(graph);
+ }
+
+ if (!mir->compilingWasm()) {
+ AutoTraceLog log(logger, TraceLogger_AddKeepAliveInstructions);
+ if (!AddKeepAliveInstructions(graph))
+ return false;
+ gs.spewPass("Add KeepAlive Instructions");
+ AssertGraphCoherency(graph);
+ }
+
+ if (mir->compilingWasm()) {
+ if (!EliminateBoundsChecks(mir, graph))
+ return false;
+ gs.spewPass("Redundant Bounds Check Elimination");
+ AssertGraphCoherency(graph);
+ }
+
+ DumpMIRExpressions(graph);
+
+ return true;
+}
+
+LIRGraph*
+GenerateLIR(MIRGenerator* mir)
+{
+ MIRGraph& graph = mir->graph();
+ GraphSpewer& gs = mir->graphSpewer();
+
+ TraceLoggerThread* logger;
+ if (GetJitContext()->onMainThread())
+ logger = TraceLoggerForMainThread(GetJitContext()->runtime);
+ else
+ logger = TraceLoggerForCurrentThread();
+
+ LIRGraph* lir = mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph);
+ if (!lir || !lir->init())
+ return nullptr;
+
+ LIRGenerator lirgen(mir, graph, *lir);
+ {
+ AutoTraceLog log(logger, TraceLogger_GenerateLIR);
+ if (!lirgen.generate())
+ return nullptr;
+ gs.spewPass("Generate LIR");
+
+ if (mir->shouldCancel("Generate LIR"))
+ return nullptr;
+ }
+
+ AllocationIntegrityState integrity(*lir);
+
+ {
+ AutoTraceLog log(logger, TraceLogger_RegisterAllocation);
+
+ IonRegisterAllocator allocator = mir->optimizationInfo().registerAllocator();
+
+ switch (allocator) {
+ case RegisterAllocator_Backtracking:
+ case RegisterAllocator_Testbed: {
+#ifdef DEBUG
+ if (!integrity.record())
+ return nullptr;
+#endif
+
+ BacktrackingAllocator regalloc(mir, &lirgen, *lir,
+ allocator == RegisterAllocator_Testbed);
+ if (!regalloc.go())
+ return nullptr;
+
+#ifdef DEBUG
+ if (!integrity.check(false))
+ return nullptr;
+#endif
+
+ gs.spewPass("Allocate Registers [Backtracking]");
+ break;
+ }
+
+ case RegisterAllocator_Stupid: {
+ // Use the integrity checker to populate safepoint information, so
+ // run it in all builds.
+ if (!integrity.record())
+ return nullptr;
+
+ StupidAllocator regalloc(mir, &lirgen, *lir);
+ if (!regalloc.go())
+ return nullptr;
+ if (!integrity.check(true))
+ return nullptr;
+ gs.spewPass("Allocate Registers [Stupid]");
+ break;
+ }
+
+ default:
+ MOZ_CRASH("Bad regalloc");
+ }
+
+ if (mir->shouldCancel("Allocate Registers"))
+ return nullptr;
+ }
+
+ return lir;
+}
+
+CodeGenerator*
+GenerateCode(MIRGenerator* mir, LIRGraph* lir)
+{
+ TraceLoggerThread* logger;
+ if (GetJitContext()->onMainThread())
+ logger = TraceLoggerForMainThread(GetJitContext()->runtime);
+ else
+ logger = TraceLoggerForCurrentThread();
+ AutoTraceLog log(logger, TraceLogger_GenerateCode);
+
+ CodeGenerator* codegen = js_new<CodeGenerator>(mir, lir);
+ if (!codegen)
+ return nullptr;
+
+ if (!codegen->generate()) {
+ js_delete(codegen);
+ return nullptr;
+ }
+
+ return codegen;
+}
+
+CodeGenerator*
+CompileBackEnd(MIRGenerator* mir)
+{
+ // Everything in CompileBackEnd can potentially run on a helper thread.
+ AutoEnterIonCompilation enter(mir->safeForMinorGC());
+ AutoSpewEndFunction spewEndFunction(mir);
+
+ if (!OptimizeMIR(mir))
+ return nullptr;
+
+ LIRGraph* lir = GenerateLIR(mir);
+ if (!lir)
+ return nullptr;
+
+ return GenerateCode(mir, lir);
+}
+
+// Find a finished builder for the compartment.
+static IonBuilder*
+GetFinishedBuilder(JSContext* cx, GlobalHelperThreadState::IonBuilderVector& finished)
+{
+ for (size_t i = 0; i < finished.length(); i++) {
+ IonBuilder* testBuilder = finished[i];
+ if (testBuilder->compartment == CompileCompartment::get(cx->compartment())) {
+ HelperThreadState().remove(finished, &i);
+ return testBuilder;
+ }
+ }
+
+ return nullptr;
+}
+
+void
+AttachFinishedCompilations(JSContext* cx)
+{
+ JitCompartment* ion = cx->compartment()->jitCompartment();
+ if (!ion)
+ return;
+
+ {
+ AutoLockHelperThreadState lock;
+
+ GlobalHelperThreadState::IonBuilderVector& finished = HelperThreadState().ionFinishedList(lock);
+
+ // Incorporate any off thread compilations for the compartment which have
+ // finished, failed or have been cancelled.
+ while (true) {
+ // Find a finished builder for the compartment.
+ IonBuilder* builder = GetFinishedBuilder(cx, finished);
+ if (!builder)
+ break;
+
+ JSScript* script = builder->script();
+ MOZ_ASSERT(script->hasBaselineScript());
+ script->baselineScript()->setPendingIonBuilder(cx->runtime(), script, builder);
+ cx->runtime()->ionLazyLinkListAdd(builder);
+
+ // Don't keep more than 100 lazy link builders.
+ // Link the oldest ones immediately.
+ while (cx->runtime()->ionLazyLinkListSize() > 100) {
+ jit::IonBuilder* builder = cx->runtime()->ionLazyLinkList().getLast();
+ RootedScript script(cx, builder->script());
+
+ AutoUnlockHelperThreadState unlock(lock);
+ AutoCompartment ac(cx, script->compartment());
+ jit::LinkIonScript(cx, script);
+ }
+
+ continue;
+ }
+ }
+}
+
+static void
+TrackAllProperties(JSContext* cx, JSObject* obj)
+{
+ MOZ_ASSERT(obj->isSingleton());
+
+ for (Shape::Range<NoGC> range(obj->as<NativeObject>().lastProperty()); !range.empty(); range.popFront())
+ EnsureTrackPropertyTypes(cx, obj, range.front().propid());
+}
+
+static void
+TrackPropertiesForSingletonScopes(JSContext* cx, JSScript* script, BaselineFrame* baselineFrame)
+{
+ // Ensure that all properties of singleton call objects which the script
+ // could access are tracked. These are generally accessed through
+ // ALIASEDVAR operations in baseline and will not be tracked even if they
+ // have been accessed in baseline code.
+ JSObject* environment = script->functionNonDelazifying()
+ ? script->functionNonDelazifying()->environment()
+ : nullptr;
+
+ while (environment && !environment->is<GlobalObject>()) {
+ if (environment->is<CallObject>() && environment->isSingleton())
+ TrackAllProperties(cx, environment);
+ environment = environment->enclosingEnvironment();
+ }
+
+ if (baselineFrame) {
+ JSObject* scope = baselineFrame->environmentChain();
+ if (scope->is<CallObject>() && scope->isSingleton())
+ TrackAllProperties(cx, scope);
+ }
+}
+
+static void
+TrackIonAbort(JSContext* cx, JSScript* script, jsbytecode* pc, const char* message)
+{
+ if (!cx->runtime()->jitRuntime()->isOptimizationTrackingEnabled(cx->runtime()))
+ return;
+
+ // Only bother tracking aborts of functions we're attempting to
+ // Ion-compile after successfully running in Baseline.
+ if (!script->hasBaselineScript())
+ return;
+
+ JitcodeGlobalTable* table = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ void* ptr = script->baselineScript()->method()->raw();
+ JitcodeGlobalEntry& entry = table->lookupInfallible(ptr);
+ entry.baselineEntry().trackIonAbort(pc, message);
+}
+
+static void
+TrackAndSpewIonAbort(JSContext* cx, JSScript* script, const char* message)
+{
+ JitSpew(JitSpew_IonAbort, "%s", message);
+ TrackIonAbort(cx, script, script->code(), message);
+}
+
+static AbortReason
+IonCompile(JSContext* cx, JSScript* script,
+ BaselineFrame* baselineFrame, jsbytecode* osrPc,
+ bool recompile, OptimizationLevel optimizationLevel)
+{
+ TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
+ TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script);
+ AutoTraceLog logScript(logger, event);
+ AutoTraceLog logCompile(logger, TraceLogger_IonCompilation);
+
+ // Make sure the script's canonical function isn't lazy. We can't de-lazify
+ // it in a helper thread.
+ script->ensureNonLazyCanonicalFunction(cx);
+
+ TrackPropertiesForSingletonScopes(cx, script, baselineFrame);
+
+ LifoAlloc* alloc = cx->new_<LifoAlloc>(TempAllocator::PreferredLifoChunkSize);
+ if (!alloc)
+ return AbortReason_Alloc;
+
+ ScopedJSDeletePtr<LifoAlloc> autoDelete(alloc);
+
+ TempAllocator* temp = alloc->new_<TempAllocator>(alloc);
+ if (!temp)
+ return AbortReason_Alloc;
+
+ JitContext jctx(cx, temp);
+
+ if (!cx->compartment()->ensureJitCompartmentExists(cx))
+ return AbortReason_Alloc;
+
+ if (!cx->compartment()->jitCompartment()->ensureIonStubsExist(cx))
+ return AbortReason_Alloc;
+
+ MIRGraph* graph = alloc->new_<MIRGraph>(temp);
+ if (!graph)
+ return AbortReason_Alloc;
+
+ InlineScriptTree* inlineScriptTree = InlineScriptTree::New(temp, nullptr, nullptr, script);
+ if (!inlineScriptTree)
+ return AbortReason_Alloc;
+
+ CompileInfo* info = alloc->new_<CompileInfo>(script, script->functionNonDelazifying(), osrPc,
+ Analysis_None,
+ script->needsArgsObj(), inlineScriptTree);
+ if (!info)
+ return AbortReason_Alloc;
+
+ BaselineInspector* inspector = alloc->new_<BaselineInspector>(script);
+ if (!inspector)
+ return AbortReason_Alloc;
+
+ BaselineFrameInspector* baselineFrameInspector = nullptr;
+ if (baselineFrame) {
+ baselineFrameInspector = NewBaselineFrameInspector(temp, baselineFrame, info);
+ if (!baselineFrameInspector)
+ return AbortReason_Alloc;
+ }
+
+ CompilerConstraintList* constraints = NewCompilerConstraintList(*temp);
+ if (!constraints)
+ return AbortReason_Alloc;
+
+ const OptimizationInfo* optimizationInfo = IonOptimizations.get(optimizationLevel);
+ const JitCompileOptions options(cx);
+
+ IonBuilder* builder = alloc->new_<IonBuilder>((JSContext*) nullptr,
+ CompileCompartment::get(cx->compartment()),
+ options, temp, graph, constraints,
+ inspector, info, optimizationInfo,
+ baselineFrameInspector);
+ if (!builder)
+ return AbortReason_Alloc;
+
+ if (cx->runtime()->gc.storeBuffer.cancelIonCompilations())
+ builder->setNotSafeForMinorGC();
+
+ MOZ_ASSERT(recompile == builder->script()->hasIonScript());
+ MOZ_ASSERT(builder->script()->canIonCompile());
+
+ RootedScript builderScript(cx, builder->script());
+
+ if (recompile)
+ builderScript->ionScript()->setRecompiling();
+
+ SpewBeginFunction(builder, builderScript);
+
+ bool succeeded;
+ {
+ AutoEnterAnalysis enter(cx);
+ succeeded = builder->build();
+ builder->clearForBackEnd();
+ }
+
+ if (!succeeded) {
+ AbortReason reason = builder->abortReason();
+ builder->graphSpewer().endFunction();
+ if (reason == AbortReason_PreliminaryObjects) {
+ // Some group was accessed which has associated preliminary objects
+ // to analyze. Do this now and we will try to build again shortly.
+ const MIRGenerator::ObjectGroupVector& groups = builder->abortedPreliminaryGroups();
+ for (size_t i = 0; i < groups.length(); i++) {
+ ObjectGroup* group = groups[i];
+ if (group->newScript()) {
+ if (!group->newScript()->maybeAnalyze(cx, group, nullptr, /* force = */ true))
+ return AbortReason_Alloc;
+ } else if (group->maybePreliminaryObjects()) {
+ group->maybePreliminaryObjects()->maybeAnalyze(cx, group, /* force = */ true);
+ } else {
+ MOZ_CRASH("Unexpected aborted preliminary group");
+ }
+ }
+ }
+
+ if (builder->hadActionableAbort()) {
+ JSScript* abortScript;
+ jsbytecode* abortPc;
+ const char* abortMessage;
+ builder->actionableAbortLocationAndMessage(&abortScript, &abortPc, &abortMessage);
+ TrackIonAbort(cx, abortScript, abortPc, abortMessage);
+ }
+
+ if (cx->isThrowingOverRecursed()) {
+ // Non-analysis compilations should never fail with stack overflow.
+ MOZ_CRASH("Stack overflow during compilation");
+ }
+
+ return reason;
+ }
+
+ // If possible, compile the script off thread.
+ if (options.offThreadCompilationAvailable()) {
+ JitSpew(JitSpew_IonSyncLogs, "Can't log script %s:%" PRIuSIZE
+ ". (Compiled on background thread.)",
+ builderScript->filename(), builderScript->lineno());
+
+ if (!CreateMIRRootList(*builder))
+ return AbortReason_Alloc;
+
+ if (!StartOffThreadIonCompile(cx, builder)) {
+ JitSpew(JitSpew_IonAbort, "Unable to start off-thread ion compilation.");
+ builder->graphSpewer().endFunction();
+ return AbortReason_Alloc;
+ }
+
+ if (!recompile)
+ builderScript->setIonScript(cx->runtime(), ION_COMPILING_SCRIPT);
+
+ // The allocator and associated data will be destroyed after being
+ // processed in the finishedOffThreadCompilations list.
+ autoDelete.forget();
+
+ return AbortReason_NoAbort;
+ }
+
+ {
+ ScopedJSDeletePtr<CodeGenerator> codegen;
+ AutoEnterAnalysis enter(cx);
+ codegen = CompileBackEnd(builder);
+ if (!codegen) {
+ JitSpew(JitSpew_IonAbort, "Failed during back-end compilation.");
+ if (cx->isExceptionPending())
+ return AbortReason_Error;
+ return AbortReason_Disable;
+ }
+
+ succeeded = LinkCodeGen(cx, builder, codegen);
+ }
+
+ if (succeeded)
+ return AbortReason_NoAbort;
+ if (cx->isExceptionPending())
+ return AbortReason_Error;
+ return AbortReason_Disable;
+}
+
+static bool
+CheckFrame(JSContext* cx, BaselineFrame* frame)
+{
+ MOZ_ASSERT(!frame->script()->isGenerator());
+ MOZ_ASSERT(!frame->isDebuggerEvalFrame());
+ MOZ_ASSERT(!frame->isEvalFrame());
+
+ // This check is to not overrun the stack.
+ if (frame->isFunctionFrame()) {
+ if (TooManyActualArguments(frame->numActualArgs())) {
+ TrackAndSpewIonAbort(cx, frame->script(), "too many actual arguments");
+ return false;
+ }
+
+ if (TooManyFormalArguments(frame->numFormalArgs())) {
+ TrackAndSpewIonAbort(cx, frame->script(), "too many arguments");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool
+CheckScript(JSContext* cx, JSScript* script, bool osr)
+{
+ if (script->isForEval()) {
+ // Eval frames are not yet supported. Supporting this will require new
+ // logic in pushBailoutFrame to deal with linking prev.
+ // Additionally, JSOP_DEFVAR support will require baking in isEvalFrame().
+ TrackAndSpewIonAbort(cx, script, "eval script");
+ return false;
+ }
+
+ if (script->isGenerator()) {
+ TrackAndSpewIonAbort(cx, script, "generator script");
+ return false;
+ }
+
+ if (script->hasNonSyntacticScope() && !script->functionNonDelazifying()) {
+ // Support functions with a non-syntactic global scope but not other
+ // scripts. For global scripts, IonBuilder currently uses the global
+ // object as scope chain, this is not valid when the script has a
+ // non-syntactic global scope.
+ TrackAndSpewIonAbort(cx, script, "has non-syntactic global scope");
+ return false;
+ }
+
+ if (script->functionHasExtraBodyVarScope() &&
+ script->functionExtraBodyVarScope()->hasEnvironment())
+ {
+ // This restriction will be lifted when intra-function scope chains
+ // are compilable by Ion. See bug 1273858.
+ TrackAndSpewIonAbort(cx, script, "has extra var environment");
+ return false;
+ }
+
+ if (script->nTypeSets() >= UINT16_MAX) {
+ // In this case multiple bytecode ops can share a single observed
+ // TypeSet (see bug 1303710).
+ TrackAndSpewIonAbort(cx, script, "too many typesets");
+ return false;
+ }
+
+ return true;
+}
+
+static MethodStatus
+CheckScriptSize(JSContext* cx, JSScript* script)
+{
+ if (!JitOptions.limitScriptSize)
+ return Method_Compiled;
+
+ uint32_t numLocalsAndArgs = NumLocalsAndArgs(script);
+
+ if (script->length() > MAX_MAIN_THREAD_SCRIPT_SIZE ||
+ numLocalsAndArgs > MAX_MAIN_THREAD_LOCALS_AND_ARGS)
+ {
+ if (!OffThreadCompilationAvailable(cx)) {
+ JitSpew(JitSpew_IonAbort, "Script too large (%" PRIuSIZE " bytes) (%u locals/args)",
+ script->length(), numLocalsAndArgs);
+ TrackIonAbort(cx, script, script->code(), "too large");
+ return Method_CantCompile;
+ }
+ }
+
+ return Method_Compiled;
+}
+
+bool
+CanIonCompileScript(JSContext* cx, JSScript* script, bool osr)
+{
+ if (!script->canIonCompile() || !CheckScript(cx, script, osr))
+ return false;
+
+ return CheckScriptSize(cx, script) == Method_Compiled;
+}
+
+static OptimizationLevel
+GetOptimizationLevel(HandleScript script, jsbytecode* pc)
+{
+ return IonOptimizations.levelForScript(script, pc);
+}
+
+static MethodStatus
+Compile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc,
+ bool forceRecompile = false)
+{
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(jit::IsBaselineEnabled(cx));
+ MOZ_ASSERT_IF(osrPc != nullptr, LoopEntryCanIonOsr(osrPc));
+
+ if (!script->hasBaselineScript())
+ return Method_Skipped;
+
+ if (script->isDebuggee() || (osrFrame && osrFrame->isDebuggee())) {
+ TrackAndSpewIonAbort(cx, script, "debugging");
+ return Method_Skipped;
+ }
+
+ if (!CheckScript(cx, script, bool(osrPc))) {
+ JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno());
+ return Method_CantCompile;
+ }
+
+ MethodStatus status = CheckScriptSize(cx, script);
+ if (status != Method_Compiled) {
+ JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno());
+ return status;
+ }
+
+ bool recompile = false;
+ OptimizationLevel optimizationLevel = GetOptimizationLevel(script, osrPc);
+ if (optimizationLevel == OptimizationLevel::DontCompile)
+ return Method_Skipped;
+
+ if (!CanLikelyAllocateMoreExecutableMemory()) {
+ script->resetWarmUpCounter();
+ return Method_Skipped;
+ }
+
+ if (script->hasIonScript()) {
+ IonScript* scriptIon = script->ionScript();
+ if (!scriptIon->method())
+ return Method_CantCompile;
+
+ // Don't recompile/overwrite higher optimized code,
+ // with a lower optimization level.
+ if (optimizationLevel <= scriptIon->optimizationLevel() && !forceRecompile)
+ return Method_Compiled;
+
+ // Don't start compiling if already compiling
+ if (scriptIon->isRecompiling())
+ return Method_Compiled;
+
+ if (osrPc)
+ scriptIon->resetOsrPcMismatchCounter();
+
+ recompile = true;
+ }
+
+ if (script->baselineScript()->hasPendingIonBuilder()) {
+ IonBuilder* buildIon = script->baselineScript()->pendingIonBuilder();
+ if (optimizationLevel <= buildIon->optimizationInfo().level() && !forceRecompile)
+ return Method_Compiled;
+
+ recompile = true;
+ }
+
+ AbortReason reason = IonCompile(cx, script, osrFrame, osrPc, recompile, optimizationLevel);
+ if (reason == AbortReason_Error)
+ return Method_Error;
+
+ if (reason == AbortReason_Disable)
+ return Method_CantCompile;
+
+ if (reason == AbortReason_Alloc) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+
+ // Compilation succeeded or we invalidated right away or an inlining/alloc abort
+ if (script->hasIonScript())
+ return Method_Compiled;
+ return Method_Skipped;
+}
+
+} // namespace jit
+} // namespace js
+
+bool
+jit::OffThreadCompilationAvailable(JSContext* cx)
+{
+ // Even if off thread compilation is enabled, compilation must still occur
+ // on the main thread in some cases.
+ //
+ // Require cpuCount > 1 so that Ion compilation jobs and main-thread
+ // execution are not competing for the same resources.
+ return cx->runtime()->canUseOffthreadIonCompilation()
+ && HelperThreadState().cpuCount > 1
+ && CanUseExtraThreads();
+}
+
+MethodStatus
+jit::CanEnter(JSContext* cx, RunState& state)
+{
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+
+ JSScript* script = state.script();
+
+ // Skip if the script has been disabled.
+ if (!script->canIonCompile())
+ return Method_Skipped;
+
+ // Skip if the script is being compiled off thread.
+ if (script->isIonCompilingOffThread())
+ return Method_Skipped;
+
+ // Skip if the code is expected to result in a bailout.
+ if (script->hasIonScript() && script->ionScript()->bailoutExpected())
+ return Method_Skipped;
+
+ RootedScript rscript(cx, script);
+
+ // If constructing, allocate a new |this| object before building Ion.
+ // Creating |this| is done before building Ion because it may change the
+ // type information and invalidate compilation results.
+ if (state.isInvoke()) {
+ InvokeState& invoke = *state.asInvoke();
+
+ if (TooManyActualArguments(invoke.args().length())) {
+ TrackAndSpewIonAbort(cx, script, "too many actual args");
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ if (TooManyFormalArguments(invoke.args().callee().as<JSFunction>().nargs())) {
+ TrackAndSpewIonAbort(cx, script, "too many args");
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ if (!state.maybeCreateThisForConstructor(cx)) {
+ if (cx->isThrowingOutOfMemory()) {
+ cx->recoverFromOutOfMemory();
+ return Method_Skipped;
+ }
+ return Method_Error;
+ }
+ }
+
+ // If --ion-eager is used, compile with Baseline first, so that we
+ // can directly enter IonMonkey.
+ if (JitOptions.eagerCompilation && !rscript->hasBaselineScript()) {
+ MethodStatus status = CanEnterBaselineMethod(cx, state);
+ if (status != Method_Compiled)
+ return status;
+ }
+
+ // Skip if the script is being compiled off thread or can't be
+ // Ion-compiled (again). MaybeCreateThisForConstructor could have
+ // started an Ion compilation or marked the script as uncompilable.
+ if (rscript->isIonCompilingOffThread() || !rscript->canIonCompile())
+ return Method_Skipped;
+
+ // Attempt compilation. Returns Method_Compiled if already compiled.
+ MethodStatus status = Compile(cx, rscript, nullptr, nullptr);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile)
+ ForbidCompilation(cx, rscript);
+ return status;
+ }
+
+ if (state.script()->baselineScript()->hasPendingIonBuilder()) {
+ LinkIonScript(cx, state.script());
+ if (!state.script()->hasIonScript())
+ return jit::Method_Skipped;
+ }
+
+ return Method_Compiled;
+}
+
+static MethodStatus
+BaselineCanEnterAtEntry(JSContext* cx, HandleScript script, BaselineFrame* frame)
+{
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(frame->callee()->nonLazyScript()->canIonCompile());
+ MOZ_ASSERT(!frame->callee()->nonLazyScript()->isIonCompilingOffThread());
+ MOZ_ASSERT(!frame->callee()->nonLazyScript()->hasIonScript());
+ MOZ_ASSERT(frame->isFunctionFrame());
+
+ // Mark as forbidden if frame can't be handled.
+ if (!CheckFrame(cx, frame)) {
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ // Attempt compilation. Returns Method_Compiled if already compiled.
+ MethodStatus status = Compile(cx, script, frame, nullptr);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile)
+ ForbidCompilation(cx, script);
+ return status;
+ }
+
+ return Method_Compiled;
+}
+
+// Decide if a transition from baseline execution to Ion code should occur.
+// May compile or recompile the target JSScript.
+static MethodStatus
+BaselineCanEnterAtBranch(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* pc)
+{
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT((JSOp)*pc == JSOP_LOOPENTRY);
+ MOZ_ASSERT(LoopEntryCanIonOsr(pc));
+
+ // Skip if the script has been disabled.
+ if (!script->canIonCompile())
+ return Method_Skipped;
+
+ // Skip if the script is being compiled off thread.
+ if (script->isIonCompilingOffThread())
+ return Method_Skipped;
+
+ // Skip if the code is expected to result in a bailout.
+ if (script->hasIonScript() && script->ionScript()->bailoutExpected())
+ return Method_Skipped;
+
+ // Optionally ignore on user request.
+ if (!JitOptions.osr)
+ return Method_Skipped;
+
+ // Mark as forbidden if frame can't be handled.
+ if (!CheckFrame(cx, osrFrame)) {
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ // Check if the jitcode still needs to get linked and do this
+ // to have a valid IonScript.
+ if (script->baselineScript()->hasPendingIonBuilder())
+ LinkIonScript(cx, script);
+
+ // By default a recompilation doesn't happen on osr mismatch.
+ // Decide if we want to force a recompilation if this happens too much.
+ bool force = false;
+ if (script->hasIonScript() && pc != script->ionScript()->osrPc()) {
+ uint32_t count = script->ionScript()->incrOsrPcMismatchCounter();
+ if (count <= JitOptions.osrPcMismatchesBeforeRecompile)
+ return Method_Skipped;
+ force = true;
+ }
+
+ // Attempt compilation.
+ // - Returns Method_Compiled if the right ionscript is present
+ // (Meaning it was present or a sequantial compile finished)
+ // - Returns Method_Skipped if pc doesn't match
+ // (This means a background thread compilation with that pc could have started or not.)
+ RootedScript rscript(cx, script);
+ MethodStatus status = Compile(cx, rscript, osrFrame, pc, force);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile)
+ ForbidCompilation(cx, script);
+ return status;
+ }
+
+ // Return the compilation was skipped when the osr pc wasn't adjusted.
+ // This can happen when there was still an IonScript available and a
+ // background compilation started, but hasn't finished yet.
+ // Or when we didn't force a recompile.
+ if (script->hasIonScript() && pc != script->ionScript()->osrPc())
+ return Method_Skipped;
+
+ return Method_Compiled;
+}
+
+bool
+jit::IonCompileScriptForBaseline(JSContext* cx, BaselineFrame* frame, jsbytecode* pc)
+{
+ // A TI OOM will disable TI and Ion.
+ if (!jit::IsIonEnabled(cx))
+ return true;
+
+ RootedScript script(cx, frame->script());
+ bool isLoopEntry = JSOp(*pc) == JSOP_LOOPENTRY;
+
+ MOZ_ASSERT(!isLoopEntry || LoopEntryCanIonOsr(pc));
+
+ if (!script->canIonCompile()) {
+ // TODO: ASSERT that ion-compilation-disabled checker stub doesn't exist.
+ // TODO: Clear all optimized stubs.
+ // TODO: Add a ion-compilation-disabled checker IC stub
+ script->resetWarmUpCounter();
+ return true;
+ }
+
+ MOZ_ASSERT(!script->isIonCompilingOffThread());
+
+ // If Ion script exists, but PC is not at a loop entry, then Ion will be entered for
+ // this script at an appropriate LOOPENTRY or the next time this function is called.
+ if (script->hasIonScript() && !isLoopEntry) {
+ JitSpew(JitSpew_BaselineOSR, "IonScript exists, but not at loop entry!");
+ // TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist.
+ // TODO: Clear all optimized stubs.
+ // TODO: Add a ion-script-already-exists checker stub.
+ return true;
+ }
+
+ // Ensure that Ion-compiled code is available.
+ JitSpew(JitSpew_BaselineOSR,
+ "WarmUpCounter for %s:%" PRIuSIZE " reached %d at pc %p, trying to switch to Ion!",
+ script->filename(), script->lineno(), (int) script->getWarmUpCount(), (void*) pc);
+
+ MethodStatus stat;
+ if (isLoopEntry) {
+ MOZ_ASSERT(LoopEntryCanIonOsr(pc));
+ JitSpew(JitSpew_BaselineOSR, " Compile at loop entry!");
+ stat = BaselineCanEnterAtBranch(cx, script, frame, pc);
+ } else if (frame->isFunctionFrame()) {
+ JitSpew(JitSpew_BaselineOSR, " Compile function from top for later entry!");
+ stat = BaselineCanEnterAtEntry(cx, script, frame);
+ } else {
+ return true;
+ }
+
+ if (stat == Method_Error) {
+ JitSpew(JitSpew_BaselineOSR, " Compile with Ion errored!");
+ return false;
+ }
+
+ if (stat == Method_CantCompile)
+ JitSpew(JitSpew_BaselineOSR, " Can't compile with Ion!");
+ else if (stat == Method_Skipped)
+ JitSpew(JitSpew_BaselineOSR, " Skipped compile with Ion!");
+ else if (stat == Method_Compiled)
+ JitSpew(JitSpew_BaselineOSR, " Compiled with Ion!");
+ else
+ MOZ_CRASH("Invalid MethodStatus!");
+
+ // Failed to compile. Reset warm-up counter and return.
+ if (stat != Method_Compiled) {
+ // TODO: If stat == Method_CantCompile, insert stub that just skips the
+ // warm-up counter entirely, instead of resetting it.
+ bool bailoutExpected = script->hasIonScript() && script->ionScript()->bailoutExpected();
+ if (stat == Method_CantCompile || bailoutExpected) {
+ JitSpew(JitSpew_BaselineOSR, " Reset WarmUpCounter cantCompile=%s bailoutExpected=%s!",
+ stat == Method_CantCompile ? "yes" : "no",
+ bailoutExpected ? "yes" : "no");
+ script->resetWarmUpCounter();
+ }
+ return true;
+ }
+
+ return true;
+}
+
+
+MethodStatus
+jit::Recompile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc,
+ bool force)
+{
+ MOZ_ASSERT(script->hasIonScript());
+ if (script->ionScript()->isRecompiling())
+ return Method_Compiled;
+
+ MethodStatus status = Compile(cx, script, osrFrame, osrPc, force);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile)
+ ForbidCompilation(cx, script);
+ return status;
+ }
+
+ return Method_Compiled;
+}
+
+MethodStatus
+jit::CanEnterUsingFastInvoke(JSContext* cx, HandleScript script, uint32_t numActualArgs)
+{
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+
+ // Skip if the code is expected to result in a bailout.
+ if (!script->hasIonScript() || script->ionScript()->bailoutExpected())
+ return Method_Skipped;
+
+ // Don't handle arguments underflow, to make this work we would have to pad
+ // missing arguments with |undefined|.
+ if (numActualArgs < script->functionNonDelazifying()->nargs())
+ return Method_Skipped;
+
+ if (!cx->compartment()->ensureJitCompartmentExists(cx))
+ return Method_Error;
+
+ // This can GC, so afterward, script->ion is not guaranteed to be valid.
+ if (!cx->runtime()->jitRuntime()->enterIon())
+ return Method_Error;
+
+ if (!script->hasIonScript())
+ return Method_Skipped;
+
+ return Method_Compiled;
+}
+
+static JitExecStatus
+EnterIon(JSContext* cx, EnterJitData& data)
+{
+ JS_CHECK_RECURSION(cx, return JitExec_Aborted);
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(!data.osrFrame);
+
+#ifdef DEBUG
+ // See comment in EnterBaseline.
+ mozilla::Maybe<JS::AutoAssertNoGC> nogc;
+ nogc.emplace(cx);
+#endif
+
+ EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon();
+
+ // Caller must construct |this| before invoking the Ion function.
+ MOZ_ASSERT_IF(data.constructing,
+ data.maxArgv[0].isObject() || data.maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));
+
+ data.result.setInt32(data.numActualArgs);
+ {
+ AssertCompartmentUnchanged pcc(cx);
+ ActivationEntryMonitor entryMonitor(cx, data.calleeToken);
+ JitActivation activation(cx);
+
+#ifdef DEBUG
+ nogc.reset();
+#endif
+ CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv, /* osrFrame = */nullptr, data.calleeToken,
+ /* envChain = */ nullptr, 0, data.result.address());
+ }
+
+ MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride());
+
+ // Jit callers wrap primitive constructor return, except for derived class constructors.
+ if (!data.result.isMagic() && data.constructing &&
+ data.result.isPrimitive())
+ {
+ MOZ_ASSERT(data.maxArgv[0].isObject());
+ data.result = data.maxArgv[0];
+ }
+
+ // Release temporary buffer used for OSR into Ion.
+ cx->runtime()->getJitRuntime(cx)->freeOsrTempData();
+
+ MOZ_ASSERT_IF(data.result.isMagic(), data.result.isMagic(JS_ION_ERROR));
+ return data.result.isMagic() ? JitExec_Error : JitExec_Ok;
+}
+
+bool
+jit::SetEnterJitData(JSContext* cx, EnterJitData& data, RunState& state,
+ MutableHandle<GCVector<Value>> vals)
+{
+ data.osrFrame = nullptr;
+
+ if (state.isInvoke()) {
+ const CallArgs& args = state.asInvoke()->args();
+ unsigned numFormals = state.script()->functionNonDelazifying()->nargs();
+ data.constructing = state.asInvoke()->constructing();
+ data.numActualArgs = args.length();
+ data.maxArgc = Max(args.length(), numFormals) + 1;
+ data.envChain = nullptr;
+ data.calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), data.constructing);
+
+ if (data.numActualArgs >= numFormals) {
+ data.maxArgv = args.base() + 1;
+ } else {
+ MOZ_ASSERT(vals.empty());
+ unsigned numPushedArgs = Max(args.length(), numFormals);
+ if (!vals.reserve(numPushedArgs + 1 + data.constructing))
+ return false;
+
+ // Append |this| and any provided arguments.
+ for (size_t i = 1; i < args.length() + 2; ++i)
+ vals.infallibleAppend(args.base()[i]);
+
+ // Pad missing arguments with |undefined|.
+ while (vals.length() < numFormals + 1)
+ vals.infallibleAppend(UndefinedValue());
+
+ if (data.constructing)
+ vals.infallibleAppend(args.newTarget());
+
+ MOZ_ASSERT(vals.length() >= numFormals + 1 + data.constructing);
+ data.maxArgv = vals.begin();
+ }
+ } else {
+ data.constructing = false;
+ data.numActualArgs = 0;
+ data.maxArgc = 0;
+ data.maxArgv = nullptr;
+ data.envChain = state.asExecute()->environmentChain();
+
+ data.calleeToken = CalleeToToken(state.script());
+
+ if (state.script()->isForEval() && state.script()->isDirectEvalInFunction()) {
+ // Push newTarget onto the stack.
+ if (!vals.reserve(1))
+ return false;
+
+ data.maxArgc = 1;
+ data.maxArgv = vals.begin();
+ if (state.asExecute()->newTarget().isNull()) {
+ ScriptFrameIter iter(cx);
+ vals.infallibleAppend(iter.newTarget());
+ } else {
+ vals.infallibleAppend(state.asExecute()->newTarget());
+ }
+ }
+ }
+
+ return true;
+}
+
+JitExecStatus
+jit::IonCannon(JSContext* cx, RunState& state)
+{
+ IonScript* ion = state.script()->ionScript();
+
+ EnterJitData data(cx);
+ data.jitcode = ion->method()->raw();
+
+ Rooted<GCVector<Value>> vals(cx, GCVector<Value>(cx));
+ if (!SetEnterJitData(cx, data, state, &vals))
+ return JitExec_Error;
+
+ JitExecStatus status = EnterIon(cx, data);
+
+ if (status == JitExec_Ok)
+ state.setReturnValue(data.result);
+
+ return status;
+}
+
+JitExecStatus
+jit::FastInvoke(JSContext* cx, HandleFunction fun, CallArgs& args)
+{
+ JS_CHECK_RECURSION(cx, return JitExec_Error);
+
+ RootedScript script(cx, fun->nonLazyScript());
+
+ if (!Debugger::checkNoExecute(cx, script))
+ return JitExec_Error;
+
+#ifdef DEBUG
+ // See comment in EnterBaseline.
+ mozilla::Maybe<JS::AutoAssertNoGC> nogc;
+ nogc.emplace(cx);
+#endif
+
+ IonScript* ion = script->ionScript();
+ JitCode* code = ion->method();
+ void* jitcode = code->raw();
+
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(!ion->bailoutExpected());
+
+ ActivationEntryMonitor entryMonitor(cx, CalleeToToken(script));
+ JitActivation activation(cx);
+
+ EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon();
+ void* calleeToken = CalleeToToken(fun, /* constructing = */ false);
+
+ RootedValue result(cx, Int32Value(args.length()));
+ MOZ_ASSERT(args.length() >= fun->nargs());
+
+#ifdef DEBUG
+ nogc.reset();
+#endif
+ CALL_GENERATED_CODE(enter, jitcode, args.length() + 1, args.array() - 1, /* osrFrame = */nullptr,
+ calleeToken, /* envChain = */ nullptr, 0, result.address());
+
+ MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride());
+
+ args.rval().set(result);
+
+ MOZ_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR));
+ return result.isMagic() ? JitExec_Error : JitExec_Ok;
+}
+
+static void
+InvalidateActivation(FreeOp* fop, const JitActivationIterator& activations, bool invalidateAll)
+{
+ JitSpew(JitSpew_IonInvalidate, "BEGIN invalidating activation");
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (JitOptions.checkOsiPointRegisters)
+ activations->asJit()->setCheckRegs(false);
+#endif
+
+ size_t frameno = 1;
+
+ for (JitFrameIterator it(activations); !it.done(); ++it, ++frameno) {
+ MOZ_ASSERT_IF(frameno == 1, it.isExitFrame() || it.type() == JitFrame_Bailout);
+
+#ifdef JS_JITSPEW
+ switch (it.type()) {
+ case JitFrame_Exit:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " exit frame @ %p", frameno, it.fp());
+ break;
+ case JitFrame_BaselineJS:
+ case JitFrame_IonJS:
+ case JitFrame_Bailout:
+ {
+ MOZ_ASSERT(it.isScripted());
+ const char* type = "Unknown";
+ if (it.isIonJS())
+ type = "Optimized";
+ else if (it.isBaselineJS())
+ type = "Baseline";
+ else if (it.isBailoutJS())
+ type = "Bailing";
+ JitSpew(JitSpew_IonInvalidate,
+ "#%" PRIuSIZE " %s JS frame @ %p, %s:%" PRIuSIZE " (fun: %p, script: %p, pc %p)",
+ frameno, type, it.fp(), it.script()->maybeForwardedFilename(),
+ it.script()->lineno(), it.maybeCallee(), (JSScript*)it.script(),
+ it.returnAddressToFp());
+ break;
+ }
+ case JitFrame_IonStub:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion stub frame @ %p", frameno, it.fp());
+ break;
+ case JitFrame_BaselineStub:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " baseline stub frame @ %p", frameno, it.fp());
+ break;
+ case JitFrame_Rectifier:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " rectifier frame @ %p", frameno, it.fp());
+ break;
+ case JitFrame_IonAccessorIC:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion IC getter/setter frame @ %p", frameno, it.fp());
+ break;
+ case JitFrame_Entry:
+ JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " entry frame @ %p", frameno, it.fp());
+ break;
+ }
+#endif // JS_JITSPEW
+
+ if (!it.isIonScripted())
+ continue;
+
+ bool calledFromLinkStub = false;
+ JitCode* lazyLinkStub = fop->runtime()->jitRuntime()->lazyLinkStub();
+ if (it.returnAddressToFp() >= lazyLinkStub->raw() &&
+ it.returnAddressToFp() < lazyLinkStub->rawEnd())
+ {
+ calledFromLinkStub = true;
+ }
+
+ // See if the frame has already been invalidated.
+ if (!calledFromLinkStub && it.checkInvalidation())
+ continue;
+
+ JSScript* script = it.script();
+ if (!script->hasIonScript())
+ continue;
+
+ if (!invalidateAll && !script->ionScript()->invalidated())
+ continue;
+
+ IonScript* ionScript = script->ionScript();
+
+ // Purge ICs before we mark this script as invalidated. This will
+ // prevent lastJump_ from appearing to be a bogus pointer, just
+ // in case anyone tries to read it.
+ ionScript->purgeCaches();
+ ionScript->purgeOptimizedStubs(script->zone());
+
+ // Clean up any pointers from elsewhere in the runtime to this IonScript
+ // which is about to become disconnected from its JSScript.
+ ionScript->unlinkFromRuntime(fop);
+
+ // This frame needs to be invalidated. We do the following:
+ //
+ // 1. Increment the reference counter to keep the ionScript alive
+ // for the invalidation bailout or for the exception handler.
+ // 2. Determine safepoint that corresponds to the current call.
+ // 3. From safepoint, get distance to the OSI-patchable offset.
+ // 4. From the IonScript, determine the distance between the
+ // call-patchable offset and the invalidation epilogue.
+ // 5. Patch the OSI point with a call-relative to the
+ // invalidation epilogue.
+ //
+ // The code generator ensures that there's enough space for us
+ // to patch in a call-relative operation at each invalidation
+ // point.
+ //
+ // Note: you can't simplify this mechanism to "just patch the
+ // instruction immediately after the call" because things may
+ // need to move into a well-defined register state (using move
+ // instructions after the call) in to capture an appropriate
+ // snapshot after the call occurs.
+
+ ionScript->incrementInvalidationCount();
+
+ JitCode* ionCode = ionScript->method();
+
+ JS::Zone* zone = script->zone();
+ if (zone->needsIncrementalBarrier()) {
+ // We're about to remove edges from the JSScript to gcthings
+ // embedded in the JitCode. Perform one final trace of the
+ // JitCode for the incremental GC, as it must know about
+ // those edges.
+ ionCode->traceChildren(zone->barrierTracer());
+ }
+ ionCode->setInvalidated();
+
+ // Don't adjust OSI points in the linkStub (which don't exist), or in a
+ // bailout path.
+ if (calledFromLinkStub || it.isBailoutJS())
+ continue;
+
+ // Write the delta (from the return address offset to the
+ // IonScript pointer embedded into the invalidation epilogue)
+ // where the safepointed call instruction used to be. We rely on
+ // the call sequence causing the safepoint being >= the size of
+ // a uint32, which is checked during safepoint index
+ // construction.
+ AutoWritableJitCode awjc(ionCode);
+ const SafepointIndex* si = ionScript->getSafepointIndex(it.returnAddressToFp());
+ CodeLocationLabel dataLabelToMunge(it.returnAddressToFp());
+ ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() -
+ (it.returnAddressToFp() - ionCode->raw());
+ Assembler::PatchWrite_Imm32(dataLabelToMunge, Imm32(delta));
+
+ CodeLocationLabel osiPatchPoint = SafepointReader::InvalidationPatchPoint(ionScript, si);
+ CodeLocationLabel invalidateEpilogue(ionCode, CodeOffset(ionScript->invalidateEpilogueOffset()));
+
+ JitSpew(JitSpew_IonInvalidate, " ! Invalidate ionScript %p (inv count %" PRIuSIZE ") -> patching osipoint %p",
+ ionScript, ionScript->invalidationCount(), (void*) osiPatchPoint.raw());
+ Assembler::PatchWrite_NearCall(osiPatchPoint, invalidateEpilogue);
+ }
+
+ JitSpew(JitSpew_IonInvalidate, "END invalidating activation");
+}
+
+void
+jit::InvalidateAll(FreeOp* fop, Zone* zone)
+{
+ // The caller should previously have cancelled off thread compilation.
+#ifdef DEBUG
+ for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
+ MOZ_ASSERT(!HasOffThreadIonCompile(comp));
+#endif
+
+ for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) {
+ if (iter->compartment()->zone() == zone) {
+ JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC");
+ InvalidateActivation(fop, iter, true);
+ }
+ }
+}
+
+
+void
+jit::Invalidate(TypeZone& types, FreeOp* fop,
+ const RecompileInfoVector& invalid, bool resetUses,
+ bool cancelOffThread)
+{
+ JitSpew(JitSpew_IonInvalidate, "Start invalidation.");
+
+ // Add an invalidation reference to all invalidated IonScripts to indicate
+ // to the traversal which frames have been invalidated.
+ size_t numInvalidations = 0;
+ for (size_t i = 0; i < invalid.length(); i++) {
+ const CompilerOutput* co = invalid[i].compilerOutput(types);
+ if (!co)
+ continue;
+ MOZ_ASSERT(co->isValid());
+
+ if (cancelOffThread)
+ CancelOffThreadIonCompile(co->script());
+
+ if (!co->ion())
+ continue;
+
+ JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%" PRIuSIZE ", IonScript %p",
+ co->script()->filename(), co->script()->lineno(), co->ion());
+
+ // Keep the ion script alive during the invalidation and flag this
+ // ionScript as being invalidated. This increment is removed by the
+ // loop after the calls to InvalidateActivation.
+ co->ion()->incrementInvalidationCount();
+ numInvalidations++;
+ }
+
+ if (!numInvalidations) {
+ JitSpew(JitSpew_IonInvalidate, " No IonScript invalidation.");
+ return;
+ }
+
+ for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter)
+ InvalidateActivation(fop, iter, false);
+
+ // Drop the references added above. If a script was never active, its
+ // IonScript will be immediately destroyed. Otherwise, it will be held live
+ // until its last invalidated frame is destroyed.
+ for (size_t i = 0; i < invalid.length(); i++) {
+ CompilerOutput* co = invalid[i].compilerOutput(types);
+ if (!co)
+ continue;
+ MOZ_ASSERT(co->isValid());
+
+ JSScript* script = co->script();
+ IonScript* ionScript = co->ion();
+ if (!ionScript)
+ continue;
+
+ script->setIonScript(nullptr, nullptr);
+ ionScript->decrementInvalidationCount(fop);
+ co->invalidate();
+ numInvalidations--;
+
+ // Wait for the scripts to get warm again before doing another
+ // compile, unless we are recompiling *because* a script got hot
+ // (resetUses is false).
+ if (resetUses)
+ script->resetWarmUpCounter();
+ }
+
+ // Make sure we didn't leak references by invalidating the same IonScript
+ // multiple times in the above loop.
+ MOZ_ASSERT(!numInvalidations);
+}
+
+void
+jit::Invalidate(JSContext* cx, const RecompileInfoVector& invalid, bool resetUses,
+ bool cancelOffThread)
+{
+ jit::Invalidate(cx->zone()->types, cx->runtime()->defaultFreeOp(), invalid, resetUses,
+ cancelOffThread);
+}
+
+void
+jit::IonScript::invalidate(JSContext* cx, bool resetUses, const char* reason)
+{
+ JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason);
+
+ // RecompileInfoVector has inline space for at least one element.
+ RecompileInfoVector list;
+ MOZ_RELEASE_ASSERT(list.reserve(1));
+ list.infallibleAppend(recompileInfo());
+
+ Invalidate(cx, list, resetUses, true);
+}
+
+void
+jit::Invalidate(JSContext* cx, JSScript* script, bool resetUses, bool cancelOffThread)
+{
+ MOZ_ASSERT(script->hasIonScript());
+
+ if (cx->runtime()->spsProfiler.enabled()) {
+ // Register invalidation with profiler.
+ // Format of event payload string:
+ // "<filename>:<lineno>"
+
+ // Get the script filename, if any, and its length.
+ const char* filename = script->filename();
+ if (filename == nullptr)
+ filename = "<unknown>";
+
+ // Construct the descriptive string.
+ char* buf = JS_smprintf("Invalidate %s:%" PRIuSIZE, filename, script->lineno());
+
+ // Ignore the event on allocation failure.
+ if (buf) {
+ cx->runtime()->spsProfiler.markEvent(buf);
+ JS_smprintf_free(buf);
+ }
+ }
+
+ // RecompileInfoVector has inline space for at least one element.
+ RecompileInfoVector scripts;
+ MOZ_ASSERT(script->hasIonScript());
+ MOZ_RELEASE_ASSERT(scripts.reserve(1));
+ scripts.infallibleAppend(script->ionScript()->recompileInfo());
+
+ Invalidate(cx, scripts, resetUses, cancelOffThread);
+}
+
+static void
+FinishInvalidationOf(FreeOp* fop, JSScript* script, IonScript* ionScript)
+{
+ TypeZone& types = script->zone()->types;
+
+ // Note: If the script is about to be swept, the compiler output may have
+ // already been destroyed.
+ if (CompilerOutput* output = ionScript->recompileInfo().compilerOutput(types))
+ output->invalidate();
+
+ // If this script has Ion code on the stack, invalidated() will return
+ // true. In this case we have to wait until destroying it.
+ if (!ionScript->invalidated())
+ jit::IonScript::Destroy(fop, ionScript);
+}
+
+void
+jit::FinishInvalidation(FreeOp* fop, JSScript* script)
+{
+ // In all cases, nullptr out script->ion to avoid re-entry.
+ if (script->hasIonScript()) {
+ IonScript* ion = script->ionScript();
+ script->setIonScript(nullptr, nullptr);
+ FinishInvalidationOf(fop, script, ion);
+ }
+}
+
+void
+jit::ForbidCompilation(JSContext* cx, JSScript* script)
+{
+ JitSpew(JitSpew_IonAbort, "Disabling Ion compilation of script %s:%" PRIuSIZE,
+ script->filename(), script->lineno());
+
+ CancelOffThreadIonCompile(script);
+
+ if (script->hasIonScript())
+ Invalidate(cx, script, false);
+
+ script->setIonScript(cx->runtime(), ION_DISABLED_SCRIPT);
+}
+
+AutoFlushICache*
+PerThreadData::autoFlushICache() const
+{
+ return autoFlushICache_;
+}
+
+void
+PerThreadData::setAutoFlushICache(AutoFlushICache* afc)
+{
+ autoFlushICache_ = afc;
+}
+
+// Set the range for the merging of flushes. The flushing is deferred until the end of
+// the AutoFlushICache context. Subsequent flushing within this range will is also
+// deferred. This is only expected to be defined once for each AutoFlushICache
+// context. It assumes the range will be flushed is required to be within an
+// AutoFlushICache context.
+void
+AutoFlushICache::setRange(uintptr_t start, size_t len)
+{
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
+ MOZ_ASSERT(afc);
+ MOZ_ASSERT(!afc->start_);
+ JitSpewCont(JitSpew_CacheFlush, "(%" PRIxPTR " %" PRIxSIZE "):", start, len);
+
+ uintptr_t stop = start + len;
+ afc->start_ = start;
+ afc->stop_ = stop;
+#endif
+}
+
+// Flush the instruction cache.
+//
+// If called within a dynamic AutoFlushICache context and if the range is already pending
+// flushing for this AutoFlushICache context then the request is ignored with the
+// understanding that it will be flushed on exit from the AutoFlushICache context.
+// Otherwise the range is flushed immediately.
+//
+// Updates outside the current code object are typically the exception so they are flushed
+// immediately rather than attempting to merge them.
+//
+// For efficiency it is expected that all large ranges will be flushed within an
+// AutoFlushICache, so check. If this assertion is hit then it does not necessarily
+// indicate a program fault but it might indicate a lost opportunity to merge cache
+// flushing. It can be corrected by wrapping the call in an AutoFlushICache to context.
+//
+// Note this can be called without TLS PerThreadData defined so this case needs
+// to be guarded against. E.g. when patching instructions from the exception
+// handler on MacOS running the ARM simulator.
+void
+AutoFlushICache::flush(uintptr_t start, size_t len)
+{
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ PerThreadData* pt = TlsPerThreadData.get();
+ AutoFlushICache* afc = pt ? pt->PerThreadData::autoFlushICache() : nullptr;
+ if (!afc) {
+ JitSpewCont(JitSpew_CacheFlush, "#");
+ ExecutableAllocator::cacheFlush((void*)start, len);
+ MOZ_ASSERT(len <= 32);
+ return;
+ }
+
+ uintptr_t stop = start + len;
+ if (start >= afc->start_ && stop <= afc->stop_) {
+ // Update is within the pending flush range, so defer to the end of the context.
+ JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "-" : "=");
+ return;
+ }
+
+ JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "x" : "*");
+ ExecutableAllocator::cacheFlush((void*)start, len);
+#endif
+}
+
+// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths
+// where the changes are being abandoned.
+void
+AutoFlushICache::setInhibit()
+{
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
+ MOZ_ASSERT(afc);
+ MOZ_ASSERT(afc->start_);
+ JitSpewCont(JitSpew_CacheFlush, "I");
+ afc->inhibit_ = true;
+#endif
+}
+
+// The common use case is merging cache flushes when preparing a code object. In this
+// case the entire range of the code object is being flushed and as the code is patched
+// smaller redundant flushes could occur. The design allows an AutoFlushICache dynamic
+// thread local context to be declared in which the range of the code object can be set
+// which defers flushing until the end of this dynamic context. The redundant flushing
+// within this code range is also deferred avoiding redundant flushing. Flushing outside
+// this code range is not affected and proceeds immediately.
+//
+// In some cases flushing is not necessary, such as when compiling an wasm module which
+// is flushed again when dynamically linked, and also in error paths that abandon the
+// code. Flushing within the set code range can be inhibited within the AutoFlushICache
+// dynamic context by setting an inhibit flag.
+//
+// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and
+// it is assumed that code being assembled or patched is not executed before the exit of
+// the respective AutoFlushICache dynamic context.
+//
+AutoFlushICache::AutoFlushICache(const char* nonce, bool inhibit)
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ : start_(0),
+ stop_(0),
+ name_(nonce),
+ inhibit_(inhibit)
+#endif
+{
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ PerThreadData* pt = TlsPerThreadData.get();
+ AutoFlushICache* afc = pt->PerThreadData::autoFlushICache();
+ if (afc)
+ JitSpew(JitSpew_CacheFlush, "<%s,%s%s ", nonce, afc->name_, inhibit ? " I" : "");
+ else
+ JitSpewCont(JitSpew_CacheFlush, "<%s%s ", nonce, inhibit ? " I" : "");
+
+ prev_ = afc;
+ pt->PerThreadData::setAutoFlushICache(this);
+#endif
+}
+
+AutoFlushICache::~AutoFlushICache()
+{
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ PerThreadData* pt = TlsPerThreadData.get();
+ MOZ_ASSERT(pt->PerThreadData::autoFlushICache() == this);
+
+ if (!inhibit_ && start_)
+ ExecutableAllocator::cacheFlush((void*)start_, size_t(stop_ - start_));
+
+ JitSpewCont(JitSpew_CacheFlush, "%s%s>", name_, start_ ? "" : " U");
+ JitSpewFin(JitSpew_CacheFlush);
+ pt->PerThreadData::setAutoFlushICache(prev_);
+#endif
+}
+
+void
+jit::PurgeCaches(JSScript* script)
+{
+ if (script->hasIonScript())
+ script->ionScript()->purgeCaches();
+}
+
+size_t
+jit::SizeOfIonData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf)
+{
+ size_t result = 0;
+
+ if (script->hasIonScript())
+ result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf);
+
+ return result;
+}
+
+void
+jit::DestroyJitScripts(FreeOp* fop, JSScript* script)
+{
+ if (script->hasIonScript())
+ jit::IonScript::Destroy(fop, script->ionScript());
+
+ if (script->hasBaselineScript())
+ jit::BaselineScript::Destroy(fop, script->baselineScript());
+}
+
+void
+jit::TraceJitScripts(JSTracer* trc, JSScript* script)
+{
+ if (script->hasIonScript())
+ jit::IonScript::Trace(trc, script->ionScript());
+
+ if (script->hasBaselineScript())
+ jit::BaselineScript::Trace(trc, script->baselineScript());
+}
+
+bool
+jit::JitSupportsFloatingPoint()
+{
+ return js::jit::MacroAssembler::SupportsFloatingPoint();
+}
+
+bool
+jit::JitSupportsUnalignedAccesses()
+{
+ return js::jit::MacroAssembler::SupportsUnalignedAccesses();
+}
+
+bool
+jit::JitSupportsSimd()
+{
+ return js::jit::MacroAssembler::SupportsSimd();
+}
+
+bool
+jit::JitSupportsAtomics()
+{
+#if defined(JS_CODEGEN_ARM)
+ // Bug 1146902, bug 1077318: Enable Ion inlining of Atomics
+ // operations on ARM only when the CPU has byte, halfword, and
+ // doubleword load-exclusive and store-exclusive instructions,
+ // until we can add support for systems that don't have those.
+ return js::jit::HasLDSTREXBHD();
+#else
+ return true;
+#endif
+}
+
+// If you change these, please also change the comment in TempAllocator.
+/* static */ const size_t TempAllocator::BallastSize = 16 * 1024;
+/* static */ const size_t TempAllocator::PreferredLifoChunkSize = 32 * 1024;