diff options
Diffstat (limited to 'js/src/jit/arm64/Trampoline-arm64.cpp')
| -rw-r--r-- | js/src/jit/arm64/Trampoline-arm64.cpp | 1229 |
1 files changed, 1229 insertions, 0 deletions
diff --git a/js/src/jit/arm64/Trampoline-arm64.cpp b/js/src/jit/arm64/Trampoline-arm64.cpp new file mode 100644 index 000000000..547bdd927 --- /dev/null +++ b/js/src/jit/arm64/Trampoline-arm64.cpp @@ -0,0 +1,1229 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Bailouts.h" +#include "jit/JitCompartment.h" +#include "jit/JitFrames.h" +#include "jit/Linker.h" +#ifdef JS_ION_PERF +# include "jit/PerfSpewer.h" +#endif +#include "jit/arm64/SharedICHelpers-arm64.h" +#include "jit/VMFunctions.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +// All registers to save and restore. This includes the stack pointer, since we +// use the ability to reference register values on the stack by index. +static const LiveRegisterSet AllRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllMask & ~(1 << 31 | 1 << 30 | 1 << 29| 1 << 28)), + FloatRegisterSet(FloatRegisters::AllMask)); + +/* This method generates a trampoline on ARM64 for a c++ function with + * the following signature: + * bool blah(void* code, int argc, Value* argv, JSObject* scopeChain, Value* vp) + * ...using standard AArch64 calling convention + */ +JitCode* +JitRuntime::generateEnterJIT(JSContext* cx, EnterJitType type) +{ + MacroAssembler masm(cx); + + const Register reg_code = IntArgReg0; // EnterJitData::jitcode. + const Register reg_argc = IntArgReg1; // EnterJitData::maxArgc. + const Register reg_argv = IntArgReg2; // EnterJitData::maxArgv. + const Register reg_osrFrame = IntArgReg3; // EnterJitData::osrFrame. + const Register reg_callee = IntArgReg4; // EnterJitData::calleeToken. + const Register reg_scope = IntArgReg5; // EnterJitData::scopeChain. + const Register reg_osrNStack = IntArgReg6; // EnterJitData::osrNumStackValues. + const Register reg_vp = IntArgReg7; // Address of EnterJitData::result. + + MOZ_ASSERT(OsrFrameReg == IntArgReg3); + + // During the pushes below, use the normal stack pointer. + masm.SetStackPointer64(sp); + + // Save old frame pointer and return address; set new frame pointer. + masm.push(r29, r30); + masm.moveStackPtrTo(r29); + + // Save callee-save integer registers. + // Also save x7 (reg_vp) and x30 (lr), for use later. + masm.push(r19, r20, r21, r22); + masm.push(r23, r24, r25, r26); + masm.push(r27, r28, r7, r30); + + // Save callee-save floating-point registers. + // AArch64 ABI specifies that only the lower 64 bits must be saved. + masm.push(d8, d9, d10, d11); + masm.push(d12, d13, d14, d15); + +#ifdef DEBUG + // Emit stack canaries. + masm.movePtr(ImmWord(0xdeadd00d), r23); + masm.movePtr(ImmWord(0xdeadd11d), r24); + masm.push(r23, r24); +#endif + + // Common code below attempts to push single registers at a time, + // which breaks the stack pointer's 16-byte alignment requirement. + // Note that movePtr() is invalid because StackPointer is treated as xzr. + // + // FIXME: After testing, this entire function should be rewritten to not + // use the PseudoStackPointer: since the amount of data pushed is precalculated, + // we can just allocate the whole frame header at once and index off sp. + // This will save a significant number of instructions where Push() updates sp. + masm.Mov(PseudoStackPointer64, sp); + masm.SetStackPointer64(PseudoStackPointer64); + + // Save the stack pointer at this point for Baseline OSR. + masm.moveStackPtrTo(BaselineFrameReg); + // Remember stack depth without padding and arguments. + masm.moveStackPtrTo(r19); + + // If constructing, include newTarget in argument vector. + { + Label noNewTarget; + Imm32 constructingToken(CalleeToken_FunctionConstructing); + masm.branchTest32(Assembler::Zero, reg_callee, constructingToken, &noNewTarget); + masm.add32(Imm32(1), reg_argc); + masm.bind(&noNewTarget); + } + + // JitFrameLayout is as follows (higher is higher in memory): + // N*8 - [ JS argument vector ] (base 16-byte aligned) + // 8 - numActualArgs + // 8 - calleeToken (16-byte aligned) + // 8 - frameDescriptor + // 8 - returnAddress (16-byte aligned, pushed by callee) + + // Push the argument vector onto the stack. + // WARNING: destructively modifies reg_argv + { + vixl::UseScratchRegisterScope temps(&masm.asVIXL()); + + const ARMRegister tmp_argc = temps.AcquireX(); + const ARMRegister tmp_sp = temps.AcquireX(); + + Label noArguments; + Label loopHead; + + masm.movePtr(reg_argc, tmp_argc.asUnsized()); + + // sp -= 8 + // Since we're using PostIndex Str below, this is necessary to avoid overwriting + // the SPS mark pushed above. + masm.subFromStackPtr(Imm32(8)); + + // sp -= 8 * argc + masm.Sub(PseudoStackPointer64, PseudoStackPointer64, Operand(tmp_argc, vixl::SXTX, 3)); + + // Give sp 16-byte alignment and sync stack pointers. + masm.andToStackPtr(Imm32(~0xff)); + masm.moveStackPtrTo(tmp_sp.asUnsized()); + + masm.branchTestPtr(Assembler::Zero, reg_argc, reg_argc, &noArguments); + + // Begin argument-pushing loop. + // This could be optimized using Ldp and Stp. + { + masm.bind(&loopHead); + + // Load an argument from argv, then increment argv by 8. + masm.Ldr(x24, MemOperand(ARMRegister(reg_argv, 64), Operand(8), vixl::PostIndex)); + + // Store the argument to tmp_sp, then increment tmp_sp by 8. + masm.Str(x24, MemOperand(tmp_sp, Operand(8), vixl::PostIndex)); + + // Set the condition codes for |cmp tmp_argc, 2| (using the old value). + masm.Subs(tmp_argc, tmp_argc, Operand(1)); + + // Branch if arguments remain. + masm.B(&loopHead, vixl::Condition::ge); + } + + masm.bind(&noArguments); + } + masm.checkStackAlignment(); + + // Push the number of actual arguments and the calleeToken. + // The result address is used to store the actual number of arguments + // without adding an argument to EnterJIT. + masm.unboxInt32(Address(reg_vp, 0x0), ip0); + masm.push(ip0, reg_callee); + masm.checkStackAlignment(); + + // Calculate the number of bytes pushed so far. + masm.subStackPtrFrom(r19); + + // Push the frameDescriptor. + masm.makeFrameDescriptor(r19, JitFrame_Entry, JitFrameLayout::Size()); + masm.Push(r19); + + Label osrReturnPoint; + if (type == EnterJitBaseline) { + // Check for OSR. + Label notOsr; + masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); + + // Push return address and previous frame pointer. + masm.Adr(ScratchReg2_64, &osrReturnPoint); + masm.push(ScratchReg2, BaselineFrameReg); + + // Reserve frame. + masm.subFromStackPtr(Imm32(BaselineFrame::Size())); + masm.moveStackPtrTo(BaselineFrameReg); + + // Reserve space for locals and stack values. + masm.Lsl(w19, ARMRegister(reg_osrNStack, 32), 3); // w19 = num_stack_values * sizeof(Value). + masm.subFromStackPtr(r19); + + // Enter exit frame. + masm.addPtr(Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), r19); + masm.makeFrameDescriptor(r19, JitFrame_BaselineJS, ExitFrameLayout::Size()); + masm.asVIXL().Push(x19, xzr); // Push xzr for a fake return address. + // No GC things to mark: push a bare token. + masm.enterFakeExitFrame(ExitFrameLayoutBareToken); + + masm.push(BaselineFrameReg, reg_code); + + // Initialize the frame, including filling in the slots. + masm.setupUnalignedABICall(r19); + masm.passABIArg(BaselineFrameReg); // BaselineFrame. + masm.passABIArg(reg_osrFrame); // InterpreterFrame. + masm.passABIArg(reg_osrNStack); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, jit::InitBaselineFrameForOsr)); + + masm.pop(r19, BaselineFrameReg); + MOZ_ASSERT(r19 != ReturnReg); + + masm.addToStackPtr(Imm32(ExitFrameLayout::SizeWithFooter())); + masm.addPtr(Imm32(BaselineFrame::Size()), BaselineFrameReg); + + Label error; + masm.branchIfFalseBool(ReturnReg, &error); + + masm.jump(r19); + + // OOM: load error value, discard return address and previous frame + // pointer, and return. + masm.bind(&error); + masm.Add(masm.GetStackPointer64(), BaselineFrameReg64, Operand(2 * sizeof(uintptr_t))); + masm.syncStackPtr(); + masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + masm.B(&osrReturnPoint); + + masm.bind(¬Osr); + masm.movePtr(reg_scope, R1_); + } + + // Call function. + // Since AArch64 doesn't have the pc register available, the callee must push lr. + masm.callJitNoProfiler(reg_code); + + // Baseline OSR will return here. + if (type == EnterJitBaseline) + masm.bind(&osrReturnPoint); + + // Return back to SP. + masm.Pop(r19); + masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(), + Operand(x19, vixl::LSR, FRAMESIZE_SHIFT)); + masm.syncStackPtr(); + masm.SetStackPointer64(sp); + +#ifdef DEBUG + // Check that canaries placed on function entry are still present. + masm.pop(r24, r23); + Label x23OK, x24OK; + + masm.branchPtr(Assembler::Equal, r23, ImmWord(0xdeadd00d), &x23OK); + masm.breakpoint(); + masm.bind(&x23OK); + + masm.branchPtr(Assembler::Equal, r24, ImmWord(0xdeadd11d), &x24OK); + masm.breakpoint(); + masm.bind(&x24OK); +#endif + + // Restore callee-save floating-point registers. + masm.pop(d15, d14, d13, d12); + masm.pop(d11, d10, d9, d8); + + // Restore callee-save integer registers. + // Also restore x7 (reg_vp) and x30 (lr). + masm.pop(r30, r7, r28, r27); + masm.pop(r26, r25, r24, r23); + masm.pop(r22, r21, r20, r19); + + // Store return value (in JSReturnReg = x2 to just-popped reg_vp). + masm.storeValue(JSReturnOperand, Address(reg_vp, 0)); + + // Restore old frame pointer. + masm.pop(r30, r29); + + // Return using the value popped into x30. + masm.abiret(); + + Linker linker(masm); + AutoFlushICache afc("EnterJIT"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "EnterJIT"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateInvalidator(JSContext* cx) +{ + MacroAssembler masm; + + masm.push(r0, r1, r2, r3); + + masm.PushRegsInMask(AllRegs); + masm.moveStackPtrTo(r0); + + masm.Sub(x1, masm.GetStackPointer64(), Operand(sizeof(size_t))); + masm.Sub(x2, masm.GetStackPointer64(), Operand(sizeof(size_t) + sizeof(void*))); + masm.moveToStackPtr(r2); + + masm.setupUnalignedABICall(r10); + masm.passABIArg(r0); + masm.passABIArg(r1); + masm.passABIArg(r2); + + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, InvalidationBailout)); + + masm.pop(r2, r1); + + masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(), x1); + masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(), + Operand(sizeof(InvalidationBailoutStack))); + masm.syncStackPtr(); + + JitCode* bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail(); + masm.branch(bailoutTail); + + Linker linker(masm); + AutoFlushICache afc("Invalidator"); + return linker.newCode<NoGC>(cx, OTHER_CODE); +} + +JitCode* +JitRuntime::generateArgumentsRectifier(JSContext* cx, void** returnAddrOut) +{ + MacroAssembler masm; + + // Save the return address for later. + masm.push(lr); + + // Load the information that the rectifier needs from the stack. + masm.Ldr(w0, MemOperand(masm.GetStackPointer64(), RectifierFrameLayout::offsetOfNumActualArgs())); + masm.Ldr(x1, MemOperand(masm.GetStackPointer64(), RectifierFrameLayout::offsetOfCalleeToken())); + + // Extract a JSFunction pointer from the callee token and keep the + // intermediary to avoid later recalculation. + masm.And(x5, x1, Operand(CalleeTokenMask)); + + // Get the arguments from the function object. + masm.Ldrh(x6, MemOperand(x5, JSFunction::offsetOfNargs())); + + static_assert(CalleeToken_FunctionConstructing == 0x1, "Constructing must be low-order bit"); + masm.And(x4, x1, Operand(CalleeToken_FunctionConstructing)); + masm.Add(x7, x6, x4); + + // Calculate the position that our arguments are at before sp gets modified. + MOZ_ASSERT(ArgumentsRectifierReg == r8, "x8 used for argc in Arguments Rectifier"); + masm.Add(x3, masm.GetStackPointer64(), Operand(x8, vixl::LSL, 3)); + masm.Add(x3, x3, Operand(sizeof(RectifierFrameLayout))); + + // Pad to a multiple of 16 bytes. This neglects the |this| value, + // which will also be pushed, because the rest of the frame will + // round off that value. See pushes of |argc|, |callee| and |desc| below. + Label noPadding; + masm.Tbnz(x7, 0, &noPadding); + masm.asVIXL().Push(xzr); + masm.Add(x7, x7, Operand(1)); + masm.bind(&noPadding); + + { + Label notConstructing; + masm.Cbz(x4, ¬Constructing); + + // new.target lives at the end of the pushed args + // NB: The arg vector holder starts at the beginning of the last arg, + // add a value to get to argv[argc] + masm.loadPtr(Address(r3, sizeof(Value)), r4); + masm.Push(r4); + + masm.bind(¬Constructing); + } + + // Calculate the number of undefineds that need to be pushed. + masm.Sub(w2, w6, w8); + + // Put an undefined in a register so it can be pushed. + masm.moveValue(UndefinedValue(), r4); + + // Push undefined N times. + { + Label undefLoopTop; + masm.bind(&undefLoopTop); + masm.Push(r4); + masm.Subs(w2, w2, Operand(1)); + masm.B(&undefLoopTop, Assembler::NonZero); + } + + // Arguments copy loop. Copy for x8 >= 0 to include |this|. + { + Label copyLoopTop; + masm.bind(©LoopTop); + masm.Ldr(x4, MemOperand(x3, -sizeof(Value), vixl::PostIndex)); + masm.Push(r4); + masm.Subs(x8, x8, Operand(1)); + masm.B(©LoopTop, Assembler::NotSigned); + } + + // Fix up the size of the stack frame. +1 accounts for |this|. + masm.Add(x6, x7, Operand(1)); + masm.Lsl(x6, x6, 3); + + // Make that into a frame descriptor. + masm.makeFrameDescriptor(r6, JitFrame_Rectifier, JitFrameLayout::Size()); + + masm.push(r0, // Number of actual arguments. + r1, // Callee token. + r6); // Frame descriptor. + + // Load the address of the code that is getting called. + masm.Ldr(x3, MemOperand(x5, JSFunction::offsetOfNativeOrScript())); + masm.loadBaselineOrIonRaw(r3, r3, nullptr); + uint32_t returnOffset = masm.callJitNoProfiler(r3); + + // Clean up! + // Get the size of the stack frame, and clean up the later fixed frame. + masm.Ldr(x4, MemOperand(masm.GetStackPointer64(), 24, vixl::PostIndex)); + + // Now that the size of the stack frame sans the fixed frame has been loaded, + // add that onto the stack pointer. + masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(), + Operand(x4, vixl::LSR, FRAMESIZE_SHIFT)); + + // Pop the return address from earlier and branch. + masm.ret(); + + Linker linker(masm); + AutoFlushICache afc("ArgumentsRectifier"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + + if (returnAddrOut) + *returnAddrOut = (void*) (code->raw() + returnOffset); + + return code; +} + +static void +PushBailoutFrame(MacroAssembler& masm, uint32_t frameClass, Register spArg) +{ + // the stack should look like: + // [IonFrame] + // bailoutFrame.registersnapshot + // bailoutFrame.fpsnapshot + // bailoutFrame.snapshotOffset + // bailoutFrame.frameSize + + // STEP 1a: Save our register sets to the stack so Bailout() can read + // everything. + // sp % 8 == 0 + + // We don't have to push everything, but this is likely easier. + // Setting regs_. + masm.subFromStackPtr(Imm32(Registers::TotalPhys * sizeof(void*))); + for (uint32_t i = 0; i < Registers::TotalPhys; i += 2) { + masm.Stp(ARMRegister::XRegFromCode(i), + ARMRegister::XRegFromCode(i + 1), + MemOperand(masm.GetStackPointer64(), i * sizeof(void*))); + } + + // Since our datastructures for stack inspection are compile-time fixed, + // if there are only 16 double registers, then we need to reserve + // space on the stack for the missing 16. + masm.subFromStackPtr(Imm32(FloatRegisters::TotalPhys * sizeof(double))); + for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i += 2) { + masm.Stp(ARMFPRegister::DRegFromCode(i), + ARMFPRegister::DRegFromCode(i + 1), + MemOperand(masm.GetStackPointer64(), i * sizeof(void*))); + } + + // STEP 1b: Push both the "return address" of the function call (the address + // of the instruction after the call that we used to get here) as + // well as the callee token onto the stack. The return address is + // currently in r14. We will proceed by loading the callee token + // into a sacrificial register <= r14, then pushing both onto the + // stack. + + // Now place the frameClass onto the stack, via a register. + masm.Mov(x9, frameClass); + + // And onto the stack. Since the stack is full, we need to put this one past + // the end of the current stack. Sadly, the ABI says that we need to always + // point to the lowest place that has been written. The OS is free to do + // whatever it wants below sp. + masm.push(r30, r9); + masm.moveStackPtrTo(spArg); +} + +static void +GenerateBailoutThunk(JSContext* cx, MacroAssembler& masm, uint32_t frameClass) +{ + PushBailoutFrame(masm, frameClass, r0); + + // SP % 8 == 4 + // STEP 1c: Call the bailout function, giving a pointer to the + // structure we just blitted onto the stack. + // Make space for the BaselineBailoutInfo* outparam. + const int sizeOfBailoutInfo = sizeof(void*) * 2; + masm.reserveStack(sizeOfBailoutInfo); + masm.moveStackPtrTo(r1); + + masm.setupUnalignedABICall(r2); + masm.passABIArg(r0); + masm.passABIArg(r1); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, Bailout)); + + masm.Ldr(x2, MemOperand(masm.GetStackPointer64(), 0)); + masm.addToStackPtr(Imm32(sizeOfBailoutInfo)); + + static const uint32_t BailoutDataSize = sizeof(void*) * Registers::Total + + sizeof(double) * FloatRegisters::TotalPhys; + + if (frameClass == NO_FRAME_SIZE_CLASS_ID) { + vixl::UseScratchRegisterScope temps(&masm.asVIXL()); + const ARMRegister scratch64 = temps.AcquireX(); + + masm.Ldr(scratch64, MemOperand(masm.GetStackPointer64(), sizeof(uintptr_t))); + masm.addToStackPtr(Imm32(BailoutDataSize + 32)); + masm.addToStackPtr(scratch64.asUnsized()); + } else { + uint32_t frameSize = FrameSizeClass::FromClass(frameClass).frameSize(); + masm.addToStackPtr(Imm32(frameSize + BailoutDataSize + sizeof(void*))); + } + + // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9. + JitCode* bailoutTail = cx->runtime()->jitRuntime()->getBailoutTail(); + masm.branch(bailoutTail); +} + +JitCode* +JitRuntime::generateBailoutTable(JSContext* cx, uint32_t frameClass) +{ + // FIXME: Implement. + MacroAssembler masm; + masm.breakpoint(); + Linker linker(masm); + AutoFlushICache afc("BailoutTable"); + return linker.newCode<NoGC>(cx, OTHER_CODE); +} + +JitCode* +JitRuntime::generateBailoutHandler(JSContext* cx) +{ + MacroAssembler masm(cx); + GenerateBailoutThunk(cx, masm, NO_FRAME_SIZE_CLASS_ID); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "BailoutHandler"); +#endif + + Linker linker(masm); + AutoFlushICache afc("BailoutHandler"); + return linker.newCode<NoGC>(cx, OTHER_CODE); +} + +JitCode* +JitRuntime::generateVMWrapper(JSContext* cx, const VMFunction& f) +{ + MOZ_ASSERT(functionWrappers_); + MOZ_ASSERT(functionWrappers_->initialized()); + VMWrapperMap::AddPtr p = functionWrappers_->lookupForAdd(&f); + if (p) + return p->value(); + + MacroAssembler masm(cx); + + // Avoid conflicts with argument registers while discarding the result after + // the function call. + AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask); + + // Wrapper register set is a superset of the Volatile register set. + JS_STATIC_ASSERT((Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0); + + // Unlike on other platforms, it is the responsibility of the VM *callee* to + // push the return address, while the caller must ensure that the address + // is stored in lr on entry. This allows the VM wrapper to work with both direct + // calls and tail calls. + masm.push(lr); + + // First argument is the JSContext. + Register reg_cx = IntArgReg0; + regs.take(reg_cx); + + // Stack is: + // ... frame ... + // +12 [args] + // +8 descriptor + // +0 returnAddress (pushed by this function, caller sets as lr) + // + // We're aligned to an exit frame, so link it up. + masm.enterExitFrame(&f); + masm.loadJSContext(reg_cx); + + // Save the current stack pointer as the base for copying arguments. + Register argsBase = InvalidReg; + if (f.explicitArgs) { + // argsBase can't be an argument register. Bad things would happen if + // the MoveResolver didn't throw an assertion failure first. + argsBase = r8; + regs.take(argsBase); + masm.Add(ARMRegister(argsBase, 64), masm.GetStackPointer64(), + Operand(ExitFrameLayout::SizeWithFooter())); + } + + // Reserve space for any outparameter. + Register outReg = InvalidReg; + switch (f.outParam) { + case Type_Value: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(Value)); + masm.moveStackPtrTo(outReg); + break; + + case Type_Handle: + outReg = regs.takeAny(); + masm.PushEmptyRooted(f.outParamRootType); + masm.moveStackPtrTo(outReg); + break; + + case Type_Int32: + case Type_Bool: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(int64_t)); + masm.moveStackPtrTo(outReg); + break; + + case Type_Double: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(double)); + masm.moveStackPtrTo(outReg); + break; + + case Type_Pointer: + outReg = regs.takeAny(); + masm.reserveStack(sizeof(uintptr_t)); + masm.moveStackPtrTo(outReg); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + if (!generateTLEnterVM(cx, masm, f)) + return nullptr; + + masm.setupUnalignedABICall(regs.getAny()); + masm.passABIArg(reg_cx); + + size_t argDisp = 0; + + // Copy arguments. + for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) { + MoveOperand from; + switch (f.argProperties(explicitArg)) { + case VMFunction::WordByValue: + masm.passABIArg(MoveOperand(argsBase, argDisp), + (f.argPassedInFloatReg(explicitArg) ? MoveOp::DOUBLE : MoveOp::GENERAL)); + argDisp += sizeof(void*); + break; + + case VMFunction::WordByRef: + masm.passABIArg(MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS), + MoveOp::GENERAL); + argDisp += sizeof(void*); + break; + + case VMFunction::DoubleByValue: + case VMFunction::DoubleByRef: + MOZ_CRASH("NYI: AArch64 callVM should not be used with 128bit values."); + } + } + + // Copy the semi-implicit outparam, if any. + // It is not a C++-abi outparam, which would get passed in the + // outparam register, but a real parameter to the function, which + // was stack-allocated above. + if (outReg != InvalidReg) + masm.passABIArg(outReg); + + masm.callWithABI(f.wrapped); + + if (!generateTLExitVM(cx, masm, f)) + return nullptr; + + // SP is used to transfer stack across call boundaries. + if (!masm.GetStackPointer64().Is(vixl::sp)) + masm.Mov(masm.GetStackPointer64(), vixl::sp); + + // Test for failure. + switch (f.failType()) { + case Type_Object: + masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel()); + break; + case Type_Bool: + masm.branchIfFalseBool(r0, masm.failureLabel()); + break; + default: + MOZ_CRASH("unknown failure kind"); + } + + // Load the outparam and free any allocated stack. + switch (f.outParam) { + case Type_Value: + masm.Ldr(ARMRegister(JSReturnReg, 64), MemOperand(masm.GetStackPointer64())); + masm.freeStack(sizeof(Value)); + break; + + case Type_Handle: + masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand); + break; + + case Type_Int32: + masm.Ldr(ARMRegister(ReturnReg, 32), MemOperand(masm.GetStackPointer64())); + masm.freeStack(sizeof(int64_t)); + break; + + case Type_Bool: + masm.Ldrb(ARMRegister(ReturnReg, 32), MemOperand(masm.GetStackPointer64())); + masm.freeStack(sizeof(int64_t)); + break; + + case Type_Double: + MOZ_ASSERT(cx->runtime()->jitSupportsFloatingPoint); + masm.Ldr(ARMFPRegister(ReturnDoubleReg, 64), MemOperand(masm.GetStackPointer64())); + masm.freeStack(sizeof(double)); + break; + + case Type_Pointer: + masm.Ldr(ARMRegister(ReturnReg, 64), MemOperand(masm.GetStackPointer64())); + masm.freeStack(sizeof(uintptr_t)); + break; + + default: + MOZ_ASSERT(f.outParam == Type_Void); + break; + } + + masm.leaveExitFrame(); + masm.retn(Imm32(sizeof(ExitFrameLayout) + + f.explicitStackSlots() * sizeof(void*) + + f.extraValuesToPop * sizeof(Value))); + + Linker linker(masm); + AutoFlushICache afc("VMWrapper"); + JitCode* wrapper = linker.newCode<NoGC>(cx, OTHER_CODE); + if (!wrapper) + return nullptr; + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(wrapper, "VMWrapper"); +#endif + + // linker.newCode may trigger a GC and sweep functionWrappers_ so we have to + // use relookupOrAdd instead of add. + if (!functionWrappers_->relookupOrAdd(p, &f, wrapper)) + return nullptr; + + return wrapper; +} + +JitCode* +JitRuntime::generatePreBarrier(JSContext* cx, MIRType type) +{ + MacroAssembler masm(cx); + + LiveRegisterSet regs = LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask), + FloatRegisterSet(FloatRegisters::VolatileMask)); + + // Also preserve the return address. + regs.add(lr); + + masm.PushRegsInMask(regs); + + MOZ_ASSERT(PreBarrierReg == r1); + masm.movePtr(ImmPtr(cx->runtime()), r3); + + masm.setupUnalignedABICall(r0); + masm.passABIArg(r3); + masm.passABIArg(PreBarrierReg); + masm.callWithABI(IonMarkFunction(type)); + + // Pop the volatile regs and restore LR. + masm.PopRegsInMask(regs); + + masm.abiret(); + + Linker linker(masm); + AutoFlushICache afc("PreBarrier"); + return linker.newCode<NoGC>(cx, OTHER_CODE); +} + +typedef bool (*HandleDebugTrapFn)(JSContext*, BaselineFrame*, uint8_t*, bool*); +static const VMFunction HandleDebugTrapInfo = + FunctionInfo<HandleDebugTrapFn>(HandleDebugTrap, "HandleDebugTrap"); + +JitCode* +JitRuntime::generateDebugTrapHandler(JSContext* cx) +{ + MacroAssembler masm(cx); +#ifndef JS_USE_LINK_REGISTER + // The first value contains the return addres, + // which we pull into ICTailCallReg for tail calls. + masm.setFramePushed(sizeof(intptr_t)); +#endif + + Register scratch1 = r0; + Register scratch2 = r1; + + // Load BaselineFrame pointer into scratch1. + masm.Sub(ARMRegister(scratch1, 64), BaselineFrameReg64, Operand(BaselineFrame::Size())); + + // Enter a stub frame and call the HandleDebugTrap VM function. Ensure the + // stub frame has a nullptr ICStub pointer, since this pointer is marked + // during GC. + masm.movePtr(ImmPtr(nullptr), ICStubReg); + EmitBaselineEnterStubFrame(masm, scratch2); + + JitCode* code = cx->runtime()->jitRuntime()->getVMWrapper(HandleDebugTrapInfo); + if (!code) + return nullptr; + + masm.asVIXL().Push(vixl::lr, ARMRegister(scratch1, 64)); + EmitBaselineCallVM(code, masm); + + EmitBaselineLeaveStubFrame(masm); + + // If the stub returns |true|, we have to perform a forced return (return + // from the JS frame). If the stub returns |false|, just return from the + // trap stub so that execution continues at the current pc. + Label forcedReturn; + masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn); + masm.abiret(); + + masm.bind(&forcedReturn); + masm.loadValue(Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfReturnValue()), + JSReturnOperand); + masm.Mov(masm.GetStackPointer64(), BaselineFrameReg64); + + masm.pop(BaselineFrameReg, lr); + masm.syncStackPtr(); + masm.abiret(); + + Linker linker(masm); + AutoFlushICache afc("DebugTrapHandler"); + JitCode* codeDbg = linker.newCode<NoGC>(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler"); +#endif + + return codeDbg; +} + +JitCode* +JitRuntime::generateExceptionTailStub(JSContext* cx, void* handler) +{ + MacroAssembler masm(cx); + + masm.handleFailureWithHandlerTail(handler); + + Linker linker(masm); + AutoFlushICache afc("ExceptionTailStub"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "ExceptionTailStub"); +#endif + + return code; +} + +JitCode* +JitRuntime::generateBailoutTailStub(JSContext* cx) +{ + MacroAssembler masm(cx); + + masm.generateBailoutTail(r1, r2); + + Linker linker(masm); + AutoFlushICache afc("BailoutTailStub"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "BailoutTailStub"); +#endif + + return code; +} +JitCode* +JitRuntime::generateProfilerExitFrameTailStub(JSContext* cx) +{ + MacroAssembler masm; + + Register scratch1 = r8; + Register scratch2 = r9; + Register scratch3 = r10; + Register scratch4 = r11; + + // + // The code generated below expects that the current stack pointer points + // to an Ion or Baseline frame, at the state it would be immediately + // before a ret(). Thus, after this stub's business is done, it executes + // a ret() and returns directly to the caller script, on behalf of the + // callee script that jumped to this code. + // + // Thus the expected stack is: + // + // StackPointer ----+ + // v + // ..., ActualArgc, CalleeToken, Descriptor, ReturnAddr + // MEM-HI MEM-LOW + // + // + // The generated jitcode is responsible for overwriting the + // jitActivation->lastProfilingFrame field with a pointer to the previous + // Ion or Baseline jit-frame that was pushed before this one. It is also + // responsible for overwriting jitActivation->lastProfilingCallSite with + // the return address into that frame. The frame could either be an + // immediate "caller" frame, or it could be a frame in a previous + // JitActivation (if the current frame was entered from C++, and the C++ + // was entered by some caller jit-frame further down the stack). + // + // So this jitcode is responsible for "walking up" the jit stack, finding + // the previous Ion or Baseline JS frame, and storing its address and the + // return address into the appropriate fields on the current jitActivation. + // + // There are a fixed number of different path types that can lead to the + // current frame, which is either a baseline or ion frame: + // + // <Baseline-Or-Ion> + // ^ + // | + // ^--- Ion + // | + // ^--- Baseline Stub <---- Baseline + // | + // ^--- Argument Rectifier + // | ^ + // | | + // | ^--- Ion + // | | + // | ^--- Baseline Stub <---- Baseline + // | + // ^--- Entry Frame (From C++) + // + Register actReg = scratch4; + AbsoluteAddress activationAddr(GetJitContext()->runtime->addressOfProfilingActivation()); + masm.loadPtr(activationAddr, actReg); + + Address lastProfilingFrame(actReg, JitActivation::offsetOfLastProfilingFrame()); + Address lastProfilingCallSite(actReg, JitActivation::offsetOfLastProfilingCallSite()); + +#ifdef DEBUG + // Ensure that frame we are exiting is current lastProfilingFrame + { + masm.loadPtr(lastProfilingFrame, scratch1); + Label checkOk; + masm.branchPtr(Assembler::Equal, scratch1, ImmWord(0), &checkOk); + masm.branchStackPtr(Assembler::Equal, scratch1, &checkOk); + masm.assumeUnreachable("Mismatch between stored lastProfilingFrame and current stack pointer."); + masm.bind(&checkOk); + } +#endif + + // Load the frame descriptor into |scratch1|, figure out what to do depending on its type. + masm.loadPtr(Address(masm.getStackPointer(), JitFrameLayout::offsetOfDescriptor()), scratch1); + + // Going into the conditionals, we will have: + // FrameDescriptor.size in scratch1 + // FrameDescriptor.type in scratch2 + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1, scratch2); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1); + + // Handling of each case is dependent on FrameDescriptor.type + Label handle_IonJS; + Label handle_BaselineStub; + Label handle_Rectifier; + Label handle_IonAccessorIC; + Label handle_Entry; + Label end; + + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_IonJS), &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_BaselineJS), &handle_IonJS); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_BaselineStub), &handle_BaselineStub); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_Rectifier), &handle_Rectifier); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_IonAccessorIC), &handle_IonAccessorIC); + masm.branch32(Assembler::Equal, scratch2, Imm32(JitFrame_Entry), &handle_Entry); + + masm.assumeUnreachable("Invalid caller frame type when exiting from Ion frame."); + + // + // JitFrame_IonJS + // + // Stack layout: + // ... + // Ion-Descriptor + // Prev-FP ---> Ion-ReturnAddr + // ... previous frame data ... |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_IonJS); + { + // |scratch1| contains Descriptor.size + + // returning directly to an IonJS frame. Store return addr to frame + // in lastProfilingCallSite. + masm.loadPtr(Address(masm.getStackPointer(), JitFrameLayout::offsetOfReturnAddress()), + scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + // Store return frame in lastProfilingFrame. + // scratch2 := masm.getStackPointer() + Descriptor.size*1 + JitFrameLayout::Size(); + masm.addPtr(masm.getStackPointer(), scratch1, scratch2); + masm.syncStackPtr(); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2, scratch2); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // + // JitFrame_BaselineStub + // + // Look past the stub and store the frame pointer to + // the baselineJS frame prior to it. + // + // Stack layout: + // ... + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-PrevFramePointer + // | ... BL-FrameData ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Descriptor.Size + // ... arguments ... | + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + // We take advantage of the fact that the stub frame saves the frame + // pointer pointing to the baseline frame, so a bunch of calculation can + // be avoided. + // + masm.bind(&handle_BaselineStub); + { + masm.addPtr(masm.getStackPointer(), scratch1, scratch3); + masm.syncStackPtr(); + Address stubFrameReturnAddr(scratch3, + JitFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr(scratch3, + JitFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); // Skip past BL-PrevFramePtr. + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + + // + // JitFrame_Rectifier + // + // The rectifier frame can be preceded by either an IonJS or a + // BaselineStub frame. + // + // Stack layout if caller of rectifier was Ion: + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- Rect-Descriptor.Size + // < COMMON LAYOUT > + // + // Stack layout if caller of rectifier was Baseline: + // + // BL-Descriptor + // Prev-FP ---> BL-ReturnAddr + // +-----> BL-SavedFramePointer + // | ... baseline frame data ... + // | BLStub-Descriptor + // | BLStub-ReturnAddr + // | BLStub-StubPointer | + // +------ BLStub-SavedFramePointer |- Rect-Descriptor.Size + // ... args to rectifier ... | + // < COMMON LAYOUT > + // + // Common stack layout: + // + // ActualArgc | + // CalleeToken |- IonRectitiferFrameLayout::Size() + // Rect-Descriptor | + // Rect-ReturnAddr | + // ... rectifier data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + // + masm.bind(&handle_Rectifier); + { + // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size(); + masm.addPtr(masm.getStackPointer(), scratch1, scratch2); + masm.syncStackPtr(); + masm.add32(Imm32(JitFrameLayout::Size()), scratch2); + masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()), scratch3); + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3, scratch1); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3); + + // Now |scratch1| contains Rect-Descriptor.Size + // and |scratch2| points to Rectifier frame + // and |scratch3| contains Rect-Descriptor.Type + + // Check for either Ion or BaselineStub frame. + Label handle_Rectifier_BaselineStub; + masm.branch32(Assembler::NotEqual, scratch3, Imm32(JitFrame_IonJS), + &handle_Rectifier_BaselineStub); + + // Handle Rectifier <- IonJS + // scratch3 := RectFrame[ReturnAddr] + masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfReturnAddress()), scratch3); + masm.storePtr(scratch3, lastProfilingCallSite); + + // scratch3 := RectFrame + Rect-Descriptor.Size + RectifierFrameLayout::Size() + masm.addPtr(scratch2, scratch1, scratch3); + masm.add32(Imm32(RectifierFrameLayout::Size()), scratch3); + masm.storePtr(scratch3, lastProfilingFrame); + masm.ret(); + + // Handle Rectifier <- BaselineStub <- BaselineJS + masm.bind(&handle_Rectifier_BaselineStub); +#ifdef DEBUG + { + Label checkOk; + masm.branch32(Assembler::Equal, scratch3, Imm32(JitFrame_BaselineStub), &checkOk); + masm.assumeUnreachable("Unrecognized frame preceding baselineStub."); + masm.bind(&checkOk); + } +#endif + masm.addPtr(scratch2, scratch1, scratch3); + Address stubFrameReturnAddr(scratch3, RectifierFrameLayout::Size() + + BaselineStubFrameLayout::offsetOfReturnAddress()); + masm.loadPtr(stubFrameReturnAddr, scratch2); + masm.storePtr(scratch2, lastProfilingCallSite); + + Address stubFrameSavedFramePtr(scratch3, + RectifierFrameLayout::Size() - (2 * sizeof(void*))); + masm.loadPtr(stubFrameSavedFramePtr, scratch2); + masm.addPtr(Imm32(sizeof(void*)), scratch2); + masm.storePtr(scratch2, lastProfilingFrame); + masm.ret(); + } + + // JitFrame_IonAccessorIC + // + // The caller is always an IonJS frame. + // + // Ion-Descriptor + // Ion-ReturnAddr + // ... ion frame data ... |- AccFrame-Descriptor.Size + // StubCode | + // AccFrame-Descriptor |- IonAccessorICFrameLayout::Size() + // AccFrame-ReturnAddr | + // ... accessor frame data & args ... |- Descriptor.Size + // ActualArgc | + // CalleeToken |- JitFrameLayout::Size() + // Descriptor | + // FP -----> ReturnAddr | + masm.bind(&handle_IonAccessorIC); + { + // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size() + masm.addPtr(masm.getStackPointer(), scratch1, scratch2); + masm.syncStackPtr(); + masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2); + + // scratch3 := AccFrame-Descriptor.Size + masm.loadPtr(Address(scratch2, IonAccessorICFrameLayout::offsetOfDescriptor()), scratch3); +#ifdef DEBUG + // Assert previous frame is an IonJS frame. + masm.movePtr(scratch3, scratch1); + masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1); + { + Label checkOk; + masm.branch32(Assembler::Equal, scratch1, Imm32(JitFrame_IonJS), &checkOk); + masm.assumeUnreachable("IonAccessorIC frame must be preceded by IonJS frame"); + masm.bind(&checkOk); + } +#endif + masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3); + + // lastProfilingCallSite := AccFrame-ReturnAddr + masm.loadPtr(Address(scratch2, IonAccessorICFrameLayout::offsetOfReturnAddress()), scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + + // lastProfilingFrame := AccessorFrame + AccFrame-Descriptor.Size + + // IonAccessorICFrameLayout::Size() + masm.addPtr(scratch2, scratch3, scratch1); + masm.addPtr(Imm32(IonAccessorICFrameLayout::Size()), scratch1); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } + + // + // JitFrame_Entry + // + // If at an entry frame, store null into both fields. + // + masm.bind(&handle_Entry); + { + masm.movePtr(ImmPtr(nullptr), scratch1); + masm.storePtr(scratch1, lastProfilingCallSite); + masm.storePtr(scratch1, lastProfilingFrame); + masm.ret(); + } + + Linker linker(masm); + AutoFlushICache afc("ProfilerExitFrameTailStub"); + JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); + +#ifdef JS_ION_PERF + writePerfSpewerJitCodeProfile(code, "ProfilerExitFrameStub"); +#endif + + return code; +} |