diff options
Diffstat (limited to 'js/src/jit/x86/MacroAssembler-x86.cpp')
| -rw-r--r-- | js/src/jit/x86/MacroAssembler-x86.cpp | 1028 |
1 files changed, 1028 insertions, 0 deletions
diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp new file mode 100644 index 000000000..754b29c2d --- /dev/null +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -0,0 +1,1028 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86/MacroAssembler-x86.h" + +#include "mozilla/Alignment.h" +#include "mozilla/Casting.h" + +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/JitFrames.h" +#include "jit/MacroAssembler.h" +#include "jit/MoveEmitter.h" + +#include "jsscriptinlines.h" +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +// vpunpckldq requires 16-byte boundary for memory operand. +// See convertUInt64ToDouble for the details. +MOZ_ALIGNED_DECL(static const uint64_t, 16) TO_DOUBLE[4] = { + 0x4530000043300000LL, + 0x0LL, + 0x4330000000000000LL, + 0x4530000000000000LL +}; + +static const double TO_DOUBLE_HIGH_SCALE = 0x100000000; + +bool +MacroAssemblerX86::convertUInt64ToDoubleNeedsTemp() +{ + return HasSSE3(); +} + +void +MacroAssemblerX86::convertUInt64ToDouble(Register64 src, FloatRegister dest, Register temp) +{ + // SUBPD needs SSE2, HADDPD needs SSE3. + if (!HasSSE3()) { + MOZ_ASSERT(temp == Register::Invalid()); + + // Zero the dest register to break dependencies, see convertInt32ToDouble. + zeroDouble(dest); + + asMasm().Push(src.high); + asMasm().Push(src.low); + fild(Operand(esp, 0)); + + Label notNegative; + asMasm().branch32(Assembler::NotSigned, src.high, Imm32(0), ¬Negative); + double add_constant = 18446744073709551616.0; // 2^64 + store64(Imm64(mozilla::BitwiseCast<uint64_t>(add_constant)), Address(esp, 0)); + fld(Operand(esp, 0)); + faddp(); + bind(¬Negative); + + fstp(Operand(esp, 0)); + vmovsd(Address(esp, 0), dest); + asMasm().freeStack(2*sizeof(intptr_t)); + return; + } + + // Following operation uses entire 128-bit of dest XMM register. + // Currently higher 64-bit is free when we have access to lower 64-bit. + MOZ_ASSERT(dest.size() == 8); + FloatRegister dest128 = FloatRegister(dest.encoding(), FloatRegisters::Simd128); + + // Assume that src is represented as following: + // src = 0x HHHHHHHH LLLLLLLL + + // Move src to dest (=dest128) and ScratchInt32x4Reg (=scratch): + // dest = 0x 00000000 00000000 00000000 LLLLLLLL + // scratch = 0x 00000000 00000000 00000000 HHHHHHHH + vmovd(src.low, dest128); + vmovd(src.high, ScratchSimd128Reg); + + // Unpack and interleave dest and scratch to dest: + // dest = 0x 00000000 00000000 HHHHHHHH LLLLLLLL + vpunpckldq(ScratchSimd128Reg, dest128, dest128); + + // Unpack and interleave dest and a constant C1 to dest: + // C1 = 0x 00000000 00000000 45300000 43300000 + // dest = 0x 45300000 HHHHHHHH 43300000 LLLLLLLL + // here, each 64-bit part of dest represents following double: + // HI(dest) = 0x 1.00000HHHHHHHH * 2**84 == 2**84 + 0x HHHHHHHH 00000000 + // LO(dest) = 0x 1.00000LLLLLLLL * 2**52 == 2**52 + 0x 00000000 LLLLLLLL + movePtr(ImmWord((uintptr_t)TO_DOUBLE), temp); + vpunpckldq(Operand(temp, 0), dest128, dest128); + + // Subtract a constant C2 from dest, for each 64-bit part: + // C2 = 0x 45300000 00000000 43300000 00000000 + // here, each 64-bit part of C2 represents following double: + // HI(C2) = 0x 1.0000000000000 * 2**84 == 2**84 + // LO(C2) = 0x 1.0000000000000 * 2**52 == 2**52 + // after the operation each 64-bit part of dest represents following: + // HI(dest) = double(0x HHHHHHHH 00000000) + // LO(dest) = double(0x 00000000 LLLLLLLL) + vsubpd(Operand(temp, sizeof(uint64_t) * 2), dest128, dest128); + + // Add HI(dest) and LO(dest) in double and store it into LO(dest), + // LO(dest) = double(0x HHHHHHHH 00000000) + double(0x 00000000 LLLLLLLL) + // = double(0x HHHHHHHH LLLLLLLL) + // = double(src) + vhaddpd(dest128, dest128); +} + +void +MacroAssemblerX86::loadConstantDouble(wasm::RawF64 d, FloatRegister dest) +{ + if (maybeInlineDouble(d, dest)) + return; + Double* dbl = getDouble(d); + if (!dbl) + return; + masm.vmovsd_mr(nullptr, dest.encoding()); + propagateOOM(dbl->uses.append(CodeOffset(masm.size()))); +} + +void +MacroAssemblerX86::loadConstantDouble(double d, FloatRegister dest) +{ + loadConstantDouble(wasm::RawF64(d), dest); +} + +void +MacroAssemblerX86::loadConstantFloat32(wasm::RawF32 f, FloatRegister dest) +{ + if (maybeInlineFloat(f, dest)) + return; + Float* flt = getFloat(f); + if (!flt) + return; + masm.vmovss_mr(nullptr, dest.encoding()); + propagateOOM(flt->uses.append(CodeOffset(masm.size()))); +} + +void +MacroAssemblerX86::loadConstantFloat32(float f, FloatRegister dest) +{ + loadConstantFloat32(wasm::RawF32(f), dest); +} + +void +MacroAssemblerX86::loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest) +{ + if (maybeInlineSimd128Int(v, dest)) + return; + SimdData* i4 = getSimdData(v); + if (!i4) + return; + masm.vmovdqa_mr(nullptr, dest.encoding()); + propagateOOM(i4->uses.append(CodeOffset(masm.size()))); +} + +void +MacroAssemblerX86::loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest) +{ + if (maybeInlineSimd128Float(v, dest)) + return; + SimdData* f4 = getSimdData(v); + if (!f4) + return; + masm.vmovaps_mr(nullptr, dest.encoding()); + propagateOOM(f4->uses.append(CodeOffset(masm.size()))); +} + +void +MacroAssemblerX86::finish() +{ + if (!doubles_.empty()) + masm.haltingAlign(sizeof(double)); + for (const Double& d : doubles_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : d.uses) + addCodeLabel(CodeLabel(use, cst)); + masm.int64Constant(d.value); + if (!enoughMemory_) + return; + } + + if (!floats_.empty()) + masm.haltingAlign(sizeof(float)); + for (const Float& f : floats_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : f.uses) + addCodeLabel(CodeLabel(use, cst)); + masm.int32Constant(f.value); + if (!enoughMemory_) + return; + } + + // SIMD memory values must be suitably aligned. + if (!simds_.empty()) + masm.haltingAlign(SimdMemoryAlignment); + for (const SimdData& v : simds_) { + CodeOffset cst(masm.currentOffset()); + for (CodeOffset use : v.uses) + addCodeLabel(CodeLabel(use, cst)); + masm.simd128Constant(v.value.bytes()); + if (!enoughMemory_) + return; + } +} + +void +MacroAssemblerX86::handleFailureWithHandlerTail(void* handler) +{ + // Reserve space for exception information. + subl(Imm32(sizeof(ResumeFromException)), esp); + movl(esp, eax); + + // Call the handler. + asMasm().setupUnalignedABICall(ecx); + asMasm().passABIArg(eax); + asMasm().callWithABI(handler); + + Label entryFrame; + Label catch_; + Label finally; + Label return_; + Label bailout; + + loadPtr(Address(esp, offsetof(ResumeFromException, kind)), eax); + asMasm().branch32(Assembler::Equal, eax, Imm32(ResumeFromException::RESUME_ENTRY_FRAME), + &entryFrame); + asMasm().branch32(Assembler::Equal, eax, Imm32(ResumeFromException::RESUME_CATCH), &catch_); + asMasm().branch32(Assembler::Equal, eax, Imm32(ResumeFromException::RESUME_FINALLY), &finally); + asMasm().branch32(Assembler::Equal, eax, Imm32(ResumeFromException::RESUME_FORCED_RETURN), + &return_); + asMasm().branch32(Assembler::Equal, eax, Imm32(ResumeFromException::RESUME_BAILOUT), &bailout); + + breakpoint(); // Invalid kind. + + // No exception handler. Load the error value, load the new stack pointer + // and return from the entry frame. + bind(&entryFrame); + moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); + loadPtr(Address(esp, offsetof(ResumeFromException, stackPointer)), esp); + ret(); + + // If we found a catch handler, this must be a baseline frame. Restore state + // and jump to the catch block. + bind(&catch_); + loadPtr(Address(esp, offsetof(ResumeFromException, target)), eax); + loadPtr(Address(esp, offsetof(ResumeFromException, framePointer)), ebp); + loadPtr(Address(esp, offsetof(ResumeFromException, stackPointer)), esp); + jmp(Operand(eax)); + + // If we found a finally block, this must be a baseline frame. Push + // two values expected by JSOP_RETSUB: BooleanValue(true) and the + // exception. + bind(&finally); + ValueOperand exception = ValueOperand(ecx, edx); + loadValue(Address(esp, offsetof(ResumeFromException, exception)), exception); + + loadPtr(Address(esp, offsetof(ResumeFromException, target)), eax); + loadPtr(Address(esp, offsetof(ResumeFromException, framePointer)), ebp); + loadPtr(Address(esp, offsetof(ResumeFromException, stackPointer)), esp); + + pushValue(BooleanValue(true)); + pushValue(exception); + jmp(Operand(eax)); + + // Only used in debug mode. Return BaselineFrame->returnValue() to the caller. + bind(&return_); + loadPtr(Address(esp, offsetof(ResumeFromException, framePointer)), ebp); + loadPtr(Address(esp, offsetof(ResumeFromException, stackPointer)), esp); + loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); + movl(ebp, esp); + pop(ebp); + + // If profiling is enabled, then update the lastProfilingFrame to refer to caller + // frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled(GetJitContext()->runtime->spsProfiler().addressOfEnabled()); + asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + profilerExitFrame(); + bind(&skipProfilingInstrumentation); + } + + ret(); + + // If we are bailing out to baseline to handle an exception, jump to + // the bailout tail stub. + bind(&bailout); + loadPtr(Address(esp, offsetof(ResumeFromException, bailoutInfo)), ecx); + movl(Imm32(BAILOUT_RETURN_OK), eax); + jmp(Operand(esp, offsetof(ResumeFromException, target))); +} + +void +MacroAssemblerX86::profilerEnterFrame(Register framePtr, Register scratch) +{ + AbsoluteAddress activation(GetJitContext()->runtime->addressOfProfilingActivation()); + loadPtr(activation, scratch); + storePtr(framePtr, Address(scratch, JitActivation::offsetOfLastProfilingFrame())); + storePtr(ImmPtr(nullptr), Address(scratch, JitActivation::offsetOfLastProfilingCallSite())); +} + +void +MacroAssemblerX86::profilerExitFrame() +{ + jmp(GetJitContext()->runtime->jitRuntime()->getProfilerExitFrameTail()); +} + +MacroAssembler& +MacroAssemblerX86::asMasm() +{ + return *static_cast<MacroAssembler*>(this); +} + +const MacroAssembler& +MacroAssemblerX86::asMasm() const +{ + return *static_cast<const MacroAssembler*>(this); +} + +void +MacroAssembler::subFromStackPtr(Imm32 imm32) +{ + if (imm32.value) { + // On windows, we cannot skip very far down the stack without touching the + // memory pages in-between. This is a corner-case code for situations where the + // Ion frame data for a piece of code is very large. To handle this special case, + // for frames over 1k in size we allocate memory on the stack incrementally, touching + // it as we go. + uint32_t amountLeft = imm32.value; + while (amountLeft > 4096) { + subl(Imm32(4096), StackPointer); + store32(Imm32(0), Address(StackPointer, 0)); + amountLeft -= 4096; + } + subl(Imm32(amountLeft), StackPointer); + } +} + +//{{{ check_macroassembler_style +// =============================================================== +// ABI function calls. + +void +MacroAssembler::setupUnalignedABICall(Register scratch) +{ + setupABICall(); + dynamicAlignment_ = true; + + movl(esp, scratch); + andl(Imm32(~(ABIStackAlignment - 1)), esp); + push(scratch); +} + +void +MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) +{ + MOZ_ASSERT(inCall_); + uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar(); + + if (dynamicAlignment_) { + // sizeof(intptr_t) accounts for the saved stack pointer pushed by + // setupUnalignedABICall. + stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t), + ABIStackAlignment); + } else { + uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0; + stackForCall += ComputeByteAlignment(stackForCall + framePushed() + alignmentAtPrologue, + ABIStackAlignment); + } + + *stackAdjust = stackForCall; + reserveStack(stackForCall); + + // Position all arguments. + { + enoughMemory_ &= moveResolver_.resolve(); + if (!enoughMemory_) + return; + + MoveEmitter emitter(*this); + emitter.emit(moveResolver_); + emitter.finish(); + } + + assertStackAlignment(ABIStackAlignment); +} + +void +MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result) +{ + freeStack(stackAdjust); + if (result == MoveOp::DOUBLE) { + reserveStack(sizeof(double)); + fstp(Operand(esp, 0)); + loadDouble(Operand(esp, 0), ReturnDoubleReg); + freeStack(sizeof(double)); + } else if (result == MoveOp::FLOAT32) { + reserveStack(sizeof(float)); + fstp32(Operand(esp, 0)); + loadFloat32(Operand(esp, 0), ReturnFloat32Reg); + freeStack(sizeof(float)); + } + if (dynamicAlignment_) + pop(esp); + +#ifdef DEBUG + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif +} + +void +MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) +{ + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(fun); + callWithABIPost(stackAdjust, result); +} + +void +MacroAssembler::callWithABINoProfiler(const Address& fun, MoveOp::Type result) +{ + uint32_t stackAdjust; + callWithABIPre(&stackAdjust); + call(fun); + callWithABIPost(stackAdjust, result); +} + +// =============================================================== +// Branch functions + +void +MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp, + Label* label) +{ + MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86. + MOZ_ASSERT(ptr != temp); + movePtr(ptr, temp); + branchPtrInNurseryChunkImpl(cond, temp, label); +} + +void +MacroAssembler::branchPtrInNurseryChunk(Condition cond, const Address& address, Register temp, + Label* label) +{ + MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86. + loadPtr(address, temp); + branchPtrInNurseryChunkImpl(cond, temp, label); +} + +void +MacroAssembler::branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + orPtr(Imm32(gc::ChunkMask), ptr); + branch32(cond, Address(ptr, gc::ChunkLocationOffsetFromLastByte), + Imm32(int32_t(gc::ChunkLocation::Nursery)), label); +} + +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, + Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label); + branchPtrInNurseryChunk(cond, address, temp, label); + + bind(&done); +} + +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, + Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label); + branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label); + + bind(&done); +} + +void +MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs, + const Value& rhs, Label* label) +{ + MOZ_ASSERT(cond == Equal || cond == NotEqual); + if (rhs.isMarkable()) + cmpPtr(lhs.payloadReg(), ImmGCPtr(rhs.toMarkablePointer())); + else + cmpPtr(lhs.payloadReg(), ImmWord(rhs.toNunboxPayload())); + + if (cond == Equal) { + Label done; + j(NotEqual, &done); + { + cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag())); + j(Equal, label); + } + bind(&done); + } else { + j(NotEqual, label); + + cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag())); + j(NotEqual, label); + } +} + +// ======================================================================== +// Memory access primitives. +template <typename T> +void +MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType, + const T& dest, MIRType slotType) +{ + if (valueType == MIRType::Double) { + storeDouble(value.reg().typedReg().fpu(), dest); + return; + } + + // Store the type tag if needed. + if (valueType != slotType) + storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), Operand(dest)); + + // Store the payload. + if (value.constant()) + storePayload(value.value(), Operand(dest)); + else + storePayload(value.reg().typedReg().gpr(), Operand(dest)); +} + +template void +MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType, + const Address& dest, MIRType slotType); +template void +MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType, + const BaseIndex& dest, MIRType slotType); + +// wasm specific methods, used in both the wasm baseline compiler and ion. + +void +MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr, AnyRegister out) +{ + memoryBarrier(access.barrierBefore()); + + size_t loadOffset = size(); + switch (access.type()) { + case Scalar::Int8: + movsblWithPatch(srcAddr, out.gpr()); + break; + case Scalar::Uint8: + movzblWithPatch(srcAddr, out.gpr()); + break; + case Scalar::Int16: + movswlWithPatch(srcAddr, out.gpr()); + break; + case Scalar::Uint16: + movzwlWithPatch(srcAddr, out.gpr()); + break; + case Scalar::Int32: + case Scalar::Uint32: + movlWithPatch(srcAddr, out.gpr()); + break; + case Scalar::Float32: + vmovssWithPatch(srcAddr, out.fpu()); + break; + case Scalar::Float64: + vmovsdWithPatch(srcAddr, out.fpu()); + break; + case Scalar::Float32x4: + switch (access.numSimdElems()) { + // In memory-to-register mode, movss zeroes out the high lanes. + case 1: vmovssWithPatch(srcAddr, out.fpu()); break; + // See comment above, which also applies to movsd. + case 2: vmovsdWithPatch(srcAddr, out.fpu()); break; + case 4: vmovupsWithPatch(srcAddr, out.fpu()); break; + default: MOZ_CRASH("unexpected size for partial load"); + } + break; + case Scalar::Int32x4: + switch (access.numSimdElems()) { + // In memory-to-register mode, movd zeroes out the high lanes. + case 1: vmovdWithPatch(srcAddr, out.fpu()); break; + // See comment above, which also applies to movq. + case 2: vmovqWithPatch(srcAddr, out.fpu()); break; + case 4: vmovdquWithPatch(srcAddr, out.fpu()); break; + default: MOZ_CRASH("unexpected size for partial load"); + } + break; + case Scalar::Int8x16: + MOZ_ASSERT(access.numSimdElems() == 16, "unexpected partial load"); + vmovdquWithPatch(srcAddr, out.fpu()); + break; + case Scalar::Int16x8: + MOZ_ASSERT(access.numSimdElems() == 8, "unexpected partial load"); + vmovdquWithPatch(srcAddr, out.fpu()); + break; + case Scalar::Int64: + case Scalar::Uint8Clamped: + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected type"); + } + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + memoryBarrier(access.barrierAfter()); +} + +void +MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAddr, Register64 out) +{ + MOZ_ASSERT(!access.isAtomic()); + MOZ_ASSERT(!access.isSimd()); + + size_t loadOffset = size(); + switch (access.type()) { + case Scalar::Int8: + MOZ_ASSERT(out == Register64(edx, eax)); + movsblWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + cdq(); + break; + case Scalar::Uint8: + movzblWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + xorl(out.high, out.high); + break; + case Scalar::Int16: + MOZ_ASSERT(out == Register64(edx, eax)); + movswlWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + cdq(); + break; + case Scalar::Uint16: + movzwlWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + xorl(out.high, out.high); + break; + case Scalar::Int32: + MOZ_ASSERT(out == Register64(edx, eax)); + movlWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + cdq(); + break; + case Scalar::Uint32: + movlWithPatch(srcAddr, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + xorl(out.high, out.high); + break; + case Scalar::Int64: + if (srcAddr.kind() == Operand::MEM_ADDRESS32) { + Operand low(PatchedAbsoluteAddress(uint32_t(srcAddr.address()) + INT64LOW_OFFSET)); + Operand high(PatchedAbsoluteAddress(uint32_t(srcAddr.address()) + INT64HIGH_OFFSET)); + + movlWithPatch(low, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + loadOffset = size(); + movlWithPatch(high, out.high); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + } else { + MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP); + Address addr = srcAddr.toAddress(); + Operand low(addr.base, addr.offset + INT64LOW_OFFSET); + Operand high(addr.base, addr.offset + INT64HIGH_OFFSET); + + if (addr.base != out.low) { + movlWithPatch(low, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + loadOffset = size(); + movlWithPatch(high, out.high); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + } else { + MOZ_ASSERT(addr.base != out.high); + movlWithPatch(high, out.high); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + + loadOffset = size(); + movlWithPatch(low, out.low); + append(wasm::MemoryPatch(size())); + append(access, loadOffset, framePushed()); + } + } + break; + case Scalar::Float32: + case Scalar::Float64: + case Scalar::Float32x4: + case Scalar::Int8x16: + case Scalar::Int16x8: + case Scalar::Int32x4: + MOZ_CRASH("non-int64 loads should use load()"); + case Scalar::Uint8Clamped: + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected array type"); + } +} + +void +MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value, Operand dstAddr) +{ + memoryBarrier(access.barrierBefore()); + + size_t storeOffset = size(); + switch (access.type()) { + case Scalar::Int8: + case Scalar::Uint8Clamped: + case Scalar::Uint8: + movbWithPatch(value.gpr(), dstAddr); + break; + case Scalar::Int16: + case Scalar::Uint16: + movwWithPatch(value.gpr(), dstAddr); + break; + case Scalar::Int32: + case Scalar::Uint32: + movlWithPatch(value.gpr(), dstAddr); + break; + case Scalar::Float32: + vmovssWithPatch(value.fpu(), dstAddr); + break; + case Scalar::Float64: + vmovsdWithPatch(value.fpu(), dstAddr); + break; + case Scalar::Float32x4: + switch (access.numSimdElems()) { + // In memory-to-register mode, movss zeroes out the high lanes. + case 1: vmovssWithPatch(value.fpu(), dstAddr); break; + // See comment above, which also applies to movsd. + case 2: vmovsdWithPatch(value.fpu(), dstAddr); break; + case 4: vmovupsWithPatch(value.fpu(), dstAddr); break; + default: MOZ_CRASH("unexpected size for partial load"); + } + break; + case Scalar::Int32x4: + switch (access.numSimdElems()) { + // In memory-to-register mode, movd zeroes out the high lanes. + case 1: vmovdWithPatch(value.fpu(), dstAddr); break; + // See comment above, which also applies to movsd. + case 2: vmovqWithPatch(value.fpu(), dstAddr); break; + case 4: vmovdquWithPatch(value.fpu(), dstAddr); break; + default: MOZ_CRASH("unexpected size for partial load"); + } + break; + case Scalar::Int8x16: + MOZ_ASSERT(access.numSimdElems() == 16, "unexpected partial store"); + vmovdquWithPatch(value.fpu(), dstAddr); + break; + case Scalar::Int16x8: + MOZ_ASSERT(access.numSimdElems() == 8, "unexpected partial store"); + vmovdquWithPatch(value.fpu(), dstAddr); + break; + case Scalar::Int64: + MOZ_CRASH("Should be handled in storeI64."); + case Scalar::MaxTypedArrayViewType: + MOZ_CRASH("unexpected type"); + } + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + + memoryBarrier(access.barrierAfter()); +} + +void +MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value, Operand dstAddr) +{ + MOZ_ASSERT(!access.isAtomic()); + MOZ_ASSERT(!access.isSimd()); + + size_t storeOffset = size(); + if (dstAddr.kind() == Operand::MEM_ADDRESS32) { + Operand low(PatchedAbsoluteAddress(uint32_t(dstAddr.address()) + INT64LOW_OFFSET)); + Operand high(PatchedAbsoluteAddress(uint32_t(dstAddr.address()) + INT64HIGH_OFFSET)); + + movlWithPatch(value.low, low); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + + storeOffset = size(); + movlWithPatch(value.high, high); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + } else { + MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP); + Address addr = dstAddr.toAddress(); + Operand low(addr.base, addr.offset + INT64LOW_OFFSET); + Operand high(addr.base, addr.offset + INT64HIGH_OFFSET); + + if (addr.base != value.low) { + movlWithPatch(value.low, low); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + + storeOffset = size(); + movlWithPatch(value.high, high); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + } else { + MOZ_ASSERT(addr.base != value.high); + + movlWithPatch(value.high, high); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + + storeOffset = size(); + movlWithPatch(value.low, low); + append(wasm::MemoryPatch(size())); + append(access, storeOffset, framePushed()); + } + } +} + +void +MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input, Register output, Label* oolEntry) +{ + Label done; + vcvttsd2si(input, output); + branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done); + + loadConstantDouble(double(int32_t(0x80000000)), ScratchDoubleReg); + addDouble(input, ScratchDoubleReg); + vcvttsd2si(ScratchDoubleReg, output); + + branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry); + or32(Imm32(0x80000000), output); + + bind(&done); +} + +void +MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input, Register output, Label* oolEntry) +{ + Label done; + vcvttss2si(input, output); + branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done); + + loadConstantFloat32(float(int32_t(0x80000000)), ScratchFloat32Reg); + addFloat32(input, ScratchFloat32Reg); + vcvttss2si(ScratchFloat32Reg, output); + + branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry); + or32(Imm32(0x80000000), output); + + bind(&done); +} + +//}}} check_macroassembler_style + +void +MacroAssemblerX86::convertInt64ToDouble(Register64 input, FloatRegister output) +{ + // Zero the output register to break dependencies, see convertInt32ToDouble. + zeroDouble(output); + + asMasm().Push(input.high); + asMasm().Push(input.low); + fild(Operand(esp, 0)); + + fstp(Operand(esp, 0)); + vmovsd(Address(esp, 0), output); + asMasm().freeStack(2*sizeof(intptr_t)); +} + +void +MacroAssemblerX86::convertInt64ToFloat32(Register64 input, FloatRegister output) +{ + convertInt64ToDouble(input, output); + convertDoubleToFloat32(output, output); +} + +void +MacroAssemblerX86::convertUInt64ToFloat32(Register64 input, FloatRegister output, Register temp) +{ + convertUInt64ToDouble(input, output.asDouble(), temp); + convertDoubleToFloat32(output, output); +} + +void +MacroAssemblerX86::wasmTruncateDoubleToInt64(FloatRegister input, Register64 output, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) +{ + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in (u)int64. + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeDouble(input, Operand(esp, 0)); + asMasm().branchDoubleNotInInt64Range(Address(esp, 0), temp, &fail); + jump(&convert); + + // Handle failure in ool. + bind(&fail); + asMasm().freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + bind(oolRejoin); + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeDouble(input, Operand(esp, 0)); + + // Convert the double/float to int64. + bind(&convert); + asMasm().truncateDoubleToInt64(Address(esp, 0), Address(esp, 0), temp); + + // Load value into int64 register. + load64(Address(esp, 0), output); + asMasm().freeStack(2 * sizeof(int32_t)); +} + +void +MacroAssemblerX86::wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) +{ + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in (u)int64. + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeFloat32(input, Operand(esp, 0)); + asMasm().branchFloat32NotInInt64Range(Address(esp, 0), temp, &fail); + jump(&convert); + + // Handle failure in ool. + bind(&fail); + asMasm().freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + bind(oolRejoin); + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeFloat32(input, Operand(esp, 0)); + + // Convert the double/float to int64. + bind(&convert); + asMasm().truncateFloat32ToInt64(Address(esp, 0), Address(esp, 0), temp); + + // Load value into int64 register. + load64(Address(esp, 0), output); + asMasm().freeStack(2 * sizeof(int32_t)); +} + +void +MacroAssemblerX86::wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) +{ + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in (u)int64. + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeDouble(input, Operand(esp, 0)); + asMasm().branchDoubleNotInUInt64Range(Address(esp, 0), temp, &fail); + jump(&convert); + + // Handle failure in ool. + bind(&fail); + asMasm().freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + bind(oolRejoin); + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeDouble(input, Operand(esp, 0)); + + // Convert the double/float to int64. + bind(&convert); + asMasm().truncateDoubleToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg); + + // Load value into int64 register. + load64(Address(esp, 0), output); + asMasm().freeStack(2 * sizeof(int32_t)); +} + +void +MacroAssemblerX86::wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output, Label* oolEntry, + Label* oolRejoin, FloatRegister tempReg) +{ + Label fail, convert; + Register temp = output.high; + + // Make sure input fits in (u)int64. + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeFloat32(input, Operand(esp, 0)); + asMasm().branchFloat32NotInUInt64Range(Address(esp, 0), temp, &fail); + jump(&convert); + + // Handle failure in ool. + bind(&fail); + asMasm().freeStack(2 * sizeof(int32_t)); + jump(oolEntry); + bind(oolRejoin); + asMasm().reserveStack(2 * sizeof(int32_t)); + asMasm().storeFloat32(input, Operand(esp, 0)); + + // Convert the double/float to int64. + bind(&convert); + asMasm().truncateFloat32ToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg); + + // Load value into int64 register. + load64(Address(esp, 0), output); + asMasm().freeStack(2 * sizeof(int32_t)); +} + |