diff options
Diffstat (limited to 'js/src/jit/x86-shared/Assembler-x86-shared.h')
| -rw-r--r-- | js/src/jit/x86-shared/Assembler-x86-shared.h | 3652 |
1 files changed, 3652 insertions, 0 deletions
diff --git a/js/src/jit/x86-shared/Assembler-x86-shared.h b/js/src/jit/x86-shared/Assembler-x86-shared.h new file mode 100644 index 000000000..510ce9a99 --- /dev/null +++ b/js/src/jit/x86-shared/Assembler-x86-shared.h @@ -0,0 +1,3652 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_x86_shared_Assembler_x86_shared_h +#define jit_x86_shared_Assembler_x86_shared_h + +#include <cstddef> + +#include "jit/shared/Assembler-shared.h" + +#if defined(JS_CODEGEN_X86) +# include "jit/x86/BaseAssembler-x86.h" +#elif defined(JS_CODEGEN_X64) +# include "jit/x64/BaseAssembler-x64.h" +#else +# error "Unknown architecture!" +#endif + +namespace js { +namespace jit { + +struct ScratchFloat32Scope : public AutoFloatRegisterScope +{ + explicit ScratchFloat32Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchFloat32Reg) + { } +}; + +struct ScratchDoubleScope : public AutoFloatRegisterScope +{ + explicit ScratchDoubleScope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchDoubleReg) + { } +}; + +struct ScratchSimd128Scope : public AutoFloatRegisterScope +{ + explicit ScratchSimd128Scope(MacroAssembler& masm) + : AutoFloatRegisterScope(masm, ScratchSimd128Reg) + { } +}; + +class Operand +{ + public: + enum Kind { + REG, + MEM_REG_DISP, + FPREG, + MEM_SCALE, + MEM_ADDRESS32 + }; + + private: + Kind kind_ : 4; + // Used as a Register::Encoding and a FloatRegister::Encoding. + uint32_t base_ : 5; + Scale scale_ : 3; + // We don't use all 8 bits, of course, but GCC complains if the size of + // this field is smaller than the size of Register::Encoding. + Register::Encoding index_ : 8; + int32_t disp_; + + public: + explicit Operand(Register reg) + : kind_(REG), + base_(reg.encoding()), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(0) + { } + explicit Operand(FloatRegister reg) + : kind_(FPREG), + base_(reg.encoding()), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(0) + { } + explicit Operand(const Address& address) + : kind_(MEM_REG_DISP), + base_(address.base.encoding()), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(address.offset) + { } + explicit Operand(const BaseIndex& address) + : kind_(MEM_SCALE), + base_(address.base.encoding()), + scale_(address.scale), + index_(address.index.encoding()), + disp_(address.offset) + { } + Operand(Register base, Register index, Scale scale, int32_t disp = 0) + : kind_(MEM_SCALE), + base_(base.encoding()), + scale_(scale), + index_(index.encoding()), + disp_(disp) + { } + Operand(Register reg, int32_t disp) + : kind_(MEM_REG_DISP), + base_(reg.encoding()), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(disp) + { } + explicit Operand(AbsoluteAddress address) + : kind_(MEM_ADDRESS32), + base_(Registers::Invalid), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(X86Encoding::AddressImmediate(address.addr)) + { } + explicit Operand(PatchedAbsoluteAddress address) + : kind_(MEM_ADDRESS32), + base_(Registers::Invalid), + scale_(TimesOne), + index_(Registers::Invalid), + disp_(X86Encoding::AddressImmediate(address.addr)) + { } + + Address toAddress() const { + MOZ_ASSERT(kind() == MEM_REG_DISP); + return Address(Register::FromCode(base()), disp()); + } + + BaseIndex toBaseIndex() const { + MOZ_ASSERT(kind() == MEM_SCALE); + return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), scale(), disp()); + } + + Kind kind() const { + return kind_; + } + Register::Encoding reg() const { + MOZ_ASSERT(kind() == REG); + return Register::Encoding(base_); + } + Register::Encoding base() const { + MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); + return Register::Encoding(base_); + } + Register::Encoding index() const { + MOZ_ASSERT(kind() == MEM_SCALE); + return index_; + } + Scale scale() const { + MOZ_ASSERT(kind() == MEM_SCALE); + return scale_; + } + FloatRegister::Encoding fpu() const { + MOZ_ASSERT(kind() == FPREG); + return FloatRegister::Encoding(base_); + } + int32_t disp() const { + MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); + return disp_; + } + void* address() const { + MOZ_ASSERT(kind() == MEM_ADDRESS32); + return reinterpret_cast<void*>(disp_); + } + + bool containsReg(Register r) const { + switch (kind()) { + case REG: return r.encoding() == reg(); + case MEM_REG_DISP: return r.encoding() == base(); + case MEM_SCALE: return r.encoding() == base() || r.encoding() == index(); + default: return false; + } + } +}; + +inline Imm32 +Imm64::firstHalf() const +{ + return low(); +} + +inline Imm32 +Imm64::secondHalf() const +{ + return hi(); +} + +class CPUInfo +{ + public: + // As the SSE's were introduced in order, the presence of a later SSE implies + // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 support. + enum SSEVersion { + UnknownSSE = 0, + NoSSE = 1, + SSE = 2, + SSE2 = 3, + SSE3 = 4, + SSSE3 = 5, + SSE4_1 = 6, + SSE4_2 = 7 + }; + + static SSEVersion GetSSEVersion() { + if (maxSSEVersion == UnknownSSE) + SetSSEVersion(); + + MOZ_ASSERT(maxSSEVersion != UnknownSSE); + MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, maxSSEVersion <= maxEnabledSSEVersion); + return maxSSEVersion; + } + + static bool IsAVXPresent() { + if (MOZ_UNLIKELY(maxSSEVersion == UnknownSSE)) + SetSSEVersion(); + + MOZ_ASSERT_IF(!avxEnabled, !avxPresent); + return avxPresent; + } + + private: + static SSEVersion maxSSEVersion; + static SSEVersion maxEnabledSSEVersion; + static bool avxPresent; + static bool avxEnabled; + static bool popcntPresent; + static bool needAmdBugWorkaround; + + static void SetSSEVersion(); + + public: + static bool IsSSE2Present() { +#ifdef JS_CODEGEN_X64 + return true; +#else + return GetSSEVersion() >= SSE2; +#endif + } + static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; } + static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; } + static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; } + static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; } + static bool IsPOPCNTPresent() { return popcntPresent; } + static bool NeedAmdBugWorkaround() { return needAmdBugWorkaround; } + + static void SetSSE3Disabled() { maxEnabledSSEVersion = SSE2; avxEnabled = false; } + static void SetSSE4Disabled() { maxEnabledSSEVersion = SSSE3; avxEnabled = false; } + static void SetAVXEnabled() { avxEnabled = true; } +}; + +class AssemblerX86Shared : public AssemblerShared +{ + protected: + struct RelativePatch { + int32_t offset; + void* target; + Relocation::Kind kind; + + RelativePatch(int32_t offset, void* target, Relocation::Kind kind) + : offset(offset), + target(target), + kind(kind) + { } + }; + + Vector<RelativePatch, 8, SystemAllocPolicy> jumps_; + CompactBufferWriter jumpRelocations_; + CompactBufferWriter dataRelocations_; + CompactBufferWriter preBarriers_; + + void writeDataRelocation(ImmGCPtr ptr) { + if (ptr.value) { + if (gc::IsInsideNursery(ptr.value)) + embedsNurseryPointers_ = true; + dataRelocations_.writeUnsigned(masm.currentOffset()); + } + } + void writePrebarrierOffset(CodeOffset label) { + preBarriers_.writeUnsigned(label.offset()); + } + + protected: + X86Encoding::BaseAssemblerSpecific masm; + + typedef X86Encoding::JmpSrc JmpSrc; + typedef X86Encoding::JmpDst JmpDst; + + public: + AssemblerX86Shared() + { + if (!HasAVX()) + masm.disableVEX(); + } + + enum Condition { + Equal = X86Encoding::ConditionE, + NotEqual = X86Encoding::ConditionNE, + Above = X86Encoding::ConditionA, + AboveOrEqual = X86Encoding::ConditionAE, + Below = X86Encoding::ConditionB, + BelowOrEqual = X86Encoding::ConditionBE, + GreaterThan = X86Encoding::ConditionG, + GreaterThanOrEqual = X86Encoding::ConditionGE, + LessThan = X86Encoding::ConditionL, + LessThanOrEqual = X86Encoding::ConditionLE, + Overflow = X86Encoding::ConditionO, + CarrySet = X86Encoding::ConditionC, + CarryClear = X86Encoding::ConditionNC, + Signed = X86Encoding::ConditionS, + NotSigned = X86Encoding::ConditionNS, + Zero = X86Encoding::ConditionE, + NonZero = X86Encoding::ConditionNE, + Parity = X86Encoding::ConditionP, + NoParity = X86Encoding::ConditionNP + }; + + // If this bit is set, the vucomisd operands have to be inverted. + static const int DoubleConditionBitInvert = 0x10; + + // Bit set when a DoubleCondition does not map to a single x86 condition. + // The macro assembler has to special-case these conditions. + static const int DoubleConditionBitSpecial = 0x20; + static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial; + + enum DoubleCondition { + // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN. + DoubleOrdered = NoParity, + DoubleEqual = Equal | DoubleConditionBitSpecial, + DoubleNotEqual = NotEqual, + DoubleGreaterThan = Above, + DoubleGreaterThanOrEqual = AboveOrEqual, + DoubleLessThan = Above | DoubleConditionBitInvert, + DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert, + // If either operand is NaN, these conditions always evaluate to true. + DoubleUnordered = Parity, + DoubleEqualOrUnordered = Equal, + DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial, + DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert, + DoubleGreaterThanOrEqualOrUnordered = BelowOrEqual | DoubleConditionBitInvert, + DoubleLessThanOrUnordered = Below, + DoubleLessThanOrEqualOrUnordered = BelowOrEqual + }; + + enum NaNCond { + NaN_HandledByCond, + NaN_IsTrue, + NaN_IsFalse + }; + + // If the primary condition returned by ConditionFromDoubleCondition doesn't + // handle NaNs properly, return NaN_IsFalse if the comparison should be + // overridden to return false on NaN, NaN_IsTrue if it should be overridden + // to return true on NaN, or NaN_HandledByCond if no secondary check is + // needed. + static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) { + switch (cond) { + case DoubleOrdered: + case DoubleNotEqual: + case DoubleGreaterThan: + case DoubleGreaterThanOrEqual: + case DoubleLessThan: + case DoubleLessThanOrEqual: + case DoubleUnordered: + case DoubleEqualOrUnordered: + case DoubleGreaterThanOrUnordered: + case DoubleGreaterThanOrEqualOrUnordered: + case DoubleLessThanOrUnordered: + case DoubleLessThanOrEqualOrUnordered: + return NaN_HandledByCond; + case DoubleEqual: + return NaN_IsFalse; + case DoubleNotEqualOrUnordered: + return NaN_IsTrue; + } + + MOZ_CRASH("Unknown double condition"); + } + + static void StaticAsserts() { + // DoubleConditionBits should not interfere with x86 condition codes. + JS_STATIC_ASSERT(!((Equal | NotEqual | Above | AboveOrEqual | Below | + BelowOrEqual | Parity | NoParity) & DoubleConditionBits)); + } + + static Condition InvertCondition(Condition cond); + static Condition UnsignedCondition(Condition cond); + static Condition ConditionWithoutEqual(Condition cond); + + // Return the primary condition to test. Some primary conditions may not + // handle NaNs properly and may therefore require a secondary condition. + // Use NaNCondFromDoubleCondition to determine what else is needed. + static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) { + return static_cast<Condition>(cond & ~DoubleConditionBits); + } + + static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader); + + // MacroAssemblers hold onto gcthings, so they are traced by the GC. + void trace(JSTracer* trc); + + bool oom() const { + return AssemblerShared::oom() || + masm.oom() || + jumpRelocations_.oom() || + dataRelocations_.oom() || + preBarriers_.oom(); + } + + void setPrinter(Sprinter* sp) { + masm.setPrinter(sp); + } + + static const Register getStackPointer() { + return StackPointer; + } + + void executableCopy(void* buffer); + bool asmMergeWith(const AssemblerX86Shared& other) { + MOZ_ASSERT(other.jumps_.length() == 0); + if (!AssemblerShared::asmMergeWith(masm.size(), other)) + return false; + return masm.appendBuffer(other.masm); + } + void processCodeLabels(uint8_t* rawCode); + void copyJumpRelocationTable(uint8_t* dest); + void copyDataRelocationTable(uint8_t* dest); + void copyPreBarrierTable(uint8_t* dest); + + // Size of the instruction stream, in bytes. + size_t size() const { + return masm.size(); + } + // Size of the jump relocation table, in bytes. + size_t jumpRelocationTableBytes() const { + return jumpRelocations_.length(); + } + size_t dataRelocationTableBytes() const { + return dataRelocations_.length(); + } + size_t preBarrierTableBytes() const { + return preBarriers_.length(); + } + // Size of the data table, in bytes. + size_t bytesNeeded() const { + return size() + + jumpRelocationTableBytes() + + dataRelocationTableBytes() + + preBarrierTableBytes(); + } + + public: + void haltingAlign(int alignment) { + masm.haltingAlign(alignment); + } + void nopAlign(int alignment) { + masm.nopAlign(alignment); + } + void writeCodePointer(CodeOffset* label) { + // A CodeOffset only has one use, bake in the "end of list" value. + masm.jumpTablePointer(LabelBase::INVALID_OFFSET); + label->bind(masm.size()); + } + void cmovz(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.cmovz_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.cmovz_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.cmovz_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movl(Imm32 imm32, Register dest) { + masm.movl_i32r(imm32.value, dest.encoding()); + } + void movl(Register src, Register dest) { + masm.movl_rr(src.encoding(), dest.encoding()); + } + void movl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.movl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.movl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.movl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.movl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movl(Imm32 imm32, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.movl_i32r(imm32.value, dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.movl_i32m(imm32.value, dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + case Operand::MEM_ADDRESS32: + masm.movl_i32m(imm32.value, dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void xchgl(Register src, Register dest) { + masm.xchgl_rr(src.encoding(), dest.encoding()); + } + + // Eventually vmovapd should be overloaded to support loads and + // stores too. + void vmovapd(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovapd_rr(src.encoding(), dest.encoding()); + } + + void vmovaps(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovaps_rr(src.encoding(), dest.encoding()); + } + void vmovaps(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovaps_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + case Operand::FPREG: + masm.vmovaps_rr(src.fpu(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovaps(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovups(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovups(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovups_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + // vmovsd is only provided in load/store form since the + // register-to-register form has different semantics (it doesn't clobber + // the whole output register) and isn't needed currently. + void vmovsd(const Address& src, FloatRegister dest) { + masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding()); + } + void vmovsd(const BaseIndex& src, FloatRegister dest) { + masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); + } + void vmovsd(FloatRegister src, const Address& dest) { + masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding()); + } + void vmovsd(FloatRegister src, const BaseIndex& dest) { + masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); + } + // Although vmovss is not only provided in load/store form (for the same + // reasons as vmovsd above), the register to register form should be only + // used in contexts where we care about not clearing the higher lanes of + // the FloatRegister. + void vmovss(const Address& src, FloatRegister dest) { + masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding()); + } + void vmovss(const BaseIndex& src, FloatRegister dest) { + masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); + } + void vmovss(FloatRegister src, const Address& dest) { + masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding()); + } + void vmovss(FloatRegister src, const BaseIndex& dest) { + masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); + } + void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vmovdqu(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovdqu(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovdqa(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::FPREG: + masm.vmovdqa_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovdqa(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovdqa(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovdqa_rr(src.encoding(), dest.encoding()); + } + void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void movzbl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movzbl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movsbl(Register src, Register dest) { + masm.movsbl_rr(src.encoding(), dest.encoding()); + } + void movsbl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movsbl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movb(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movb_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movb(Imm32 src, Register dest) { + masm.movb_ir(src.value & 255, dest.encoding()); + } + void movb(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movb_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movb(Imm32 src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movb_im(src.value, dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movzwl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.movzwl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.movzwl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movzwl(Register src, Register dest) { + masm.movzwl_rr(src.encoding(), dest.encoding()); + } + void movw(const Operand& src, Register dest) { + masm.prefix_16_for_32(); + movl(src, dest); + } + void movw(Imm32 src, Register dest) { + masm.prefix_16_for_32(); + movl(src, dest); + } + void movw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movw(Imm32 src, const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.movw_im(src.value, dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void movswl(Register src, Register dest) { + masm.movswl_rr(src.encoding(), dest.encoding()); + } + void movswl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.movswl_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void leal(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.leal_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + protected: + void jSrc(Condition cond, Label* label) { + if (label->bound()) { + // The jump can be immediately encoded to the correct destination. + masm.jCC_i(static_cast<X86Encoding::Condition>(cond), JmpDst(label->offset())); + } else { + // Thread the jump list through the unpatched jump targets. + JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond)); + JmpSrc prev = JmpSrc(label->use(j.offset())); + masm.setNextJump(j, prev); + } + } + void jmpSrc(Label* label) { + if (label->bound()) { + // The jump can be immediately encoded to the correct destination. + masm.jmp_i(JmpDst(label->offset())); + } else { + // Thread the jump list through the unpatched jump targets. + JmpSrc j = masm.jmp(); + JmpSrc prev = JmpSrc(label->use(j.offset())); + masm.setNextJump(j, prev); + } + } + + // Comparison of EAX against the address given by a Label. + JmpSrc cmpSrc(Label* label) { + JmpSrc j = masm.cmp_eax(); + if (label->bound()) { + // The jump can be immediately patched to the correct destination. + masm.linkJump(j, JmpDst(label->offset())); + } else { + // Thread the jump list through the unpatched jump targets. + JmpSrc prev = JmpSrc(label->use(j.offset())); + masm.setNextJump(j, prev); + } + return j; + } + + JmpSrc jSrc(Condition cond, RepatchLabel* label) { + JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond)); + if (label->bound()) { + // The jump can be immediately patched to the correct destination. + masm.linkJump(j, JmpDst(label->offset())); + } else { + label->use(j.offset()); + } + return j; + } + JmpSrc jmpSrc(RepatchLabel* label) { + JmpSrc j = masm.jmp(); + if (label->bound()) { + // The jump can be immediately patched to the correct destination. + masm.linkJump(j, JmpDst(label->offset())); + } else { + // Thread the jump list through the unpatched jump targets. + label->use(j.offset()); + } + return j; + } + + public: + void nop() { masm.nop(); } + void nop(size_t n) { masm.insert_nop(n); } + void j(Condition cond, Label* label) { jSrc(cond, label); } + void jmp(Label* label) { jmpSrc(label); } + void j(Condition cond, RepatchLabel* label) { jSrc(cond, label); } + void jmp(RepatchLabel* label) { jmpSrc(label); } + + void j(Condition cond, wasm::TrapDesc target) { + Label l; + j(cond, &l); + bindLater(&l, target); + } + void jmp(wasm::TrapDesc target) { + Label l; + jmp(&l); + bindLater(&l, target); + } + + void jmp(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.jmp_m(op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.jmp_m(op.disp(), op.base(), op.index(), op.scale()); + break; + case Operand::REG: + masm.jmp_r(op.reg()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void cmpEAX(Label* label) { cmpSrc(label); } + void bind(Label* label) { + JmpDst dst(masm.label()); + if (label->used()) { + bool more; + JmpSrc jmp(label->offset()); + do { + JmpSrc next; + more = masm.nextJump(jmp, &next); + masm.linkJump(jmp, dst); + jmp = next; + } while (more); + } + label->bind(dst.offset()); + } + void bindLater(Label* label, wasm::TrapDesc target) { + if (label->used()) { + JmpSrc jmp(label->offset()); + do { + append(wasm::TrapSite(target, jmp.offset())); + } while (masm.nextJump(jmp, &jmp)); + } + label->reset(); + } + void bind(RepatchLabel* label) { + JmpDst dst(masm.label()); + if (label->used()) { + JmpSrc jmp(label->offset()); + masm.linkJump(jmp, dst); + } + label->bind(dst.offset()); + } + void use(CodeOffset* label) { + label->bind(currentOffset()); + } + uint32_t currentOffset() { + return masm.label().offset(); + } + + // Re-routes pending jumps to a new label. + void retarget(Label* label, Label* target) { + if (!label->used()) + return; + bool more; + JmpSrc jmp(label->offset()); + do { + JmpSrc next; + more = masm.nextJump(jmp, &next); + if (target->bound()) { + // The jump can be immediately patched to the correct destination. + masm.linkJump(jmp, JmpDst(target->offset())); + } else { + // Thread the jump list through the unpatched jump targets. + JmpSrc prev(target->use(jmp.offset())); + masm.setNextJump(jmp, prev); + } + jmp = JmpSrc(next.offset()); + } while (more); + label->reset(); + } + + static void Bind(uint8_t* raw, CodeOffset* label, const void* address) { + if (label->bound()) { + intptr_t offset = label->offset(); + X86Encoding::SetPointer(raw + offset, address); + } + } + + // See Bind and X86Encoding::setPointer. + size_t labelToPatchOffset(CodeOffset label) { + return label.offset() - sizeof(void*); + } + + void ret() { + masm.ret(); + } + void retn(Imm32 n) { + // Remove the size of the return address which is included in the frame. + masm.ret_i(n.value - sizeof(void*)); + } + CodeOffset call(Label* label) { + if (label->bound()) { + masm.linkJump(masm.call(), JmpDst(label->offset())); + } else { + JmpSrc j = masm.call(); + JmpSrc prev = JmpSrc(label->use(j.offset())); + masm.setNextJump(j, prev); + } + return CodeOffset(masm.currentOffset()); + } + CodeOffset call(Register reg) { + masm.call_r(reg.encoding()); + return CodeOffset(masm.currentOffset()); + } + void call(const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.call_r(op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.call_m(op.disp(), op.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + CodeOffset callWithPatch() { + return CodeOffset(masm.call().offset()); + } + + struct AutoPrepareForPatching : X86Encoding::AutoUnprotectAssemblerBufferRegion { + explicit AutoPrepareForPatching(AssemblerX86Shared& masm) + : X86Encoding::AutoUnprotectAssemblerBufferRegion(masm.masm, 0, masm.size()) + {} + }; + + void patchCall(uint32_t callerOffset, uint32_t calleeOffset) { + // The caller uses AutoUnprotectBuffer. + unsigned char* code = masm.data(); + X86Encoding::SetRel32(code + callerOffset, code + calleeOffset); + } + CodeOffset farJumpWithPatch() { + return CodeOffset(masm.jmp().offset()); + } + void patchFarJump(CodeOffset farJump, uint32_t targetOffset) { + // The caller uses AutoUnprotectBuffer. + unsigned char* code = masm.data(); + X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset); + } + static void repatchFarJump(uint8_t* code, uint32_t farJumpOffset, uint32_t targetOffset) { + X86Encoding::SetRel32(code + farJumpOffset, code + targetOffset); + } + + CodeOffset twoByteNop() { + return CodeOffset(masm.twoByteNop().offset()); + } + static void patchTwoByteNopToJump(uint8_t* jump, uint8_t* target) { + X86Encoding::BaseAssembler::patchTwoByteNopToJump(jump, target); + } + static void patchJumpToTwoByteNop(uint8_t* jump) { + X86Encoding::BaseAssembler::patchJumpToTwoByteNop(jump); + } + + void breakpoint() { + masm.int3(); + } + + static bool HasSSE2() { return CPUInfo::IsSSE2Present(); } + static bool HasSSE3() { return CPUInfo::IsSSE3Present(); } + static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); } + static bool HasSSE41() { return CPUInfo::IsSSE41Present(); } + static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); } + static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); } + static bool SupportsUnalignedAccesses() { return true; } + static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); } + static bool HasAVX() { return CPUInfo::IsAVXPresent(); } + + void cmpl(Register rhs, Register lhs) { + masm.cmpl_rr(rhs.encoding(), lhs.encoding()); + } + void cmpl(const Operand& rhs, Register lhs) { + switch (rhs.kind()) { + case Operand::REG: + masm.cmpl_rr(rhs.reg(), lhs.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.cmpl_mr(rhs.address(), lhs.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void cmpl(Register rhs, const Operand& lhs) { + switch (lhs.kind()) { + case Operand::REG: + masm.cmpl_rr(rhs.encoding(), lhs.reg()); + break; + case Operand::MEM_REG_DISP: + masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base()); + break; + case Operand::MEM_ADDRESS32: + masm.cmpl_rm(rhs.encoding(), lhs.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void cmpl(Imm32 rhs, Register lhs) { + masm.cmpl_ir(rhs.value, lhs.encoding()); + } + void cmpl(Imm32 rhs, const Operand& lhs) { + switch (lhs.kind()) { + case Operand::REG: + masm.cmpl_ir(rhs.value, lhs.reg()); + break; + case Operand::MEM_REG_DISP: + masm.cmpl_im(rhs.value, lhs.disp(), lhs.base()); + break; + case Operand::MEM_SCALE: + masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), lhs.scale()); + break; + case Operand::MEM_ADDRESS32: + masm.cmpl_im(rhs.value, lhs.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + CodeOffset cmplWithPatch(Imm32 rhs, Register lhs) { + masm.cmpl_i32r(rhs.value, lhs.encoding()); + return CodeOffset(masm.currentOffset()); + } + void cmpw(Register rhs, Register lhs) { + masm.cmpw_rr(rhs.encoding(), lhs.encoding()); + } + void setCC(Condition cond, Register r) { + masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding()); + } + void testb(Register rhs, Register lhs) { + MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs)); + MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs)); + masm.testb_rr(rhs.encoding(), lhs.encoding()); + } + void testw(Register rhs, Register lhs) { + masm.testw_rr(lhs.encoding(), rhs.encoding()); + } + void testl(Register rhs, Register lhs) { + masm.testl_rr(lhs.encoding(), rhs.encoding()); + } + void testl(Imm32 rhs, Register lhs) { + masm.testl_ir(rhs.value, lhs.encoding()); + } + void testl(Imm32 rhs, const Operand& lhs) { + switch (lhs.kind()) { + case Operand::REG: + masm.testl_ir(rhs.value, lhs.reg()); + break; + case Operand::MEM_REG_DISP: + masm.testl_i32m(rhs.value, lhs.disp(), lhs.base()); + break; + case Operand::MEM_ADDRESS32: + masm.testl_i32m(rhs.value, lhs.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + void addl(Imm32 imm, Register dest) { + masm.addl_ir(imm.value, dest.encoding()); + } + CodeOffset addlWithPatch(Imm32 imm, Register dest) { + masm.addl_i32r(imm.value, dest.encoding()); + return CodeOffset(masm.currentOffset()); + } + void addl(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.addl_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.addl_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_ADDRESS32: + masm.addl_im(imm.value, op.address()); + break; + case Operand::MEM_SCALE: + masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void addw(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.addw_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.addw_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_ADDRESS32: + masm.addw_im(imm.value, op.address()); + break; + case Operand::MEM_SCALE: + masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void subl(Imm32 imm, Register dest) { + masm.subl_ir(imm.value, dest.encoding()); + } + void subl(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.subl_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.subl_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void subw(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.subw_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.subw_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void addl(Register src, Register dest) { + masm.addl_rr(src.encoding(), dest.encoding()); + } + void addl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.addl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.addl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void addw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.addw_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.addw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void subl(Register src, Register dest) { + masm.subl_rr(src.encoding(), dest.encoding()); + } + void subl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.subl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.subl_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void subl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.subl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.subl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void subw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.subw_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.subw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void orl(Register reg, Register dest) { + masm.orl_rr(reg.encoding(), dest.encoding()); + } + void orl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.orl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.orl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void orw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.orw_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.orw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void orl(Imm32 imm, Register reg) { + masm.orl_ir(imm.value, reg.encoding()); + } + void orl(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.orl_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.orl_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void orw(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.orw_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.orw_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xorl(Register src, Register dest) { + masm.xorl_rr(src.encoding(), dest.encoding()); + } + void xorl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.xorl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.xorl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xorw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.xorw_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.xorw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xorl(Imm32 imm, Register reg) { + masm.xorl_ir(imm.value, reg.encoding()); + } + void xorl(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.xorl_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.xorl_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xorw(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.xorw_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.xorw_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void andl(Register src, Register dest) { + masm.andl_rr(src.encoding(), dest.encoding()); + } + void andl(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.andl_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.andl_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void andw(Register src, const Operand& dest) { + switch (dest.kind()) { + case Operand::REG: + masm.andw_rr(src.encoding(), dest.reg()); + break; + case Operand::MEM_REG_DISP: + masm.andw_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void andl(Imm32 imm, Register dest) { + masm.andl_ir(imm.value, dest.encoding()); + } + void andl(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.andl_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.andl_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void andw(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::REG: + masm.andw_ir(imm.value, op.reg()); + break; + case Operand::MEM_REG_DISP: + masm.andw_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void addl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.addl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.addl_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void orl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.orl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.orl_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xorl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.xorl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.xorl_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void andl(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.andl_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.andl_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void bsrl(const Register& src, const Register& dest) { + masm.bsrl_rr(src.encoding(), dest.encoding()); + } + void bsfl(const Register& src, const Register& dest) { + masm.bsfl_rr(src.encoding(), dest.encoding()); + } + void popcntl(const Register& src, const Register& dest) { + masm.popcntl_rr(src.encoding(), dest.encoding()); + } + void imull(Register multiplier) { + masm.imull_r(multiplier.encoding()); + } + void umull(Register multiplier) { + masm.mull_r(multiplier.encoding()); + } + void imull(Imm32 imm, Register dest) { + masm.imull_ir(imm.value, dest.encoding(), dest.encoding()); + } + void imull(Register src, Register dest) { + masm.imull_rr(src.encoding(), dest.encoding()); + } + void imull(Imm32 imm, Register src, Register dest) { + masm.imull_ir(imm.value, src.encoding(), dest.encoding()); + } + void imull(const Operand& src, Register dest) { + switch (src.kind()) { + case Operand::REG: + masm.imull_rr(src.reg(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.imull_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void negl(const Operand& src) { + switch (src.kind()) { + case Operand::REG: + masm.negl_r(src.reg()); + break; + case Operand::MEM_REG_DISP: + masm.negl_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void negl(Register reg) { + masm.negl_r(reg.encoding()); + } + void notl(const Operand& src) { + switch (src.kind()) { + case Operand::REG: + masm.notl_r(src.reg()); + break; + case Operand::MEM_REG_DISP: + masm.notl_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void notl(Register reg) { + masm.notl_r(reg.encoding()); + } + void shrl(const Imm32 imm, Register dest) { + masm.shrl_ir(imm.value, dest.encoding()); + } + void shll(const Imm32 imm, Register dest) { + masm.shll_ir(imm.value, dest.encoding()); + } + void sarl(const Imm32 imm, Register dest) { + masm.sarl_ir(imm.value, dest.encoding()); + } + void shrl_cl(Register dest) { + masm.shrl_CLr(dest.encoding()); + } + void shll_cl(Register dest) { + masm.shll_CLr(dest.encoding()); + } + void sarl_cl(Register dest) { + masm.sarl_CLr(dest.encoding()); + } + void shrdl_cl(Register src, Register dest) { + masm.shrdl_CLr(src.encoding(), dest.encoding()); + } + void shldl_cl(Register src, Register dest) { + masm.shldl_CLr(src.encoding(), dest.encoding()); + } + + void roll(const Imm32 imm, Register dest) { + masm.roll_ir(imm.value, dest.encoding()); + } + void roll_cl(Register dest) { + masm.roll_CLr(dest.encoding()); + } + void rorl(const Imm32 imm, Register dest) { + masm.rorl_ir(imm.value, dest.encoding()); + } + void rorl_cl(Register dest) { + masm.rorl_CLr(dest.encoding()); + } + + void incl(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.incl_m32(op.disp(), op.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void lock_incl(const Operand& op) { + masm.prefix_lock(); + incl(op); + } + + void decl(const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.decl_m32(op.disp(), op.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void lock_decl(const Operand& op) { + masm.prefix_lock(); + decl(op); + } + + void addb(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.addb_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + void addb(Register src, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.addb_rm(src.encoding(), op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + void subb(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.subb_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + void subb(Register src, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.subb_rm(src.encoding(), op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + void andb(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.andb_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + void andb(Register src, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.andb_rm(src.encoding(), op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + void orb(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.orb_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + void orb(Register src, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.orb_rm(src.encoding(), op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + void xorb(Imm32 imm, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.xorb_im(imm.value, op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + void xorb(Register src, const Operand& op) { + switch (op.kind()) { + case Operand::MEM_REG_DISP: + masm.xorb_rm(src.encoding(), op.disp(), op.base()); + break; + case Operand::MEM_SCALE: + masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + break; + } + } + + template<typename T> + void lock_addb(T src, const Operand& op) { + masm.prefix_lock(); + addb(src, op); + } + template<typename T> + void lock_subb(T src, const Operand& op) { + masm.prefix_lock(); + subb(src, op); + } + template<typename T> + void lock_andb(T src, const Operand& op) { + masm.prefix_lock(); + andb(src, op); + } + template<typename T> + void lock_orb(T src, const Operand& op) { + masm.prefix_lock(); + orb(src, op); + } + template<typename T> + void lock_xorb(T src, const Operand& op) { + masm.prefix_lock(); + xorb(src, op); + } + + template<typename T> + void lock_addw(T src, const Operand& op) { + masm.prefix_lock(); + addw(src, op); + } + template<typename T> + void lock_subw(T src, const Operand& op) { + masm.prefix_lock(); + subw(src, op); + } + template<typename T> + void lock_andw(T src, const Operand& op) { + masm.prefix_lock(); + andw(src, op); + } + template<typename T> + void lock_orw(T src, const Operand& op) { + masm.prefix_lock(); + orw(src, op); + } + template<typename T> + void lock_xorw(T src, const Operand& op) { + masm.prefix_lock(); + xorw(src, op); + } + + // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems, + // among other things. Do not optimize, replace by XADDL, or similar. + template<typename T> + void lock_addl(T src, const Operand& op) { + masm.prefix_lock(); + addl(src, op); + } + template<typename T> + void lock_subl(T src, const Operand& op) { + masm.prefix_lock(); + subl(src, op); + } + template<typename T> + void lock_andl(T src, const Operand& op) { + masm.prefix_lock(); + andl(src, op); + } + template<typename T> + void lock_orl(T src, const Operand& op) { + masm.prefix_lock(); + orl(src, op); + } + template<typename T> + void lock_xorl(T src, const Operand& op) { + masm.prefix_lock(); + xorl(src, op); + } + + void lock_cmpxchgb(Register src, const Operand& mem) { + masm.prefix_lock(); + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.cmpxchgb(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void lock_cmpxchgw(Register src, const Operand& mem) { + masm.prefix_lock(); + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.cmpxchgw(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void lock_cmpxchgl(Register src, const Operand& mem) { + masm.prefix_lock(); + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.cmpxchgl(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void xchgb(Register src, const Operand& mem) { + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.xchgb_rm(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xchgw(Register src, const Operand& mem) { + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.xchgw_rm(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void xchgl(Register src, const Operand& mem) { + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.xchgl_rm(src.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void lock_xaddb(Register srcdest, const Operand& mem) { + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void lock_xaddw(Register srcdest, const Operand& mem) { + masm.prefix_16_for_32(); + lock_xaddl(srcdest, mem); + } + void lock_xaddl(Register srcdest, const Operand& mem) { + switch (mem.kind()) { + case Operand::MEM_REG_DISP: + masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base()); + break; + case Operand::MEM_SCALE: + masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void push(const Imm32 imm) { + masm.push_i(imm.value); + } + + void push(const Operand& src) { + switch (src.kind()) { + case Operand::REG: + masm.push_r(src.reg()); + break; + case Operand::MEM_REG_DISP: + masm.push_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void push(Register src) { + masm.push_r(src.encoding()); + } + void push(const Address& src) { + masm.push_m(src.offset, src.base.encoding()); + } + + void pop(const Operand& src) { + switch (src.kind()) { + case Operand::REG: + masm.pop_r(src.reg()); + break; + case Operand::MEM_REG_DISP: + masm.pop_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void pop(Register src) { + masm.pop_r(src.encoding()); + } + void pop(const Address& src) { + masm.pop_m(src.offset, src.base.encoding()); + } + + void pushFlags() { + masm.push_flags(); + } + void popFlags() { + masm.pop_flags(); + } + +#ifdef JS_CODEGEN_X86 + void pushAllRegs() { + masm.pusha(); + } + void popAllRegs() { + masm.popa(); + } +#endif + + // Zero-extend byte to 32-bit integer. + void movzbl(Register src, Register dest) { + masm.movzbl_rr(src.encoding(), dest.encoding()); + } + + void cdq() { + masm.cdq(); + } + void idiv(Register divisor) { + masm.idivl_r(divisor.encoding()); + } + void udiv(Register divisor) { + masm.divl_r(divisor.encoding()); + } + + void vpinsrb(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vpinsrb_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpinsrw(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { + masm.vpinsrw_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); + } + + void vpinsrd(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); + } + + void vpextrb(unsigned lane, FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE41()); + masm.vpextrb_irr(lane, src.encoding(), dest.encoding()); + } + void vpextrw(unsigned lane, FloatRegister src, Register dest) { + masm.vpextrw_irr(lane, src.encoding(), dest.encoding()); + } + void vpextrd(unsigned lane, FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE41()); + masm.vpextrd_irr(lane, src.encoding(), dest.encoding()); + } + void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding()); + } + void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding()); + } + void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding()); + } + void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpslld_ir(count.value, src0.encoding(), dest.encoding()); + } + void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding()); + } + void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding()); + } + + void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding()); + } + void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding()); + } + void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding()); + } + + void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::REG: + masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vcvttsd2si(FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvttsd2si_rr(src.encoding(), dest.encoding()); + } + void vcvttss2si(FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvttss2si_rr(src.encoding(), dest.encoding()); + } + void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::REG: + masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vcvttps2dq(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvttps2dq_rr(src.encoding(), dest.encoding()); + } + void vcvtdq2ps(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vcvtdq2ps_rr(src.encoding(), dest.encoding()); + } + void vmovmskpd(FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovmskpd_rr(src.encoding(), dest.encoding()); + } + void vmovmskps(FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovmskps_rr(src.encoding(), dest.encoding()); + } + void vptest(FloatRegister rhs, FloatRegister lhs) { + MOZ_ASSERT(HasSSE41()); + masm.vptest_rr(rhs.encoding(), lhs.encoding()); + } + void vucomisd(FloatRegister rhs, FloatRegister lhs) { + MOZ_ASSERT(HasSSE2()); + masm.vucomisd_rr(rhs.encoding(), lhs.encoding()); + } + void vucomiss(FloatRegister rhs, FloatRegister lhs) { + MOZ_ASSERT(HasSSE2()); + masm.vucomiss_rr(rhs.encoding(), lhs.encoding()); + } + + void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (rhs.kind()) { + case Operand::FPREG: + masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void vcmpps(uint8_t order, Operand src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + // :TODO: (Bug 1132894) See LIRGeneratorX86Shared::lowerForFPU + // FIXME: This logic belongs in the MacroAssembler. + if (!HasAVX() && !src0.aliases(dest)) { + if (src1.kind() == Operand::FPREG && + dest.aliases(FloatRegister::FromCode(src1.fpu()))) + { + vmovdqa(src1, ScratchSimd128Reg); + src1 = Operand(ScratchSimd128Reg); + } + vmovdqa(src0, dest); + src0 = dest; + } + switch (src1.kind()) { + case Operand::FPREG: + masm.vcmpps_rr(order, src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vcmpps_mr(order, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vcmpps_mr(order, src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vcmpeqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_EQ, src1, src0, dest); + } + void vcmpltps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_LT, src1, src0, dest); + } + void vcmpleps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_LE, src1, src0, dest); + } + void vcmpunordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_UNORD, src1, src0, dest); + } + void vcmpneqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_NEQ, src1, src0, dest); + } + void vcmpordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + vcmpps(X86Encoding::ConditionCmp_ORD, src1, src0, dest); + } + void vrcpps(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::FPREG: + masm.vrcpps_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vrcpps_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vrcpps_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vsqrtps(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::FPREG: + masm.vsqrtps_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vsqrtps_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vrsqrtps(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::FPREG: + masm.vrsqrtps_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vrsqrtps_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovd(Register src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_rr(src.encoding(), dest.encoding()); + } + void vmovd(FloatRegister src, Register dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovd_rr(src.encoding(), dest.encoding()); + } + void vmovd(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovd(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovd_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_rm(src.encoding(), dest.address()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovq(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_mr(src.disp(), src.base(), dest.encoding()); + break; + case Operand::MEM_SCALE: + masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmovq_mr(src.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovq(FloatRegister src, const Operand& dest) { + MOZ_ASSERT(HasSSE2()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.vmovq_rm(src.encoding(), dest.disp(), dest.base()); + break; + case Operand::MEM_SCALE: + masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + // Negates bits of dest and then applies AND + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpshufd_irr(mask, src.encoding(), dest.encoding()); + } + void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vpshufd_irr(mask, src1.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vpshufd_imr(mask, src1.address(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpshuflw_irr(mask, src.encoding(), dest.encoding()); + } + void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vpshufhw_irr(mask, src.encoding(), dest.encoding()); + } + void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSSE3()); + masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding()); + } + void vmovddup(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE3()); + masm.vmovddup_rr(src.encoding(), dest.encoding()); + } + void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); + } + void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vshufps_imr(mask, src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_ADDRESS32: + masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); + } + void vroundss(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vroundss_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); + } + unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, unsigned zeroMask = 0) + { + // Note that the sourceLane bits are ignored in the case of a source + // memory operand, and the source is the given 32-bits memory location. + MOZ_ASSERT(zeroMask < 16); + unsigned ret = zeroMask ; + ret |= destLane << 4; + ret |= sourceLane << 6; + MOZ_ASSERT(ret < 256); + return ret; + } + void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); + } + void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + unsigned blendpsMask(bool x, bool y, bool z, bool w) { + return (x << 0) | (y << 1) | (z << 2) | (w << 3); + } + void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); + } + void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), dest.encoding()); + } + void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE41()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovsldup(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE3()); + masm.vmovsldup_rr(src.encoding(), dest.encoding()); + } + void vmovsldup(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE3()); + switch (src.kind()) { + case Operand::FPREG: + masm.vmovsldup_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmovshdup(FloatRegister src, FloatRegister dest) { + MOZ_ASSERT(HasSSE3()); + masm.vmovshdup_rr(src.encoding(), dest.encoding()); + } + void vmovshdup(const Operand& src, FloatRegister dest) { + MOZ_ASSERT(HasSSE3()); + switch (src.kind()) { + case Operand::FPREG: + masm.vmovshdup_rr(src.fpu(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + switch (src1.kind()) { + case Operand::FPREG: + masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); + break; + case Operand::MEM_REG_DISP: + masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { + MOZ_ASSERT(HasSSE2()); + masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding()); + } + void fisttp(const Operand& dest) { + MOZ_ASSERT(HasSSE3()); + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fisttp_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fistp(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fistp_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fnstcw(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fnstcw_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fldcw(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fldcw_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fnstsw(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fnstsw_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fld(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fld_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fld32(const Operand& dest) { + switch (dest.kind()) { + case Operand::MEM_REG_DISP: + masm.fld32_m(dest.disp(), dest.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fstp(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fstp_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + void fstp32(const Operand& src) { + switch (src.kind()) { + case Operand::MEM_REG_DISP: + masm.fstp32_m(src.disp(), src.base()); + break; + default: + MOZ_CRASH("unexpected operand kind"); + } + } + + // Defined for compatibility with ARM's assembler + uint32_t actualIndex(uint32_t x) { + return x; + } + + void flushBuffer() { + } + + // Patching. + + static size_t PatchWrite_NearCallSize() { + return 5; + } + static uintptr_t GetPointer(uint8_t* instPtr) { + uintptr_t* ptr = ((uintptr_t*) instPtr) - 1; + return *ptr; + } + // Write a relative call at the start location |dataLabel|. + // Note that this DOES NOT patch data that comes before |label|. + static void PatchWrite_NearCall(CodeLocationLabel startLabel, CodeLocationLabel target) { + uint8_t* start = startLabel.raw(); + *start = 0xE8; + ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize(); + MOZ_ASSERT(int32_t(offset) == offset); + *((int32_t*) (start + 1)) = offset; + } + + static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) { + *((int32_t*) dataLabel.raw() - 1) = toWrite.value; + } + + static void PatchDataWithValueCheck(CodeLocationLabel data, PatchedImmPtr newData, + PatchedImmPtr expectedData) { + // The pointer given is a pointer to *after* the data. + uintptr_t* ptr = ((uintptr_t*) data.raw()) - 1; + MOZ_ASSERT(*ptr == (uintptr_t)expectedData.value); + *ptr = (uintptr_t)newData.value; + } + static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, ImmPtr expectedData) { + PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), PatchedImmPtr(expectedData.value)); + } + + static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm) { + MOZ_CRASH("Unused."); + } + + static uint32_t NopSize() { + return 1; + } + static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) { + MOZ_CRASH("nextInstruction NYI on x86"); + } + + // Toggle a jmp or cmp emitted by toggledJump(). + static void ToggleToJmp(CodeLocationLabel inst) { + uint8_t* ptr = (uint8_t*)inst.raw(); + MOZ_ASSERT(*ptr == 0x3D); + *ptr = 0xE9; + } + static void ToggleToCmp(CodeLocationLabel inst) { + uint8_t* ptr = (uint8_t*)inst.raw(); + MOZ_ASSERT(*ptr == 0xE9); + *ptr = 0x3D; + } + static void ToggleCall(CodeLocationLabel inst, bool enabled) { + uint8_t* ptr = (uint8_t*)inst.raw(); + MOZ_ASSERT(*ptr == 0x3D || // CMP + *ptr == 0xE8); // CALL + *ptr = enabled ? 0xE8 : 0x3D; + } + + MOZ_COLD void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, + const Disassembler::HeapAccess& heapAccess); +}; + +} // namespace jit +} // namespace js + +#endif /* jit_x86_shared_Assembler_x86_shared_h */ |