/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef jit_x86_shared_Assembler_x86_shared_h #define jit_x86_shared_Assembler_x86_shared_h #include #include "jit/shared/Assembler-shared.h" #if defined(JS_CODEGEN_X86) # include "jit/x86/BaseAssembler-x86.h" #elif defined(JS_CODEGEN_X64) # include "jit/x64/BaseAssembler-x64.h" #else # error "Unknown architecture!" #endif namespace js { namespace jit { struct ScratchFloat32Scope : public AutoFloatRegisterScope { explicit ScratchFloat32Scope(MacroAssembler& masm) : AutoFloatRegisterScope(masm, ScratchFloat32Reg) { } }; struct ScratchDoubleScope : public AutoFloatRegisterScope { explicit ScratchDoubleScope(MacroAssembler& masm) : AutoFloatRegisterScope(masm, ScratchDoubleReg) { } }; struct ScratchSimd128Scope : public AutoFloatRegisterScope { explicit ScratchSimd128Scope(MacroAssembler& masm) : AutoFloatRegisterScope(masm, ScratchSimd128Reg) { } }; class Operand { public: enum Kind { REG, MEM_REG_DISP, FPREG, MEM_SCALE, MEM_ADDRESS32 }; private: Kind kind_ : 4; // Used as a Register::Encoding and a FloatRegister::Encoding. uint32_t base_ : 5; Scale scale_ : 3; // We don't use all 8 bits, of course, but GCC complains if the size of // this field is smaller than the size of Register::Encoding. Register::Encoding index_ : 8; int32_t disp_; public: explicit Operand(Register reg) : kind_(REG), base_(reg.encoding()), scale_(TimesOne), index_(Registers::Invalid), disp_(0) { } explicit Operand(FloatRegister reg) : kind_(FPREG), base_(reg.encoding()), scale_(TimesOne), index_(Registers::Invalid), disp_(0) { } explicit Operand(const Address& address) : kind_(MEM_REG_DISP), base_(address.base.encoding()), scale_(TimesOne), index_(Registers::Invalid), disp_(address.offset) { } explicit Operand(const BaseIndex& address) : kind_(MEM_SCALE), base_(address.base.encoding()), scale_(address.scale), index_(address.index.encoding()), disp_(address.offset) { } Operand(Register base, Register index, Scale scale, int32_t disp = 0) : kind_(MEM_SCALE), base_(base.encoding()), scale_(scale), index_(index.encoding()), disp_(disp) { } Operand(Register reg, int32_t disp) : kind_(MEM_REG_DISP), base_(reg.encoding()), scale_(TimesOne), index_(Registers::Invalid), disp_(disp) { } explicit Operand(AbsoluteAddress address) : kind_(MEM_ADDRESS32), base_(Registers::Invalid), scale_(TimesOne), index_(Registers::Invalid), disp_(X86Encoding::AddressImmediate(address.addr)) { } explicit Operand(PatchedAbsoluteAddress address) : kind_(MEM_ADDRESS32), base_(Registers::Invalid), scale_(TimesOne), index_(Registers::Invalid), disp_(X86Encoding::AddressImmediate(address.addr)) { } Address toAddress() const { MOZ_ASSERT(kind() == MEM_REG_DISP); return Address(Register::FromCode(base()), disp()); } BaseIndex toBaseIndex() const { MOZ_ASSERT(kind() == MEM_SCALE); return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), scale(), disp()); } Kind kind() const { return kind_; } Register::Encoding reg() const { MOZ_ASSERT(kind() == REG); return Register::Encoding(base_); } Register::Encoding base() const { MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); return Register::Encoding(base_); } Register::Encoding index() const { MOZ_ASSERT(kind() == MEM_SCALE); return index_; } Scale scale() const { MOZ_ASSERT(kind() == MEM_SCALE); return scale_; } FloatRegister::Encoding fpu() const { MOZ_ASSERT(kind() == FPREG); return FloatRegister::Encoding(base_); } int32_t disp() const { MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); return disp_; } void* address() const { MOZ_ASSERT(kind() == MEM_ADDRESS32); return reinterpret_cast(disp_); } bool containsReg(Register r) const { switch (kind()) { case REG: return r.encoding() == reg(); case MEM_REG_DISP: return r.encoding() == base(); case MEM_SCALE: return r.encoding() == base() || r.encoding() == index(); default: return false; } } }; inline Imm32 Imm64::firstHalf() const { return low(); } inline Imm32 Imm64::secondHalf() const { return hi(); } class CPUInfo { public: // As the SSE's were introduced in order, the presence of a later SSE implies // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 support. enum SSEVersion { UnknownSSE = 0, NoSSE = 1, SSE = 2, SSE2 = 3, SSE3 = 4, SSSE3 = 5, SSE4_1 = 6, SSE4_2 = 7 }; static SSEVersion GetSSEVersion() { if (maxSSEVersion == UnknownSSE) SetSSEVersion(); MOZ_ASSERT(maxSSEVersion != UnknownSSE); MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, maxSSEVersion <= maxEnabledSSEVersion); return maxSSEVersion; } static bool IsAVXPresent() { if (MOZ_UNLIKELY(maxSSEVersion == UnknownSSE)) SetSSEVersion(); MOZ_ASSERT_IF(!avxEnabled, !avxPresent); return avxPresent; } private: static SSEVersion maxSSEVersion; static SSEVersion maxEnabledSSEVersion; static bool avxPresent; static bool avxEnabled; static bool popcntPresent; static bool needAmdBugWorkaround; static void SetSSEVersion(); public: static bool IsSSE2Present() { #ifdef JS_CODEGEN_X64 return true; #else return GetSSEVersion() >= SSE2; #endif } static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; } static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; } static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; } static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; } static bool IsPOPCNTPresent() { return popcntPresent; } static bool NeedAmdBugWorkaround() { return needAmdBugWorkaround; } static void SetSSE3Disabled() { maxEnabledSSEVersion = SSE2; avxEnabled = false; } static void SetSSE4Disabled() { maxEnabledSSEVersion = SSSE3; avxEnabled = false; } static void SetAVXEnabled() { avxEnabled = true; } }; class AssemblerX86Shared : public AssemblerShared { protected: struct RelativePatch { int32_t offset; void* target; Relocation::Kind kind; RelativePatch(int32_t offset, void* target, Relocation::Kind kind) : offset(offset), target(target), kind(kind) { } }; Vector jumps_; CompactBufferWriter jumpRelocations_; CompactBufferWriter dataRelocations_; CompactBufferWriter preBarriers_; void writeDataRelocation(ImmGCPtr ptr) { if (ptr.value) { if (gc::IsInsideNursery(ptr.value)) embedsNurseryPointers_ = true; dataRelocations_.writeUnsigned(masm.currentOffset()); } } void writePrebarrierOffset(CodeOffset label) { preBarriers_.writeUnsigned(label.offset()); } protected: X86Encoding::BaseAssemblerSpecific masm; typedef X86Encoding::JmpSrc JmpSrc; typedef X86Encoding::JmpDst JmpDst; public: AssemblerX86Shared() { if (!HasAVX()) masm.disableVEX(); } enum Condition { Equal = X86Encoding::ConditionE, NotEqual = X86Encoding::ConditionNE, Above = X86Encoding::ConditionA, AboveOrEqual = X86Encoding::ConditionAE, Below = X86Encoding::ConditionB, BelowOrEqual = X86Encoding::ConditionBE, GreaterThan = X86Encoding::ConditionG, GreaterThanOrEqual = X86Encoding::ConditionGE, LessThan = X86Encoding::ConditionL, LessThanOrEqual = X86Encoding::ConditionLE, Overflow = X86Encoding::ConditionO, CarrySet = X86Encoding::ConditionC, CarryClear = X86Encoding::ConditionNC, Signed = X86Encoding::ConditionS, NotSigned = X86Encoding::ConditionNS, Zero = X86Encoding::ConditionE, NonZero = X86Encoding::ConditionNE, Parity = X86Encoding::ConditionP, NoParity = X86Encoding::ConditionNP }; // If this bit is set, the vucomisd operands have to be inverted. static const int DoubleConditionBitInvert = 0x10; // Bit set when a DoubleCondition does not map to a single x86 condition. // The macro assembler has to special-case these conditions. static const int DoubleConditionBitSpecial = 0x20; static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial; enum DoubleCondition { // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN. DoubleOrdered = NoParity, DoubleEqual = Equal | DoubleConditionBitSpecial, DoubleNotEqual = NotEqual, DoubleGreaterThan = Above, DoubleGreaterThanOrEqual = AboveOrEqual, DoubleLessThan = Above | DoubleConditionBitInvert, DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert, // If either operand is NaN, these conditions always evaluate to true. DoubleUnordered = Parity, DoubleEqualOrUnordered = Equal, DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial, DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert, DoubleGreaterThanOrEqualOrUnordered = BelowOrEqual | DoubleConditionBitInvert, DoubleLessThanOrUnordered = Below, DoubleLessThanOrEqualOrUnordered = BelowOrEqual }; enum NaNCond { NaN_HandledByCond, NaN_IsTrue, NaN_IsFalse }; // If the primary condition returned by ConditionFromDoubleCondition doesn't // handle NaNs properly, return NaN_IsFalse if the comparison should be // overridden to return false on NaN, NaN_IsTrue if it should be overridden // to return true on NaN, or NaN_HandledByCond if no secondary check is // needed. static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) { switch (cond) { case DoubleOrdered: case DoubleNotEqual: case DoubleGreaterThan: case DoubleGreaterThanOrEqual: case DoubleLessThan: case DoubleLessThanOrEqual: case DoubleUnordered: case DoubleEqualOrUnordered: case DoubleGreaterThanOrUnordered: case DoubleGreaterThanOrEqualOrUnordered: case DoubleLessThanOrUnordered: case DoubleLessThanOrEqualOrUnordered: return NaN_HandledByCond; case DoubleEqual: return NaN_IsFalse; case DoubleNotEqualOrUnordered: return NaN_IsTrue; } MOZ_CRASH("Unknown double condition"); } static void StaticAsserts() { // DoubleConditionBits should not interfere with x86 condition codes. JS_STATIC_ASSERT(!((Equal | NotEqual | Above | AboveOrEqual | Below | BelowOrEqual | Parity | NoParity) & DoubleConditionBits)); } static Condition InvertCondition(Condition cond); static Condition UnsignedCondition(Condition cond); static Condition ConditionWithoutEqual(Condition cond); // Return the primary condition to test. Some primary conditions may not // handle NaNs properly and may therefore require a secondary condition. // Use NaNCondFromDoubleCondition to determine what else is needed. static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) { return static_cast(cond & ~DoubleConditionBits); } static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader); // MacroAssemblers hold onto gcthings, so they are traced by the GC. void trace(JSTracer* trc); bool oom() const { return AssemblerShared::oom() || masm.oom() || jumpRelocations_.oom() || dataRelocations_.oom() || preBarriers_.oom(); } void setPrinter(Sprinter* sp) { masm.setPrinter(sp); } static const Register getStackPointer() { return StackPointer; } void executableCopy(void* buffer); bool asmMergeWith(const AssemblerX86Shared& other) { MOZ_ASSERT(other.jumps_.length() == 0); if (!AssemblerShared::asmMergeWith(masm.size(), other)) return false; return masm.appendBuffer(other.masm); } void processCodeLabels(uint8_t* rawCode); void copyJumpRelocationTable(uint8_t* dest); void copyDataRelocationTable(uint8_t* dest); void copyPreBarrierTable(uint8_t* dest); // Size of the instruction stream, in bytes. size_t size() const { return masm.size(); } // Size of the jump relocation table, in bytes. size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); } size_t dataRelocationTableBytes() const { return dataRelocations_.length(); } size_t preBarrierTableBytes() const { return preBarriers_.length(); } // Size of the data table, in bytes. size_t bytesNeeded() const { return size() + jumpRelocationTableBytes() + dataRelocationTableBytes() + preBarrierTableBytes(); } public: void haltingAlign(int alignment) { masm.haltingAlign(alignment); } void nopAlign(int alignment) { masm.nopAlign(alignment); } void writeCodePointer(CodeOffset* label) { // A CodeOffset only has one use, bake in the "end of list" value. masm.jumpTablePointer(LabelBase::INVALID_OFFSET); label->bind(masm.size()); } void cmovz(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.cmovz_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.cmovz_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.cmovz_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movl(Imm32 imm32, Register dest) { masm.movl_i32r(imm32.value, dest.encoding()); } void movl(Register src, Register dest) { masm.movl_rr(src.encoding(), dest.encoding()); } void movl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.movl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.movl_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.movl_mr(src.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.movl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.movl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; case Operand::MEM_ADDRESS32: masm.movl_rm(src.encoding(), dest.address()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movl(Imm32 imm32, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.movl_i32r(imm32.value, dest.reg()); break; case Operand::MEM_REG_DISP: masm.movl_i32m(imm32.value, dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale()); break; case Operand::MEM_ADDRESS32: masm.movl_i32m(imm32.value, dest.address()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xchgl(Register src, Register dest) { masm.xchgl_rr(src.encoding(), dest.encoding()); } // Eventually vmovapd should be overloaded to support loads and // stores too. void vmovapd(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovapd_rr(src.encoding(), dest.encoding()); } void vmovaps(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovaps_rr(src.encoding(), dest.encoding()); } void vmovaps(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::MEM_REG_DISP: masm.vmovaps_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; case Operand::FPREG: masm.vmovaps_rr(src.fpu(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovaps(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovups(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::MEM_REG_DISP: masm.vmovups_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovups(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovups_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } // vmovsd is only provided in load/store form since the // register-to-register form has different semantics (it doesn't clobber // the whole output register) and isn't needed currently. void vmovsd(const Address& src, FloatRegister dest) { masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding()); } void vmovsd(const BaseIndex& src, FloatRegister dest) { masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); } void vmovsd(FloatRegister src, const Address& dest) { masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding()); } void vmovsd(FloatRegister src, const BaseIndex& dest) { masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); } // Although vmovss is not only provided in load/store form (for the same // reasons as vmovsd above), the register to register form should be only // used in contexts where we care about not clearing the higher lanes of // the FloatRegister. void vmovss(const Address& src, FloatRegister dest) { masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding()); } void vmovss(const BaseIndex& src, FloatRegister dest) { masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); } void vmovss(FloatRegister src, const Address& dest) { masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding()); } void vmovss(FloatRegister src, const BaseIndex& dest) { masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); } void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vmovdqu(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::MEM_REG_DISP: masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovdqu(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovdqa(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::FPREG: masm.vmovdqa_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovdqa(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovdqa(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovdqa_rr(src.encoding(), dest.encoding()); } void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void movzbl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.movzbl_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movsbl(Register src, Register dest) { masm.movsbl_rr(src.encoding(), dest.encoding()); } void movsbl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.movsbl_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movb(const Operand& src, Register dest) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.movb_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movb(Imm32 src, Register dest) { masm.movb_ir(src.value & 255, dest.encoding()); } void movb(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.movb_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movb(Imm32 src, const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.movb_im(src.value, dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movzwl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.movzwl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.movzwl_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movzwl(Register src, Register dest) { masm.movzwl_rr(src.encoding(), dest.encoding()); } void movw(const Operand& src, Register dest) { masm.prefix_16_for_32(); movl(src, dest); } void movw(Imm32 src, Register dest) { masm.prefix_16_for_32(); movl(src, dest); } void movw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.movw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movw(Imm32 src, const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.movw_im(src.value, dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void movswl(Register src, Register dest) { masm.movswl_rr(src.encoding(), dest.encoding()); } void movswl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.movswl_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void leal(const Operand& src, Register dest) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.leal_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } protected: void jSrc(Condition cond, Label* label) { if (label->bound()) { // The jump can be immediately encoded to the correct destination. masm.jCC_i(static_cast(cond), JmpDst(label->offset())); } else { // Thread the jump list through the unpatched jump targets. JmpSrc j = masm.jCC(static_cast(cond)); JmpSrc prev = JmpSrc(label->use(j.offset())); masm.setNextJump(j, prev); } } void jmpSrc(Label* label) { if (label->bound()) { // The jump can be immediately encoded to the correct destination. masm.jmp_i(JmpDst(label->offset())); } else { // Thread the jump list through the unpatched jump targets. JmpSrc j = masm.jmp(); JmpSrc prev = JmpSrc(label->use(j.offset())); masm.setNextJump(j, prev); } } // Comparison of EAX against the address given by a Label. JmpSrc cmpSrc(Label* label) { JmpSrc j = masm.cmp_eax(); if (label->bound()) { // The jump can be immediately patched to the correct destination. masm.linkJump(j, JmpDst(label->offset())); } else { // Thread the jump list through the unpatched jump targets. JmpSrc prev = JmpSrc(label->use(j.offset())); masm.setNextJump(j, prev); } return j; } JmpSrc jSrc(Condition cond, RepatchLabel* label) { JmpSrc j = masm.jCC(static_cast(cond)); if (label->bound()) { // The jump can be immediately patched to the correct destination. masm.linkJump(j, JmpDst(label->offset())); } else { label->use(j.offset()); } return j; } JmpSrc jmpSrc(RepatchLabel* label) { JmpSrc j = masm.jmp(); if (label->bound()) { // The jump can be immediately patched to the correct destination. masm.linkJump(j, JmpDst(label->offset())); } else { // Thread the jump list through the unpatched jump targets. label->use(j.offset()); } return j; } public: void nop() { masm.nop(); } void nop(size_t n) { masm.insert_nop(n); } void j(Condition cond, Label* label) { jSrc(cond, label); } void jmp(Label* label) { jmpSrc(label); } void j(Condition cond, RepatchLabel* label) { jSrc(cond, label); } void jmp(RepatchLabel* label) { jmpSrc(label); } void j(Condition cond, wasm::TrapDesc target) { Label l; j(cond, &l); bindLater(&l, target); } void jmp(wasm::TrapDesc target) { Label l; jmp(&l); bindLater(&l, target); } void jmp(const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.jmp_m(op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.jmp_m(op.disp(), op.base(), op.index(), op.scale()); break; case Operand::REG: masm.jmp_r(op.reg()); break; default: MOZ_CRASH("unexpected operand kind"); } } void cmpEAX(Label* label) { cmpSrc(label); } void bind(Label* label) { JmpDst dst(masm.label()); if (label->used()) { bool more; JmpSrc jmp(label->offset()); do { JmpSrc next; more = masm.nextJump(jmp, &next); masm.linkJump(jmp, dst); jmp = next; } while (more); } label->bind(dst.offset()); } void bindLater(Label* label, wasm::TrapDesc target) { if (label->used()) { JmpSrc jmp(label->offset()); do { append(wasm::TrapSite(target, jmp.offset())); } while (masm.nextJump(jmp, &jmp)); } label->reset(); } void bind(RepatchLabel* label) { JmpDst dst(masm.label()); if (label->used()) { JmpSrc jmp(label->offset()); masm.linkJump(jmp, dst); } label->bind(dst.offset()); } void use(CodeOffset* label) { label->bind(currentOffset()); } uint32_t currentOffset() { return masm.label().offset(); } // Re-routes pending jumps to a new label. void retarget(Label* label, Label* target) { if (!label->used()) return; bool more; JmpSrc jmp(label->offset()); do { JmpSrc next; more = masm.nextJump(jmp, &next); if (target->bound()) { // The jump can be immediately patched to the correct destination. masm.linkJump(jmp, JmpDst(target->offset())); } else { // Thread the jump list through the unpatched jump targets. JmpSrc prev(target->use(jmp.offset())); masm.setNextJump(jmp, prev); } jmp = JmpSrc(next.offset()); } while (more); label->reset(); } static void Bind(uint8_t* raw, CodeOffset* label, const void* address) { if (label->bound()) { intptr_t offset = label->offset(); X86Encoding::SetPointer(raw + offset, address); } } // See Bind and X86Encoding::setPointer. size_t labelToPatchOffset(CodeOffset label) { return label.offset() - sizeof(void*); } void ret() { masm.ret(); } void retn(Imm32 n) { // Remove the size of the return address which is included in the frame. masm.ret_i(n.value - sizeof(void*)); } CodeOffset call(Label* label) { if (label->bound()) { masm.linkJump(masm.call(), JmpDst(label->offset())); } else { JmpSrc j = masm.call(); JmpSrc prev = JmpSrc(label->use(j.offset())); masm.setNextJump(j, prev); } return CodeOffset(masm.currentOffset()); } CodeOffset call(Register reg) { masm.call_r(reg.encoding()); return CodeOffset(masm.currentOffset()); } void call(const Operand& op) { switch (op.kind()) { case Operand::REG: masm.call_r(op.reg()); break; case Operand::MEM_REG_DISP: masm.call_m(op.disp(), op.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } CodeOffset callWithPatch() { return CodeOffset(masm.call().offset()); } struct AutoPrepareForPatching : X86Encoding::AutoUnprotectAssemblerBufferRegion { explicit AutoPrepareForPatching(AssemblerX86Shared& masm) : X86Encoding::AutoUnprotectAssemblerBufferRegion(masm.masm, 0, masm.size()) {} }; void patchCall(uint32_t callerOffset, uint32_t calleeOffset) { // The caller uses AutoUnprotectBuffer. unsigned char* code = masm.data(); X86Encoding::SetRel32(code + callerOffset, code + calleeOffset); } CodeOffset farJumpWithPatch() { return CodeOffset(masm.jmp().offset()); } void patchFarJump(CodeOffset farJump, uint32_t targetOffset) { // The caller uses AutoUnprotectBuffer. unsigned char* code = masm.data(); X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset); } static void repatchFarJump(uint8_t* code, uint32_t farJumpOffset, uint32_t targetOffset) { X86Encoding::SetRel32(code + farJumpOffset, code + targetOffset); } CodeOffset twoByteNop() { return CodeOffset(masm.twoByteNop().offset()); } static void patchTwoByteNopToJump(uint8_t* jump, uint8_t* target) { X86Encoding::BaseAssembler::patchTwoByteNopToJump(jump, target); } static void patchJumpToTwoByteNop(uint8_t* jump) { X86Encoding::BaseAssembler::patchJumpToTwoByteNop(jump); } void breakpoint() { masm.int3(); } static bool HasSSE2() { return CPUInfo::IsSSE2Present(); } static bool HasSSE3() { return CPUInfo::IsSSE3Present(); } static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); } static bool HasSSE41() { return CPUInfo::IsSSE41Present(); } static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); } static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); } static bool SupportsUnalignedAccesses() { return true; } static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); } static bool HasAVX() { return CPUInfo::IsAVXPresent(); } void cmpl(Register rhs, Register lhs) { masm.cmpl_rr(rhs.encoding(), lhs.encoding()); } void cmpl(const Operand& rhs, Register lhs) { switch (rhs.kind()) { case Operand::REG: masm.cmpl_rr(rhs.reg(), lhs.encoding()); break; case Operand::MEM_REG_DISP: masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding()); break; case Operand::MEM_ADDRESS32: masm.cmpl_mr(rhs.address(), lhs.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void cmpl(Register rhs, const Operand& lhs) { switch (lhs.kind()) { case Operand::REG: masm.cmpl_rr(rhs.encoding(), lhs.reg()); break; case Operand::MEM_REG_DISP: masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base()); break; case Operand::MEM_ADDRESS32: masm.cmpl_rm(rhs.encoding(), lhs.address()); break; default: MOZ_CRASH("unexpected operand kind"); } } void cmpl(Imm32 rhs, Register lhs) { masm.cmpl_ir(rhs.value, lhs.encoding()); } void cmpl(Imm32 rhs, const Operand& lhs) { switch (lhs.kind()) { case Operand::REG: masm.cmpl_ir(rhs.value, lhs.reg()); break; case Operand::MEM_REG_DISP: masm.cmpl_im(rhs.value, lhs.disp(), lhs.base()); break; case Operand::MEM_SCALE: masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), lhs.scale()); break; case Operand::MEM_ADDRESS32: masm.cmpl_im(rhs.value, lhs.address()); break; default: MOZ_CRASH("unexpected operand kind"); } } CodeOffset cmplWithPatch(Imm32 rhs, Register lhs) { masm.cmpl_i32r(rhs.value, lhs.encoding()); return CodeOffset(masm.currentOffset()); } void cmpw(Register rhs, Register lhs) { masm.cmpw_rr(rhs.encoding(), lhs.encoding()); } void setCC(Condition cond, Register r) { masm.setCC_r(static_cast(cond), r.encoding()); } void testb(Register rhs, Register lhs) { MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs)); MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs)); masm.testb_rr(rhs.encoding(), lhs.encoding()); } void testw(Register rhs, Register lhs) { masm.testw_rr(lhs.encoding(), rhs.encoding()); } void testl(Register rhs, Register lhs) { masm.testl_rr(lhs.encoding(), rhs.encoding()); } void testl(Imm32 rhs, Register lhs) { masm.testl_ir(rhs.value, lhs.encoding()); } void testl(Imm32 rhs, const Operand& lhs) { switch (lhs.kind()) { case Operand::REG: masm.testl_ir(rhs.value, lhs.reg()); break; case Operand::MEM_REG_DISP: masm.testl_i32m(rhs.value, lhs.disp(), lhs.base()); break; case Operand::MEM_ADDRESS32: masm.testl_i32m(rhs.value, lhs.address()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void addl(Imm32 imm, Register dest) { masm.addl_ir(imm.value, dest.encoding()); } CodeOffset addlWithPatch(Imm32 imm, Register dest) { masm.addl_i32r(imm.value, dest.encoding()); return CodeOffset(masm.currentOffset()); } void addl(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.addl_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.addl_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_ADDRESS32: masm.addl_im(imm.value, op.address()); break; case Operand::MEM_SCALE: masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void addw(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.addw_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.addw_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_ADDRESS32: masm.addw_im(imm.value, op.address()); break; case Operand::MEM_SCALE: masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void subl(Imm32 imm, Register dest) { masm.subl_ir(imm.value, dest.encoding()); } void subl(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.subl_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.subl_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void subw(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.subw_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.subw_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void addl(Register src, Register dest) { masm.addl_rr(src.encoding(), dest.encoding()); } void addl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.addl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.addl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void addw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.addw_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.addw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void subl(Register src, Register dest) { masm.subl_rr(src.encoding(), dest.encoding()); } void subl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.subl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.subl_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void subl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.subl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.subl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void subw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.subw_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.subw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void orl(Register reg, Register dest) { masm.orl_rr(reg.encoding(), dest.encoding()); } void orl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.orl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.orl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void orw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.orw_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.orw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void orl(Imm32 imm, Register reg) { masm.orl_ir(imm.value, reg.encoding()); } void orl(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.orl_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.orl_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void orw(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.orw_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.orw_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xorl(Register src, Register dest) { masm.xorl_rr(src.encoding(), dest.encoding()); } void xorl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.xorl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.xorl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xorw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.xorw_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.xorw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xorl(Imm32 imm, Register reg) { masm.xorl_ir(imm.value, reg.encoding()); } void xorl(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.xorl_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.xorl_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xorw(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.xorw_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.xorw_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void andl(Register src, Register dest) { masm.andl_rr(src.encoding(), dest.encoding()); } void andl(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.andl_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.andl_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void andw(Register src, const Operand& dest) { switch (dest.kind()) { case Operand::REG: masm.andw_rr(src.encoding(), dest.reg()); break; case Operand::MEM_REG_DISP: masm.andw_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void andl(Imm32 imm, Register dest) { masm.andl_ir(imm.value, dest.encoding()); } void andl(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.andl_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.andl_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void andw(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::REG: masm.andw_ir(imm.value, op.reg()); break; case Operand::MEM_REG_DISP: masm.andw_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void addl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.addl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.addl_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void orl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.orl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.orl_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xorl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.xorl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.xorl_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void andl(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.andl_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.andl_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void bsrl(const Register& src, const Register& dest) { masm.bsrl_rr(src.encoding(), dest.encoding()); } void bsfl(const Register& src, const Register& dest) { masm.bsfl_rr(src.encoding(), dest.encoding()); } void popcntl(const Register& src, const Register& dest) { masm.popcntl_rr(src.encoding(), dest.encoding()); } void imull(Register multiplier) { masm.imull_r(multiplier.encoding()); } void umull(Register multiplier) { masm.mull_r(multiplier.encoding()); } void imull(Imm32 imm, Register dest) { masm.imull_ir(imm.value, dest.encoding(), dest.encoding()); } void imull(Register src, Register dest) { masm.imull_rr(src.encoding(), dest.encoding()); } void imull(Imm32 imm, Register src, Register dest) { masm.imull_ir(imm.value, src.encoding(), dest.encoding()); } void imull(const Operand& src, Register dest) { switch (src.kind()) { case Operand::REG: masm.imull_rr(src.reg(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.imull_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void negl(const Operand& src) { switch (src.kind()) { case Operand::REG: masm.negl_r(src.reg()); break; case Operand::MEM_REG_DISP: masm.negl_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void negl(Register reg) { masm.negl_r(reg.encoding()); } void notl(const Operand& src) { switch (src.kind()) { case Operand::REG: masm.notl_r(src.reg()); break; case Operand::MEM_REG_DISP: masm.notl_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void notl(Register reg) { masm.notl_r(reg.encoding()); } void shrl(const Imm32 imm, Register dest) { masm.shrl_ir(imm.value, dest.encoding()); } void shll(const Imm32 imm, Register dest) { masm.shll_ir(imm.value, dest.encoding()); } void sarl(const Imm32 imm, Register dest) { masm.sarl_ir(imm.value, dest.encoding()); } void shrl_cl(Register dest) { masm.shrl_CLr(dest.encoding()); } void shll_cl(Register dest) { masm.shll_CLr(dest.encoding()); } void sarl_cl(Register dest) { masm.sarl_CLr(dest.encoding()); } void shrdl_cl(Register src, Register dest) { masm.shrdl_CLr(src.encoding(), dest.encoding()); } void shldl_cl(Register src, Register dest) { masm.shldl_CLr(src.encoding(), dest.encoding()); } void roll(const Imm32 imm, Register dest) { masm.roll_ir(imm.value, dest.encoding()); } void roll_cl(Register dest) { masm.roll_CLr(dest.encoding()); } void rorl(const Imm32 imm, Register dest) { masm.rorl_ir(imm.value, dest.encoding()); } void rorl_cl(Register dest) { masm.rorl_CLr(dest.encoding()); } void incl(const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.incl_m32(op.disp(), op.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_incl(const Operand& op) { masm.prefix_lock(); incl(op); } void decl(const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.decl_m32(op.disp(), op.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_decl(const Operand& op) { masm.prefix_lock(); decl(op); } void addb(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.addb_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void addb(Register src, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.addb_rm(src.encoding(), op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void subb(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.subb_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void subb(Register src, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.subb_rm(src.encoding(), op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void andb(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.andb_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void andb(Register src, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.andb_rm(src.encoding(), op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void orb(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.orb_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void orb(Register src, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.orb_rm(src.encoding(), op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void xorb(Imm32 imm, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.xorb_im(imm.value, op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } void xorb(Register src, const Operand& op) { switch (op.kind()) { case Operand::MEM_REG_DISP: masm.xorb_rm(src.encoding(), op.disp(), op.base()); break; case Operand::MEM_SCALE: masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); break; default: MOZ_CRASH("unexpected operand kind"); break; } } template void lock_addb(T src, const Operand& op) { masm.prefix_lock(); addb(src, op); } template void lock_subb(T src, const Operand& op) { masm.prefix_lock(); subb(src, op); } template void lock_andb(T src, const Operand& op) { masm.prefix_lock(); andb(src, op); } template void lock_orb(T src, const Operand& op) { masm.prefix_lock(); orb(src, op); } template void lock_xorb(T src, const Operand& op) { masm.prefix_lock(); xorb(src, op); } template void lock_addw(T src, const Operand& op) { masm.prefix_lock(); addw(src, op); } template void lock_subw(T src, const Operand& op) { masm.prefix_lock(); subw(src, op); } template void lock_andw(T src, const Operand& op) { masm.prefix_lock(); andw(src, op); } template void lock_orw(T src, const Operand& op) { masm.prefix_lock(); orw(src, op); } template void lock_xorw(T src, const Operand& op) { masm.prefix_lock(); xorw(src, op); } // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems, // among other things. Do not optimize, replace by XADDL, or similar. template void lock_addl(T src, const Operand& op) { masm.prefix_lock(); addl(src, op); } template void lock_subl(T src, const Operand& op) { masm.prefix_lock(); subl(src, op); } template void lock_andl(T src, const Operand& op) { masm.prefix_lock(); andl(src, op); } template void lock_orl(T src, const Operand& op) { masm.prefix_lock(); orl(src, op); } template void lock_xorl(T src, const Operand& op) { masm.prefix_lock(); xorl(src, op); } void lock_cmpxchgb(Register src, const Operand& mem) { masm.prefix_lock(); switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.cmpxchgb(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_cmpxchgw(Register src, const Operand& mem) { masm.prefix_lock(); switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.cmpxchgw(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_cmpxchgl(Register src, const Operand& mem) { masm.prefix_lock(); switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.cmpxchgl(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xchgb(Register src, const Operand& mem) { switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.xchgb_rm(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xchgw(Register src, const Operand& mem) { switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.xchgw_rm(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void xchgl(Register src, const Operand& mem) { switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.xchgl_rm(src.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_xaddb(Register srcdest, const Operand& mem) { switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void lock_xaddw(Register srcdest, const Operand& mem) { masm.prefix_16_for_32(); lock_xaddl(srcdest, mem); } void lock_xaddl(Register srcdest, const Operand& mem) { switch (mem.kind()) { case Operand::MEM_REG_DISP: masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base()); break; case Operand::MEM_SCALE: masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void push(const Imm32 imm) { masm.push_i(imm.value); } void push(const Operand& src) { switch (src.kind()) { case Operand::REG: masm.push_r(src.reg()); break; case Operand::MEM_REG_DISP: masm.push_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void push(Register src) { masm.push_r(src.encoding()); } void push(const Address& src) { masm.push_m(src.offset, src.base.encoding()); } void pop(const Operand& src) { switch (src.kind()) { case Operand::REG: masm.pop_r(src.reg()); break; case Operand::MEM_REG_DISP: masm.pop_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void pop(Register src) { masm.pop_r(src.encoding()); } void pop(const Address& src) { masm.pop_m(src.offset, src.base.encoding()); } void pushFlags() { masm.push_flags(); } void popFlags() { masm.pop_flags(); } #ifdef JS_CODEGEN_X86 void pushAllRegs() { masm.pusha(); } void popAllRegs() { masm.popa(); } #endif // Zero-extend byte to 32-bit integer. void movzbl(Register src, Register dest) { masm.movzbl_rr(src.encoding(), dest.encoding()); } void cdq() { masm.cdq(); } void idiv(Register divisor) { masm.idivl_r(divisor.encoding()); } void udiv(Register divisor) { masm.divl_r(divisor.encoding()); } void vpinsrb(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vpinsrb_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); } void vpinsrw(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { masm.vpinsrw_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); } void vpinsrd(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); } void vpextrb(unsigned lane, FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE41()); masm.vpextrb_irr(lane, src.encoding(), dest.encoding()); } void vpextrw(unsigned lane, FloatRegister src, Register dest) { masm.vpextrw_irr(lane, src.encoding(), dest.encoding()); } void vpextrd(unsigned lane, FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE41()); masm.vpextrd_irr(lane, src.encoding(), dest.encoding()); } void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding()); } void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding()); } void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding()); } void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpslld_ir(count.value, src0.encoding(), dest.encoding()); } void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding()); } void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding()); } void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding()); } void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding()); } void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding()); } void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::REG: masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vcvttsd2si(FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE2()); masm.vcvttsd2si_rr(src.encoding(), dest.encoding()); } void vcvttss2si(FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE2()); masm.vcvttss2si_rr(src.encoding(), dest.encoding()); } void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::REG: masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vcvttps2dq(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvttps2dq_rr(src.encoding(), dest.encoding()); } void vcvtdq2ps(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vcvtdq2ps_rr(src.encoding(), dest.encoding()); } void vmovmskpd(FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE2()); masm.vmovmskpd_rr(src.encoding(), dest.encoding()); } void vmovmskps(FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE2()); masm.vmovmskps_rr(src.encoding(), dest.encoding()); } void vptest(FloatRegister rhs, FloatRegister lhs) { MOZ_ASSERT(HasSSE41()); masm.vptest_rr(rhs.encoding(), lhs.encoding()); } void vucomisd(FloatRegister rhs, FloatRegister lhs) { MOZ_ASSERT(HasSSE2()); masm.vucomisd_rr(rhs.encoding(), lhs.encoding()); } void vucomiss(FloatRegister rhs, FloatRegister lhs) { MOZ_ASSERT(HasSSE2()); masm.vucomiss_rr(rhs.encoding(), lhs.encoding()); } void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (rhs.kind()) { case Operand::FPREG: masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vcmpps(uint8_t order, Operand src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); // :TODO: (Bug 1132894) See LIRGeneratorX86Shared::lowerForFPU // FIXME: This logic belongs in the MacroAssembler. if (!HasAVX() && !src0.aliases(dest)) { if (src1.kind() == Operand::FPREG && dest.aliases(FloatRegister::FromCode(src1.fpu()))) { vmovdqa(src1, ScratchSimd128Reg); src1 = Operand(ScratchSimd128Reg); } vmovdqa(src0, dest); src0 = dest; } switch (src1.kind()) { case Operand::FPREG: masm.vcmpps_rr(order, src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vcmpps_mr(order, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vcmpps_mr(order, src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vcmpeqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_EQ, src1, src0, dest); } void vcmpltps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_LT, src1, src0, dest); } void vcmpleps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_LE, src1, src0, dest); } void vcmpunordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_UNORD, src1, src0, dest); } void vcmpneqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_NEQ, src1, src0, dest); } void vcmpordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { vcmpps(X86Encoding::ConditionCmp_ORD, src1, src0, dest); } void vrcpps(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::FPREG: masm.vrcpps_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vrcpps_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vrcpps_mr(src.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vsqrtps(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::FPREG: masm.vsqrtps_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vsqrtps_mr(src.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vrsqrtps(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::FPREG: masm.vrsqrtps_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vrsqrtps_mr(src.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovd(Register src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovd_rr(src.encoding(), dest.encoding()); } void vmovd(FloatRegister src, Register dest) { MOZ_ASSERT(HasSSE2()); masm.vmovd_rr(src.encoding(), dest.encoding()); } void vmovd(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::MEM_REG_DISP: masm.vmovd_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovd(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovd_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; case Operand::MEM_ADDRESS32: masm.vmovq_rm(src.encoding(), dest.address()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovq(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src.kind()) { case Operand::MEM_REG_DISP: masm.vmovq_mr(src.disp(), src.base(), dest.encoding()); break; case Operand::MEM_SCALE: masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vmovq_mr(src.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovq(FloatRegister src, const Operand& dest) { MOZ_ASSERT(HasSSE2()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.vmovq_rm(src.encoding(), dest.disp(), dest.base()); break; case Operand::MEM_SCALE: masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); switch (src1.kind()) { case Operand::FPREG: masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) { // Negates bits of dest and then applies AND MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpshufd_irr(mask, src.encoding(), dest.encoding()); } void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vpshufd_irr(mask, src1.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vpshufd_imr(mask, src1.address(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpshuflw_irr(mask, src.encoding(), dest.encoding()); } void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vpshufhw_irr(mask, src.encoding(), dest.encoding()); } void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSSE3()); masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding()); } void vmovddup(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE3()); masm.vmovddup_rr(src.encoding(), dest.encoding()); } void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); } void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vshufps_imr(mask, src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; case Operand::MEM_ADDRESS32: masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); } void vroundss(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vroundss_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); } unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, unsigned zeroMask = 0) { // Note that the sourceLane bits are ignored in the case of a source // memory operand, and the source is the given 32-bits memory location. MOZ_ASSERT(zeroMask < 16); unsigned ret = zeroMask ; ret |= destLane << 4; ret |= sourceLane << 6; MOZ_ASSERT(ret < 256); return ret; } void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); } void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); switch (src1.kind()) { case Operand::FPREG: masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } unsigned blendpsMask(bool x, bool y, bool z, bool w) { return (x << 0) | (y << 1) | (z << 2) | (w << 3); } void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); } void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); switch (src1.kind()) { case Operand::FPREG: masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), dest.encoding()); } void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE41()); switch (src1.kind()) { case Operand::FPREG: masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovsldup(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE3()); masm.vmovsldup_rr(src.encoding(), dest.encoding()); } void vmovsldup(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE3()); switch (src.kind()) { case Operand::FPREG: masm.vmovsldup_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmovshdup(FloatRegister src, FloatRegister dest) { MOZ_ASSERT(HasSSE3()); masm.vmovshdup_rr(src.encoding(), dest.encoding()); } void vmovshdup(const Operand& src, FloatRegister dest) { MOZ_ASSERT(HasSSE3()); switch (src.kind()) { case Operand::FPREG: masm.vmovshdup_rr(src.fpu(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); switch (src1.kind()) { case Operand::FPREG: masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); break; case Operand::MEM_REG_DISP: masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); break; default: MOZ_CRASH("unexpected operand kind"); } } void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { MOZ_ASSERT(HasSSE2()); masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding()); } void fisttp(const Operand& dest) { MOZ_ASSERT(HasSSE3()); switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fisttp_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fistp(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fistp_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fnstcw(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fnstcw_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fldcw(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fldcw_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fnstsw(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fnstsw_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fld(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fld_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fld32(const Operand& dest) { switch (dest.kind()) { case Operand::MEM_REG_DISP: masm.fld32_m(dest.disp(), dest.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fstp(const Operand& src) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.fstp_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } void fstp32(const Operand& src) { switch (src.kind()) { case Operand::MEM_REG_DISP: masm.fstp32_m(src.disp(), src.base()); break; default: MOZ_CRASH("unexpected operand kind"); } } // Defined for compatibility with ARM's assembler uint32_t actualIndex(uint32_t x) { return x; } void flushBuffer() { } // Patching. static size_t PatchWrite_NearCallSize() { return 5; } static uintptr_t GetPointer(uint8_t* instPtr) { uintptr_t* ptr = ((uintptr_t*) instPtr) - 1; return *ptr; } // Write a relative call at the start location |dataLabel|. // Note that this DOES NOT patch data that comes before |label|. static void PatchWrite_NearCall(CodeLocationLabel startLabel, CodeLocationLabel target) { uint8_t* start = startLabel.raw(); *start = 0xE8; ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize(); MOZ_ASSERT(int32_t(offset) == offset); *((int32_t*) (start + 1)) = offset; } static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) { *((int32_t*) dataLabel.raw() - 1) = toWrite.value; } static void PatchDataWithValueCheck(CodeLocationLabel data, PatchedImmPtr newData, PatchedImmPtr expectedData) { // The pointer given is a pointer to *after* the data. uintptr_t* ptr = ((uintptr_t*) data.raw()) - 1; MOZ_ASSERT(*ptr == (uintptr_t)expectedData.value); *ptr = (uintptr_t)newData.value; } static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, ImmPtr expectedData) { PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), PatchedImmPtr(expectedData.value)); } static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm) { MOZ_CRASH("Unused."); } static uint32_t NopSize() { return 1; } static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) { MOZ_CRASH("nextInstruction NYI on x86"); } // Toggle a jmp or cmp emitted by toggledJump(). static void ToggleToJmp(CodeLocationLabel inst) { uint8_t* ptr = (uint8_t*)inst.raw(); MOZ_ASSERT(*ptr == 0x3D); *ptr = 0xE9; } static void ToggleToCmp(CodeLocationLabel inst) { uint8_t* ptr = (uint8_t*)inst.raw(); MOZ_ASSERT(*ptr == 0xE9); *ptr = 0x3D; } static void ToggleCall(CodeLocationLabel inst, bool enabled) { uint8_t* ptr = (uint8_t*)inst.raw(); MOZ_ASSERT(*ptr == 0x3D || // CMP *ptr == 0xE8); // CALL *ptr = enabled ? 0xE8 : 0x3D; } MOZ_COLD void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, const Disassembler::HeapAccess& heapAccess); }; } // namespace jit } // namespace js #endif /* jit_x86_shared_Assembler_x86_shared_h */