/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "jit/x86-shared/CodeGenerator-x86-shared.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/MathAlgorithms.h"
#include "jsmath.h"
#include "jit/JitCompartment.h"
#include "jit/JitFrames.h"
#include "jit/Linker.h"
#include "jit/RangeAnalysis.h"
#include "vm/TraceLogging.h"
#include "jit/MacroAssembler-inl.h"
#include "jit/shared/CodeGenerator-shared-inl.h"
using namespace js;
using namespace js::jit;
using mozilla::Abs;
using mozilla::BitwiseCast;
using mozilla::DebugOnly;
using mozilla::FloatingPoint;
using mozilla::FloorLog2;
using mozilla::NegativeInfinity;
using mozilla::SpecificNaN;
using JS::GenericNaN;
namespace js {
namespace jit {
CodeGeneratorX86Shared::CodeGeneratorX86Shared(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
: CodeGeneratorShared(gen, graph, masm)
{
}
#ifdef JS_PUNBOX64
Operand
CodeGeneratorX86Shared::ToOperandOrRegister64(const LInt64Allocation input)
{
return ToOperand(input.value());
}
#else
Register64
CodeGeneratorX86Shared::ToOperandOrRegister64(const LInt64Allocation input)
{
return ToRegister64(input);
}
#endif
void
OutOfLineBailout::accept(CodeGeneratorX86Shared* codegen)
{
codegen->visitOutOfLineBailout(this);
}
void
CodeGeneratorX86Shared::emitBranch(Assembler::Condition cond, MBasicBlock* mirTrue,
MBasicBlock* mirFalse, Assembler::NaNCond ifNaN)
{
if (ifNaN == Assembler::NaN_IsFalse)
jumpToBlock(mirFalse, Assembler::Parity);
else if (ifNaN == Assembler::NaN_IsTrue)
jumpToBlock(mirTrue, Assembler::Parity);
if (isNextBlock(mirFalse->lir())) {
jumpToBlock(mirTrue, cond);
} else {
jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
jumpToBlock(mirTrue);
}
}
void
CodeGeneratorX86Shared::visitDouble(LDouble* ins)
{
const LDefinition* out = ins->getDef(0);
masm.loadConstantDouble(ins->getDouble(), ToFloatRegister(out));
}
void
CodeGeneratorX86Shared::visitFloat32(LFloat32* ins)
{
const LDefinition* out = ins->getDef(0);
masm.loadConstantFloat32(ins->getFloat(), ToFloatRegister(out));
}
void
CodeGeneratorX86Shared::visitTestIAndBranch(LTestIAndBranch* test)
{
Register input = ToRegister(test->input());
masm.test32(input, input);
emitBranch(Assembler::NonZero, test->ifTrue(), test->ifFalse());
}
void
CodeGeneratorX86Shared::visitTestDAndBranch(LTestDAndBranch* test)
{
const LAllocation* opd = test->input();
// vucomisd flags:
// Z P C
// ---------
// NaN 1 1 1
// > 0 0 0
// < 0 0 1
// = 1 0 0
//
// NaN is falsey, so comparing against 0 and then using the Z flag is
// enough to determine which branch to take.
ScratchDoubleScope scratch(masm);
masm.zeroDouble(scratch);
masm.vucomisd(scratch, ToFloatRegister(opd));
emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
}
void
CodeGeneratorX86Shared::visitTestFAndBranch(LTestFAndBranch* test)
{
const LAllocation* opd = test->input();
// vucomiss flags are the same as doubles; see comment above
{
ScratchFloat32Scope scratch(masm);
masm.zeroFloat32(scratch);
masm.vucomiss(scratch, ToFloatRegister(opd));
}
emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
}
void
CodeGeneratorX86Shared::visitBitAndAndBranch(LBitAndAndBranch* baab)
{
if (baab->right()->isConstant())
masm.test32(ToRegister(baab->left()), Imm32(ToInt32(baab->right())));
else
masm.test32(ToRegister(baab->left()), ToRegister(baab->right()));
emitBranch(Assembler::NonZero, baab->ifTrue(), baab->ifFalse());
}
void
CodeGeneratorX86Shared::emitCompare(MCompare::CompareType type, const LAllocation* left, const LAllocation* right)
{
#ifdef JS_CODEGEN_X64
if (type == MCompare::Compare_Object) {
masm.cmpPtr(ToRegister(left), ToOperand(right));
return;
}
#endif
if (right->isConstant())
masm.cmp32(ToRegister(left), Imm32(ToInt32(right)));
else
masm.cmp32(ToRegister(left), ToOperand(right));
}
void
CodeGeneratorX86Shared::visitCompare(LCompare* comp)
{
MCompare* mir = comp->mir();
emitCompare(mir->compareType(), comp->left(), comp->right());
masm.emitSet(JSOpToCondition(mir->compareType(), comp->jsop()), ToRegister(comp->output()));
}
void
CodeGeneratorX86Shared::visitCompareAndBranch(LCompareAndBranch* comp)
{
MCompare* mir = comp->cmpMir();
emitCompare(mir->compareType(), comp->left(), comp->right());
Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
emitBranch(cond, comp->ifTrue(), comp->ifFalse());
}
void
CodeGeneratorX86Shared::visitCompareD(LCompareD* comp)
{
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
if (comp->mir()->operandsAreNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
masm.compareDouble(cond, lhs, rhs);
masm.emitSet(Assembler::ConditionFromDoubleCondition(cond), ToRegister(comp->output()), nanCond);
}
void
CodeGeneratorX86Shared::visitCompareF(LCompareF* comp)
{
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
if (comp->mir()->operandsAreNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
masm.compareFloat(cond, lhs, rhs);
masm.emitSet(Assembler::ConditionFromDoubleCondition(cond), ToRegister(comp->output()), nanCond);
}
void
CodeGeneratorX86Shared::visitNotI(LNotI* ins)
{
masm.cmp32(ToRegister(ins->input()), Imm32(0));
masm.emitSet(Assembler::Equal, ToRegister(ins->output()));
}
void
CodeGeneratorX86Shared::visitNotD(LNotD* ins)
{
FloatRegister opd = ToFloatRegister(ins->input());
// Not returns true if the input is a NaN. We don't have to worry about
// it if we know the input is never NaN though.
Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
if (ins->mir()->operandIsNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
ScratchDoubleScope scratch(masm);
masm.zeroDouble(scratch);
masm.compareDouble(Assembler::DoubleEqualOrUnordered, opd, scratch);
masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
}
void
CodeGeneratorX86Shared::visitNotF(LNotF* ins)
{
FloatRegister opd = ToFloatRegister(ins->input());
// Not returns true if the input is a NaN. We don't have to worry about
// it if we know the input is never NaN though.
Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
if (ins->mir()->operandIsNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
ScratchFloat32Scope scratch(masm);
masm.zeroFloat32(scratch);
masm.compareFloat(Assembler::DoubleEqualOrUnordered, opd, scratch);
masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
}
void
CodeGeneratorX86Shared::visitCompareDAndBranch(LCompareDAndBranch* comp)
{
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->cmpMir()->jsop());
Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
if (comp->cmpMir()->operandsAreNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
masm.compareDouble(cond, lhs, rhs);
emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(), comp->ifFalse(), nanCond);
}
void
CodeGeneratorX86Shared::visitCompareFAndBranch(LCompareFAndBranch* comp)
{
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->cmpMir()->jsop());
Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
if (comp->cmpMir()->operandsAreNeverNaN())
nanCond = Assembler::NaN_HandledByCond;
masm.compareFloat(cond, lhs, rhs);
emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(), comp->ifFalse(), nanCond);
}
void
CodeGeneratorX86Shared::visitWasmStackArg(LWasmStackArg* ins)
{
const MWasmStackArg* mir = ins->mir();
Address dst(StackPointer, mir->spOffset());
if (ins->arg()->isConstant()) {
masm.storePtr(ImmWord(ToInt32(ins->arg())), dst);
} else if (ins->arg()->isGeneralReg()) {
masm.storePtr(ToRegister(ins->arg()), dst);
} else {
switch (mir->input()->type()) {
case MIRType::Double:
masm.storeDouble(ToFloatRegister(ins->arg()), dst);
return;
case MIRType::Float32:
masm.storeFloat32(ToFloatRegister(ins->arg()), dst);
return;
// StackPointer is SIMD-aligned and ABIArgGenerator guarantees
// stack offsets are SIMD-aligned.
case MIRType::Int32x4:
case MIRType::Bool32x4:
masm.storeAlignedSimd128Int(ToFloatRegister(ins->arg()), dst);
return;
case MIRType::Float32x4:
masm.storeAlignedSimd128Float(ToFloatRegister(ins->arg()), dst);
return;
default: break;
}
MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("unexpected mir type in WasmStackArg");
}
}
void
CodeGeneratorX86Shared::visitWasmStackArgI64(LWasmStackArgI64* ins)
{
const MWasmStackArg* mir = ins->mir();
Address dst(StackPointer, mir->spOffset());
if (IsConstant(ins->arg()))
masm.store64(Imm64(ToInt64(ins->arg())), dst);
else
masm.store64(ToRegister64(ins->arg()), dst);
}
void
CodeGeneratorX86Shared::visitWasmSelect(LWasmSelect* ins)
{
MIRType mirType = ins->mir()->type();
Register cond = ToRegister(ins->condExpr());
Operand falseExpr = ToOperand(ins->falseExpr());
masm.test32(cond, cond);
if (mirType == MIRType::Int32) {
Register out = ToRegister(ins->output());
MOZ_ASSERT(ToRegister(ins->trueExpr()) == out, "true expr input is reused for output");
masm.cmovz(falseExpr, out);
return;
}
FloatRegister out = ToFloatRegister(ins->output());
MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out, "true expr input is reused for output");
Label done;
masm.j(Assembler::NonZero, &done);
if (mirType == MIRType::Float32) {
if (falseExpr.kind() == Operand::FPREG)
masm.moveFloat32(ToFloatRegister(ins->falseExpr()), out);
else
masm.loadFloat32(falseExpr, out);
} else if (mirType == MIRType::Double) {
if (falseExpr.kind() == Operand::FPREG)
masm.moveDouble(ToFloatRegister(ins->falseExpr()), out);
else
masm.loadDouble(falseExpr, out);
} else {
MOZ_CRASH("unhandled type in visitWasmSelect!");
}
masm.bind(&done);
return;
}
void
CodeGeneratorX86Shared::visitWasmReinterpret(LWasmReinterpret* lir)
{
MOZ_ASSERT(gen->compilingWasm());
MWasmReinterpret* ins = lir->mir();
MIRType to = ins->type();
#ifdef DEBUG
MIRType from = ins->input()->type();
#endif
switch (to) {
case MIRType::Int32:
MOZ_ASSERT(from == MIRType::Float32);
masm.vmovd(ToFloatRegister(lir->input()), ToRegister(lir->output()));
break;
case MIRType::Float32:
MOZ_ASSERT(from == MIRType::Int32);
masm.vmovd(ToRegister(lir->input()), ToFloatRegister(lir->output()));
break;
case MIRType::Double:
case MIRType::Int64:
MOZ_CRASH("not handled by this LIR opcode");
default:
MOZ_CRASH("unexpected WasmReinterpret");
}
}
void
CodeGeneratorX86Shared::visitOutOfLineLoadTypedArrayOutOfBounds(OutOfLineLoadTypedArrayOutOfBounds* ool)
{
switch (ool->viewType()) {
case Scalar::Int64:
case Scalar::Float32x4:
case Scalar::Int8x16:
case Scalar::Int16x8:
case Scalar::Int32x4:
case Scalar::MaxTypedArrayViewType:
MOZ_CRASH("unexpected array type");
case Scalar::Float32:
masm.loadConstantFloat32(float(GenericNaN()), ool->dest().fpu());
break;
case Scalar::Float64:
masm.loadConstantDouble(GenericNaN(), ool->dest().fpu());
break;
case Scalar::Int8:
case Scalar::Uint8:
case Scalar::Int16:
case Scalar::Uint16:
case Scalar::Int32:
case Scalar::Uint32:
case Scalar::Uint8Clamped:
Register destReg = ool->dest().gpr();
masm.mov(ImmWord(0), destReg);
break;
}
masm.jmp(ool->rejoin());
}
void
CodeGeneratorX86Shared::visitWasmAddOffset(LWasmAddOffset* lir)
{
MWasmAddOffset* mir = lir->mir();
Register base = ToRegister(lir->base());
Register out = ToRegister(lir->output());
if (base != out)
masm.move32(base, out);
masm.add32(Imm32(mir->offset()), out);
masm.j(Assembler::CarrySet, trap(mir, wasm::Trap::OutOfBounds));
}
void
CodeGeneratorX86Shared::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
Register output = ToRegister(lir->output());
MWasmTruncateToInt32* mir = lir->mir();
MIRType inputType = mir->input()->type();
MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input);
addOutOfLineCode(ool, mir);
Label* oolEntry = ool->entry();
if (mir->isUnsigned()) {
if (inputType == MIRType::Double)
masm.wasmTruncateDoubleToUInt32(input, output, oolEntry);
else if (inputType == MIRType::Float32)
masm.wasmTruncateFloat32ToUInt32(input, output, oolEntry);
else
MOZ_CRASH("unexpected type");
return;
}
if (inputType == MIRType::Double)
masm.wasmTruncateDoubleToInt32(input, output, oolEntry);
else if (inputType == MIRType::Float32)
masm.wasmTruncateFloat32ToInt32(input, output, oolEntry);
else
MOZ_CRASH("unexpected type");
masm.bind(ool->rejoin());
}
bool
CodeGeneratorX86Shared::generateOutOfLineCode()
{
if (!CodeGeneratorShared::generateOutOfLineCode())
return false;
if (deoptLabel_.used()) {
// All non-table-based bailouts will go here.
masm.bind(&deoptLabel_);
// Push the frame size, so the handler can recover the IonScript.
masm.push(Imm32(frameSize()));
JitCode* handler = gen->jitRuntime()->getGenericBailoutHandler();
masm.jmp(ImmPtr(handler->raw()), Relocation::JITCODE);
}
return !masm.oom();
}
class BailoutJump {
Assembler::Condition cond_;
public:
explicit BailoutJump(Assembler::Condition cond) : cond_(cond)
{ }
#ifdef JS_CODEGEN_X86
void operator()(MacroAssembler& masm, uint8_t* code) const {
masm.j(cond_, ImmPtr(code), Relocation::HARDCODED);
}
#endif
void operator()(MacroAssembler& masm, Label* label) const {
masm.j(cond_, label);
}
};
class BailoutLabel {
Label* label_;
public:
explicit BailoutLabel(Label* label) : label_(label)
{ }
#ifdef JS_CODEGEN_X86
void operator()(MacroAssembler& masm, uint8_t* code) const {
masm.retarget(label_, ImmPtr(code), Relocation::HARDCODED);
}
#endif
void operator()(MacroAssembler& masm, Label* label) const {
masm.retarget(label_, label);
}
};
template <typename T> void
CodeGeneratorX86Shared::bailout(const T& binder, LSnapshot* snapshot)
{
encode(snapshot);
// Though the assembler doesn't track all frame pushes, at least make sure
// the known value makes sense. We can't use bailout tables if the stack
// isn't properly aligned to the static frame size.
MOZ_ASSERT_IF(frameClass_ != FrameSizeClass::None() && deoptTable_,
frameClass_.frameSize() == masm.framePushed());
#ifdef JS_CODEGEN_X86
// On x64, bailout tables are pointless, because 16 extra bytes are
// reserved per external jump, whereas it takes only 10 bytes to encode a
// a non-table based bailout.
if (assignBailoutId(snapshot)) {
binder(masm, deoptTable_->raw() + snapshot->bailoutId() * BAILOUT_TABLE_ENTRY_SIZE);
return;
}
#endif
// We could not use a jump table, either because all bailout IDs were
// reserved, or a jump table is not optimal for this frame size or
// platform. Whatever, we will generate a lazy bailout.
//
// All bailout code is associated with the bytecodeSite of the block we are
// bailing out from.
InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
OutOfLineBailout* ool = new(alloc()) OutOfLineBailout(snapshot);
addOutOfLineCode(ool, new(alloc()) BytecodeSite(tree, tree->script()->code()));
binder(masm, ool->entry());
}
void
CodeGeneratorX86Shared::bailoutIf(Assembler::Condition condition, LSnapshot* snapshot)
{
bailout(BailoutJump(condition), snapshot);
}
void
CodeGeneratorX86Shared::bailoutIf(Assembler::DoubleCondition condition, LSnapshot* snapshot)
{
MOZ_ASSERT(Assembler::NaNCondFromDoubleCondition(condition) == Assembler::NaN_HandledByCond);
bailoutIf(Assembler::ConditionFromDoubleCondition(condition), snapshot);
}
void
CodeGeneratorX86Shared::bailoutFrom(Label* label, LSnapshot* snapshot)
{
MOZ_ASSERT(label->used() && !label->bound());
bailout(BailoutLabel(label), snapshot);
}
void
CodeGeneratorX86Shared::bailout(LSnapshot* snapshot)
{
Label label;
masm.jump(&label);
bailoutFrom(&label, snapshot);
}
void
CodeGeneratorX86Shared::visitOutOfLineBailout(OutOfLineBailout* ool)
{
masm.push(Imm32(ool->snapshot()->snapshotOffset()));
masm.jmp(&deoptLabel_);
}
void
CodeGeneratorX86Shared::visitMinMaxD(LMinMaxD* ins)
{
FloatRegister first = ToFloatRegister(ins->first());
FloatRegister second = ToFloatRegister(ins->second());
#ifdef DEBUG
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT(first == output);
#endif
bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
if (ins->mir()->isMax())
masm.maxDouble(second, first, handleNaN);
else
masm.minDouble(second, first, handleNaN);
}
void
CodeGeneratorX86Shared::visitMinMaxF(LMinMaxF* ins)
{
FloatRegister first = ToFloatRegister(ins->first());
FloatRegister second = ToFloatRegister(ins->second());
#ifdef DEBUG
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT(first == output);
#endif
bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
if (ins->mir()->isMax())
masm.maxFloat32(second, first, handleNaN);
else
masm.minFloat32(second, first, handleNaN);
}
void
CodeGeneratorX86Shared::visitAbsD(LAbsD* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
MOZ_ASSERT(input == ToFloatRegister(ins->output()));
// Load a value which is all ones except for the sign bit.
ScratchDoubleScope scratch(masm);
masm.loadConstantDouble(SpecificNaN<double>(0, FloatingPoint<double>::kSignificandBits), scratch);
masm.vandpd(scratch, input, input);
}
void
CodeGeneratorX86Shared::visitAbsF(LAbsF* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
MOZ_ASSERT(input == ToFloatRegister(ins->output()));
// Same trick as visitAbsD above.
ScratchFloat32Scope scratch(masm);
masm.loadConstantFloat32(SpecificNaN<float>(0, FloatingPoint<float>::kSignificandBits), scratch);
masm.vandps(scratch, input, input);
}
void
CodeGeneratorX86Shared::visitClzI(LClzI* ins)
{
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
bool knownNotZero = ins->mir()->operandIsNeverZero();
masm.clz32(input, output, knownNotZero);
}
void
CodeGeneratorX86Shared::visitCtzI(LCtzI* ins)
{
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
bool knownNotZero = ins->mir()->operandIsNeverZero();
masm.ctz32(input, output, knownNotZero);
}
void
CodeGeneratorX86Shared::visitPopcntI(LPopcntI* ins)
{
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
Register temp = ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
masm.popcnt32(input, output, temp);
}
void
CodeGeneratorX86Shared::visitSqrtD(LSqrtD* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
masm.vsqrtsd(input, output, output);
}
void
CodeGeneratorX86Shared::visitSqrtF(LSqrtF* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
masm.vsqrtss(input, output, output);
}
void
CodeGeneratorX86Shared::visitPowHalfD(LPowHalfD* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
ScratchDoubleScope scratch(masm);
Label done, sqrt;
if (!ins->mir()->operandIsNeverNegativeInfinity()) {
// Branch if not -Infinity.
masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
Assembler::DoubleCondition cond = Assembler::DoubleNotEqualOrUnordered;
if (ins->mir()->operandIsNeverNaN())
cond = Assembler::DoubleNotEqual;
masm.branchDouble(cond, input, scratch, &sqrt);
// Math.pow(-Infinity, 0.5) == Infinity.
masm.zeroDouble(output);
masm.subDouble(scratch, output);
masm.jump(&done);
masm.bind(&sqrt);
}
if (!ins->mir()->operandIsNeverNegativeZero()) {
// Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5). Adding 0 converts any -0 to 0.
masm.zeroDouble(scratch);
masm.addDouble(input, scratch);
masm.vsqrtsd(scratch, output, output);
} else {
masm.vsqrtsd(input, output, output);
}
masm.bind(&done);
}
class OutOfLineUndoALUOperation : public OutOfLineCodeBase<CodeGeneratorX86Shared>
{
LInstruction* ins_;
public:
explicit OutOfLineUndoALUOperation(LInstruction* ins)
: ins_(ins)
{ }
virtual void accept(CodeGeneratorX86Shared* codegen) {
codegen->visitOutOfLineUndoALUOperation(this);
}
LInstruction* ins() const {
return ins_;
}
};
void
CodeGeneratorX86Shared::visitAddI(LAddI* ins)
{
if (ins->rhs()->isConstant())
masm.addl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
else
masm.addl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
if (ins->snapshot()) {
if (ins->recoversInput()) {
OutOfLineUndoALUOperation* ool = new(alloc()) OutOfLineUndoALUOperation(ins);
addOutOfLineCode(ool, ins->mir());
masm.j(Assembler::Overflow, ool->entry());
} else {
bailoutIf(Assembler::Overflow, ins->snapshot());
}
}
}
void
CodeGeneratorX86Shared::visitAddI64(LAddI64* lir)
{
const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
if (IsConstant(rhs)) {
masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
return;
}
masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
}
void
CodeGeneratorX86Shared::visitSubI(LSubI* ins)
{
if (ins->rhs()->isConstant())
masm.subl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
else
masm.subl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
if (ins->snapshot()) {
if (ins->recoversInput()) {
OutOfLineUndoALUOperation* ool = new(alloc()) OutOfLineUndoALUOperation(ins);
addOutOfLineCode(ool, ins->mir());
masm.j(Assembler::Overflow, ool->entry());
} else {
bailoutIf(Assembler::Overflow, ins->snapshot());
}
}
}
void
CodeGeneratorX86Shared::visitSubI64(LSubI64* lir)
{
const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
if (IsConstant(rhs)) {
masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
return;
}
masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
}
void
CodeGeneratorX86Shared::visitOutOfLineUndoALUOperation(OutOfLineUndoALUOperation* ool)
{
LInstruction* ins = ool->ins();
Register reg = ToRegister(ins->getDef(0));
DebugOnly<LAllocation*> lhs = ins->getOperand(0);
LAllocation* rhs = ins->getOperand(1);
MOZ_ASSERT(reg == ToRegister(lhs));
MOZ_ASSERT_IF(rhs->isGeneralReg(), reg != ToRegister(rhs));
// Undo the effect of the ALU operation, which was performed on the output
// register and overflowed. Writing to the output register clobbered an
// input reg, and the original value of the input needs to be recovered
// to satisfy the constraint imposed by any RECOVERED_INPUT operands to
// the bailout snapshot.
if (rhs->isConstant()) {
Imm32 constant(ToInt32(rhs));
if (ins->isAddI())
masm.subl(constant, reg);
else
masm.addl(constant, reg);
} else {
if (ins->isAddI())
masm.subl(ToOperand(rhs), reg);
else
masm.addl(ToOperand(rhs), reg);
}
bailout(ool->ins()->snapshot());
}
class MulNegativeZeroCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared>
{
LMulI* ins_;
public:
explicit MulNegativeZeroCheck(LMulI* ins)
: ins_(ins)
{ }
virtual void accept(CodeGeneratorX86Shared* codegen) {
codegen->visitMulNegativeZeroCheck(this);
}
LMulI* ins() const {
return ins_;
}
};
void
CodeGeneratorX86Shared::visitMulI(LMulI* ins)
{
const LAllocation* lhs = ins->lhs();
const LAllocation* rhs = ins->rhs();
MMul* mul = ins->mir();
MOZ_ASSERT_IF(mul->mode() == MMul::Integer, !mul->canBeNegativeZero() && !mul->canOverflow());
if (rhs->isConstant()) {
// Bailout on -0.0
int32_t constant = ToInt32(rhs);
if (mul->canBeNegativeZero() && constant <= 0) {
Assembler::Condition bailoutCond = (constant == 0) ? Assembler::Signed : Assembler::Equal;
masm.test32(ToRegister(lhs), ToRegister(lhs));
bailoutIf(bailoutCond, ins->snapshot());
}
switch (constant) {
case -1:
masm.negl(ToOperand(lhs));
break;
case 0:
masm.xorl(ToOperand(lhs), ToRegister(lhs));
return; // escape overflow check;
case 1:
// nop
return; // escape overflow check;
case 2:
masm.addl(ToOperand(lhs), ToRegister(lhs));
break;
default:
if (!mul->canOverflow() && constant > 0) {
// Use shift if cannot overflow and constant is power of 2
int32_t shift = FloorLog2(constant);
if ((1 << shift) == constant) {
masm.shll(Imm32(shift), ToRegister(lhs));
return;
}
}
masm.imull(Imm32(ToInt32(rhs)), ToRegister(lhs));
}
// Bailout on overflow
if (mul->canOverflow())
bailoutIf(Assembler::Overflow, ins->snapshot());
} else {
masm.imull(ToOperand(rhs), ToRegister(lhs));
// Bailout on overflow
if (mul->canOverflow())
bailoutIf(Assembler::Overflow, ins->snapshot());
if (mul->canBeNegativeZero()) {
// Jump to an OOL path if the result is 0.
MulNegativeZeroCheck* ool = new(alloc()) MulNegativeZeroCheck(ins);
addOutOfLineCode(ool, mul);
masm.test32(ToRegister(lhs), ToRegister(lhs));
masm.j(Assembler::Zero, ool->entry());
masm.bind(ool->rejoin());
}
}
}
void
CodeGeneratorX86Shared::visitMulI64(LMulI64* lir)
{
const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));
if (IsConstant(rhs)) {
int64_t constant = ToInt64(rhs);
switch (constant) {
case -1:
masm.neg64(ToRegister64(lhs));
return;
case 0:
masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
return;
case 1:
// nop
return;
case 2:
masm.add64(ToRegister64(lhs), ToRegister64(lhs));
return;
default:
if (constant > 0) {
// Use shift if constant is power of 2.
int32_t shift = mozilla::FloorLog2(constant);
if (int64_t(1) << shift == constant) {
masm.lshift64(Imm32(shift), ToRegister64(lhs));
return;
}
}
Register temp = ToTempRegisterOrInvalid(lir->temp());
masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
}
} else {
Register temp = ToTempRegisterOrInvalid(lir->temp());
masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
}
}
class ReturnZero : public OutOfLineCodeBase<CodeGeneratorX86Shared>
{
Register reg_;
public:
explicit ReturnZero(Register reg)
: reg_(reg)
{ }
virtual void accept(CodeGeneratorX86Shared* codegen) {
codegen->visitReturnZero(this);
}
Register reg() const {
return reg_;
}
};
void
CodeGeneratorX86Shared::visitReturnZero(ReturnZero* ool)
{
masm.mov(ImmWord(0), ool->reg());
masm.jmp(ool->rejoin());
}
void
CodeGeneratorX86Shared::visitUDivOrMod(LUDivOrMod* ins)
{
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register output = ToRegister(ins->output());
MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
MOZ_ASSERT(rhs != edx);
MOZ_ASSERT_IF(output == eax, ToRegister(ins->remainder()) == edx);
ReturnZero* ool = nullptr;
// Put the lhs in eax.
if (lhs != eax)
masm.mov(lhs, eax);
// Prevent divide by zero.
if (ins->canBeDivideByZero()) {
masm.test32(rhs, rhs);
if (ins->mir()->isTruncated()) {
if (ins->trapOnError()) {
masm.j(Assembler::Zero, trap(ins, wasm::Trap::IntegerDivideByZero));
} else {
ool = new(alloc()) ReturnZero(output);
masm.j(Assembler::Zero, ool->entry());
}
} else {
bailoutIf(Assembler::Zero, ins->snapshot());
}
}
// Zero extend the lhs into edx to make (edx:eax), since udiv is 64-bit.
masm.mov(ImmWord(0), edx);
masm.udiv(rhs);
// If the remainder is > 0, bailout since this must be a double.
if (ins->mir()->isDiv() && !ins->mir()->toDiv()->canTruncateRemainder()) {
Register remainder = ToRegister(ins->remainder());
masm.test32(remainder, remainder);
bailoutIf(Assembler::NonZero, ins->snapshot());
}
// Unsigned div or mod can return a value that's not a signed int32.
// If our users aren't expecting that, bail.
if (!ins->mir()->isTruncated()) {
masm.test32(output, output);
bailoutIf(Assembler::Signed, ins->snapshot());
}
if (ool) {
addOutOfLineCode(ool, ins->mir());
masm.bind(ool->rejoin());
}
}
void
CodeGeneratorX86Shared::visitUDivOrModConstant(LUDivOrModConstant *ins) {
Register lhs = ToRegister(ins->numerator());
Register output = ToRegister(ins->output());
uint32_t d = ins->denominator();
// This emits the division answer into edx or the modulus answer into eax.
MOZ_ASSERT(output == eax || output == edx);
MOZ_ASSERT(lhs != eax && lhs != edx);
bool isDiv = (output == edx);
if (d == 0) {
if (ins->mir()->isTruncated()) {
if (ins->trapOnError())
masm.jump(trap(ins, wasm::Trap::IntegerDivideByZero));
else
masm.xorl(output, output);
} else {
bailout(ins->snapshot());
}
return;
}
// The denominator isn't a power of 2 (see LDivPowTwoI and LModPowTwoI).
MOZ_ASSERT((d & (d - 1)) != 0);
ReciprocalMulConstants rmc = computeDivisionConstants(d, /* maxLog = */ 32);
// We first compute (M * n) >> 32, where M = rmc.multiplier.
masm.movl(Imm32(rmc.multiplier), eax);
masm.umull(lhs);
if (rmc.multiplier > UINT32_MAX) {
// M >= 2^32 and shift == 0 is impossible, as d >= 2 implies that
// ((M * n) >> (32 + shift)) >= n > floor(n/d) whenever n >= d, contradicting
// the proof of correctness in computeDivisionConstants.
MOZ_ASSERT(rmc.shiftAmount > 0);
MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 33));
// We actually computed edx = ((uint32_t(M) * n) >> 32) instead. Since
// (M * n) >> (32 + shift) is the same as (edx + n) >> shift, we can
// correct for the overflow. This case is a bit trickier than the signed
// case, though, as the (edx + n) addition itself can overflow; however,
// note that (edx + n) >> shift == (((n - edx) >> 1) + edx) >> (shift - 1),
// which is overflow-free. See Hacker's Delight, section 10-8 for details.
// Compute (n - edx) >> 1 into eax.
masm.movl(lhs, eax);
masm.subl(edx, eax);
masm.shrl(Imm32(1), eax);
// Finish the computation.
masm.addl(eax, edx);
masm.shrl(Imm32(rmc.shiftAmount - 1), edx);
} else {
masm.shrl(Imm32(rmc.shiftAmount), edx);
}
// We now have the truncated division value in edx. If we're
// computing a modulus or checking whether the division resulted
// in an integer, we need to multiply the obtained value by d and
// finish the computation/check.
if (!isDiv) {
masm.imull(Imm32(d), edx, edx);
masm.movl(lhs, eax);
masm.subl(edx, eax);
// The final result of the modulus op, just computed above by the
// sub instruction, can be a number in the range [2^31, 2^32). If
// this is the case and the modulus is not truncated, we must bail
// out.
if (!ins->mir()->isTruncated())
bailoutIf(Assembler::Signed, ins->snapshot());
} else if (!ins->mir()->isTruncated()) {
masm.imull(Imm32(d), edx, eax);
masm.cmpl(lhs, eax);
bailoutIf(Assembler::NotEqual, ins->snapshot());
}
}
void
CodeGeneratorX86Shared::visitMulNegativeZeroCheck(MulNegativeZeroCheck* ool)
{
LMulI* ins = ool->ins();
Register result = ToRegister(ins->output());
Operand lhsCopy = ToOperand(ins->lhsCopy());
Operand rhs = ToOperand(ins->rhs());
MOZ_ASSERT_IF(lhsCopy.kind() == Operand::REG, lhsCopy.reg() != result.code());
// Result is -0 if lhs or rhs is negative.
masm.movl(lhsCopy, result);
masm.orl(rhs, result);
bailoutIf(Assembler::Signed, ins->snapshot());
masm.mov(ImmWord(0), result);
masm.jmp(ool->rejoin());
}
void
CodeGeneratorX86Shared::visitDivPowTwoI(LDivPowTwoI* ins)
{
Register lhs = ToRegister(ins->numerator());
DebugOnly<Register> output = ToRegister(ins->output());
int32_t shift = ins->shift();
bool negativeDivisor = ins->negativeDivisor();
MDiv* mir = ins->mir();
// We use defineReuseInput so these should always be the same, which is
// convenient since all of our instructions here are two-address.
MOZ_ASSERT(lhs == output);
if (!mir->isTruncated() && negativeDivisor) {
// 0 divided by a negative number must return a double.
masm.test32(lhs, lhs);
bailoutIf(Assembler::Zero, ins->snapshot());
}
if (shift) {
if (!mir->isTruncated()) {
// If the remainder is != 0, bailout since this must be a double.
masm.test32(lhs, Imm32(UINT32_MAX >> (32 - shift)));
bailoutIf(Assembler::NonZero, ins->snapshot());
}
if (mir->isUnsigned()) {
masm.shrl(Imm32(shift), lhs);
} else {
// Adjust the value so that shifting produces a correctly
// rounded result when the numerator is negative. See 10-1
// "Signed Division by a Known Power of 2" in Henry
// S. Warren, Jr.'s Hacker's Delight.
if (mir->canBeNegativeDividend()) {
Register lhsCopy = ToRegister(ins->numeratorCopy());
MOZ_ASSERT(lhsCopy != lhs);
if (shift > 1)
masm.sarl(Imm32(31), lhs);
masm.shrl(Imm32(32 - shift), lhs);
masm.addl(lhsCopy, lhs);
}
masm.sarl(Imm32(shift), lhs);
if (negativeDivisor)
masm.negl(lhs);
}
return;
}
if (negativeDivisor) {
// INT32_MIN / -1 overflows.
masm.negl(lhs);
if (!mir->isTruncated())
bailoutIf(Assembler::Overflow, ins->snapshot());
else if (mir->trapOnError())
masm.j(Assembler::Overflow, trap(mir, wasm::Trap::IntegerOverflow));
} else if (mir->isUnsigned() && !mir->isTruncated()) {
// Unsigned division by 1 can overflow if output is not
// truncated.
masm.test32(lhs, lhs);
bailoutIf(Assembler::Signed, ins->snapshot());
}
}
void
CodeGeneratorX86Shared::visitDivOrModConstantI(LDivOrModConstantI* ins) {
Register lhs = ToRegister(ins->numerator());
Register output = ToRegister(ins->output());
int32_t d = ins->denominator();
// This emits the division answer into edx or the modulus answer into eax.
MOZ_ASSERT(output == eax || output == edx);
MOZ_ASSERT(lhs != eax && lhs != edx);
bool isDiv = (output == edx);
// The absolute value of the denominator isn't a power of 2 (see LDivPowTwoI
// and LModPowTwoI).
MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0);
// We will first divide by Abs(d), and negate the answer if d is negative.
// If desired, this can be avoided by generalizing computeDivisionConstants.
ReciprocalMulConstants rmc = computeDivisionConstants(Abs(d), /* maxLog = */ 31);
// We first compute (M * n) >> 32, where M = rmc.multiplier.
masm.movl(Imm32(rmc.multiplier), eax);
masm.imull(lhs);
if (rmc.multiplier > INT32_MAX) {
MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 32));
// We actually computed edx = ((int32_t(M) * n) >> 32) instead. Since
// (M * n) >> 32 is the same as (edx + n), we can correct for the overflow.
// (edx + n) can't overflow, as n and edx have opposite signs because int32_t(M)
// is negative.
masm.addl(lhs, edx);
}
// (M * n) >> (32 + shift) is the truncated division answer if n is non-negative,
// as proved in the comments of computeDivisionConstants. We must add 1 later if n is
// negative to get the right answer in all cases.
masm.sarl(Imm32(rmc.shiftAmount), edx);
// We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be
// computed with just a sign-extending shift of 31 bits.
if (ins->canBeNegativeDividend()) {
masm.movl(lhs, eax);
masm.sarl(Imm32(31), eax);
masm.subl(eax, edx);
}
// After this, edx contains the correct truncated division result.
if (d < 0)
masm.negl(edx);
if (!isDiv) {
masm.imull(Imm32(-d), edx, eax);
masm.addl(lhs, eax);
}
if (!ins->mir()->isTruncated()) {
if (isDiv) {
// This is a division op. Multiply the obtained value by d to check if
// the correct answer is an integer. This cannot overflow, since |d| > 1.
masm.imull(Imm32(d), edx, eax);
masm.cmp32(lhs, eax);
bailoutIf(Assembler::NotEqual, ins->snapshot());
// If lhs is zero and the divisor is negative, the answer should have
// been -0.
if (d < 0) {
masm.test32(lhs, lhs);
bailoutIf(Assembler::Zero, ins->snapshot());
}
} else if (ins->canBeNegativeDividend()) {
// This is a mod op. If the computed value is zero and lhs
// is negative, the answer should have been -0.
Label done;
masm.cmp32(lhs, Imm32(0));
masm.j(Assembler::GreaterThanOrEqual, &done);
masm.test32(eax, eax);
bailoutIf(Assembler::Zero, ins->snapshot());
masm.bind(&done);
}
}
}
void
CodeGeneratorX86Shared::visitDivI(LDivI* ins)
{
Register remainder = ToRegister(ins->remainder());
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register output = ToRegister(ins->output());
MDiv* mir = ins->mir();
MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
MOZ_ASSERT(rhs != edx);
MOZ_ASSERT(remainder == edx);
MOZ_ASSERT(output == eax);
Label done;
ReturnZero* ool = nullptr;
// Put the lhs in eax, for either the negative overflow case or the regular
// divide case.
if (lhs != eax)
masm.mov(lhs, eax);
// Handle divide by zero.
if (mir->canBeDivideByZero()) {
masm.test32(rhs, rhs);
if (mir->trapOnError()) {
masm.j(Assembler::Zero, trap(mir, wasm::Trap::IntegerDivideByZero));
} else if (mir->canTruncateInfinities()) {
// Truncated division by zero is zero (Infinity|0 == 0)
if (!ool)
ool = new(alloc()) ReturnZero(output);
masm.j(Assembler::Zero, ool->entry());
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Zero, ins->snapshot());
}
}
// Handle an integer overflow exception from -2147483648 / -1.
if (mir->canBeNegativeOverflow()) {
Label notmin;
masm.cmp32(lhs, Imm32(INT32_MIN));
masm.j(Assembler::NotEqual, ¬min);
masm.cmp32(rhs, Imm32(-1));
if (mir->trapOnError()) {
masm.j(Assembler::Equal, trap(mir, wasm::Trap::IntegerOverflow));
} else if (mir->canTruncateOverflow()) {
// (-INT32_MIN)|0 == INT32_MIN and INT32_MIN is already in the
// output register (lhs == eax).
masm.j(Assembler::Equal, &done);
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Equal, ins->snapshot());
}
masm.bind(¬min);
}
// Handle negative 0.
if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
Label nonzero;
masm.test32(lhs, lhs);
masm.j(Assembler::NonZero, &nonzero);
masm.cmp32(rhs, Imm32(0));
bailoutIf(Assembler::LessThan, ins->snapshot());
masm.bind(&nonzero);
}
// Sign extend the lhs into edx to make (edx:eax), since idiv is 64-bit.
if (lhs != eax)
masm.mov(lhs, eax);
masm.cdq();
masm.idiv(rhs);
if (!mir->canTruncateRemainder()) {
// If the remainder is > 0, bailout since this must be a double.
masm.test32(remainder, remainder);
bailoutIf(Assembler::NonZero, ins->snapshot());
}
masm.bind(&done);
if (ool) {
addOutOfLineCode(ool, mir);
masm.bind(ool->rejoin());
}
}
void
CodeGeneratorX86Shared::visitModPowTwoI(LModPowTwoI* ins)
{
Register lhs = ToRegister(ins->getOperand(0));
int32_t shift = ins->shift();
Label negative;
if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
// Switch based on sign of the lhs.
// Positive numbers are just a bitmask
masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
}
masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
Label done;
masm.jump(&done);
// Negative numbers need a negate, bitmask, negate
masm.bind(&negative);
// Unlike in the visitModI case, we are not computing the mod by means of a
// division. Therefore, the divisor = -1 case isn't problematic (the andl
// always returns 0, which is what we expect).
//
// The negl instruction overflows if lhs == INT32_MIN, but this is also not
// a problem: shift is at most 31, and so the andl also always returns 0.
masm.negl(lhs);
masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
masm.negl(lhs);
// Since a%b has the same sign as b, and a is negative in this branch,
// an answer of 0 means the correct result is actually -0. Bail out.
if (!ins->mir()->isTruncated())
bailoutIf(Assembler::Zero, ins->snapshot());
masm.bind(&done);
}
}
class ModOverflowCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared>
{
Label done_;
LModI* ins_;
Register rhs_;
public:
explicit ModOverflowCheck(LModI* ins, Register rhs)
: ins_(ins), rhs_(rhs)
{ }
virtual void accept(CodeGeneratorX86Shared* codegen) {
codegen->visitModOverflowCheck(this);
}
Label* done() {
return &done_;
}
LModI* ins() const {
return ins_;
}
Register rhs() const {
return rhs_;
}
};
void
CodeGeneratorX86Shared::visitModOverflowCheck(ModOverflowCheck* ool)
{
masm.cmp32(ool->rhs(), Imm32(-1));
if (ool->ins()->mir()->isTruncated()) {
masm.j(Assembler::NotEqual, ool->rejoin());
masm.mov(ImmWord(0), edx);
masm.jmp(ool->done());
} else {
bailoutIf(Assembler::Equal, ool->ins()->snapshot());
masm.jmp(ool->rejoin());
}
}
void
CodeGeneratorX86Shared::visitModI(LModI* ins)
{
Register remainder = ToRegister(ins->remainder());
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
// Required to use idiv.
MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
MOZ_ASSERT(rhs != edx);
MOZ_ASSERT(remainder == edx);
MOZ_ASSERT(ToRegister(ins->getTemp(0)) == eax);
Label done;
ReturnZero* ool = nullptr;
ModOverflowCheck* overflow = nullptr;
// Set up eax in preparation for doing a div.
if (lhs != eax)
masm.mov(lhs, eax);
MMod* mir = ins->mir();
// Prevent divide by zero.
if (mir->canBeDivideByZero()) {
masm.test32(rhs, rhs);
if (mir->isTruncated()) {
if (mir->trapOnError()) {
masm.j(Assembler::Zero, trap(mir, wasm::Trap::IntegerDivideByZero));
} else {
if (!ool)
ool = new(alloc()) ReturnZero(edx);
masm.j(Assembler::Zero, ool->entry());
}
} else {
bailoutIf(Assembler::Zero, ins->snapshot());
}
}
Label negative;
// Switch based on sign of the lhs.
if (mir->canBeNegativeDividend())
masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
// If lhs >= 0 then remainder = lhs % rhs. The remainder must be positive.
{
// Check if rhs is a power-of-two.
if (mir->canBePowerOfTwoDivisor()) {
MOZ_ASSERT(rhs != remainder);
// Rhs y is a power-of-two if (y & (y-1)) == 0. Note that if
// y is any negative number other than INT32_MIN, both y and
// y-1 will have the sign bit set so these are never optimized
// as powers-of-two. If y is INT32_MIN, y-1 will be INT32_MAX
// and because lhs >= 0 at this point, lhs & INT32_MAX returns
// the correct value.
Label notPowerOfTwo;
masm.mov(rhs, remainder);
masm.subl(Imm32(1), remainder);
masm.branchTest32(Assembler::NonZero, remainder, rhs, ¬PowerOfTwo);
{
masm.andl(lhs, remainder);
masm.jmp(&done);
}
masm.bind(¬PowerOfTwo);
}
// Since lhs >= 0, the sign-extension will be 0
masm.mov(ImmWord(0), edx);
masm.idiv(rhs);
}
// Otherwise, we have to beware of two special cases:
if (mir->canBeNegativeDividend()) {
masm.jump(&done);
masm.bind(&negative);
// Prevent an integer overflow exception from -2147483648 % -1
Label notmin;
masm.cmp32(lhs, Imm32(INT32_MIN));
overflow = new(alloc()) ModOverflowCheck(ins, rhs);
masm.j(Assembler::Equal, overflow->entry());
masm.bind(overflow->rejoin());
masm.cdq();
masm.idiv(rhs);
if (!mir->isTruncated()) {
// A remainder of 0 means that the rval must be -0, which is a double.
masm.test32(remainder, remainder);
bailoutIf(Assembler::Zero, ins->snapshot());
}
}
masm.bind(&done);
if (overflow) {
addOutOfLineCode(overflow, mir);
masm.bind(overflow->done());
}
if (ool) {
addOutOfLineCode(ool, mir);
masm.bind(ool->rejoin());
}
}
void
CodeGeneratorX86Shared::visitBitNotI(LBitNotI* ins)
{
const LAllocation* input = ins->getOperand(0);
MOZ_ASSERT(!input->isConstant());
masm.notl(ToOperand(input));
}
void
CodeGeneratorX86Shared::visitBitOpI(LBitOpI* ins)
{
const LAllocation* lhs = ins->getOperand(0);
const LAllocation* rhs = ins->getOperand(1);
switch (ins->bitop()) {
case JSOP_BITOR:
if (rhs->isConstant())
masm.orl(Imm32(ToInt32(rhs)), ToOperand(lhs));
else
masm.orl(ToOperand(rhs), ToRegister(lhs));
break;
case JSOP_BITXOR:
if (rhs->isConstant())
masm.xorl(Imm32(ToInt32(rhs)), ToOperand(lhs));
else
masm.xorl(ToOperand(rhs), ToRegister(lhs));
break;
case JSOP_BITAND:
if (rhs->isConstant())
masm.andl(Imm32(ToInt32(rhs)), ToOperand(lhs));
else
masm.andl(ToOperand(rhs), ToRegister(lhs));
break;
default:
MOZ_CRASH("unexpected binary opcode");
}
}
void
CodeGeneratorX86Shared::visitBitOpI64(LBitOpI64* lir)
{
const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
switch (lir->bitop()) {
case JSOP_BITOR:
if (IsConstant(rhs))
masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
else
masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
break;
case JSOP_BITXOR:
if (IsConstant(rhs))
masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
else
masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
break;
case JSOP_BITAND:
if (IsConstant(rhs))
masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
else
masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
break;
default:
MOZ_CRASH("unexpected binary opcode");
}
}
void
CodeGeneratorX86Shared::visitShiftI(LShiftI* ins)
{
Register lhs = ToRegister(ins->lhs());
const LAllocation* rhs = ins->rhs();
if (rhs->isConstant()) {
int32_t shift = ToInt32(rhs) & 0x1F;
switch (ins->bitop()) {
case JSOP_LSH:
if (shift)
masm.shll(Imm32(shift), lhs);
break;
case JSOP_RSH:
if (shift)
masm.sarl(Imm32(shift), lhs);
break;
case JSOP_URSH:
if (shift) {
masm.shrl(Imm32(shift), lhs);
} else if (ins->mir()->toUrsh()->fallible()) {
// x >>> 0 can overflow.
masm.test32(lhs, lhs);
bailoutIf(Assembler::Signed, ins->snapshot());
}
break;
default:
MOZ_CRASH("Unexpected shift op");
}
} else {
MOZ_ASSERT(ToRegister(rhs) == ecx);
switch (ins->bitop()) {
case JSOP_LSH:
masm.shll_cl(lhs);
break;
case JSOP_RSH:
masm.sarl_cl(lhs);
break;
case JSOP_URSH:
masm.shrl_cl(lhs);
if (ins->mir()->toUrsh()->fallible()) {
// x >>> 0 can overflow.
masm.test32(lhs, lhs);
bailoutIf(Assembler::Signed, ins->snapshot());
}
break;
default:
MOZ_CRASH("Unexpected shift op");
}
}
}
void
CodeGeneratorX86Shared::visitShiftI64(LShiftI64* lir)
{
const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
if (rhs->isConstant()) {
int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
switch (lir->bitop()) {
case JSOP_LSH:
if (shift)
masm.lshift64(Imm32(shift), ToRegister64(lhs));
break;
case JSOP_RSH:
if (shift)
masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
break;
case JSOP_URSH:
if (shift)
masm.rshift64(Imm32(shift), ToRegister64(lhs));
break;
default:
MOZ_CRASH("Unexpected shift op");
}
return;
}
MOZ_ASSERT(ToRegister(rhs) == ecx);
switch (lir->bitop()) {
case JSOP_LSH:
masm.lshift64(ecx, ToRegister64(lhs));
break;
case JSOP_RSH:
masm.rshift64Arithmetic(ecx, ToRegister64(lhs));
break;
case JSOP_URSH:
masm.rshift64(ecx, ToRegister64(lhs));
break;
default:
MOZ_CRASH("Unexpected shift op");
}
}
void
CodeGeneratorX86Shared::visitUrshD(LUrshD* ins)
{
Register lhs = ToRegister(ins->lhs());
MOZ_ASSERT(ToRegister(ins->temp()) == lhs);
const LAllocation* rhs = ins->rhs();
FloatRegister out = ToFloatRegister(ins->output());
if (rhs->isConstant()) {
int32_t shift = ToInt32(rhs) & 0x1F;
if (shift)
masm.shrl(Imm32(shift), lhs);
} else {
MOZ_ASSERT(ToRegister(rhs) == ecx);
masm.shrl_cl(lhs);
}
masm.convertUInt32ToDouble(lhs, out);
}
Operand
CodeGeneratorX86Shared::ToOperand(const LAllocation& a)
{
if (a.isGeneralReg())
return Operand(a.toGeneralReg()->reg());
if (a.isFloatReg())
return Operand(a.toFloatReg()->reg());
return Operand(masm.getStackPointer(), ToStackOffset(&a));
}
Operand
CodeGeneratorX86Shared::ToOperand(const LAllocation* a)
{
return ToOperand(*a);
}
Operand
CodeGeneratorX86Shared::ToOperand(const LDefinition* def)
{
return ToOperand(def->output());
}
MoveOperand
CodeGeneratorX86Shared::toMoveOperand(LAllocation a) const
{
if (a.isGeneralReg())
return MoveOperand(ToRegister(a));
if (a.isFloatReg())
return MoveOperand(ToFloatRegister(a));
return MoveOperand(StackPointer, ToStackOffset(a));
}
class OutOfLineTableSwitch : public OutOfLineCodeBase<CodeGeneratorX86Shared>
{
MTableSwitch* mir_;
CodeLabel jumpLabel_;
void accept(CodeGeneratorX86Shared* codegen) {
codegen->visitOutOfLineTableSwitch(this);
}
public:
explicit OutOfLineTableSwitch(MTableSwitch* mir)
: mir_(mir)
{}
MTableSwitch* mir() const {
return mir_;
}
CodeLabel* jumpLabel() {
return &jumpLabel_;
}
};
void
CodeGeneratorX86Shared::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool)
{
MTableSwitch* mir = ool->mir();
masm.haltingAlign(sizeof(void*));
masm.use(ool->jumpLabel()->target());
masm.addCodeLabel(*ool->jumpLabel());
for (size_t i = 0; i < mir->numCases(); i++) {
LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
Label* caseheader = caseblock->label();
uint32_t caseoffset = caseheader->offset();
// The entries of the jump table need to be absolute addresses and thus
// must be patched after codegen is finished.
CodeLabel cl;
masm.writeCodePointer(cl.patchAt());
cl.target()->bind(caseoffset);
masm.addCodeLabel(cl);
}
}
void
CodeGeneratorX86Shared::emitTableSwitchDispatch(MTableSwitch* mir, Register index, Register base)
{
Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
// Lower value with low value
if (mir->low() != 0)
masm.subl(Imm32(mir->low()), index);
// Jump to default case if input is out of range
int32_t cases = mir->numCases();
masm.cmp32(index, Imm32(cases));
masm.j(AssemblerX86Shared::AboveOrEqual, defaultcase);
// To fill in the CodeLabels for the case entries, we need to first
// generate the case entries (we don't yet know their offsets in the
// instruction stream).
OutOfLineTableSwitch* ool = new(alloc()) OutOfLineTableSwitch(mir);
addOutOfLineCode(ool, mir);
// Compute the position where a pointer to the right case stands.
masm.mov(ool->jumpLabel()->patchAt(), base);
Operand pointer = Operand(base, index, ScalePointer);
// Jump to the right case
masm.jmp(pointer);
}
void
CodeGeneratorX86Shared::visitMathD(LMathD* math)
{
FloatRegister lhs = ToFloatRegister(math->lhs());
Operand rhs = ToOperand(math->rhs());
FloatRegister output = ToFloatRegister(math->output());
switch (math->jsop()) {
case JSOP_ADD:
masm.vaddsd(rhs, lhs, output);
break;
case JSOP_SUB:
masm.vsubsd(rhs, lhs, output);
break;
case JSOP_MUL:
masm.vmulsd(rhs, lhs, output);
break;
case JSOP_DIV:
masm.vdivsd(rhs, lhs, output);
break;
default:
MOZ_CRASH("unexpected opcode");
}
}
void
CodeGeneratorX86Shared::visitMathF(LMathF* math)
{
FloatRegister lhs = ToFloatRegister(math->lhs());
Operand rhs = ToOperand(math->rhs());
FloatRegister output = ToFloatRegister(math->output());
switch (math->jsop()) {
case JSOP_ADD:
masm.vaddss(rhs, lhs, output);
break;
case JSOP_SUB:
masm.vsubss(rhs, lhs, output);
break;
case JSOP_MUL:
masm.vmulss(rhs, lhs, output);
break;
case JSOP_DIV:
masm.vdivss(rhs, lhs, output);
break;
default:
MOZ_CRASH("unexpected opcode");
}
}
void
CodeGeneratorX86Shared::visitFloor(LFloor* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
Register output = ToRegister(lir->output());
Label bailout;
if (AssemblerX86Shared::HasSSE41()) {
// Bail on negative-zero.
masm.branchNegativeZero(input, output, &bailout);
bailoutFrom(&bailout, lir->snapshot());
// Round toward -Infinity.
{
ScratchDoubleScope scratch(masm);
masm.vroundsd(X86Encoding::RoundDown, input, scratch, scratch);
bailoutCvttsd2si(scratch, output, lir->snapshot());
}
} else {
Label negative, end;
// Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
{
ScratchDoubleScope scratch(masm);
masm.zeroDouble(scratch);
masm.branchDouble(Assembler::DoubleLessThan, input, scratch, &negative);
}
// Bail on negative-zero.
masm.branchNegativeZero(input, output, &bailout);
bailoutFrom(&bailout, lir->snapshot());
// Input is non-negative, so truncation correctly rounds.
bailoutCvttsd2si(input, output, lir->snapshot());
masm.jump(&end);
// Input is negative, but isn't -0.
// Negative values go on a comparatively expensive path, since no
// native rounding mode matches JS semantics. Still better than callVM.
masm.bind(&negative);
{
// Truncate and round toward zero.
// This is off-by-one for everything but integer-valued inputs.
bailoutCvttsd2si(input, output, lir->snapshot());
// Test whether the input double was integer-valued.
{
ScratchDoubleScope scratch(masm);
masm.convertInt32ToDouble(output, scratch);
masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch, &end);
}
// Input is not integer-valued, so we rounded off-by-one in the
// wrong direction. Correct by subtraction.
masm.subl(Imm32(1), output);
// Cannot overflow: output was already checked against INT_MIN.
}
masm.bind(&end);
}
}
void
CodeGeneratorX86Shared::visitFloorF(LFloorF* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
Register output = ToRegister(lir->output());
Label bailout;
if (AssemblerX86Shared::HasSSE41()) {
// Bail on negative-zero.
masm.branchNegativeZeroFloat32(input, output, &bailout);
bailoutFrom(&bailout, lir->snapshot());
// Round toward -Infinity.
{
ScratchFloat32Scope scratch(masm);
masm.vroundss(X86Encoding::RoundDown, input, scratch, scratch);
bailoutCvttss2si(scratch, output, lir->snapshot());
}
} else {
Label negative, end;
// Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
{
ScratchFloat32Scope scratch(masm);
masm.zeroFloat32(scratch);
masm.branchFloat(Assembler::DoubleLessThan, input, scratch, &negative);
}
// Bail on negative-zero.
masm.branchNegativeZeroFloat32(input, output, &bailout);
bailoutFrom(&bailout, lir->snapshot());
// Input is non-negative, so truncation correctly rounds.
bailoutCvttss2si(input, output, lir->snapshot());
masm.jump(&end);
// Input is negative, but isn't -0.
// Negative values go on a comparatively expensive path, since no
// native rounding mode matches JS semantics. Still better than callVM.
masm.bind(&negative);
{
// Truncate and round toward zero.
// This is off-by-one for everything but integer-valued inputs.
bailoutCvttss2si(input, output, lir->snapshot());
// Test whether the input double was integer-valued.
{
ScratchFloat32Scope scratch(masm);
masm.convertInt32ToFloat32(output, scratch);
masm.branchFloat(Assembler::DoubleEqualOrUnordered, input, scratch, &end);
}
// Input is not integer-valued, so we rounded off-by-one in the
// wrong direction. Correct by subtraction.
masm.subl(Imm32(1), output);
// Cannot overflow: output was already checked against INT_MIN.
}
masm.bind(&end);
}
}
void
CodeGeneratorX86Shared::visitCeil(LCeil* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
ScratchDoubleScope scratch(masm);
Register output = ToRegister(lir->output());
Label bailout, lessThanMinusOne;
// Bail on ]-1; -0] range
masm.loadConstantDouble(-1, scratch);
masm.branchDouble(Assembler::DoubleLessThanOrEqualOrUnordered, input,
scratch, &lessThanMinusOne);
// Test for remaining values with the sign bit set, i.e. ]-1; -0]
masm.vmovmskpd(input, output);
masm.branchTest32(Assembler::NonZero, output, Imm32(1), &bailout);
bailoutFrom(&bailout, lir->snapshot());
if (AssemblerX86Shared::HasSSE41()) {
// x <= -1 or x > -0
masm.bind(&lessThanMinusOne);
// Round toward +Infinity.
masm.vroundsd(X86Encoding::RoundUp, input, scratch, scratch);
bailoutCvttsd2si(scratch, output, lir->snapshot());
return;
}
// No SSE4.1
Label end;
// x >= 0 and x is not -0.0, we can truncate (resp. truncate and add 1) for
// integer (resp. non-integer) values.
// Will also work for values >= INT_MAX + 1, as the truncate
// operation will return INT_MIN and there'll be a bailout.
bailoutCvttsd2si(input, output, lir->snapshot());
masm.convertInt32ToDouble(output, scratch);
masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch, &end);
// Input is not integer-valued, add 1 to obtain the ceiling value
masm.addl(Imm32(1), output);
// if input > INT_MAX, output == INT_MAX so adding 1 will overflow.
bailoutIf(Assembler::Overflow, lir->snapshot());
masm.jump(&end);
// x <= -1, truncation is the way to go.
masm.bind(&lessThanMinusOne);
bailoutCvttsd2si(input, output, lir->snapshot());
masm.bind(&end);
}
void
CodeGeneratorX86Shared::visitCeilF(LCeilF* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
ScratchFloat32Scope scratch(masm);
Register output = ToRegister(lir->output());
Label bailout, lessThanMinusOne;
// Bail on ]-1; -0] range
masm.loadConstantFloat32(-1.f, scratch);
masm.branchFloat(Assembler::DoubleLessThanOrEqualOrUnordered, input,
scratch, &lessThanMinusOne);
// Test for remaining values with the sign bit set, i.e. ]-1; -0]
masm.vmovmskps(input, output);
masm.branchTest32(Assembler::NonZero, output, Imm32(1), &bailout);
bailoutFrom(&bailout, lir->snapshot());
if (AssemblerX86Shared::HasSSE41()) {
// x <= -1 or x > -0
masm.bind(&lessThanMinusOne);
// Round toward +Infinity.
masm.vroundss(X86Encoding::RoundUp, input, scratch, scratch);
bailoutCvttss2si(scratch, output, lir->snapshot());
return;
}
// No SSE4.1
Label end;
// x >= 0 and x is not -0.0, we can truncate (resp. truncate and add 1) for
// integer (resp. non-integer) values.
// Will also work for values >= INT_MAX + 1, as the truncate
// operation will return INT_MIN and there'll be a bailout.
bailoutCvttss2si(input, output, lir->snapshot());
masm.convertInt32ToFloat32(output, scratch);
masm.branchFloat(Assembler::DoubleEqualOrUnordered, input, scratch, &end);
// Input is not integer-valued, add 1 to obtain the ceiling value
masm.addl(Imm32(1), output);
// if input > INT_MAX, output == INT_MAX so adding 1 will overflow.
bailoutIf(Assembler::Overflow, lir->snapshot());
masm.jump(&end);
// x <= -1, truncation is the way to go.
masm.bind(&lessThanMinusOne);
bailoutCvttss2si(input, output, lir->snapshot());
masm.bind(&end);
}
void
CodeGeneratorX86Shared::visitRound(LRound* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
FloatRegister temp = ToFloatRegister(lir->temp());
ScratchDoubleScope scratch(masm);
Register output = ToRegister(lir->output());
Label negativeOrZero, negative, end, bailout;
// Branch to a slow path for non-positive inputs. Doesn't catch NaN.
masm.zeroDouble(scratch);
masm.loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, scratch, &negativeOrZero);
// Input is positive. Add the biggest double less than 0.5 and
// truncate, rounding down (because if the input is the biggest double less
// than 0.5, adding 0.5 would undesirably round up to 1). Note that we have
// to add the input to the temp register because we're not allowed to
// modify the input register.
masm.addDouble(input, temp);
bailoutCvttsd2si(temp, output, lir->snapshot());
masm.jump(&end);
// Input is negative, +0 or -0.
masm.bind(&negativeOrZero);
// Branch on negative input.
masm.j(Assembler::NotEqual, &negative);
// Bail on negative-zero.
masm.branchNegativeZero(input, output, &bailout, /* maybeNonZero = */ false);
bailoutFrom(&bailout, lir->snapshot());
// Input is +0
masm.xor32(output, output);
masm.jump(&end);
// Input is negative.
masm.bind(&negative);
// Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
// be added the biggest double less than 0.5.
Label loadJoin;
masm.loadConstantDouble(-0.5, scratch);
masm.branchDouble(Assembler::DoubleLessThan, input, scratch, &loadJoin);
masm.loadConstantDouble(0.5, temp);
masm.bind(&loadJoin);
if (AssemblerX86Shared::HasSSE41()) {
// Add 0.5 and round toward -Infinity. The result is stored in the temp
// register (currently contains 0.5).
masm.addDouble(input, temp);
masm.vroundsd(X86Encoding::RoundDown, temp, scratch, scratch);
// Truncate.
bailoutCvttsd2si(scratch, output, lir->snapshot());
// If the result is positive zero, then the actual result is -0. Bail.
// Otherwise, the truncation will have produced the correct negative integer.
masm.test32(output, output);
bailoutIf(Assembler::Zero, lir->snapshot());
} else {
masm.addDouble(input, temp);
// Round toward -Infinity without the benefit of ROUNDSD.
{
// If input + 0.5 >= 0, input is a negative number >= -0.5 and the result is -0.
masm.compareDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch);
bailoutIf(Assembler::DoubleGreaterThanOrEqual, lir->snapshot());
// Truncate and round toward zero.
// This is off-by-one for everything but integer-valued inputs.
bailoutCvttsd2si(temp, output, lir->snapshot());
// Test whether the truncated double was integer-valued.
masm.convertInt32ToDouble(output, scratch);
masm.branchDouble(Assembler::DoubleEqualOrUnordered, temp, scratch, &end);
// Input is not integer-valued, so we rounded off-by-one in the
// wrong direction. Correct by subtraction.
masm.subl(Imm32(1), output);
// Cannot overflow: output was already checked against INT_MIN.
}
}
masm.bind(&end);
}
void
CodeGeneratorX86Shared::visitRoundF(LRoundF* lir)
{
FloatRegister input = ToFloatRegister(lir->input());
FloatRegister temp = ToFloatRegister(lir->temp());
ScratchFloat32Scope scratch(masm);
Register output = ToRegister(lir->output());
Label negativeOrZero, negative, end, bailout;
// Branch to a slow path for non-positive inputs. Doesn't catch NaN.
masm.zeroFloat32(scratch);
masm.loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
masm.branchFloat(Assembler::DoubleLessThanOrEqual, input, scratch, &negativeOrZero);
// Input is non-negative. Add the biggest float less than 0.5 and truncate,
// rounding down (because if the input is the biggest float less than 0.5,
// adding 0.5 would undesirably round up to 1). Note that we have to add
// the input to the temp register because we're not allowed to modify the
// input register.
masm.addFloat32(input, temp);
bailoutCvttss2si(temp, output, lir->snapshot());
masm.jump(&end);
// Input is negative, +0 or -0.
masm.bind(&negativeOrZero);
// Branch on negative input.
masm.j(Assembler::NotEqual, &negative);
// Bail on negative-zero.
masm.branchNegativeZeroFloat32(input, output, &bailout);
bailoutFrom(&bailout, lir->snapshot());
// Input is +0.
masm.xor32(output, output);
masm.jump(&end);
// Input is negative.
masm.bind(&negative);
// Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
// be added the biggest double less than 0.5.
Label loadJoin;
masm.loadConstantFloat32(-0.5f, scratch);
masm.branchFloat(Assembler::DoubleLessThan, input, scratch, &loadJoin);
masm.loadConstantFloat32(0.5f, temp);
masm.bind(&loadJoin);
if (AssemblerX86Shared::HasSSE41()) {
// Add 0.5 and round toward -Infinity. The result is stored in the temp
// register (currently contains 0.5).
masm.addFloat32(input, temp);
masm.vroundss(X86Encoding::RoundDown, temp, scratch, scratch);
// Truncate.
bailoutCvttss2si(scratch, output, lir->snapshot());
// If the result is positive zero, then the actual result is -0. Bail.
// Otherwise, the truncation will have produced the correct negative integer.
masm.test32(output, output);
bailoutIf(Assembler::Zero, lir->snapshot());
} else {
masm.addFloat32(input, temp);
// Round toward -Infinity without the benefit of ROUNDSS.
{
// If input + 0.5 >= 0, input is a negative number >= -0.5 and the result is -0.
masm.compareFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch);
bailoutIf(Assembler::DoubleGreaterThanOrEqual, lir->snapshot());
// Truncate and round toward zero.
// This is off-by-one for everything but integer-valued inputs.
bailoutCvttss2si(temp, output, lir->snapshot());
// Test whether the truncated double was integer-valued.
masm.convertInt32ToFloat32(output, scratch);
masm.branchFloat(Assembler::DoubleEqualOrUnordered, temp, scratch, &end);
// Input is not integer-valued, so we rounded off-by-one in the
// wrong direction. Correct by subtraction.
masm.subl(Imm32(1), output);
// Cannot overflow: output was already checked against INT_MIN.
}
}
masm.bind(&end);
}
void
CodeGeneratorX86Shared::visitGuardShape(LGuardShape* guard)
{
Register obj = ToRegister(guard->input());
masm.cmpPtr(Operand(obj, ShapedObject::offsetOfShape()), ImmGCPtr(guard->mir()->shape()));
bailoutIf(Assembler::NotEqual, guard->snapshot());
}
void
CodeGeneratorX86Shared::visitGuardObjectGroup(LGuardObjectGroup* guard)
{
Register obj = ToRegister(guard->input());
masm.cmpPtr(Operand(obj, JSObject::offsetOfGroup()), ImmGCPtr(guard->mir()->group()));
Assembler::Condition cond =
guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
bailoutIf(cond, guard->snapshot());
}
void
CodeGeneratorX86Shared::visitGuardClass(LGuardClass* guard)
{
Register obj = ToRegister(guard->input());
Register tmp = ToRegister(guard->tempInt());
masm.loadPtr(Address(obj, JSObject::offsetOfGroup()), tmp);
masm.cmpPtr(Operand(tmp, ObjectGroup::offsetOfClasp()), ImmPtr(guard->mir()->getClass()));
bailoutIf(Assembler::NotEqual, guard->snapshot());
}
void
CodeGeneratorX86Shared::visitEffectiveAddress(LEffectiveAddress* ins)
{
const MEffectiveAddress* mir = ins->mir();
Register base = ToRegister(ins->base());
Register index = ToRegister(ins->index());
Register output = ToRegister(ins->output());
masm.leal(Operand(base, index, mir->scale(), mir->displacement()), output);
}
void
CodeGeneratorX86Shared::generateInvalidateEpilogue()
{
// Ensure that there is enough space in the buffer for the OsiPoint
// patching to occur. Otherwise, we could overwrite the invalidation
// epilogue.
for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize())
masm.nop();
masm.bind(&invalidate_);
// Push the Ion script onto the stack (when we determine what that pointer is).
invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
JitCode* thunk = gen->jitRuntime()->getInvalidationThunk();
masm.call(thunk);
// We should never reach this point in JIT code -- the invalidation thunk should
// pop the invalidated JS frame and return directly to its caller.
masm.assumeUnreachable("Should have returned directly to its caller instead of here.");
}
void
CodeGeneratorX86Shared::visitNegI(LNegI* ins)
{
Register input = ToRegister(ins->input());
MOZ_ASSERT(input == ToRegister(ins->output()));
masm.neg32(input);
}
void
CodeGeneratorX86Shared::visitNegD(LNegD* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
MOZ_ASSERT(input == ToFloatRegister(ins->output()));
masm.negateDouble(input);
}
void
CodeGeneratorX86Shared::visitNegF(LNegF* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
MOZ_ASSERT(input == ToFloatRegister(ins->output()));
masm.negateFloat(input);
}
void
CodeGeneratorX86Shared::visitSimd128Int(LSimd128Int* ins)
{
const LDefinition* out = ins->getDef(0);
masm.loadConstantSimd128Int(ins->getValue(), ToFloatRegister(out));
}
void
CodeGeneratorX86Shared::visitSimd128Float(LSimd128Float* ins)
{
const LDefinition* out = ins->getDef(0);
masm.loadConstantSimd128Float(ins->getValue(), ToFloatRegister(out));
}
void
CodeGeneratorX86Shared::visitInt32x4ToFloat32x4(LInt32x4ToFloat32x4* ins)
{
FloatRegister in = ToFloatRegister(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
masm.convertInt32x4ToFloat32x4(in, out);
}
void
CodeGeneratorX86Shared::visitFloat32x4ToInt32x4(LFloat32x4ToInt32x4* ins)
{
FloatRegister in = ToFloatRegister(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
Register temp = ToRegister(ins->temp());
masm.convertFloat32x4ToInt32x4(in, out);
auto* ool = new(alloc()) OutOfLineSimdFloatToIntCheck(temp, in, ins, ins->mir()->trapOffset());
addOutOfLineCode(ool, ins->mir());
static const SimdConstant InvalidResult = SimdConstant::SplatX4(int32_t(-2147483648));
ScratchSimd128Scope scratch(masm);
masm.loadConstantSimd128Int(InvalidResult, scratch);
masm.packedEqualInt32x4(Operand(out), scratch);
// TODO (bug 1156228): If we have SSE4.1, we can use PTEST here instead of
// the two following instructions.
masm.vmovmskps(scratch, temp);
masm.cmp32(temp, Imm32(0));
masm.j(Assembler::NotEqual, ool->entry());
masm.bind(ool->rejoin());
}
void
CodeGeneratorX86Shared::visitOutOfLineSimdFloatToIntCheck(OutOfLineSimdFloatToIntCheck *ool)
{
static const SimdConstant Int32MaxX4 = SimdConstant::SplatX4(2147483647.f);
static const SimdConstant Int32MinX4 = SimdConstant::SplatX4(-2147483648.f);
Label onConversionError;
FloatRegister input = ool->input();
Register temp = ool->temp();
ScratchSimd128Scope scratch(masm);
masm.loadConstantSimd128Float(Int32MinX4, scratch);
masm.vcmpleps(Operand(input), scratch, scratch);
masm.vmovmskps(scratch, temp);
masm.cmp32(temp, Imm32(15));
masm.j(Assembler::NotEqual, &onConversionError);
masm.loadConstantSimd128Float(Int32MaxX4, scratch);
masm.vcmpleps(Operand(input), scratch, scratch);
masm.vmovmskps(scratch, temp);
masm.cmp32(temp, Imm32(0));
masm.j(Assembler::NotEqual, &onConversionError);
masm.jump(ool->rejoin());
if (gen->compilingWasm()) {
masm.bindLater(&onConversionError, trap(ool, wasm::Trap::ImpreciseSimdConversion));
} else {
masm.bind(&onConversionError);
bailout(ool->ins()->snapshot());
}
}
// Convert Float32x4 to Uint32x4.
//
// If any input lane value is out of range or NaN, bail out.
void
CodeGeneratorX86Shared::visitFloat32x4ToUint32x4(LFloat32x4ToUint32x4* ins)
{
const MSimdConvert* mir = ins->mir();
FloatRegister in = ToFloatRegister(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
Register temp = ToRegister(ins->tempR());
FloatRegister tempF = ToFloatRegister(ins->tempF());
// Classify lane values into 4 disjoint classes:
//
// N-lanes: in <= -1.0
// A-lanes: -1.0 < in <= 0x0.ffffffp31
// B-lanes: 0x1.0p31 <= in <= 0x0.ffffffp32
// V-lanes: 0x1.0p32 <= in, or isnan(in)
//
// We need to bail out to throw a RangeError if we see any N-lanes or
// V-lanes.
//
// For A-lanes and B-lanes, we make two float -> int32 conversions:
//
// A = cvttps2dq(in)
// B = cvttps2dq(in - 0x1.0p31f)
//
// Note that the subtraction for the B computation is exact for B-lanes.
// There is no rounding, so B is the low 31 bits of the correctly converted
// result.
//
// The cvttps2dq instruction produces 0x80000000 when the input is NaN or
// out of range for a signed int32_t. This conveniently provides the missing
// high bit for B, so the desired result is A for A-lanes and A|B for
// B-lanes.
ScratchSimd128Scope scratch(masm);
// TODO: If the majority of lanes are A-lanes, it could be faster to compute
// A first, use vmovmskps to check for any non-A-lanes and handle them in
// ool code. OTOH, we we're wrong about the lane distribution, that would be
// slower.
// Compute B in |scratch|.
static const float Adjust = 0x80000000; // 0x1.0p31f for the benefit of MSVC.
static const SimdConstant Bias = SimdConstant::SplatX4(-Adjust);
masm.loadConstantSimd128Float(Bias, scratch);
masm.packedAddFloat32(Operand(in), scratch);
masm.convertFloat32x4ToInt32x4(scratch, scratch);
// Compute A in |out|. This is the last time we use |in| and the first time
// we use |out|, so we can tolerate if they are the same register.
masm.convertFloat32x4ToInt32x4(in, out);
// We can identify A-lanes by the sign bits in A: Any A-lanes will be
// positive in A, and N, B, and V-lanes will be 0x80000000 in A. Compute a
// mask of non-A-lanes into |tempF|.
masm.zeroSimd128Float(tempF);
masm.packedGreaterThanInt32x4(Operand(out), tempF);
// Clear the A-lanes in B.
masm.bitwiseAndSimd128(Operand(tempF), scratch);
// Compute the final result: A for A-lanes, A|B for B-lanes.
masm.bitwiseOrSimd128(Operand(scratch), out);
// We still need to filter out the V-lanes. They would show up as 0x80000000
// in both A and B. Since we cleared the valid A-lanes in B, the V-lanes are
// the remaining negative lanes in B.
masm.vmovmskps(scratch, temp);
masm.cmp32(temp, Imm32(0));
if (gen->compilingWasm())
masm.j(Assembler::NotEqual, trap(mir, wasm::Trap::ImpreciseSimdConversion));
else
bailoutIf(Assembler::NotEqual, ins->snapshot());
}
void
CodeGeneratorX86Shared::visitSimdValueInt32x4(LSimdValueInt32x4* ins)
{
MOZ_ASSERT(ins->mir()->type() == MIRType::Int32x4 || ins->mir()->type() == MIRType::Bool32x4);
FloatRegister output = ToFloatRegister(ins->output());
if (AssemblerX86Shared::HasSSE41()) {
masm.vmovd(ToRegister(ins->getOperand(0)), output);
for (size_t i = 1; i < 4; ++i) {
Register r = ToRegister(ins->getOperand(i));
masm.vpinsrd(i, r, output, output);
}
return;
}
masm.reserveStack(Simd128DataSize);
for (size_t i = 0; i < 4; ++i) {
Register r = ToRegister(ins->getOperand(i));
masm.store32(r, Address(StackPointer, i * sizeof(int32_t)));
}
masm.loadAlignedSimd128Int(Address(StackPointer, 0), output);
masm.freeStack(Simd128DataSize);
}
void
CodeGeneratorX86Shared::visitSimdValueFloat32x4(LSimdValueFloat32x4* ins)
{
MOZ_ASSERT(ins->mir()->type() == MIRType::Float32x4);
FloatRegister r0 = ToFloatRegister(ins->getOperand(0));
FloatRegister r1 = ToFloatRegister(ins->getOperand(1));
FloatRegister r2 = ToFloatRegister(ins->getOperand(2));
FloatRegister r3 = ToFloatRegister(ins->getOperand(3));
FloatRegister tmp = ToFloatRegister(ins->getTemp(0));
FloatRegister output = ToFloatRegister(ins->output());
FloatRegister r0Copy = masm.reusedInputFloat32x4(r0, output);
FloatRegister r1Copy = masm.reusedInputFloat32x4(r1, tmp);
masm.vunpcklps(r3, r1Copy, tmp);
masm.vunpcklps(r2, r0Copy, output);
masm.vunpcklps(tmp, output, output);
}
void
CodeGeneratorX86Shared::visitSimdSplatX16(LSimdSplatX16* ins)
{
MOZ_ASSERT(SimdTypeToLength(ins->mir()->type()) == 16);
Register input = ToRegister(ins->getOperand(0));
FloatRegister output = ToFloatRegister(ins->output());
masm.vmovd(input, output);
if (AssemblerX86Shared::HasSSSE3()) {
masm.zeroSimd128Int(ScratchSimd128Reg);
masm.vpshufb(ScratchSimd128Reg, output, output);
} else {
// Use two shifts to duplicate the low 8 bits into the low 16 bits.
masm.vpsllw(Imm32(8), output, output);
masm.vmovdqa(output, ScratchSimd128Reg);
masm.vpsrlw(Imm32(8), ScratchSimd128Reg, ScratchSimd128Reg);
masm.vpor(ScratchSimd128Reg, output, output);
// Then do an X8 splat.
masm.vpshuflw(0, output, output);
masm.vpshufd(0, output, output);
}
}
void
CodeGeneratorX86Shared::visitSimdSplatX8(LSimdSplatX8* ins)
{
MOZ_ASSERT(SimdTypeToLength(ins->mir()->type()) == 8);
Register input = ToRegister(ins->getOperand(0));
FloatRegister output = ToFloatRegister(ins->output());
masm.vmovd(input, output);
masm.vpshuflw(0, output, output);
masm.vpshufd(0, output, output);
}
void
CodeGeneratorX86Shared::visitSimdSplatX4(LSimdSplatX4* ins)
{
FloatRegister output = ToFloatRegister(ins->output());
MSimdSplat* mir = ins->mir();
MOZ_ASSERT(IsSimdType(mir->type()));
JS_STATIC_ASSERT(sizeof(float) == sizeof(int32_t));
if (mir->type() == MIRType::Float32x4) {
FloatRegister r = ToFloatRegister(ins->getOperand(0));
FloatRegister rCopy = masm.reusedInputFloat32x4(r, output);
masm.vshufps(0, rCopy, rCopy, output);
} else {
Register r = ToRegister(ins->getOperand(0));
masm.vmovd(r, output);
masm.vpshufd(0, output, output);
}
}
void
CodeGeneratorX86Shared::visitSimdReinterpretCast(LSimdReinterpretCast* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
if (input.aliases(output))
return;
if (IsIntegerSimdType(ins->mir()->type()))
masm.vmovdqa(input, output);
else
masm.vmovaps(input, output);
}
// Extract an integer lane from the 32x4 vector register |input| and place it in
// |output|.
void
CodeGeneratorX86Shared::emitSimdExtractLane32x4(FloatRegister input, Register output, unsigned lane)
{
if (lane == 0) {
// The value we want to extract is in the low double-word
masm.moveLowInt32(input, output);
} else if (AssemblerX86Shared::HasSSE41()) {
masm.vpextrd(lane, input, output);
} else {
uint32_t mask = MacroAssembler::ComputeShuffleMask(lane);
masm.shuffleInt32(mask, input, ScratchSimd128Reg);
masm.moveLowInt32(ScratchSimd128Reg, output);
}
}
// Extract an integer lane from the 16x8 vector register |input|, sign- or
// zero-extend to 32 bits and place the result in |output|.
void
CodeGeneratorX86Shared::emitSimdExtractLane16x8(FloatRegister input, Register output,
unsigned lane, SimdSign signedness)
{
// Unlike pextrd and pextrb, this is available in SSE2.
masm.vpextrw(lane, input, output);
if (signedness == SimdSign::Signed)
masm.movswl(output, output);
}
// Extract an integer lane from the 8x16 vector register |input|, sign- or
// zero-extend to 32 bits and place the result in |output|.
void
CodeGeneratorX86Shared::emitSimdExtractLane8x16(FloatRegister input, Register output,
unsigned lane, SimdSign signedness)
{
if (AssemblerX86Shared::HasSSE41()) {
masm.vpextrb(lane, input, output);
// vpextrb clears the high bits, so no further extension required.
if (signedness == SimdSign::Unsigned)
signedness = SimdSign::NotApplicable;
} else {
// Extract the relevant 16 bits containing our lane, then shift the
// right 8 bits into place.
emitSimdExtractLane16x8(input, output, lane / 2, SimdSign::Unsigned);
if (lane % 2) {
masm.shrl(Imm32(8), output);
// The shrl handles the zero-extension. Don't repeat it.
if (signedness == SimdSign::Unsigned)
signedness = SimdSign::NotApplicable;
}
}
// We have the right low 8 bits in |output|, but we may need to fix the high
// bits. Note that this requires |output| to be one of the %eax-%edx
// registers.
switch (signedness) {
case SimdSign::Signed:
masm.movsbl(output, output);
break;
case SimdSign::Unsigned:
masm.movzbl(output, output);
break;
case SimdSign::NotApplicable:
// No adjustment needed.
break;
}
}
void
CodeGeneratorX86Shared::visitSimdExtractElementB(LSimdExtractElementB* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
Register output = ToRegister(ins->output());
MSimdExtractElement* mir = ins->mir();
unsigned length = SimdTypeToLength(mir->specialization());
switch (length) {
case 4:
emitSimdExtractLane32x4(input, output, mir->lane());
break;
case 8:
// Get a lane, don't bother fixing the high bits since we'll mask below.
emitSimdExtractLane16x8(input, output, mir->lane(), SimdSign::NotApplicable);
break;
case 16:
emitSimdExtractLane8x16(input, output, mir->lane(), SimdSign::NotApplicable);
break;
default:
MOZ_CRASH("Unhandled SIMD length");
}
// We need to generate a 0/1 value. We have 0/-1 and possibly dirty high bits.
masm.and32(Imm32(1), output);
}
void
CodeGeneratorX86Shared::visitSimdExtractElementI(LSimdExtractElementI* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
Register output = ToRegister(ins->output());
MSimdExtractElement* mir = ins->mir();
unsigned length = SimdTypeToLength(mir->specialization());
switch (length) {
case 4:
emitSimdExtractLane32x4(input, output, mir->lane());
break;
case 8:
emitSimdExtractLane16x8(input, output, mir->lane(), mir->signedness());
break;
case 16:
emitSimdExtractLane8x16(input, output, mir->lane(), mir->signedness());
break;
default:
MOZ_CRASH("Unhandled SIMD length");
}
}
void
CodeGeneratorX86Shared::visitSimdExtractElementU2D(LSimdExtractElementU2D* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
Register temp = ToRegister(ins->temp());
MSimdExtractElement* mir = ins->mir();
MOZ_ASSERT(mir->specialization() == MIRType::Int32x4);
emitSimdExtractLane32x4(input, temp, mir->lane());
masm.convertUInt32ToDouble(temp, output);
}
void
CodeGeneratorX86Shared::visitSimdExtractElementF(LSimdExtractElementF* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
unsigned lane = ins->mir()->lane();
if (lane == 0) {
// The value we want to extract is in the low double-word
if (input != output)
masm.moveFloat32(input, output);
} else if (lane == 2) {
masm.moveHighPairToLowPairFloat32(input, output);
} else {
uint32_t mask = MacroAssembler::ComputeShuffleMask(lane);
masm.shuffleFloat32(mask, input, output);
}
// NaNs contained within SIMD values are not enforced to be canonical, so
// when we extract an element into a "regular" scalar JS value, we have to
// canonicalize. In wasm code, we can skip this, as wasm only has to
// canonicalize NaNs at FFI boundaries.
if (!gen->compilingWasm())
masm.canonicalizeFloat(output);
}
void
CodeGeneratorX86Shared::visitSimdInsertElementI(LSimdInsertElementI* ins)
{
FloatRegister vector = ToFloatRegister(ins->vector());
Register value = ToRegister(ins->value());
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT(vector == output); // defineReuseInput(0)
unsigned lane = ins->lane();
unsigned length = ins->length();
if (length == 8) {
// Available in SSE 2.
masm.vpinsrw(lane, value, vector, output);
return;
}
// Note that, contrarily to float32x4, we cannot use vmovd if the inserted
// value goes into the first component, as vmovd clears out the higher lanes
// of the output.
if (AssemblerX86Shared::HasSSE41()) {
// TODO: Teach Lowering that we don't need defineReuseInput if we have AVX.
switch (length) {
case 4:
masm.vpinsrd(lane, value, vector, output);
return;
case 16:
masm.vpinsrb(lane, value, vector, output);
return;
}
}
masm.reserveStack(Simd128DataSize);
masm.storeAlignedSimd128Int(vector, Address(StackPointer, 0));
switch (length) {
case 4:
masm.store32(value, Address(StackPointer, lane * sizeof(int32_t)));
break;
case 16:
// Note that this requires `value` to be in one the registers where the
// low 8 bits are addressible (%eax - %edx on x86, all of them on x86-64).
masm.store8(value, Address(StackPointer, lane * sizeof(int8_t)));
break;
default:
MOZ_CRASH("Unsupported SIMD length");
}
masm.loadAlignedSimd128Int(Address(StackPointer, 0), output);
masm.freeStack(Simd128DataSize);
}
void
CodeGeneratorX86Shared::visitSimdInsertElementF(LSimdInsertElementF* ins)
{
FloatRegister vector = ToFloatRegister(ins->vector());
FloatRegister value = ToFloatRegister(ins->value());
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT(vector == output); // defineReuseInput(0)
if (ins->lane() == 0) {
// As both operands are registers, vmovss doesn't modify the upper bits
// of the destination operand.
if (value != output)
masm.vmovss(value, vector, output);
return;
}
if (AssemblerX86Shared::HasSSE41()) {
// The input value is in the low float32 of the 'value' FloatRegister.
masm.vinsertps(masm.vinsertpsMask(0, ins->lane()), value, output, output);
return;
}
unsigned component = unsigned(ins->lane());
masm.reserveStack(Simd128DataSize);
masm.storeAlignedSimd128Float(vector, Address(StackPointer, 0));
masm.storeFloat32(value, Address(StackPointer, component * sizeof(int32_t)));
masm.loadAlignedSimd128Float(Address(StackPointer, 0), output);
masm.freeStack(Simd128DataSize);
}
void
CodeGeneratorX86Shared::visitSimdAllTrue(LSimdAllTrue* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
Register output = ToRegister(ins->output());
masm.vmovmskps(input, output);
masm.cmp32(output, Imm32(0xf));
masm.emitSet(Assembler::Zero, output);
}
void
CodeGeneratorX86Shared::visitSimdAnyTrue(LSimdAnyTrue* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
Register output = ToRegister(ins->output());
masm.vmovmskps(input, output);
masm.cmp32(output, Imm32(0x0));
masm.emitSet(Assembler::NonZero, output);
}
template <class T, class Reg> void
CodeGeneratorX86Shared::visitSimdGeneralShuffle(LSimdGeneralShuffleBase* ins, Reg tempRegister)
{
MSimdGeneralShuffle* mir = ins->mir();
unsigned numVectors = mir->numVectors();
Register laneTemp = ToRegister(ins->temp());
// This won't generate fast code, but it's fine because we expect users
// to have used constant indices (and thus MSimdGeneralShuffle to be fold
// into MSimdSwizzle/MSimdShuffle, which are fast).
// We need stack space for the numVectors inputs and for the output vector.
unsigned stackSpace = Simd128DataSize * (numVectors + 1);
masm.reserveStack(stackSpace);
for (unsigned i = 0; i < numVectors; i++) {
masm.storeAlignedVector<T>(ToFloatRegister(ins->vector(i)),
Address(StackPointer, Simd128DataSize * (1 + i)));
}
Label bail;
const Scale laneScale = ScaleFromElemWidth(sizeof(T));
for (size_t i = 0; i < mir->numLanes(); i++) {
Operand lane = ToOperand(ins->lane(i));
masm.cmp32(lane, Imm32(numVectors * mir->numLanes() - 1));
masm.j(Assembler::Above, &bail);
if (lane.kind() == Operand::REG) {
masm.loadScalar<T>(Operand(StackPointer, ToRegister(ins->lane(i)), laneScale, Simd128DataSize),
tempRegister);
} else {
masm.load32(lane, laneTemp);
masm.loadScalar<T>(Operand(StackPointer, laneTemp, laneScale, Simd128DataSize), tempRegister);
}
masm.storeScalar<T>(tempRegister, Address(StackPointer, i * sizeof(T)));
}
FloatRegister output = ToFloatRegister(ins->output());
masm.loadAlignedVector<T>(Address(StackPointer, 0), output);
Label join;
masm.jump(&join);
{
masm.bind(&bail);
masm.freeStack(stackSpace);
bailout(ins->snapshot());
}
masm.bind(&join);
masm.setFramePushed(masm.framePushed() + stackSpace);
masm.freeStack(stackSpace);
}
void
CodeGeneratorX86Shared::visitSimdGeneralShuffleI(LSimdGeneralShuffleI* ins)
{
switch (ins->mir()->type()) {
case MIRType::Int8x16:
return visitSimdGeneralShuffle<int8_t, Register>(ins, ToRegister(ins->temp()));
case MIRType::Int16x8:
return visitSimdGeneralShuffle<int16_t, Register>(ins, ToRegister(ins->temp()));
case MIRType::Int32x4:
return visitSimdGeneralShuffle<int32_t, Register>(ins, ToRegister(ins->temp()));
default:
MOZ_CRASH("unsupported type for general shuffle");
}
}
void
CodeGeneratorX86Shared::visitSimdGeneralShuffleF(LSimdGeneralShuffleF* ins)
{
ScratchFloat32Scope scratch(masm);
visitSimdGeneralShuffle<float, FloatRegister>(ins, scratch);
}
void
CodeGeneratorX86Shared::visitSimdSwizzleI(LSimdSwizzleI* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
const unsigned numLanes = ins->numLanes();
switch (numLanes) {
case 4: {
uint32_t x = ins->lane(0);
uint32_t y = ins->lane(1);
uint32_t z = ins->lane(2);
uint32_t w = ins->lane(3);
uint32_t mask = MacroAssembler::ComputeShuffleMask(x, y, z, w);
masm.shuffleInt32(mask, input, output);
return;
}
}
// In the general case, use pshufb if it is available. Convert to a
// byte-wise swizzle.
const unsigned bytesPerLane = 16 / numLanes;
int8_t bLane[16];
for (unsigned i = 0; i < numLanes; i++) {
for (unsigned b = 0; b < bytesPerLane; b++) {
bLane[i * bytesPerLane + b] = ins->lane(i) * bytesPerLane + b;
}
}
if (AssemblerX86Shared::HasSSSE3()) {
ScratchSimd128Scope scratch(masm);
masm.loadConstantSimd128Int(SimdConstant::CreateX16(bLane), scratch);
FloatRegister inputCopy = masm.reusedInputInt32x4(input, output);
masm.vpshufb(scratch, inputCopy, output);
return;
}
// Worst-case fallback for pre-SSSE3 machines. Bounce through memory.
Register temp = ToRegister(ins->getTemp(0));
masm.reserveStack(2 * Simd128DataSize);
masm.storeAlignedSimd128Int(input, Address(StackPointer, Simd128DataSize));
for (unsigned i = 0; i < 16; i++) {
masm.load8ZeroExtend(Address(StackPointer, Simd128DataSize + bLane[i]), temp);
masm.store8(temp, Address(StackPointer, i));
}
masm.loadAlignedSimd128Int(Address(StackPointer, 0), output);
masm.freeStack(2 * Simd128DataSize);
}
void
CodeGeneratorX86Shared::visitSimdSwizzleF(LSimdSwizzleF* ins)
{
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT(ins->numLanes() == 4);
uint32_t x = ins->lane(0);
uint32_t y = ins->lane(1);
uint32_t z = ins->lane(2);
uint32_t w = ins->lane(3);
if (AssemblerX86Shared::HasSSE3()) {
if (ins->lanesMatch(0, 0, 2, 2)) {
masm.vmovsldup(input, output);
return;
}
if (ins->lanesMatch(1, 1, 3, 3)) {
masm.vmovshdup(input, output);
return;
}
}
// TODO Here and below, arch specific lowering could identify this pattern
// and use defineReuseInput to avoid this move (bug 1084404)
if (ins->lanesMatch(2, 3, 2, 3)) {
FloatRegister inputCopy = masm.reusedInputFloat32x4(input, output);
masm.vmovhlps(input, inputCopy, output);
return;
}
if (ins->lanesMatch(0, 1, 0, 1)) {
if (AssemblerX86Shared::HasSSE3() && !AssemblerX86Shared::HasAVX()) {
masm.vmovddup(input, output);
return;
}
FloatRegister inputCopy = masm.reusedInputFloat32x4(input, output);
masm.vmovlhps(input, inputCopy, output);
return;
}
if (ins->lanesMatch(0, 0, 1, 1)) {
FloatRegister inputCopy = masm.reusedInputFloat32x4(input, output);
masm.vunpcklps(input, inputCopy, output);
return;
}
if (ins->lanesMatch(2, 2, 3, 3)) {
FloatRegister inputCopy = masm.reusedInputFloat32x4(input, output);
masm.vunpckhps(input, inputCopy, output);
return;
}
uint32_t mask = MacroAssembler::ComputeShuffleMask(x, y, z, w);
masm.shuffleFloat32(mask, input, output);
}
void
CodeGeneratorX86Shared::visitSimdShuffle(LSimdShuffle* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
FloatRegister rhs = ToFloatRegister(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
const unsigned numLanes = ins->numLanes();
const unsigned bytesPerLane = 16 / numLanes;
// Convert the shuffle to a byte-wise shuffle.
uint8_t bLane[16];
for (unsigned i = 0; i < numLanes; i++) {
for (unsigned b = 0; b < bytesPerLane; b++) {
bLane[i * bytesPerLane + b] = ins->lane(i) * bytesPerLane + b;
}
}
// Use pshufb if it is available.
if (AssemblerX86Shared::HasSSSE3()) {
FloatRegister scratch1 = ToFloatRegister(ins->temp());
ScratchSimd128Scope scratch2(masm);
// Use pshufb instructions to gather the lanes from each source vector.
// A negative index creates a zero lane, so the two vectors can be combined.
// Set scratch2 = lanes from lhs.
int8_t idx[16];
for (unsigned i = 0; i < 16; i++)
idx[i] = bLane[i] < 16 ? bLane[i] : -1;
masm.loadConstantSimd128Int(SimdConstant::CreateX16(idx), scratch1);
FloatRegister lhsCopy = masm.reusedInputInt32x4(lhs, scratch2);
masm.vpshufb(scratch1, lhsCopy, scratch2);
// Set output = lanes from rhs.
for (unsigned i = 0; i < 16; i++)
idx[i] = bLane[i] >= 16 ? bLane[i] - 16 : -1;
masm.loadConstantSimd128Int(SimdConstant::CreateX16(idx), scratch1);
FloatRegister rhsCopy = masm.reusedInputInt32x4(rhs, output);
masm.vpshufb(scratch1, rhsCopy, output);
// Combine.
masm.vpor(scratch2, output, output);
return;
}
// Worst-case fallback for pre-SSE3 machines. Bounce through memory.
Register temp = ToRegister(ins->getTemp(0));
masm.reserveStack(3 * Simd128DataSize);
masm.storeAlignedSimd128Int(lhs, Address(StackPointer, Simd128DataSize));
masm.storeAlignedSimd128Int(rhs, Address(StackPointer, 2 * Simd128DataSize));
for (unsigned i = 0; i < 16; i++) {
masm.load8ZeroExtend(Address(StackPointer, Simd128DataSize + bLane[i]), temp);
masm.store8(temp, Address(StackPointer, i));
}
masm.loadAlignedSimd128Int(Address(StackPointer, 0), output);
masm.freeStack(3 * Simd128DataSize);
}
void
CodeGeneratorX86Shared::visitSimdShuffleX4(LSimdShuffleX4* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister out = ToFloatRegister(ins->output());
uint32_t x = ins->lane(0);
uint32_t y = ins->lane(1);
uint32_t z = ins->lane(2);
uint32_t w = ins->lane(3);
// Check that lanes come from LHS in majority:
unsigned numLanesFromLHS = (x < 4) + (y < 4) + (z < 4) + (w < 4);
MOZ_ASSERT(numLanesFromLHS >= 2);
// When reading this method, remember that vshufps takes the two first
// inputs of the destination operand (right operand) and the two last
// inputs of the source operand (left operand).
//
// Legend for explanations:
// - L: LHS
// - R: RHS
// - T: temporary
uint32_t mask;
// If all lanes came from a single vector, we should have constructed a
// MSimdSwizzle instead.
MOZ_ASSERT(numLanesFromLHS < 4);
// If all values stay in their lane, this is a blend.
if (AssemblerX86Shared::HasSSE41()) {
if (x % 4 == 0 && y % 4 == 1 && z % 4 == 2 && w % 4 == 3) {
masm.vblendps(masm.blendpsMask(x >= 4, y >= 4, z >= 4, w >= 4), rhs, lhs, out);
return;
}
}
// One element of the second, all other elements of the first
if (numLanesFromLHS == 3) {
unsigned firstMask = -1, secondMask = -1;
// register-register vmovss preserves the high lanes.
if (ins->lanesMatch(4, 1, 2, 3) && rhs.kind() == Operand::FPREG) {
masm.vmovss(FloatRegister::FromCode(rhs.fpu()), lhs, out);
return;
}
// SSE4.1 vinsertps can handle any single element.
unsigned numLanesUnchanged = (x == 0) + (y == 1) + (z == 2) + (w == 3);
if (AssemblerX86Shared::HasSSE41() && numLanesUnchanged == 3) {
unsigned srcLane;
unsigned dstLane;
if (x >= 4) {
srcLane = x - 4;
dstLane = 0;
} else if (y >= 4) {
srcLane = y - 4;
dstLane = 1;
} else if (z >= 4) {
srcLane = z - 4;
dstLane = 2;
} else {
MOZ_ASSERT(w >= 4);
srcLane = w - 4;
dstLane = 3;
}
masm.vinsertps(masm.vinsertpsMask(srcLane, dstLane), rhs, lhs, out);
return;
}
FloatRegister rhsCopy = ToFloatRegister(ins->temp());
if (x < 4 && y < 4) {
if (w >= 4) {
w %= 4;
// T = (Rw Rw Lz Lz) = vshufps(firstMask, lhs, rhs, rhsCopy)
firstMask = MacroAssembler::ComputeShuffleMask(w, w, z, z);
// (Lx Ly Lz Rw) = (Lx Ly Tz Tx) = vshufps(secondMask, T, lhs, out)
secondMask = MacroAssembler::ComputeShuffleMask(x, y, 2, 0);
} else {
MOZ_ASSERT(z >= 4);
z %= 4;
// T = (Rz Rz Lw Lw) = vshufps(firstMask, lhs, rhs, rhsCopy)
firstMask = MacroAssembler::ComputeShuffleMask(z, z, w, w);
// (Lx Ly Rz Lw) = (Lx Ly Tx Tz) = vshufps(secondMask, T, lhs, out)
secondMask = MacroAssembler::ComputeShuffleMask(x, y, 0, 2);
}
masm.vshufps(firstMask, lhs, rhsCopy, rhsCopy);
masm.vshufps(secondMask, rhsCopy, lhs, out);
return;
}
MOZ_ASSERT(z < 4 && w < 4);
if (y >= 4) {
y %= 4;
// T = (Ry Ry Lx Lx) = vshufps(firstMask, lhs, rhs, rhsCopy)
firstMask = MacroAssembler::ComputeShuffleMask(y, y, x, x);
// (Lx Ry Lz Lw) = (Tz Tx Lz Lw) = vshufps(secondMask, lhs, T, out)
secondMask = MacroAssembler::ComputeShuffleMask(2, 0, z, w);
} else {
MOZ_ASSERT(x >= 4);
x %= 4;
// T = (Rx Rx Ly Ly) = vshufps(firstMask, lhs, rhs, rhsCopy)
firstMask = MacroAssembler::ComputeShuffleMask(x, x, y, y);
// (Rx Ly Lz Lw) = (Tx Tz Lz Lw) = vshufps(secondMask, lhs, T, out)
secondMask = MacroAssembler::ComputeShuffleMask(0, 2, z, w);
}
masm.vshufps(firstMask, lhs, rhsCopy, rhsCopy);
if (AssemblerX86Shared::HasAVX()) {
masm.vshufps(secondMask, lhs, rhsCopy, out);
} else {
masm.vshufps(secondMask, lhs, rhsCopy, rhsCopy);
masm.moveSimd128Float(rhsCopy, out);
}
return;
}
// Two elements from one vector, two other elements from the other
MOZ_ASSERT(numLanesFromLHS == 2);
// TODO Here and below, symmetric case would be more handy to avoid a move,
// but can't be reached because operands would get swapped (bug 1084404).
if (ins->lanesMatch(2, 3, 6, 7)) {
ScratchSimd128Scope scratch(masm);
if (AssemblerX86Shared::HasAVX()) {
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, scratch);
masm.vmovhlps(lhs, rhsCopy, out);
} else {
masm.loadAlignedSimd128Float(rhs, scratch);
masm.vmovhlps(lhs, scratch, scratch);
masm.moveSimd128Float(scratch, out);
}
return;
}
if (ins->lanesMatch(0, 1, 4, 5)) {
FloatRegister rhsCopy;
ScratchSimd128Scope scratch(masm);
if (rhs.kind() == Operand::FPREG) {
// No need to make an actual copy, since the operand is already
// in a register, and it won't be clobbered by the vmovlhps.
rhsCopy = FloatRegister::FromCode(rhs.fpu());
} else {
masm.loadAlignedSimd128Float(rhs, scratch);
rhsCopy = scratch;
}
masm.vmovlhps(rhsCopy, lhs, out);
return;
}
if (ins->lanesMatch(0, 4, 1, 5)) {
masm.vunpcklps(rhs, lhs, out);
return;
}
// TODO swapped case would be better (bug 1084404)
if (ins->lanesMatch(4, 0, 5, 1)) {
ScratchSimd128Scope scratch(masm);
if (AssemblerX86Shared::HasAVX()) {
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, scratch);
masm.vunpcklps(lhs, rhsCopy, out);
} else {
masm.loadAlignedSimd128Float(rhs, scratch);
masm.vunpcklps(lhs, scratch, scratch);
masm.moveSimd128Float(scratch, out);
}
return;
}
if (ins->lanesMatch(2, 6, 3, 7)) {
masm.vunpckhps(rhs, lhs, out);
return;
}
// TODO swapped case would be better (bug 1084404)
if (ins->lanesMatch(6, 2, 7, 3)) {
ScratchSimd128Scope scratch(masm);
if (AssemblerX86Shared::HasAVX()) {
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, scratch);
masm.vunpckhps(lhs, rhsCopy, out);
} else {
masm.loadAlignedSimd128Float(rhs, scratch);
masm.vunpckhps(lhs, scratch, scratch);
masm.moveSimd128Float(scratch, out);
}
return;
}
// In one vshufps
if (x < 4 && y < 4) {
mask = MacroAssembler::ComputeShuffleMask(x, y, z % 4, w % 4);
masm.vshufps(mask, rhs, lhs, out);
return;
}
// At creation, we should have explicitly swapped in this case.
MOZ_ASSERT(!(z >= 4 && w >= 4));
// In two vshufps, for the most generic case:
uint32_t firstMask[4], secondMask[4];
unsigned i = 0, j = 2, k = 0;
#define COMPUTE_MASK(lane) \
if (lane >= 4) { \
firstMask[j] = lane % 4; \
secondMask[k++] = j++; \
} else { \
firstMask[i] = lane; \
secondMask[k++] = i++; \
}
COMPUTE_MASK(x)
COMPUTE_MASK(y)
COMPUTE_MASK(z)
COMPUTE_MASK(w)
#undef COMPUTE_MASK
MOZ_ASSERT(i == 2 && j == 4 && k == 4);
mask = MacroAssembler::ComputeShuffleMask(firstMask[0], firstMask[1],
firstMask[2], firstMask[3]);
masm.vshufps(mask, rhs, lhs, lhs);
mask = MacroAssembler::ComputeShuffleMask(secondMask[0], secondMask[1],
secondMask[2], secondMask[3]);
masm.vshufps(mask, lhs, lhs, lhs);
}
void
CodeGeneratorX86Shared::visitSimdBinaryCompIx16(LSimdBinaryCompIx16* ins)
{
static const SimdConstant allOnes = SimdConstant::SplatX16(-1);
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT_IF(!Assembler::HasAVX(), output == lhs);
ScratchSimd128Scope scratch(masm);
MSimdBinaryComp::Operation op = ins->operation();
switch (op) {
case MSimdBinaryComp::greaterThan:
masm.vpcmpgtb(rhs, lhs, output);
return;
case MSimdBinaryComp::equal:
masm.vpcmpeqb(rhs, lhs, output);
return;
case MSimdBinaryComp::lessThan:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
// src := src > lhs (i.e. lhs < rhs)
// Improve by doing custom lowering (rhs is tied to the output register)
masm.vpcmpgtb(ToOperand(ins->lhs()), scratch, scratch);
masm.moveSimd128Int(scratch, output);
return;
case MSimdBinaryComp::notEqual:
// Ideally for notEqual, greaterThanOrEqual, and lessThanOrEqual, we
// should invert the comparison by, e.g. swapping the arms of a select
// if that's what it's used in.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.vpcmpeqb(rhs, lhs, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
case MSimdBinaryComp::greaterThanOrEqual:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
masm.vpcmpgtb(ToOperand(ins->lhs()), scratch, scratch);
masm.loadConstantSimd128Int(allOnes, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
case MSimdBinaryComp::lessThanOrEqual:
// lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.vpcmpgtb(rhs, lhs, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryCompIx8(LSimdBinaryCompIx8* ins)
{
static const SimdConstant allOnes = SimdConstant::SplatX8(-1);
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MOZ_ASSERT_IF(!Assembler::HasAVX(), output == lhs);
ScratchSimd128Scope scratch(masm);
MSimdBinaryComp::Operation op = ins->operation();
switch (op) {
case MSimdBinaryComp::greaterThan:
masm.vpcmpgtw(rhs, lhs, output);
return;
case MSimdBinaryComp::equal:
masm.vpcmpeqw(rhs, lhs, output);
return;
case MSimdBinaryComp::lessThan:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
// src := src > lhs (i.e. lhs < rhs)
// Improve by doing custom lowering (rhs is tied to the output register)
masm.vpcmpgtw(ToOperand(ins->lhs()), scratch, scratch);
masm.moveSimd128Int(scratch, output);
return;
case MSimdBinaryComp::notEqual:
// Ideally for notEqual, greaterThanOrEqual, and lessThanOrEqual, we
// should invert the comparison by, e.g. swapping the arms of a select
// if that's what it's used in.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.vpcmpeqw(rhs, lhs, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
case MSimdBinaryComp::greaterThanOrEqual:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
masm.vpcmpgtw(ToOperand(ins->lhs()), scratch, scratch);
masm.loadConstantSimd128Int(allOnes, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
case MSimdBinaryComp::lessThanOrEqual:
// lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.vpcmpgtw(rhs, lhs, output);
masm.bitwiseXorSimd128(Operand(scratch), output);
return;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryCompIx4(LSimdBinaryCompIx4* ins)
{
static const SimdConstant allOnes = SimdConstant::SplatX4(-1);
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
MOZ_ASSERT(ToFloatRegister(ins->output()) == lhs);
ScratchSimd128Scope scratch(masm);
MSimdBinaryComp::Operation op = ins->operation();
switch (op) {
case MSimdBinaryComp::greaterThan:
masm.packedGreaterThanInt32x4(rhs, lhs);
return;
case MSimdBinaryComp::equal:
masm.packedEqualInt32x4(rhs, lhs);
return;
case MSimdBinaryComp::lessThan:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
// src := src > lhs (i.e. lhs < rhs)
// Improve by doing custom lowering (rhs is tied to the output register)
masm.packedGreaterThanInt32x4(ToOperand(ins->lhs()), scratch);
masm.moveSimd128Int(scratch, lhs);
return;
case MSimdBinaryComp::notEqual:
// Ideally for notEqual, greaterThanOrEqual, and lessThanOrEqual, we
// should invert the comparison by, e.g. swapping the arms of a select
// if that's what it's used in.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.packedEqualInt32x4(rhs, lhs);
masm.bitwiseXorSimd128(Operand(scratch), lhs);
return;
case MSimdBinaryComp::greaterThanOrEqual:
// src := rhs
if (rhs.kind() == Operand::FPREG)
masm.moveSimd128Int(ToFloatRegister(ins->rhs()), scratch);
else
masm.loadAlignedSimd128Int(rhs, scratch);
masm.packedGreaterThanInt32x4(ToOperand(ins->lhs()), scratch);
masm.loadConstantSimd128Int(allOnes, lhs);
masm.bitwiseXorSimd128(Operand(scratch), lhs);
return;
case MSimdBinaryComp::lessThanOrEqual:
// lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
masm.loadConstantSimd128Int(allOnes, scratch);
masm.packedGreaterThanInt32x4(rhs, lhs);
masm.bitwiseXorSimd128(Operand(scratch), lhs);
return;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryCompFx4(LSimdBinaryCompFx4* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MSimdBinaryComp::Operation op = ins->operation();
switch (op) {
case MSimdBinaryComp::equal:
masm.vcmpeqps(rhs, lhs, output);
return;
case MSimdBinaryComp::lessThan:
masm.vcmpltps(rhs, lhs, output);
return;
case MSimdBinaryComp::lessThanOrEqual:
masm.vcmpleps(rhs, lhs, output);
return;
case MSimdBinaryComp::notEqual:
masm.vcmpneqps(rhs, lhs, output);
return;
case MSimdBinaryComp::greaterThanOrEqual:
case MSimdBinaryComp::greaterThan:
// We reverse these before register allocation so that we don't have to
// copy into and out of temporaries after codegen.
MOZ_CRASH("lowering should have reversed this");
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryArithIx16(LSimdBinaryArithIx16* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MSimdBinaryArith::Operation op = ins->operation();
switch (op) {
case MSimdBinaryArith::Op_add:
masm.vpaddb(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_sub:
masm.vpsubb(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_mul:
// 8x16 mul is a valid operation, but not supported in SSE or AVX.
// The operation is synthesized from 16x8 multiplies by
// MSimdBinaryArith::AddLegalized().
break;
case MSimdBinaryArith::Op_div:
case MSimdBinaryArith::Op_max:
case MSimdBinaryArith::Op_min:
case MSimdBinaryArith::Op_minNum:
case MSimdBinaryArith::Op_maxNum:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryArithIx8(LSimdBinaryArithIx8* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MSimdBinaryArith::Operation op = ins->operation();
switch (op) {
case MSimdBinaryArith::Op_add:
masm.vpaddw(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_sub:
masm.vpsubw(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_mul:
masm.vpmullw(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_div:
case MSimdBinaryArith::Op_max:
case MSimdBinaryArith::Op_min:
case MSimdBinaryArith::Op_minNum:
case MSimdBinaryArith::Op_maxNum:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryArithIx4(LSimdBinaryArithIx4* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
ScratchSimd128Scope scratch(masm);
MSimdBinaryArith::Operation op = ins->operation();
switch (op) {
case MSimdBinaryArith::Op_add:
masm.vpaddd(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_sub:
masm.vpsubd(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_mul: {
if (AssemblerX86Shared::HasSSE41()) {
masm.vpmulld(rhs, lhs, output);
return;
}
masm.loadAlignedSimd128Int(rhs, scratch);
masm.vpmuludq(lhs, scratch, scratch);
// scratch contains (Rx, _, Rz, _) where R is the resulting vector.
FloatRegister temp = ToFloatRegister(ins->temp());
masm.vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), lhs, lhs);
masm.vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), rhs, temp);
masm.vpmuludq(temp, lhs, lhs);
// lhs contains (Ry, _, Rw, _) where R is the resulting vector.
masm.vshufps(MacroAssembler::ComputeShuffleMask(0, 2, 0, 2), scratch, lhs, lhs);
// lhs contains (Ry, Rw, Rx, Rz)
masm.vshufps(MacroAssembler::ComputeShuffleMask(2, 0, 3, 1), lhs, lhs, lhs);
return;
}
case MSimdBinaryArith::Op_div:
// x86 doesn't have SIMD i32 div.
break;
case MSimdBinaryArith::Op_max:
// we can do max with a single instruction only if we have SSE4.1
// using the PMAXSD instruction.
break;
case MSimdBinaryArith::Op_min:
// we can do max with a single instruction only if we have SSE4.1
// using the PMINSD instruction.
break;
case MSimdBinaryArith::Op_minNum:
case MSimdBinaryArith::Op_maxNum:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryArithFx4(LSimdBinaryArithFx4* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
ScratchSimd128Scope scratch(masm);
MSimdBinaryArith::Operation op = ins->operation();
switch (op) {
case MSimdBinaryArith::Op_add:
masm.vaddps(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_sub:
masm.vsubps(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_mul:
masm.vmulps(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_div:
masm.vdivps(rhs, lhs, output);
return;
case MSimdBinaryArith::Op_max: {
FloatRegister lhsCopy = masm.reusedInputFloat32x4(lhs, scratch);
masm.vcmpunordps(rhs, lhsCopy, scratch);
FloatRegister tmp = ToFloatRegister(ins->temp());
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, tmp);
masm.vmaxps(Operand(lhs), rhsCopy, tmp);
masm.vmaxps(rhs, lhs, output);
masm.vandps(tmp, output, output);
masm.vorps(scratch, output, output); // or in the all-ones NaNs
return;
}
case MSimdBinaryArith::Op_min: {
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, scratch);
masm.vminps(Operand(lhs), rhsCopy, scratch);
masm.vminps(rhs, lhs, output);
masm.vorps(scratch, output, output); // NaN or'd with arbitrary bits is NaN
return;
}
case MSimdBinaryArith::Op_minNum: {
FloatRegister tmp = ToFloatRegister(ins->temp());
masm.loadConstantSimd128Int(SimdConstant::SplatX4(int32_t(0x80000000)), tmp);
FloatRegister mask = scratch;
FloatRegister tmpCopy = masm.reusedInputFloat32x4(tmp, scratch);
masm.vpcmpeqd(Operand(lhs), tmpCopy, mask);
masm.vandps(tmp, mask, mask);
FloatRegister lhsCopy = masm.reusedInputFloat32x4(lhs, tmp);
masm.vminps(rhs, lhsCopy, tmp);
masm.vorps(mask, tmp, tmp);
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, mask);
masm.vcmpneqps(rhs, rhsCopy, mask);
if (AssemblerX86Shared::HasAVX()) {
masm.vblendvps(mask, lhs, tmp, output);
} else {
// Emulate vblendvps.
// With SSE.4.1 we could use blendvps, however it's awkward since
// it requires the mask to be in xmm0.
if (lhs != output)
masm.moveSimd128Float(lhs, output);
masm.vandps(Operand(mask), output, output);
masm.vandnps(Operand(tmp), mask, mask);
masm.vorps(Operand(mask), output, output);
}
return;
}
case MSimdBinaryArith::Op_maxNum: {
FloatRegister mask = scratch;
masm.loadConstantSimd128Int(SimdConstant::SplatX4(0), mask);
masm.vpcmpeqd(Operand(lhs), mask, mask);
FloatRegister tmp = ToFloatRegister(ins->temp());
masm.loadConstantSimd128Int(SimdConstant::SplatX4(int32_t(0x80000000)), tmp);
masm.vandps(tmp, mask, mask);
FloatRegister lhsCopy = masm.reusedInputFloat32x4(lhs, tmp);
masm.vmaxps(rhs, lhsCopy, tmp);
masm.vandnps(Operand(tmp), mask, mask);
// Ensure tmp always contains the temporary result
mask = tmp;
tmp = scratch;
FloatRegister rhsCopy = masm.reusedInputAlignedFloat32x4(rhs, mask);
masm.vcmpneqps(rhs, rhsCopy, mask);
if (AssemblerX86Shared::HasAVX()) {
masm.vblendvps(mask, lhs, tmp, output);
} else {
// Emulate vblendvps.
// With SSE.4.1 we could use blendvps, however it's awkward since
// it requires the mask to be in xmm0.
if (lhs != output)
masm.moveSimd128Float(lhs, output);
masm.vandps(Operand(mask), output, output);
masm.vandnps(Operand(tmp), mask, mask);
masm.vorps(Operand(mask), output, output);
}
return;
}
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinarySaturating(LSimdBinarySaturating* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
SimdSign sign = ins->signedness();
MOZ_ASSERT(sign != SimdSign::NotApplicable);
switch (ins->type()) {
case MIRType::Int8x16:
switch (ins->operation()) {
case MSimdBinarySaturating::add:
if (sign == SimdSign::Signed)
masm.vpaddsb(rhs, lhs, output);
else
masm.vpaddusb(rhs, lhs, output);
return;
case MSimdBinarySaturating::sub:
if (sign == SimdSign::Signed)
masm.vpsubsb(rhs, lhs, output);
else
masm.vpsubusb(rhs, lhs, output);
return;
}
break;
case MIRType::Int16x8:
switch (ins->operation()) {
case MSimdBinarySaturating::add:
if (sign == SimdSign::Signed)
masm.vpaddsw(rhs, lhs, output);
else
masm.vpaddusw(rhs, lhs, output);
return;
case MSimdBinarySaturating::sub:
if (sign == SimdSign::Signed)
masm.vpsubsw(rhs, lhs, output);
else
masm.vpsubusw(rhs, lhs, output);
return;
}
break;
default:
break;
}
MOZ_CRASH("unsupported type for SIMD saturating arithmetic");
}
void
CodeGeneratorX86Shared::visitSimdUnaryArithIx16(LSimdUnaryArithIx16* ins)
{
Operand in = ToOperand(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
static const SimdConstant allOnes = SimdConstant::SplatX16(-1);
switch (ins->operation()) {
case MSimdUnaryArith::neg:
masm.zeroSimd128Int(out);
masm.packedSubInt8(in, out);
return;
case MSimdUnaryArith::not_:
masm.loadConstantSimd128Int(allOnes, out);
masm.bitwiseXorSimd128(in, out);
return;
case MSimdUnaryArith::abs:
case MSimdUnaryArith::reciprocalApproximation:
case MSimdUnaryArith::reciprocalSqrtApproximation:
case MSimdUnaryArith::sqrt:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdUnaryArithIx8(LSimdUnaryArithIx8* ins)
{
Operand in = ToOperand(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
static const SimdConstant allOnes = SimdConstant::SplatX8(-1);
switch (ins->operation()) {
case MSimdUnaryArith::neg:
masm.zeroSimd128Int(out);
masm.packedSubInt16(in, out);
return;
case MSimdUnaryArith::not_:
masm.loadConstantSimd128Int(allOnes, out);
masm.bitwiseXorSimd128(in, out);
return;
case MSimdUnaryArith::abs:
case MSimdUnaryArith::reciprocalApproximation:
case MSimdUnaryArith::reciprocalSqrtApproximation:
case MSimdUnaryArith::sqrt:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdUnaryArithIx4(LSimdUnaryArithIx4* ins)
{
Operand in = ToOperand(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
static const SimdConstant allOnes = SimdConstant::SplatX4(-1);
switch (ins->operation()) {
case MSimdUnaryArith::neg:
masm.zeroSimd128Int(out);
masm.packedSubInt32(in, out);
return;
case MSimdUnaryArith::not_:
masm.loadConstantSimd128Int(allOnes, out);
masm.bitwiseXorSimd128(in, out);
return;
case MSimdUnaryArith::abs:
case MSimdUnaryArith::reciprocalApproximation:
case MSimdUnaryArith::reciprocalSqrtApproximation:
case MSimdUnaryArith::sqrt:
break;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdUnaryArithFx4(LSimdUnaryArithFx4* ins)
{
Operand in = ToOperand(ins->input());
FloatRegister out = ToFloatRegister(ins->output());
// All ones but the sign bit
float signMask = SpecificNaN<float>(0, FloatingPoint<float>::kSignificandBits);
static const SimdConstant signMasks = SimdConstant::SplatX4(signMask);
// All ones including the sign bit
float ones = SpecificNaN<float>(1, FloatingPoint<float>::kSignificandBits);
static const SimdConstant allOnes = SimdConstant::SplatX4(ones);
// All zeros but the sign bit
static const SimdConstant minusZero = SimdConstant::SplatX4(-0.f);
switch (ins->operation()) {
case MSimdUnaryArith::abs:
masm.loadConstantSimd128Float(signMasks, out);
masm.bitwiseAndSimd128(in, out);
return;
case MSimdUnaryArith::neg:
masm.loadConstantSimd128Float(minusZero, out);
masm.bitwiseXorSimd128(in, out);
return;
case MSimdUnaryArith::not_:
masm.loadConstantSimd128Float(allOnes, out);
masm.bitwiseXorSimd128(in, out);
return;
case MSimdUnaryArith::reciprocalApproximation:
masm.packedRcpApproximationFloat32x4(in, out);
return;
case MSimdUnaryArith::reciprocalSqrtApproximation:
masm.packedRcpSqrtApproximationFloat32x4(in, out);
return;
case MSimdUnaryArith::sqrt:
masm.packedSqrtFloat32x4(in, out);
return;
}
MOZ_CRASH("unexpected SIMD op");
}
void
CodeGeneratorX86Shared::visitSimdBinaryBitwise(LSimdBinaryBitwise* ins)
{
FloatRegister lhs = ToFloatRegister(ins->lhs());
Operand rhs = ToOperand(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
MSimdBinaryBitwise::Operation op = ins->operation();
switch (op) {
case MSimdBinaryBitwise::and_:
if (ins->type() == MIRType::Float32x4)
masm.vandps(rhs, lhs, output);
else
masm.vpand(rhs, lhs, output);
return;
case MSimdBinaryBitwise::or_:
if (ins->type() == MIRType::Float32x4)
masm.vorps(rhs, lhs, output);
else
masm.vpor(rhs, lhs, output);
return;
case MSimdBinaryBitwise::xor_:
if (ins->type() == MIRType::Float32x4)
masm.vxorps(rhs, lhs, output);
else
masm.vpxor(rhs, lhs, output);
return;
}
MOZ_CRASH("unexpected SIMD bitwise op");
}
void
CodeGeneratorX86Shared::visitSimdShift(LSimdShift* ins)
{
FloatRegister out = ToFloatRegister(ins->output());
MOZ_ASSERT(ToFloatRegister(ins->vector()) == out); // defineReuseInput(0);
// The shift amount is masked to the number of bits in a lane.
uint32_t shiftmask = (128u / SimdTypeToLength(ins->type())) - 1;
// Note that SSE doesn't have instructions for shifting 8x16 vectors.
// These shifts are synthesized by the MSimdShift::AddLegalized() function.
const LAllocation* val = ins->value();
if (val->isConstant()) {
MOZ_ASSERT(ins->temp()->isBogusTemp());
Imm32 count(uint32_t(ToInt32(val)) & shiftmask);
switch (ins->type()) {
case MIRType::Int16x8:
switch (ins->operation()) {
case MSimdShift::lsh:
masm.packedLeftShiftByScalarInt16x8(count, out);
return;
case MSimdShift::rsh:
masm.packedRightShiftByScalarInt16x8(count, out);
return;
case MSimdShift::ursh:
masm.packedUnsignedRightShiftByScalarInt16x8(count, out);
return;
}
break;
case MIRType::Int32x4:
switch (ins->operation()) {
case MSimdShift::lsh:
masm.packedLeftShiftByScalarInt32x4(count, out);
return;
case MSimdShift::rsh:
masm.packedRightShiftByScalarInt32x4(count, out);
return;
case MSimdShift::ursh:
masm.packedUnsignedRightShiftByScalarInt32x4(count, out);
return;
}
break;
default:
MOZ_CRASH("unsupported type for SIMD shifts");
}
MOZ_CRASH("unexpected SIMD bitwise op");
}
// Truncate val to 5 bits. We should have a temp register for that.
MOZ_ASSERT(val->isRegister());
Register count = ToRegister(ins->temp());
masm.mov(ToRegister(val), count);
masm.andl(Imm32(shiftmask), count);
ScratchFloat32Scope scratch(masm);
masm.vmovd(count, scratch);
switch (ins->type()) {
case MIRType::Int16x8:
switch (ins->operation()) {
case MSimdShift::lsh:
masm.packedLeftShiftByScalarInt16x8(scratch, out);
return;
case MSimdShift::rsh:
masm.packedRightShiftByScalarInt16x8(scratch, out);
return;
case MSimdShift::ursh:
masm.packedUnsignedRightShiftByScalarInt16x8(scratch, out);
return;
}
break;
case MIRType::Int32x4:
switch (ins->operation()) {
case MSimdShift::lsh:
masm.packedLeftShiftByScalarInt32x4(scratch, out);
return;
case MSimdShift::rsh:
masm.packedRightShiftByScalarInt32x4(scratch, out);
return;
case MSimdShift::ursh:
masm.packedUnsignedRightShiftByScalarInt32x4(scratch, out);
return;
}
break;
default:
MOZ_CRASH("unsupported type for SIMD shifts");
}
MOZ_CRASH("unexpected SIMD bitwise op");
}
void
CodeGeneratorX86Shared::visitSimdSelect(LSimdSelect* ins)
{
FloatRegister mask = ToFloatRegister(ins->mask());
FloatRegister onTrue = ToFloatRegister(ins->lhs());
FloatRegister onFalse = ToFloatRegister(ins->rhs());
FloatRegister output = ToFloatRegister(ins->output());
FloatRegister temp = ToFloatRegister(ins->temp());
if (onTrue != output)
masm.vmovaps(onTrue, output);
if (mask != temp)
masm.vmovaps(mask, temp);
MSimdSelect* mir = ins->mir();
unsigned lanes = SimdTypeToLength(mir->type());
if (AssemblerX86Shared::HasAVX() && lanes == 4) {
// TBD: Use vpblendvb for lanes > 4, HasAVX.
masm.vblendvps(mask, onTrue, onFalse, output);
return;
}
// SSE4.1 has plain blendvps which can do this, but it is awkward
// to use because it requires the mask to be in xmm0.
masm.bitwiseAndSimd128(Operand(temp), output);
masm.bitwiseAndNotSimd128(Operand(onFalse), temp);
masm.bitwiseOrSimd128(Operand(temp), output);
}
void
CodeGeneratorX86Shared::visitCompareExchangeTypedArrayElement(LCompareExchangeTypedArrayElement* lir)
{
Register elements = ToRegister(lir->elements());
AnyRegister output = ToAnyRegister(lir->output());
Register temp = lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
Register oldval = ToRegister(lir->oldval());
Register newval = ToRegister(lir->newval());
Scalar::Type arrayType = lir->mir()->arrayType();
int width = Scalar::byteSize(arrayType);
if (lir->index()->isConstant()) {
Address dest(elements, ToInt32(lir->index()) * width);
masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output);
} else {
BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
masm.compareExchangeToTypedIntArray(arrayType, dest, oldval, newval, temp, output);
}
}
void
CodeGeneratorX86Shared::visitAtomicExchangeTypedArrayElement(LAtomicExchangeTypedArrayElement* lir)
{
Register elements = ToRegister(lir->elements());
AnyRegister output = ToAnyRegister(lir->output());
Register temp = lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
Register value = ToRegister(lir->value());
Scalar::Type arrayType = lir->mir()->arrayType();
int width = Scalar::byteSize(arrayType);
if (lir->index()->isConstant()) {
Address dest(elements, ToInt32(lir->index()) * width);
masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output);
} else {
BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
masm.atomicExchangeToTypedIntArray(arrayType, dest, value, temp, output);
}
}
template<typename S, typename T>
void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value,
const T& mem, Register temp1, Register temp2, AnyRegister output)
{
switch (arrayType) {
case Scalar::Int8:
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd8SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchSubOp:
masm.atomicFetchSub8SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd8SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchOrOp:
masm.atomicFetchOr8SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchXorOp:
masm.atomicFetchXor8SignExtend(value, mem, temp1, output.gpr());
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Uint8:
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd8ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchSubOp:
masm.atomicFetchSub8ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd8ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchOrOp:
masm.atomicFetchOr8ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchXorOp:
masm.atomicFetchXor8ZeroExtend(value, mem, temp1, output.gpr());
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Int16:
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd16SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchSubOp:
masm.atomicFetchSub16SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd16SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchOrOp:
masm.atomicFetchOr16SignExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchXorOp:
masm.atomicFetchXor16SignExtend(value, mem, temp1, output.gpr());
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Uint16:
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd16ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchSubOp:
masm.atomicFetchSub16ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd16ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchOrOp:
masm.atomicFetchOr16ZeroExtend(value, mem, temp1, output.gpr());
break;
case AtomicFetchXorOp:
masm.atomicFetchXor16ZeroExtend(value, mem, temp1, output.gpr());
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Int32:
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd32(value, mem, temp1, output.gpr());
break;
case AtomicFetchSubOp:
masm.atomicFetchSub32(value, mem, temp1, output.gpr());
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd32(value, mem, temp1, output.gpr());
break;
case AtomicFetchOrOp:
masm.atomicFetchOr32(value, mem, temp1, output.gpr());
break;
case AtomicFetchXorOp:
masm.atomicFetchXor32(value, mem, temp1, output.gpr());
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Uint32:
// At the moment, the code in MCallOptimize.cpp requires the output
// type to be double for uint32 arrays. See bug 1077305.
MOZ_ASSERT(output.isFloat());
switch (op) {
case AtomicFetchAddOp:
masm.atomicFetchAdd32(value, mem, InvalidReg, temp1);
break;
case AtomicFetchSubOp:
masm.atomicFetchSub32(value, mem, InvalidReg, temp1);
break;
case AtomicFetchAndOp:
masm.atomicFetchAnd32(value, mem, temp2, temp1);
break;
case AtomicFetchOrOp:
masm.atomicFetchOr32(value, mem, temp2, temp1);
break;
case AtomicFetchXorOp:
masm.atomicFetchXor32(value, mem, temp2, temp1);
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
masm.convertUInt32ToDouble(temp1, output.fpu());
break;
default:
MOZ_CRASH("Invalid typed array type");
}
}
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Imm32& value, const Address& mem,
Register temp1, Register temp2, AnyRegister output);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Imm32& value, const BaseIndex& mem,
Register temp1, Register temp2, AnyRegister output);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Register& value, const Address& mem,
Register temp1, Register temp2, AnyRegister output);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Register& value, const BaseIndex& mem,
Register temp1, Register temp2, AnyRegister output);
// Binary operation for effect, result discarded.
template<typename S, typename T>
void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType, const S& value,
const T& mem)
{
switch (arrayType) {
case Scalar::Int8:
case Scalar::Uint8:
switch (op) {
case AtomicFetchAddOp:
masm.atomicAdd8(value, mem);
break;
case AtomicFetchSubOp:
masm.atomicSub8(value, mem);
break;
case AtomicFetchAndOp:
masm.atomicAnd8(value, mem);
break;
case AtomicFetchOrOp:
masm.atomicOr8(value, mem);
break;
case AtomicFetchXorOp:
masm.atomicXor8(value, mem);
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Int16:
case Scalar::Uint16:
switch (op) {
case AtomicFetchAddOp:
masm.atomicAdd16(value, mem);
break;
case AtomicFetchSubOp:
masm.atomicSub16(value, mem);
break;
case AtomicFetchAndOp:
masm.atomicAnd16(value, mem);
break;
case AtomicFetchOrOp:
masm.atomicOr16(value, mem);
break;
case AtomicFetchXorOp:
masm.atomicXor16(value, mem);
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
case Scalar::Int32:
case Scalar::Uint32:
switch (op) {
case AtomicFetchAddOp:
masm.atomicAdd32(value, mem);
break;
case AtomicFetchSubOp:
masm.atomicSub32(value, mem);
break;
case AtomicFetchAndOp:
masm.atomicAnd32(value, mem);
break;
case AtomicFetchOrOp:
masm.atomicOr32(value, mem);
break;
case AtomicFetchXorOp:
masm.atomicXor32(value, mem);
break;
default:
MOZ_CRASH("Invalid typed array atomic operation");
}
break;
default:
MOZ_CRASH("Invalid typed array type");
}
}
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Imm32& value, const Address& mem);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Imm32& value, const BaseIndex& mem);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Register& value, const Address& mem);
template void
CodeGeneratorX86Shared::atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
const Register& value, const BaseIndex& mem);
template <typename T>
static inline void
AtomicBinopToTypedArray(CodeGeneratorX86Shared* cg, AtomicOp op,
Scalar::Type arrayType, const LAllocation* value, const T& mem,
Register temp1, Register temp2, AnyRegister output)
{
if (value->isConstant())
cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem, temp1, temp2, output);
else
cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem, temp1, temp2, output);
}
void
CodeGeneratorX86Shared::visitAtomicTypedArrayElementBinop(LAtomicTypedArrayElementBinop* lir)
{
MOZ_ASSERT(lir->mir()->hasUses());
AnyRegister output = ToAnyRegister(lir->output());
Register elements = ToRegister(lir->elements());
Register temp1 = lir->temp1()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp1());
Register temp2 = lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2());
const LAllocation* value = lir->value();
Scalar::Type arrayType = lir->mir()->arrayType();
int width = Scalar::byteSize(arrayType);
if (lir->index()->isConstant()) {
Address mem(elements, ToInt32(lir->index()) * width);
AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output);
} else {
BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem, temp1, temp2, output);
}
}
template <typename T>
static inline void
AtomicBinopToTypedArray(CodeGeneratorX86Shared* cg, AtomicOp op,
Scalar::Type arrayType, const LAllocation* value, const T& mem)
{
if (value->isConstant())
cg->atomicBinopToTypedIntArray(op, arrayType, Imm32(ToInt32(value)), mem);
else
cg->atomicBinopToTypedIntArray(op, arrayType, ToRegister(value), mem);
}
void
CodeGeneratorX86Shared::visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedArrayElementBinopForEffect* lir)
{
MOZ_ASSERT(!lir->mir()->hasUses());
Register elements = ToRegister(lir->elements());
const LAllocation* value = lir->value();
Scalar::Type arrayType = lir->mir()->arrayType();
int width = Scalar::byteSize(arrayType);
if (lir->index()->isConstant()) {
Address mem(elements, ToInt32(lir->index()) * width);
AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem);
} else {
BaseIndex mem(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
AtomicBinopToTypedArray(this, lir->mir()->operation(), arrayType, value, mem);
}
}
void
CodeGeneratorX86Shared::visitMemoryBarrier(LMemoryBarrier* ins)
{
if (ins->type() & MembarStoreLoad)
masm.storeLoadFence();
}
void
CodeGeneratorX86Shared::setReturnDoubleRegs(LiveRegisterSet* regs)
{
MOZ_ASSERT(ReturnFloat32Reg.encoding() == X86Encoding::xmm0);
MOZ_ASSERT(ReturnDoubleReg.encoding() == X86Encoding::xmm0);
MOZ_ASSERT(ReturnSimd128Reg.encoding() == X86Encoding::xmm0);
regs->add(ReturnFloat32Reg);
regs->add(ReturnDoubleReg);
regs->add(ReturnSimd128Reg);
}
void
CodeGeneratorX86Shared::visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool)
{
FloatRegister input = ool->input();
MIRType fromType = ool->fromType();
MIRType toType = ool->toType();
Label* oolRejoin = ool->rejoin();
bool isUnsigned = ool->isUnsigned();
wasm::TrapOffset off = ool->trapOffset();
if (fromType == MIRType::Float32) {
if (toType == MIRType::Int32)
masm.outOfLineWasmTruncateFloat32ToInt32(input, isUnsigned, off, oolRejoin);
else if (toType == MIRType::Int64)
masm.outOfLineWasmTruncateFloat32ToInt64(input, isUnsigned, off, oolRejoin);
else
MOZ_CRASH("unexpected type");
} else if (fromType == MIRType::Double) {
if (toType == MIRType::Int32)
masm.outOfLineWasmTruncateDoubleToInt32(input, isUnsigned, off, oolRejoin);
else if (toType == MIRType::Int64)
masm.outOfLineWasmTruncateDoubleToInt64(input, isUnsigned, off, oolRejoin);
else
MOZ_CRASH("unexpected type");
} else {
MOZ_CRASH("unexpected type");
}
}
void
CodeGeneratorX86Shared::canonicalizeIfDeterministic(Scalar::Type type, const LAllocation* value)
{
#ifdef JS_MORE_DETERMINISTIC
switch (type) {
case Scalar::Float32: {
FloatRegister in = ToFloatRegister(value);
masm.canonicalizeFloatIfDeterministic(in);
break;
}
case Scalar::Float64: {
FloatRegister in = ToFloatRegister(value);
masm.canonicalizeDoubleIfDeterministic(in);
break;
}
case Scalar::Float32x4: {
FloatRegister in = ToFloatRegister(value);
MOZ_ASSERT(in.isSimd128());
FloatRegister scratch = in != xmm0.asSimd128() ? xmm0 : xmm1;
masm.push(scratch);
masm.canonicalizeFloat32x4(in, scratch);
masm.pop(scratch);
break;
}
default: {
// Other types don't need canonicalization.
break;
}
}
#endif // JS_MORE_DETERMINISTIC
}
void
CodeGeneratorX86Shared::visitCopySignF(LCopySignF* lir)
{
FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
FloatRegister out = ToFloatRegister(lir->output());
if (lhs == rhs) {
if (lhs != out)
masm.moveFloat32(lhs, out);
return;
}
ScratchFloat32Scope scratch(masm);
float clearSignMask = BitwiseCast<float>(INT32_MAX);
masm.loadConstantFloat32(clearSignMask, scratch);
masm.vandps(scratch, lhs, out);
float keepSignMask = BitwiseCast<float>(INT32_MIN);
masm.loadConstantFloat32(keepSignMask, scratch);
masm.vandps(rhs, scratch, scratch);
masm.vorps(scratch, out, out);
}
void
CodeGeneratorX86Shared::visitCopySignD(LCopySignD* lir)
{
FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
FloatRegister out = ToFloatRegister(lir->output());
if (lhs == rhs) {
if (lhs != out)
masm.moveDouble(lhs, out);
return;
}
ScratchDoubleScope scratch(masm);
double clearSignMask = BitwiseCast<double>(INT64_MAX);
masm.loadConstantDouble(clearSignMask, scratch);
masm.vandpd(scratch, lhs, out);
double keepSignMask = BitwiseCast<double>(INT64_MIN);
masm.loadConstantDouble(keepSignMask, scratch);
masm.vandpd(rhs, scratch, scratch);
masm.vorpd(scratch, out, out);
}
void
CodeGeneratorX86Shared::visitRotateI64(LRotateI64* lir)
{
MRotate* mir = lir->mir();
LAllocation* count = lir->count();
Register64 input = ToRegister64(lir->input());
Register64 output = ToOutRegister64(lir);
Register temp = ToTempRegisterOrInvalid(lir->temp());
MOZ_ASSERT(input == output);
if (count->isConstant()) {
int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
if (!c)
return;
if (mir->isLeftRotate())
masm.rotateLeft64(Imm32(c), input, output, temp);
else
masm.rotateRight64(Imm32(c), input, output, temp);
} else {
if (mir->isLeftRotate())
masm.rotateLeft64(ToRegister(count), input, output, temp);
else
masm.rotateRight64(ToRegister(count), input, output, temp);
}
}
void
CodeGeneratorX86Shared::visitPopcntI64(LPopcntI64* lir)
{
Register64 input = ToRegister64(lir->getInt64Operand(0));
Register64 output = ToOutRegister64(lir);
Register temp = InvalidReg;
if (!AssemblerX86Shared::HasPOPCNT())
temp = ToRegister(lir->getTemp(0));
masm.popcnt64(input, output, temp);
}
} // namespace jit
} // namespace js