/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
* vim: set ts=8 sts=4 et sw=4 tw=99:
*
* Copyright 2016 Mozilla Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* WebAssembly baseline compiler ("RabaldrMonkey")
*
* General status notes:
*
* "FIXME" indicates a known or suspected bug. Always has a bug#.
*
* "TODO" indicates an opportunity for a general improvement, with an additional
* tag to indicate the area of improvement. Usually has a bug#.
*
* Unimplemented functionality:
*
* - Tiered compilation (bug 1277562)
* - profiler support / devtools (bug 1286948)
* - SIMD
* - Atomics
*
* There are lots of machine dependencies here but they are pretty well isolated
* to a segment of the compiler. Many dependencies will eventually be factored
* into the MacroAssembler layer and shared with other code generators.
*
*
* High-value compiler performance improvements:
*
* - (Bug 1316802) The specific-register allocator (the needI32(r), needI64(r)
* etc methods) can avoid syncing the value stack if the specific register is
* in use but there is a free register to shuffle the specific register into.
* (This will also improve the generated code.) The sync happens often enough
* here to show up in profiles, because it is triggered by integer multiply
* and divide.
*
*
* High-value code generation improvements:
*
* - (Bug 1316803) Opportunities for cheaply folding in a constant rhs to
* arithmetic operations, we do this already for I32 add and shift operators,
* this reduces register pressure and instruction count.
*
* - (Bug 1286816) Opportunities for cheaply folding in a constant rhs to
* conditionals.
*
* - (Bug 1286816) Boolean evaluation for control can be optimized by pushing a
* bool-generating operation onto the value stack in the same way that we now
* push latent constants and local lookups, or (easier) by remembering the
* operation in a side location if the next Op will consume it.
*
* - (Bug 1286816) brIf pessimizes by branching over code that performs stack
* cleanup and a branch. If no cleanup is needed we can just branch
* conditionally to the target.
*
* - (Bug 1316804) brTable pessimizes by always dispatching to code that pops
* the stack and then jumps to the code for the target case. If no cleanup is
* needed we could just branch conditionally to the target; if the same amount
* of cleanup is needed for all cases then the cleanup can be done before the
* dispatch. Both are highly likely.
*
* - (Bug 1316806) Register management around calls: At the moment we sync the
* value stack unconditionally (this is simple) but there are probably many
* common cases where we could instead save/restore live caller-saves
* registers and perform parallel assignment into argument registers. This
* may be important if we keep some locals in registers.
*
* - (Bug 1316808) Allocate some locals to registers on machines where there are
* enough registers. This is probably hard to do well in a one-pass compiler
* but it might be that just keeping register arguments and the first few
* locals in registers is a viable strategy; another (more general) strategy
* is caching locals in registers in straight-line code. Such caching could
* also track constant values in registers, if that is deemed valuable. A
* combination of techniques may be desirable: parameters and the first few
* locals could be cached on entry to the function but not statically assigned
* to registers throughout.
*
* (On a large corpus of code it should be possible to compute, for every
* signature comprising the types of parameters and locals, and using a static
* weight for loops, a list in priority order of which parameters and locals
* that should be assigned to registers. Or something like that. Wasm makes
* this simple. Static assignments are desirable because they are not flushed
* to memory by the pre-block sync() call.)
*/
#include "wasm/WasmBaselineCompile.h"
#include "mozilla/MathAlgorithms.h"
#include "jit/AtomicOp.h"
#include "jit/IonTypes.h"
#include "jit/JitAllocPolicy.h"
#include "jit/Label.h"
#include "jit/MacroAssembler.h"
#include "jit/MIR.h"
#include "jit/Registers.h"
#include "jit/RegisterSets.h"
#if defined(JS_CODEGEN_ARM)
# include "jit/arm/Assembler-arm.h"
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
# include "jit/x86-shared/Architecture-x86-shared.h"
# include "jit/x86-shared/Assembler-x86-shared.h"
#endif
#include "wasm/WasmBinaryFormat.h"
#include "wasm/WasmBinaryIterator.h"
#include "wasm/WasmGenerator.h"
#include "wasm/WasmSignalHandlers.h"
#include "jit/MacroAssembler-inl.h"
using mozilla::DebugOnly;
using mozilla::FloatingPoint;
using mozilla::IsPowerOfTwo;
using mozilla::SpecificNaN;
namespace js {
namespace wasm {
using namespace js::jit;
using JS::GenericNaN;
struct BaseCompilePolicy : OpIterPolicy
{
static const bool Output = true;
// The baseline compiler tracks values on a stack of its own -- it
// needs to scan that stack for spilling -- and thus has no need
// for the values maintained by the iterator.
//
// The baseline compiler tracks control items on a stack of its
// own as well.
//
// TODO / REDUNDANT (Bug 1316814): It would be nice if we could
// make use of the iterator's ControlItems and not require our own
// stack for that.
};
typedef OpIter<BaseCompilePolicy> BaseOpIter;
typedef bool IsUnsigned;
typedef bool IsSigned;
typedef bool ZeroOnOverflow;
typedef bool IsKnownNotZero;
typedef bool HandleNaNSpecially;
typedef unsigned ByteSize;
typedef unsigned BitSize;
// UseABI::Wasm implies that the Tls/Heap/Global registers are nonvolatile,
// except when InterModule::True is also set, when they are volatile.
//
// UseABI::System implies that the Tls/Heap/Global registers are volatile.
// Additionally, the parameter passing mechanism may be slightly different from
// the UseABI::Wasm convention.
//
// When the Tls/Heap/Global registers are not volatile, the baseline compiler
// will restore the Tls register from its save slot before the call, since the
// baseline compiler uses the Tls register for other things.
//
// When those registers are volatile, the baseline compiler will reload them
// after the call (it will restore the Tls register from the save slot and load
// the other two from the Tls data).
enum class UseABI { Wasm, System };
enum class InterModule { False = false, True = true };
#ifdef JS_CODEGEN_ARM64
// FIXME: This is not correct, indeed for ARM64 there is no reliable
// StackPointer and we'll need to change the abstractions that use
// SP-relative addressing. There's a guard in emitFunction() below to
// prevent this workaround from having any consequence. This hack
// exists only as a stopgap; there is no ARM64 JIT support yet.
static const Register StackPointer = RealStackPointer;
#endif
#ifdef JS_CODEGEN_X86
// The selection of EBX here steps gingerly around: the need for EDX
// to be allocatable for multiply/divide; ECX to be allocatable for
// shift/rotate; EAX (= ReturnReg) to be allocatable as the joinreg;
// EBX not being one of the WasmTableCall registers; and needing a
// temp register for load/store that has a single-byte persona.
static const Register ScratchRegX86 = ebx;
# define INT_DIV_I64_CALLOUT
#endif
#ifdef JS_CODEGEN_ARM
// We need a temp for funcPtrCall. It can't be any of the
// WasmTableCall registers, an argument register, or a scratch
// register, and probably should not be ReturnReg.
static const Register FuncPtrCallTemp = CallTempReg1;
// We use our own scratch register, because the macro assembler uses
// the regular scratch register(s) pretty liberally. We could
// work around that in several cases but the mess does not seem
// worth it yet. CallTempReg2 seems safe.
static const Register ScratchRegARM = CallTempReg2;
# define INT_DIV_I64_CALLOUT
# define I64_TO_FLOAT_CALLOUT
# define FLOAT_TO_I64_CALLOUT
#endif
class BaseCompiler
{
// We define our own ScratchRegister abstractions, deferring to
// the platform's when possible.
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
typedef ScratchDoubleScope ScratchF64;
#else
class ScratchF64
{
public:
ScratchF64(BaseCompiler& b) {}
operator FloatRegister() const {
MOZ_CRASH("BaseCompiler platform hook - ScratchF64");
}
};
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
typedef ScratchFloat32Scope ScratchF32;
#else
class ScratchF32
{
public:
ScratchF32(BaseCompiler& b) {}
operator FloatRegister() const {
MOZ_CRASH("BaseCompiler platform hook - ScratchF32");
}
};
#endif
#if defined(JS_CODEGEN_X64)
typedef ScratchRegisterScope ScratchI32;
#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
class ScratchI32
{
# ifdef DEBUG
BaseCompiler& bc;
public:
explicit ScratchI32(BaseCompiler& bc) : bc(bc) {
MOZ_ASSERT(!bc.scratchRegisterTaken());
bc.setScratchRegisterTaken(true);
}
~ScratchI32() {
MOZ_ASSERT(bc.scratchRegisterTaken());
bc.setScratchRegisterTaken(false);
}
# else
public:
explicit ScratchI32(BaseCompiler& bc) {}
# endif
operator Register() const {
# ifdef JS_CODEGEN_X86
return ScratchRegX86;
# else
return ScratchRegARM;
# endif
}
};
#else
class ScratchI32
{
public:
ScratchI32(BaseCompiler& bc) {}
operator Register() const {
MOZ_CRASH("BaseCompiler platform hook - ScratchI32");
}
};
#endif
// A Label in the code, allocated out of a temp pool in the
// TempAllocator attached to the compilation.
struct PooledLabel : public Label, public TempObject, public InlineListNode<PooledLabel>
{
PooledLabel() : f(nullptr) {}
explicit PooledLabel(BaseCompiler* f) : f(f) {}
BaseCompiler* f;
};
typedef Vector<PooledLabel*, 8, SystemAllocPolicy> LabelVector;
struct UniquePooledLabelFreePolicy
{
void operator()(PooledLabel* p) {
p->f->freeLabel(p);
}
};
typedef UniquePtr<PooledLabel, UniquePooledLabelFreePolicy> UniquePooledLabel;
// The strongly typed register wrappers have saved my bacon a few
// times; though they are largely redundant they stay, for now.
// TODO / INVESTIGATE (Bug 1316815): Things would probably be
// simpler if these inherited from Register, Register64, and
// FloatRegister.
struct RegI32
{
RegI32() : reg(Register::Invalid()) {}
explicit RegI32(Register reg) : reg(reg) {}
Register reg;
bool operator==(const RegI32& that) { return reg == that.reg; }
bool operator!=(const RegI32& that) { return reg != that.reg; }
};
struct RegI64
{
RegI64() : reg(Register64::Invalid()) {}
explicit RegI64(Register64 reg) : reg(reg) {}
Register64 reg;
bool operator==(const RegI64& that) { return reg == that.reg; }
bool operator!=(const RegI64& that) { return reg != that.reg; }
};
struct RegF32
{
RegF32() {}
explicit RegF32(FloatRegister reg) : reg(reg) {}
FloatRegister reg;
bool operator==(const RegF32& that) { return reg == that.reg; }
bool operator!=(const RegF32& that) { return reg != that.reg; }
};
struct RegF64
{
RegF64() {}
explicit RegF64(FloatRegister reg) : reg(reg) {}
FloatRegister reg;
bool operator==(const RegF64& that) { return reg == that.reg; }
bool operator!=(const RegF64& that) { return reg != that.reg; }
};
struct AnyReg
{
AnyReg() { tag = NONE; }
explicit AnyReg(RegI32 r) { tag = I32; i32_ = r; }
explicit AnyReg(RegI64 r) { tag = I64; i64_ = r; }
explicit AnyReg(RegF32 r) { tag = F32; f32_ = r; }
explicit AnyReg(RegF64 r) { tag = F64; f64_ = r; }
RegI32 i32() {
MOZ_ASSERT(tag == I32);
return i32_;
}
RegI64 i64() {
MOZ_ASSERT(tag == I64);
return i64_;
}
RegF32 f32() {
MOZ_ASSERT(tag == F32);
return f32_;
}
RegF64 f64() {
MOZ_ASSERT(tag == F64);
return f64_;
}
AnyRegister any() {
switch (tag) {
case F32: return AnyRegister(f32_.reg);
case F64: return AnyRegister(f64_.reg);
case I32: return AnyRegister(i32_.reg);
case I64:
#ifdef JS_PUNBOX64
return AnyRegister(i64_.reg.reg);
#else
// The compiler is written so that this is never needed: any() is called
// on arbitrary registers for asm.js but asm.js does not have 64-bit ints.
// For wasm, any() is called on arbitrary registers only on 64-bit platforms.
MOZ_CRASH("AnyReg::any() on 32-bit platform");
#endif
case NONE:
MOZ_CRASH("AnyReg::any() on NONE");
}
// Work around GCC 5 analysis/warning bug.
MOZ_CRASH("AnyReg::any(): impossible case");
}
union {
RegI32 i32_;
RegI64 i64_;
RegF32 f32_;
RegF64 f64_;
};
enum { NONE, I32, I64, F32, F64 } tag;
};
struct Local
{
Local() : type_(MIRType::None), offs_(UINT32_MAX) {}
Local(MIRType type, uint32_t offs) : type_(type), offs_(offs) {}
void init(MIRType type_, uint32_t offs_) {
this->type_ = type_;
this->offs_ = offs_;
}
MIRType type_; // Type of the value, or MIRType::None
uint32_t offs_; // Zero-based frame offset of value, or UINT32_MAX
MIRType type() const { MOZ_ASSERT(type_ != MIRType::None); return type_; }
uint32_t offs() const { MOZ_ASSERT(offs_ != UINT32_MAX); return offs_; }
};
// Control node, representing labels and stack heights at join points.
struct Control
{
Control(uint32_t framePushed, uint32_t stackSize)
: label(nullptr),
otherLabel(nullptr),
framePushed(framePushed),
stackSize(stackSize),
deadOnArrival(false),
deadThenBranch(false)
{}
PooledLabel* label;
PooledLabel* otherLabel; // Used for the "else" branch of if-then-else
uint32_t framePushed; // From masm
uint32_t stackSize; // Value stack height
bool deadOnArrival; // deadCode_ was set on entry to the region
bool deadThenBranch; // deadCode_ was set on exit from "then"
};
// Volatile registers except ReturnReg.
static LiveRegisterSet VolatileReturnGPR;
// The baseline compiler will use OOL code more sparingly than
// Baldr since our code is not high performance and frills like
// code density and branch prediction friendliness will be less
// important.
class OutOfLineCode : public TempObject
{
private:
Label entry_;
Label rejoin_;
uint32_t framePushed_;
public:
OutOfLineCode() : framePushed_(UINT32_MAX) {}
Label* entry() { return &entry_; }
Label* rejoin() { return &rejoin_; }
void setFramePushed(uint32_t framePushed) {
MOZ_ASSERT(framePushed_ == UINT32_MAX);
framePushed_ = framePushed;
}
void bind(MacroAssembler& masm) {
MOZ_ASSERT(framePushed_ != UINT32_MAX);
masm.bind(&entry_);
masm.setFramePushed(framePushed_);
}
// Save volatile registers but not ReturnReg.
void saveVolatileReturnGPR(MacroAssembler& masm) {
masm.PushRegsInMask(BaseCompiler::VolatileReturnGPR);
}
// Restore volatile registers but not ReturnReg.
void restoreVolatileReturnGPR(MacroAssembler& masm) {
masm.PopRegsInMask(BaseCompiler::VolatileReturnGPR);
}
// The generate() method must be careful about register use
// because it will be invoked when there is a register
// assignment in the BaseCompiler that does not correspond
// to the available registers when the generated OOL code is
// executed. The register allocator *must not* be called.
//
// The best strategy is for the creator of the OOL object to
// allocate all temps that the OOL code will need.
//
// Input, output, and temp registers are embedded in the OOL
// object and are known to the code generator.
//
// Scratch registers are available to use in OOL code.
//
// All other registers must be explicitly saved and restored
// by the OOL code before being used.
virtual void generate(MacroAssembler& masm) = 0;
};
const ModuleGeneratorData& mg_;
BaseOpIter iter_;
const FuncBytes& func_;
size_t lastReadCallSite_;
TempAllocator& alloc_;
const ValTypeVector& locals_; // Types of parameters and locals
int32_t localSize_; // Size of local area in bytes (stable after beginFunction)
int32_t varLow_; // Low byte offset of local area for true locals (not parameters)
int32_t varHigh_; // High byte offset + 1 of local area for true locals
int32_t maxFramePushed_; // Max value of masm.framePushed() observed
bool deadCode_; // Flag indicating we should decode & discard the opcode
ValTypeVector SigI64I64_;
ValTypeVector SigDD_;
ValTypeVector SigD_;
ValTypeVector SigF_;
ValTypeVector SigI_;
ValTypeVector Sig_;
Label returnLabel_;
Label outOfLinePrologue_;
Label bodyLabel_;
TrapOffset prologueTrapOffset_;
FuncCompileResults& compileResults_;
MacroAssembler& masm; // No '_' suffix - too tedious...
AllocatableGeneralRegisterSet availGPR_;
AllocatableFloatRegisterSet availFPU_;
#ifdef DEBUG
bool scratchRegisterTaken_;
#endif
TempObjectPool<PooledLabel> labelPool_;
Vector<Local, 8, SystemAllocPolicy> localInfo_;
Vector<OutOfLineCode*, 8, SystemAllocPolicy> outOfLine_;
// Index into localInfo_ of the special local used for saving the TLS
// pointer. This follows the function's real arguments and locals.
uint32_t tlsSlot_;
// On specific platforms we sometimes need to use specific registers.
#ifdef JS_CODEGEN_X64
RegI64 specific_rax;
RegI64 specific_rcx;
RegI64 specific_rdx;
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
RegI32 specific_eax;
RegI32 specific_ecx;
RegI32 specific_edx;
#endif
#if defined(JS_CODEGEN_X86)
AllocatableGeneralRegisterSet singleByteRegs_;
#endif
#if defined(JS_NUNBOX32)
RegI64 abiReturnRegI64;
#endif
// The join registers are used to carry values out of blocks.
// JoinRegI32 and joinRegI64 must overlap: emitBrIf and
// emitBrTable assume that.
RegI32 joinRegI32;
RegI64 joinRegI64;
RegF32 joinRegF32;
RegF64 joinRegF64;
// More members: see the stk_ and ctl_ vectors, defined below.
public:
BaseCompiler(const ModuleGeneratorData& mg,
Decoder& decoder,
const FuncBytes& func,
const ValTypeVector& locals,
FuncCompileResults& compileResults);
MOZ_MUST_USE bool init();
void finish();
MOZ_MUST_USE bool emitFunction();
// Used by some of the ScratchRegister implementations.
operator MacroAssembler&() const { return masm; }
#ifdef DEBUG
bool scratchRegisterTaken() const {
return scratchRegisterTaken_;
}
void setScratchRegisterTaken(bool state) {
scratchRegisterTaken_ = state;
}
#endif
private:
////////////////////////////////////////////////////////////
//
// Out of line code management.
MOZ_MUST_USE OutOfLineCode* addOutOfLineCode(OutOfLineCode* ool) {
if (!ool || !outOfLine_.append(ool))
return nullptr;
ool->setFramePushed(masm.framePushed());
return ool;
}
MOZ_MUST_USE bool generateOutOfLineCode() {
for (uint32_t i = 0; i < outOfLine_.length(); i++) {
OutOfLineCode* ool = outOfLine_[i];
ool->bind(masm);
ool->generate(masm);
}
return !masm.oom();
}
////////////////////////////////////////////////////////////
//
// The stack frame.
// SP-relative load and store.
int32_t localOffsetToSPOffset(int32_t offset) {
return masm.framePushed() - offset;
}
void storeToFrameI32(Register r, int32_t offset) {
masm.store32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
}
void storeToFrameI64(Register64 r, int32_t offset) {
masm.store64(r, Address(StackPointer, localOffsetToSPOffset(offset)));
}
void storeToFramePtr(Register r, int32_t offset) {
masm.storePtr(r, Address(StackPointer, localOffsetToSPOffset(offset)));
}
void storeToFrameF64(FloatRegister r, int32_t offset) {
masm.storeDouble(r, Address(StackPointer, localOffsetToSPOffset(offset)));
}
void storeToFrameF32(FloatRegister r, int32_t offset) {
masm.storeFloat32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
}
void loadFromFrameI32(Register r, int32_t offset) {
masm.load32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
}
void loadFromFrameI64(Register64 r, int32_t offset) {
masm.load64(Address(StackPointer, localOffsetToSPOffset(offset)), r);
}
void loadFromFramePtr(Register r, int32_t offset) {
masm.loadPtr(Address(StackPointer, localOffsetToSPOffset(offset)), r);
}
void loadFromFrameF64(FloatRegister r, int32_t offset) {
masm.loadDouble(Address(StackPointer, localOffsetToSPOffset(offset)), r);
}
void loadFromFrameF32(FloatRegister r, int32_t offset) {
masm.loadFloat32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
}
// Stack-allocated local slots.
int32_t pushLocal(size_t nbytes) {
if (nbytes == 8)
localSize_ = AlignBytes(localSize_, 8u);
else if (nbytes == 16)
localSize_ = AlignBytes(localSize_, 16u);
localSize_ += nbytes;
return localSize_; // Locals grow down so capture base address
}
int32_t frameOffsetFromSlot(uint32_t slot, MIRType type) {
MOZ_ASSERT(localInfo_[slot].type() == type);
return localInfo_[slot].offs();
}
////////////////////////////////////////////////////////////
//
// Low-level register allocation.
bool isAvailable(Register r) {
return availGPR_.has(r);
}
bool hasGPR() {
return !availGPR_.empty();
}
void allocGPR(Register r) {
MOZ_ASSERT(isAvailable(r));
availGPR_.take(r);
}
Register allocGPR() {
MOZ_ASSERT(hasGPR());
return availGPR_.takeAny();
}
void freeGPR(Register r) {
availGPR_.add(r);
}
bool isAvailable(Register64 r) {
#ifdef JS_PUNBOX64
return isAvailable(r.reg);
#else
return isAvailable(r.low) && isAvailable(r.high);
#endif
}
bool hasInt64() {
#ifdef JS_PUNBOX64
return !availGPR_.empty();
#else
if (availGPR_.empty())
return false;
Register r = allocGPR();
bool available = !availGPR_.empty();
freeGPR(r);
return available;
#endif
}
void allocInt64(Register64 r) {
MOZ_ASSERT(isAvailable(r));
#ifdef JS_PUNBOX64
availGPR_.take(r.reg);
#else
availGPR_.take(r.low);
availGPR_.take(r.high);
#endif
}
Register64 allocInt64() {
MOZ_ASSERT(hasInt64());
#ifdef JS_PUNBOX64
return Register64(availGPR_.takeAny());
#else
Register high = availGPR_.takeAny();
Register low = availGPR_.takeAny();
return Register64(high, low);
#endif
}
void freeInt64(Register64 r) {
#ifdef JS_PUNBOX64
availGPR_.add(r.reg);
#else
availGPR_.add(r.low);
availGPR_.add(r.high);
#endif
}
// Notes on float register allocation.
//
// The general rule in SpiderMonkey is that float registers can
// alias double registers, but there are predicates to handle
// exceptions to that rule: hasUnaliasedDouble() and
// hasMultiAlias(). The way aliasing actually works is platform
// dependent and exposed through the aliased(n, &r) predicate,
// etc.
//
// - hasUnaliasedDouble(): on ARM VFPv3-D32 there are double
// registers that cannot be treated as float.
// - hasMultiAlias(): on ARM and MIPS a double register aliases
// two float registers.
// - notes in Architecture-arm.h indicate that when we use a
// float register that aliases a double register we only use
// the low float register, never the high float register. I
// think those notes lie, or at least are confusing.
// - notes in Architecture-mips32.h suggest that the MIPS port
// will use both low and high float registers except on the
// Longsoon, which may be the only MIPS that's being tested, so
// who knows what's working.
// - SIMD is not yet implemented on ARM or MIPS so constraints
// may change there.
//
// On some platforms (x86, x64, ARM64) but not all (ARM)
// ScratchFloat32Register is the same as ScratchDoubleRegister.
//
// It's a basic invariant of the AllocatableRegisterSet that it
// deals properly with aliasing of registers: if s0 or s1 are
// allocated then d0 is not allocatable; if s0 and s1 are freed
// individually then d0 becomes allocatable.
template<MIRType t>
FloatRegisters::SetType maskFromTypeFPU() {
static_assert(t == MIRType::Float32 || t == MIRType::Double, "Float mask type");
if (t == MIRType::Float32)
return FloatRegisters::AllSingleMask;
return FloatRegisters::AllDoubleMask;
}
template<MIRType t>
bool hasFPU() {
return !!(availFPU_.bits() & maskFromTypeFPU<t>());
}
bool isAvailable(FloatRegister r) {
return availFPU_.has(r);
}
void allocFPU(FloatRegister r) {
MOZ_ASSERT(isAvailable(r));
availFPU_.take(r);
}
template<MIRType t>
FloatRegister allocFPU() {
MOZ_ASSERT(hasFPU<t>());
FloatRegister r =
FloatRegisterSet::Intersect(FloatRegisterSet(availFPU_.bits()),
FloatRegisterSet(maskFromTypeFPU<t>())).getAny();
availFPU_.take(r);
return r;
}
void freeFPU(FloatRegister r) {
availFPU_.add(r);
}
////////////////////////////////////////////////////////////
//
// Value stack and high-level register allocation.
//
// The value stack facilitates some on-the-fly register allocation
// and immediate-constant use. It tracks constants, latent
// references to locals, register contents, and values on the CPU
// stack.
//
// The stack can be flushed to memory using sync(). This is handy
// to avoid problems with control flow and messy register usage
// patterns.
struct Stk
{
enum Kind
{
// The Mem opcodes are all clustered at the beginning to
// allow for a quick test within sync().
MemI32, // 32-bit integer stack value ("offs")
MemI64, // 64-bit integer stack value ("offs")
MemF32, // 32-bit floating stack value ("offs")
MemF64, // 64-bit floating stack value ("offs")
// The Local opcodes follow the Mem opcodes for a similar
// quick test within hasLocal().
LocalI32, // Local int32 var ("slot")
LocalI64, // Local int64 var ("slot")
LocalF32, // Local float32 var ("slot")
LocalF64, // Local double var ("slot")
RegisterI32, // 32-bit integer register ("i32reg")
RegisterI64, // 64-bit integer register ("i64reg")
RegisterF32, // 32-bit floating register ("f32reg")
RegisterF64, // 64-bit floating register ("f64reg")
ConstI32, // 32-bit integer constant ("i32val")
ConstI64, // 64-bit integer constant ("i64val")
ConstF32, // 32-bit floating constant ("f32val")
ConstF64, // 64-bit floating constant ("f64val")
None // Uninitialized or void
};
Kind kind_;
static const Kind MemLast = MemF64;
static const Kind LocalLast = LocalF64;
union {
RegI32 i32reg_;
RegI64 i64reg_;
RegF32 f32reg_;
RegF64 f64reg_;
int32_t i32val_;
int64_t i64val_;
RawF32 f32val_;
RawF64 f64val_;
uint32_t slot_;
uint32_t offs_;
};
Stk() { kind_ = None; }
Kind kind() const { return kind_; }
bool isMem() const { return kind_ <= MemLast; }
RegI32 i32reg() const { MOZ_ASSERT(kind_ == RegisterI32); return i32reg_; }
RegI64 i64reg() const { MOZ_ASSERT(kind_ == RegisterI64); return i64reg_; }
RegF32 f32reg() const { MOZ_ASSERT(kind_ == RegisterF32); return f32reg_; }
RegF64 f64reg() const { MOZ_ASSERT(kind_ == RegisterF64); return f64reg_; }
int32_t i32val() const { MOZ_ASSERT(kind_ == ConstI32); return i32val_; }
int64_t i64val() const { MOZ_ASSERT(kind_ == ConstI64); return i64val_; }
RawF32 f32val() const { MOZ_ASSERT(kind_ == ConstF32); return f32val_; }
RawF64 f64val() const { MOZ_ASSERT(kind_ == ConstF64); return f64val_; }
uint32_t slot() const { MOZ_ASSERT(kind_ > MemLast && kind_ <= LocalLast); return slot_; }
uint32_t offs() const { MOZ_ASSERT(isMem()); return offs_; }
void setI32Reg(RegI32 r) { kind_ = RegisterI32; i32reg_ = r; }
void setI64Reg(RegI64 r) { kind_ = RegisterI64; i64reg_ = r; }
void setF32Reg(RegF32 r) { kind_ = RegisterF32; f32reg_ = r; }
void setF64Reg(RegF64 r) { kind_ = RegisterF64; f64reg_ = r; }
void setI32Val(int32_t v) { kind_ = ConstI32; i32val_ = v; }
void setI64Val(int64_t v) { kind_ = ConstI64; i64val_ = v; }
void setF32Val(RawF32 v) { kind_ = ConstF32; f32val_ = v; }
void setF64Val(RawF64 v) { kind_ = ConstF64; f64val_ = v; }
void setSlot(Kind k, uint32_t v) { MOZ_ASSERT(k > MemLast && k <= LocalLast); kind_ = k; slot_ = v; }
void setOffs(Kind k, uint32_t v) { MOZ_ASSERT(k <= MemLast); kind_ = k; offs_ = v; }
};
Vector<Stk, 8, SystemAllocPolicy> stk_;
Stk& push() {
stk_.infallibleEmplaceBack(Stk());
return stk_.back();
}
Register64 invalidRegister64() {
return Register64::Invalid();
}
RegI32 invalidI32() {
return RegI32(Register::Invalid());
}
RegI64 invalidI64() {
return RegI64(invalidRegister64());
}
RegF64 invalidF64() {
return RegF64(InvalidFloatReg);
}
RegI32 fromI64(RegI64 r) {
return RegI32(lowPart(r));
}
RegI64 widenI32(RegI32 r) {
MOZ_ASSERT(!isAvailable(r.reg));
#ifdef JS_PUNBOX64
return RegI64(Register64(r.reg));
#else
RegI32 high = needI32();
return RegI64(Register64(high.reg, r.reg));
#endif
}
Register lowPart(RegI64 r) {
#ifdef JS_PUNBOX64
return r.reg.reg;
#else
return r.reg.low;
#endif
}
Register maybeHighPart(RegI64 r) {
#ifdef JS_PUNBOX64
return Register::Invalid();
#else
return r.reg.high;
#endif
}
void maybeClearHighPart(RegI64 r) {
#ifdef JS_NUNBOX32
masm.move32(Imm32(0), r.reg.high);
#endif
}
void freeI32(RegI32 r) {
freeGPR(r.reg);
}
void freeI64(RegI64 r) {
freeInt64(r.reg);
}
void freeI64Except(RegI64 r, RegI32 except) {
#ifdef JS_PUNBOX64
MOZ_ASSERT(r.reg.reg == except.reg);
#else
MOZ_ASSERT(r.reg.high == except.reg || r.reg.low == except.reg);
freeI64(r);
needI32(except);
#endif
}
void freeF64(RegF64 r) {
freeFPU(r.reg);
}
void freeF32(RegF32 r) {
freeFPU(r.reg);
}
MOZ_MUST_USE RegI32 needI32() {
if (!hasGPR())
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
return RegI32(allocGPR());
}
void needI32(RegI32 specific) {
if (!isAvailable(specific.reg))
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
allocGPR(specific.reg);
}
// TODO / OPTIMIZE: need2xI32() can be optimized along with needI32()
// to avoid sync(). (Bug 1316802)
void need2xI32(RegI32 r0, RegI32 r1) {
needI32(r0);
needI32(r1);
}
MOZ_MUST_USE RegI64 needI64() {
if (!hasInt64())
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
return RegI64(allocInt64());
}
void needI64(RegI64 specific) {
if (!isAvailable(specific.reg))
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
allocInt64(specific.reg);
}
void need2xI64(RegI64 r0, RegI64 r1) {
needI64(r0);
needI64(r1);
}
MOZ_MUST_USE RegF32 needF32() {
if (!hasFPU<MIRType::Float32>())
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
return RegF32(allocFPU<MIRType::Float32>());
}
void needF32(RegF32 specific) {
if (!isAvailable(specific.reg))
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
allocFPU(specific.reg);
}
MOZ_MUST_USE RegF64 needF64() {
if (!hasFPU<MIRType::Double>())
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
return RegF64(allocFPU<MIRType::Double>());
}
void needF64(RegF64 specific) {
if (!isAvailable(specific.reg))
sync(); // TODO / OPTIMIZE: improve this (Bug 1316802)
allocFPU(specific.reg);
}
void moveI32(RegI32 src, RegI32 dest) {
if (src != dest)
masm.move32(src.reg, dest.reg);
}
void moveI64(RegI64 src, RegI64 dest) {
if (src != dest)
masm.move64(src.reg, dest.reg);
}
void moveF64(RegF64 src, RegF64 dest) {
if (src != dest)
masm.moveDouble(src.reg, dest.reg);
}
void moveF32(RegF32 src, RegF32 dest) {
if (src != dest)
masm.moveFloat32(src.reg, dest.reg);
}
void setI64(int64_t v, RegI64 r) {
masm.move64(Imm64(v), r.reg);
}
void loadConstI32(Register r, Stk& src) {
masm.mov(ImmWord((uint32_t)src.i32val() & 0xFFFFFFFFU), r);
}
void loadMemI32(Register r, Stk& src) {
loadFromFrameI32(r, src.offs());
}
void loadLocalI32(Register r, Stk& src) {
loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int32));
}
void loadRegisterI32(Register r, Stk& src) {
if (src.i32reg().reg != r)
masm.move32(src.i32reg().reg, r);
}
void loadI32(Register r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI32:
loadConstI32(r, src);
break;
case Stk::MemI32:
loadMemI32(r, src);
break;
case Stk::LocalI32:
loadLocalI32(r, src);
break;
case Stk::RegisterI32:
loadRegisterI32(r, src);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: Expected int on stack");
}
}
// TODO / OPTIMIZE: Refactor loadI64, loadF64, and loadF32 in the
// same way as loadI32 to avoid redundant dispatch in callers of
// these load() functions. (Bug 1316816, also see annotations on
// popI64 et al below.)
void loadI64(Register64 r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
masm.move64(Imm64(src.i64val()), r);
break;
case Stk::MemI64:
loadFromFrameI64(r, src.offs());
break;
case Stk::LocalI64:
loadFromFrameI64(r, frameOffsetFromSlot(src.slot(), MIRType::Int64));
break;
case Stk::RegisterI64:
if (src.i64reg().reg != r)
masm.move64(src.i64reg().reg, r);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: Expected int on stack");
}
}
#ifdef JS_NUNBOX32
void loadI64Low(Register r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
masm.move32(Imm64(src.i64val()).low(), r);
break;
case Stk::MemI64:
loadFromFrameI32(r, src.offs() - INT64LOW_OFFSET);
break;
case Stk::LocalI64:
loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64LOW_OFFSET);
break;
case Stk::RegisterI64:
if (src.i64reg().reg.low != r)
masm.move32(src.i64reg().reg.low, r);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: Expected int on stack");
}
}
void loadI64High(Register r, Stk& src) {
switch (src.kind()) {
case Stk::ConstI64:
masm.move32(Imm64(src.i64val()).hi(), r);
break;
case Stk::MemI64:
loadFromFrameI32(r, src.offs() - INT64HIGH_OFFSET);
break;
case Stk::LocalI64:
loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64HIGH_OFFSET);
break;
case Stk::RegisterI64:
if (src.i64reg().reg.high != r)
masm.move32(src.i64reg().reg.high, r);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: Expected int on stack");
}
}
#endif
void loadF64(FloatRegister r, Stk& src) {
switch (src.kind()) {
case Stk::ConstF64:
masm.loadConstantDouble(src.f64val(), r);
break;
case Stk::MemF64:
loadFromFrameF64(r, src.offs());
break;
case Stk::LocalF64:
loadFromFrameF64(r, frameOffsetFromSlot(src.slot(), MIRType::Double));
break;
case Stk::RegisterF64:
if (src.f64reg().reg != r)
masm.moveDouble(src.f64reg().reg, r);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: expected double on stack");
}
}
void loadF32(FloatRegister r, Stk& src) {
switch (src.kind()) {
case Stk::ConstF32:
masm.loadConstantFloat32(src.f32val(), r);
break;
case Stk::MemF32:
loadFromFrameF32(r, src.offs());
break;
case Stk::LocalF32:
loadFromFrameF32(r, frameOffsetFromSlot(src.slot(), MIRType::Float32));
break;
case Stk::RegisterF32:
if (src.f32reg().reg != r)
masm.moveFloat32(src.f32reg().reg, r);
break;
case Stk::None:
break;
default:
MOZ_CRASH("Compiler bug: expected float on stack");
}
}
// Flush all local and register value stack elements to memory.
//
// TODO / OPTIMIZE: As this is fairly expensive and causes worse
// code to be emitted subsequently, it is useful to avoid calling
// it. (Bug 1316802)
//
// Some optimization has been done already. Remaining
// opportunities:
//
// - It would be interesting to see if we can specialize it
// before calls with particularly simple signatures, or where
// we can do parallel assignment of register arguments, or
// similar. See notes in emitCall().
//
// - Operations that need specific registers: multiply, quotient,
// remainder, will tend to sync because the registers we need
// will tend to be allocated. We may be able to avoid that by
// prioritizing registers differently (takeLast instead of
// takeFirst) but we may also be able to allocate an unused
// register on demand to free up one we need, thus avoiding the
// sync. That type of fix would go into needI32().
void sync() {
size_t start = 0;
size_t lim = stk_.length();
for (size_t i = lim; i > 0; i--) {
// Memory opcodes are first in the enum, single check against MemLast is fine.
if (stk_[i - 1].kind() <= Stk::MemLast) {
start = i;
break;
}
}
for (size_t i = start; i < lim; i++) {
Stk& v = stk_[i];
switch (v.kind()) {
case Stk::LocalI32: {
ScratchI32 scratch(*this);
loadLocalI32(scratch, v);
masm.Push(scratch);
v.setOffs(Stk::MemI32, masm.framePushed());
break;
}
case Stk::RegisterI32: {
masm.Push(v.i32reg().reg);
freeI32(v.i32reg());
v.setOffs(Stk::MemI32, masm.framePushed());
break;
}
case Stk::LocalI64: {
ScratchI32 scratch(*this);
#ifdef JS_PUNBOX64
loadI64(Register64(scratch), v);
masm.Push(scratch);
#else
int32_t offset = frameOffsetFromSlot(v.slot(), MIRType::Int64);
loadFromFrameI32(scratch, offset - INT64HIGH_OFFSET);
masm.Push(scratch);
loadFromFrameI32(scratch, offset - INT64LOW_OFFSET);
masm.Push(scratch);
#endif
v.setOffs(Stk::MemI64, masm.framePushed());
break;
}
case Stk::RegisterI64: {
#ifdef JS_PUNBOX64
masm.Push(v.i64reg().reg.reg);
freeI64(v.i64reg());
#else
masm.Push(v.i64reg().reg.high);
masm.Push(v.i64reg().reg.low);
freeI64(v.i64reg());
#endif
v.setOffs(Stk::MemI64, masm.framePushed());
break;
}
case Stk::LocalF64: {
ScratchF64 scratch(*this);
loadF64(scratch, v);
masm.Push(scratch);
v.setOffs(Stk::MemF64, masm.framePushed());
break;
}
case Stk::RegisterF64: {
masm.Push(v.f64reg().reg);
freeF64(v.f64reg());
v.setOffs(Stk::MemF64, masm.framePushed());
break;
}
case Stk::LocalF32: {
ScratchF32 scratch(*this);
loadF32(scratch, v);
masm.Push(scratch);
v.setOffs(Stk::MemF32, masm.framePushed());
break;
}
case Stk::RegisterF32: {
masm.Push(v.f32reg().reg);
freeF32(v.f32reg());
v.setOffs(Stk::MemF32, masm.framePushed());
break;
}
default: {
break;
}
}
}
maxFramePushed_ = Max(maxFramePushed_, int32_t(masm.framePushed()));
}
// This is an optimization used to avoid calling sync() for
// setLocal(): if the local does not exist unresolved on the stack
// then we can skip the sync.
bool hasLocal(uint32_t slot) {
for (size_t i = stk_.length(); i > 0; i--) {
// Memory opcodes are first in the enum, single check against MemLast is fine.
Stk::Kind kind = stk_[i-1].kind();
if (kind <= Stk::MemLast)
return false;
// Local opcodes follow memory opcodes in the enum, single check against
// LocalLast is sufficient.
if (kind <= Stk::LocalLast && stk_[i-1].slot() == slot)
return true;
}
return false;
}
void syncLocal(uint32_t slot) {
if (hasLocal(slot))
sync(); // TODO / OPTIMIZE: Improve this? (Bug 1316817)
}
// Push the register r onto the stack.
void pushI32(RegI32 r) {
MOZ_ASSERT(!isAvailable(r.reg));
Stk& x = push();
x.setI32Reg(r);
}
void pushI64(RegI64 r) {
MOZ_ASSERT(!isAvailable(r.reg));
Stk& x = push();
x.setI64Reg(r);
}
void pushF64(RegF64 r) {
MOZ_ASSERT(!isAvailable(r.reg));
Stk& x = push();
x.setF64Reg(r);
}
void pushF32(RegF32 r) {
MOZ_ASSERT(!isAvailable(r.reg));
Stk& x = push();
x.setF32Reg(r);
}
// Push the value onto the stack.
void pushI32(int32_t v) {
Stk& x = push();
x.setI32Val(v);
}
void pushI64(int64_t v) {
Stk& x = push();
x.setI64Val(v);
}
void pushF64(RawF64 v) {
Stk& x = push();
x.setF64Val(v);
}
void pushF32(RawF32 v) {
Stk& x = push();
x.setF32Val(v);
}
// Push the local slot onto the stack. The slot will not be read
// here; it will be read when it is consumed, or when a side
// effect to the slot forces its value to be saved.
void pushLocalI32(uint32_t slot) {
Stk& x = push();
x.setSlot(Stk::LocalI32, slot);
}
void pushLocalI64(uint32_t slot) {
Stk& x = push();
x.setSlot(Stk::LocalI64, slot);
}
void pushLocalF64(uint32_t slot) {
Stk& x = push();
x.setSlot(Stk::LocalF64, slot);
}
void pushLocalF32(uint32_t slot) {
Stk& x = push();
x.setSlot(Stk::LocalF32, slot);
}
// PRIVATE. Call only from other popI32() variants.
// v must be the stack top.
void popI32(Stk& v, RegI32 r) {
switch (v.kind()) {
case Stk::ConstI32:
loadConstI32(r.reg, v);
break;
case Stk::LocalI32:
loadLocalI32(r.reg, v);
break;
case Stk::MemI32:
masm.Pop(r.reg);
break;
case Stk::RegisterI32:
moveI32(v.i32reg(), r);
break;
case Stk::None:
// This case crops up in situations where there's unreachable code that
// the type system interprets as "generating" a value of the correct type:
//
// (if (return) E1 E2) type is type(E1) meet type(E2)
// (if E (unreachable) (i32.const 1)) type is int
// (if E (i32.const 1) (unreachable)) type is int
//
// It becomes silly to handle this throughout the code, so just handle it
// here even if that means weaker run-time checking.
break;
default:
MOZ_CRASH("Compiler bug: expected int on stack");
}
}
MOZ_MUST_USE RegI32 popI32() {
Stk& v = stk_.back();
RegI32 r;
if (v.kind() == Stk::RegisterI32)
r = v.i32reg();
else
popI32(v, (r = needI32()));
stk_.popBack();
return r;
}
RegI32 popI32(RegI32 specific) {
Stk& v = stk_.back();
if (!(v.kind() == Stk::RegisterI32 && v.i32reg() == specific)) {
needI32(specific);
popI32(v, specific);
if (v.kind() == Stk::RegisterI32)
freeI32(v.i32reg());
}
stk_.popBack();
return specific;
}
// PRIVATE. Call only from other popI64() variants.
// v must be the stack top.
void popI64(Stk& v, RegI64 r) {
// TODO / OPTIMIZE: avoid loadI64() here. (Bug 1316816)
switch (v.kind()) {
case Stk::ConstI64:
case Stk::LocalI64:
loadI64(r.reg, v);
break;
case Stk::MemI64:
#ifdef JS_PUNBOX64
masm.Pop(r.reg.reg);
#else
masm.Pop(r.reg.low);
masm.Pop(r.reg.high);
#endif
break;
case Stk::RegisterI64:
moveI64(v.i64reg(), r);
break;
case Stk::None:
// See popI32()
break;
default:
MOZ_CRASH("Compiler bug: expected long on stack");
}
}
MOZ_MUST_USE RegI64 popI64() {
Stk& v = stk_.back();
RegI64 r;
if (v.kind() == Stk::RegisterI64)
r = v.i64reg();
else
popI64(v, (r = needI64()));
stk_.popBack();
return r;
}
// Note, the stack top can be in one half of "specific" on 32-bit
// systems. We can optimize, but for simplicity, if the register
// does not match exactly, then just force the stack top to memory
// and then read it back in.
RegI64 popI64(RegI64 specific) {
Stk& v = stk_.back();
if (!(v.kind() == Stk::RegisterI64 && v.i64reg() == specific)) {
needI64(specific);
popI64(v, specific);
if (v.kind() == Stk::RegisterI64)
freeI64(v.i64reg());
}
stk_.popBack();
return specific;
}
// PRIVATE. Call only from other popF64() variants.
// v must be the stack top.
void popF64(Stk& v, RegF64 r) {
// TODO / OPTIMIZE: avoid loadF64 here. (Bug 1316816)
switch (v.kind()) {
case Stk::ConstF64:
case Stk::LocalF64:
loadF64(r.reg, v);
break;
case Stk::MemF64:
masm.Pop(r.reg);
break;
case Stk::RegisterF64:
moveF64(v.f64reg(), r);
break;
case Stk::None:
// See popI32()
break;
default:
MOZ_CRASH("Compiler bug: expected double on stack");
}
}
MOZ_MUST_USE RegF64 popF64() {
Stk& v = stk_.back();
RegF64 r;
if (v.kind() == Stk::RegisterF64)
r = v.f64reg();
else
popF64(v, (r = needF64()));
stk_.popBack();
return r;
}
RegF64 popF64(RegF64 specific) {
Stk& v = stk_.back();
if (!(v.kind() == Stk::RegisterF64 && v.f64reg() == specific)) {
needF64(specific);
popF64(v, specific);
if (v.kind() == Stk::RegisterF64)
freeF64(v.f64reg());
}
stk_.popBack();
return specific;
}
// PRIVATE. Call only from other popF32() variants.
// v must be the stack top.
void popF32(Stk& v, RegF32 r) {
// TODO / OPTIMIZE: avoid loadF32 here. (Bug 1316816)
switch (v.kind()) {
case Stk::ConstF32:
case Stk::LocalF32:
loadF32(r.reg, v);
break;
case Stk::MemF32:
masm.Pop(r.reg);
break;
case Stk::RegisterF32:
moveF32(v.f32reg(), r);
break;
case Stk::None:
// See popI32()
break;
default:
MOZ_CRASH("Compiler bug: expected float on stack");
}
}
MOZ_MUST_USE RegF32 popF32() {
Stk& v = stk_.back();
RegF32 r;
if (v.kind() == Stk::RegisterF32)
r = v.f32reg();
else
popF32(v, (r = needF32()));
stk_.popBack();
return r;
}
RegF32 popF32(RegF32 specific) {
Stk& v = stk_.back();
if (!(v.kind() == Stk::RegisterF32 && v.f32reg() == specific)) {
needF32(specific);
popF32(v, specific);
if (v.kind() == Stk::RegisterF32)
freeF32(v.f32reg());
}
stk_.popBack();
return specific;
}
MOZ_MUST_USE bool popConstI32(int32_t& c) {
Stk& v = stk_.back();
if (v.kind() != Stk::ConstI32)
return false;
c = v.i32val();
stk_.popBack();
return true;
}
// TODO / OPTIMIZE (Bug 1316818): At the moment we use ReturnReg
// for JoinReg. It is possible other choices would lead to better
// register allocation, as ReturnReg is often first in the
// register set and will be heavily wanted by the register
// allocator that uses takeFirst().
//
// Obvious options:
// - pick a register at the back of the register set
// - pick a random register per block (different blocks have
// different join regs)
//
// On the other hand, we sync() before every block and only the
// JoinReg is live out of the block. But on the way out, we
// currently pop the JoinReg before freeing regs to be discarded,
// so there is a real risk of some pointless shuffling there. If
// we instead integrate the popping of the join reg into the
// popping of the stack we can just use the JoinReg as it will
// become available in that process.
MOZ_MUST_USE AnyReg popJoinReg() {
switch (stk_.back().kind()) {
case Stk::RegisterI32:
case Stk::ConstI32:
case Stk::MemI32:
case Stk::LocalI32:
return AnyReg(popI32(joinRegI32));
case Stk::RegisterI64:
case Stk::ConstI64:
case Stk::MemI64:
case Stk::LocalI64:
return AnyReg(popI64(joinRegI64));
case Stk::RegisterF64:
case Stk::ConstF64:
case Stk::MemF64:
case Stk::LocalF64:
return AnyReg(popF64(joinRegF64));
case Stk::RegisterF32:
case Stk::ConstF32:
case Stk::MemF32:
case Stk::LocalF32:
return AnyReg(popF32(joinRegF32));
case Stk::None:
stk_.popBack();
return AnyReg();
default:
MOZ_CRASH("Compiler bug: unexpected value on stack");
}
}
MOZ_MUST_USE AnyReg allocJoinReg(ExprType type) {
switch (type) {
case ExprType::I32:
allocGPR(joinRegI32.reg);
return AnyReg(joinRegI32);
case ExprType::I64:
allocInt64(joinRegI64.reg);
return AnyReg(joinRegI64);
case ExprType::F32:
allocFPU(joinRegF32.reg);
return AnyReg(joinRegF32);
case ExprType::F64:
allocFPU(joinRegF64.reg);
return AnyReg(joinRegF64);
case ExprType::Void:
MOZ_CRASH("Compiler bug: allocating void join reg");
default:
MOZ_CRASH("Compiler bug: unexpected type");
}
}
void pushJoinReg(AnyReg r) {
switch (r.tag) {
case AnyReg::NONE:
MOZ_CRASH("Compile bug: attempting to push void");
break;
case AnyReg::I32:
pushI32(r.i32());
break;
case AnyReg::I64:
pushI64(r.i64());
break;
case AnyReg::F64:
pushF64(r.f64());
break;
case AnyReg::F32:
pushF32(r.f32());
break;
}
}
void freeJoinReg(AnyReg r) {
switch (r.tag) {
case AnyReg::NONE:
MOZ_CRASH("Compile bug: attempting to free void reg");
break;
case AnyReg::I32:
freeI32(r.i32());
break;
case AnyReg::I64:
freeI64(r.i64());
break;
case AnyReg::F64:
freeF64(r.f64());
break;
case AnyReg::F32:
freeF32(r.f32());
break;
}
}
void maybeReserveJoinRegI(ExprType type) {
if (type == ExprType::I32)
needI32(joinRegI32);
else if (type == ExprType::I64)
needI64(joinRegI64);
}
void maybeUnreserveJoinRegI(ExprType type) {
if (type == ExprType::I32)
freeI32(joinRegI32);
else if (type == ExprType::I64)
freeI64(joinRegI64);
}
// Return the amount of execution stack consumed by the top numval
// values on the value stack.
size_t stackConsumed(size_t numval) {
size_t size = 0;
MOZ_ASSERT(numval <= stk_.length());
for (uint32_t i = stk_.length() - 1; numval > 0; numval--, i--) {
// The size computations come from the implementation of Push() in
// MacroAssembler-x86-shared.cpp and MacroAssembler-arm-shared.cpp,
// and from VFPRegister::size() in Architecture-arm.h.
//
// On ARM unlike on x86 we push a single for float.
Stk& v = stk_[i];
switch (v.kind()) {
case Stk::MemI32:
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
size += sizeof(intptr_t);
#else
MOZ_CRASH("BaseCompiler platform hook: stackConsumed I32");
#endif
break;
case Stk::MemI64:
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
size += sizeof(int64_t);
#else
MOZ_CRASH("BaseCompiler platform hook: stackConsumed I64");
#endif
break;
case Stk::MemF64:
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
size += sizeof(double);
#else
MOZ_CRASH("BaseCompiler platform hook: stackConsumed F64");
#endif
break;
case Stk::MemF32:
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
size += sizeof(double);
#elif defined(JS_CODEGEN_ARM)
size += sizeof(float);
#else
MOZ_CRASH("BaseCompiler platform hook: stackConsumed F32");
#endif
break;
default:
break;
}
}
return size;
}
void popValueStackTo(uint32_t stackSize) {
for (uint32_t i = stk_.length(); i > stackSize; i--) {
Stk& v = stk_[i-1];
switch (v.kind()) {
case Stk::RegisterI32:
freeI32(v.i32reg());
break;
case Stk::RegisterI64:
freeI64(v.i64reg());
break;
case Stk::RegisterF64:
freeF64(v.f64reg());
break;
case Stk::RegisterF32:
freeF32(v.f32reg());
break;
default:
break;
}
}
stk_.shrinkTo(stackSize);
}
void popValueStackBy(uint32_t items) {
popValueStackTo(stk_.length() - items);
}
// Before branching to an outer control label, pop the execution
// stack to the level expected by that region, but do not free the
// stack as that will happen as compilation leaves the block.
void popStackBeforeBranch(uint32_t framePushed) {
uint32_t frameHere = masm.framePushed();
if (frameHere > framePushed)
masm.addPtr(ImmWord(frameHere - framePushed), StackPointer);
}
// Before exiting a nested control region, pop the execution stack
// to the level expected by the nesting region, and free the
// stack.
void popStackOnBlockExit(uint32_t framePushed) {
uint32_t frameHere = masm.framePushed();
if (frameHere > framePushed) {
if (deadCode_)
masm.adjustStack(frameHere - framePushed);
else
masm.freeStack(frameHere - framePushed);
}
}
void popStackIfMemory() {
if (peek(0).isMem())
masm.freeStack(stackConsumed(1));
}
// Peek at the stack, for calls.
Stk& peek(uint32_t relativeDepth) {
return stk_[stk_.length()-1-relativeDepth];
}
////////////////////////////////////////////////////////////
//
// Control stack
Vector<Control, 8, SystemAllocPolicy> ctl_;
MOZ_MUST_USE bool pushControl(UniquePooledLabel* label, UniquePooledLabel* otherLabel = nullptr)
{
uint32_t framePushed = masm.framePushed();
uint32_t stackSize = stk_.length();
if (!ctl_.emplaceBack(Control(framePushed, stackSize)))
return false;
if (label)
ctl_.back().label = label->release();
if (otherLabel)
ctl_.back().otherLabel = otherLabel->release();
ctl_.back().deadOnArrival = deadCode_;
return true;
}
void popControl() {
Control last = ctl_.popCopy();
if (last.label)
freeLabel(last.label);
if (last.otherLabel)
freeLabel(last.otherLabel);
if (deadCode_ && !ctl_.empty())
popValueStackTo(ctl_.back().stackSize);
}
Control& controlItem(uint32_t relativeDepth) {
return ctl_[ctl_.length() - 1 - relativeDepth];
}
MOZ_MUST_USE PooledLabel* newLabel() {
// TODO / INVESTIGATE (Bug 1316819): allocate() is fallible, but we can
// probably rely on an infallible allocator here. That would simplify
// code later.
PooledLabel* candidate = labelPool_.allocate();
if (!candidate)
return nullptr;
return new (candidate) PooledLabel(this);
}
void freeLabel(PooledLabel* label) {
label->~PooledLabel();
labelPool_.free(label);
}
//////////////////////////////////////////////////////////////////////
//
// Function prologue and epilogue.
void beginFunction() {
JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code");
SigIdDesc sigId = mg_.funcSigs[func_.index()]->id;
GenerateFunctionPrologue(masm, localSize_, sigId, &compileResults_.offsets());
MOZ_ASSERT(masm.framePushed() == uint32_t(localSize_));
maxFramePushed_ = localSize_;
// We won't know until after we've generated code how big the
// frame will be (we may need arbitrary spill slots and
// outgoing param slots) so branch to code emitted after the
// function body that will perform the check.
//
// Code there will also assume that the fixed-size stack frame
// has been allocated.
masm.jump(&outOfLinePrologue_);
masm.bind(&bodyLabel_);
// Copy arguments from registers to stack.
const ValTypeVector& args = func_.sig().args();
for (ABIArgIter<const ValTypeVector> i(args); !i.done(); i++) {
Local& l = localInfo_[i.index()];
switch (i.mirType()) {
case MIRType::Int32:
if (i->argInRegister())
storeToFrameI32(i->gpr(), l.offs());
break;
case MIRType::Int64:
if (i->argInRegister())
storeToFrameI64(i->gpr64(), l.offs());
break;
case MIRType::Double:
if (i->argInRegister())
storeToFrameF64(i->fpu(), l.offs());
break;
case MIRType::Float32:
if (i->argInRegister())
storeToFrameF32(i->fpu(), l.offs());
break;
default:
MOZ_CRASH("Function argument type");
}
}
// The TLS pointer is always passed as a hidden argument in WasmTlsReg.
// Save it into its assigned local slot.
storeToFramePtr(WasmTlsReg, localInfo_[tlsSlot_].offs());
// Initialize the stack locals to zero.
//
// The following are all Bug 1316820:
//
// TODO / OPTIMIZE: on x64, at least, scratch will be a 64-bit
// register and we can move 64 bits at a time.
//
// TODO / OPTIMIZE: On SSE2 or better SIMD systems we may be
// able to store 128 bits at a time. (I suppose on some
// systems we have 512-bit SIMD for that matter.)
//
// TODO / OPTIMIZE: if we have only one initializing store
// then it's better to store a zero literal, probably.
if (varLow_ < varHigh_) {
ScratchI32 scratch(*this);
masm.mov(ImmWord(0), scratch);
for (int32_t i = varLow_ ; i < varHigh_ ; i += 4)
storeToFrameI32(scratch, i + 4);
}
}
bool endFunction() {
// Out-of-line prologue. Assumes that the in-line prologue has
// been executed and that a frame of size = localSize_ + sizeof(Frame)
// has been allocated.
masm.bind(&outOfLinePrologue_);
MOZ_ASSERT(maxFramePushed_ >= localSize_);
// ABINonArgReg0 != ScratchReg, which can be used by branchPtr().
masm.movePtr(masm.getStackPointer(), ABINonArgReg0);
if (maxFramePushed_ - localSize_)
masm.subPtr(Imm32(maxFramePushed_ - localSize_), ABINonArgReg0);
masm.branchPtr(Assembler::Below,
Address(WasmTlsReg, offsetof(TlsData, stackLimit)),
ABINonArgReg0,
&bodyLabel_);
// Since we just overflowed the stack, to be on the safe side, pop the
// stack so that, when the trap exit stub executes, it is a safe
// distance away from the end of the native stack.
if (localSize_)
masm.addToStackPtr(Imm32(localSize_));
masm.jump(TrapDesc(prologueTrapOffset_, Trap::StackOverflow, /* framePushed = */ 0));
masm.bind(&returnLabel_);
// Restore the TLS register in case it was overwritten by the function.
loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
GenerateFunctionEpilogue(masm, localSize_, &compileResults_.offsets());
#if defined(JS_ION_PERF)
// FIXME - profiling code missing. Bug 1286948.
// Note the end of the inline code and start of the OOL code.
//gen->perfSpewer().noteEndInlineCode(masm);
#endif
if (!generateOutOfLineCode())
return false;
masm.wasmEmitTrapOutOfLineCode();
compileResults_.offsets().end = masm.currentOffset();
// A frame greater than 256KB is implausible, probably an attack,
// so fail the compilation.
if (maxFramePushed_ > 256 * 1024)
return false;
return true;
}
//////////////////////////////////////////////////////////////////////
//
// Calls.
struct FunctionCall
{
explicit FunctionCall(uint32_t lineOrBytecode)
: lineOrBytecode(lineOrBytecode),
reloadMachineStateAfter(false),
usesSystemAbi(false),
loadTlsBefore(false),
#ifdef JS_CODEGEN_ARM
hardFP(true),
#endif
frameAlignAdjustment(0),
stackArgAreaSize(0)
{}
uint32_t lineOrBytecode;
ABIArgGenerator abi;
bool reloadMachineStateAfter;
bool usesSystemAbi;
bool loadTlsBefore;
#ifdef JS_CODEGEN_ARM
bool hardFP;
#endif
size_t frameAlignAdjustment;
size_t stackArgAreaSize;
};
void beginCall(FunctionCall& call, UseABI useABI, InterModule interModule)
{
call.reloadMachineStateAfter = interModule == InterModule::True || useABI == UseABI::System;
call.usesSystemAbi = useABI == UseABI::System;
call.loadTlsBefore = useABI == UseABI::Wasm;
if (call.usesSystemAbi) {
// Call-outs need to use the appropriate system ABI.
#if defined(JS_CODEGEN_ARM)
# if defined(JS_SIMULATOR_ARM)
call.hardFP = UseHardFpABI();
# elif defined(JS_CODEGEN_ARM_HARDFP)
call.hardFP = true;
# else
call.hardFP = false;
# endif
call.abi.setUseHardFp(call.hardFP);
#endif
}
call.frameAlignAdjustment = ComputeByteAlignment(masm.framePushed() + sizeof(Frame),
JitStackAlignment);
}
void endCall(FunctionCall& call)
{
size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
if (adjustment)
masm.freeStack(adjustment);
if (call.reloadMachineStateAfter) {
loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
masm.loadWasmPinnedRegsFromTls();
}
}
// TODO / OPTIMIZE (Bug 1316820): This is expensive; let's roll the iterator
// walking into the walking done for passArg. See comments in passArg.
size_t stackArgAreaSize(const ValTypeVector& args) {
ABIArgIter<const ValTypeVector> i(args);
while (!i.done())
i++;
return AlignBytes(i.stackBytesConsumedSoFar(), 16u);
}
void startCallArgs(FunctionCall& call, size_t stackArgAreaSize)
{
call.stackArgAreaSize = stackArgAreaSize;
size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
if (adjustment)
masm.reserveStack(adjustment);
}
const ABIArg reservePointerArgument(FunctionCall& call) {
return call.abi.next(MIRType::Pointer);
}
// TODO / OPTIMIZE (Bug 1316820): Note passArg is used only in one place.
// (Or it was, until Luke wandered through, but that can be fixed again.)
// I'm not saying we should manually inline it, but we could hoist the
// dispatch into the caller and have type-specific implementations of
// passArg: passArgI32(), etc. Then those might be inlined, at least in PGO
// builds.
//
// The bulk of the work here (60%) is in the next() call, though.
//
// Notably, since next() is so expensive, stackArgAreaSize() becomes
// expensive too.
//
// Somehow there could be a trick here where the sequence of
// argument types (read from the input stream) leads to a cached
// entry for stackArgAreaSize() and for how to pass arguments...
//
// But at least we could reduce the cost of stackArgAreaSize() by
// first reading the argument types into a (reusable) vector, then
// we have the outgoing size at low cost, and then we can pass
// args based on the info we read.
void passArg(FunctionCall& call, ValType type, Stk& arg) {
switch (type) {
case ValType::I32: {
ABIArg argLoc = call.abi.next(MIRType::Int32);
if (argLoc.kind() == ABIArg::Stack) {
ScratchI32 scratch(*this);
loadI32(scratch, arg);
masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
} else {
loadI32(argLoc.gpr(), arg);
}
break;
}
case ValType::I64: {
ABIArg argLoc = call.abi.next(MIRType::Int64);
if (argLoc.kind() == ABIArg::Stack) {
ScratchI32 scratch(*this);
#if defined(JS_CODEGEN_X64)
loadI64(Register64(scratch), arg);
masm.movq(scratch, Operand(StackPointer, argLoc.offsetFromArgBase()));
#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
loadI64Low(scratch, arg);
masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase() + INT64LOW_OFFSET));
loadI64High(scratch, arg);
masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase() + INT64HIGH_OFFSET));
#else
MOZ_CRASH("BaseCompiler platform hook: passArg I64");
#endif
} else {
loadI64(argLoc.gpr64(), arg);
}
break;
}
case ValType::F64: {
ABIArg argLoc = call.abi.next(MIRType::Double);
switch (argLoc.kind()) {
case ABIArg::Stack: {
ScratchF64 scratch(*this);
loadF64(scratch, arg);
masm.storeDouble(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
break;
}
#if defined(JS_CODEGEN_REGISTER_PAIR)
case ABIArg::GPR_PAIR: {
# ifdef JS_CODEGEN_ARM
ScratchF64 scratch(*this);
loadF64(scratch, arg);
masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr());
break;
# else
MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair");
# endif
}
#endif
case ABIArg::FPU: {
loadF64(argLoc.fpu(), arg);
break;
}
case ABIArg::GPR: {
MOZ_CRASH("Unexpected parameter passing discipline");
}
}
break;
}
case ValType::F32: {
ABIArg argLoc = call.abi.next(MIRType::Float32);
switch (argLoc.kind()) {
case ABIArg::Stack: {
ScratchF32 scratch(*this);
loadF32(scratch, arg);
masm.storeFloat32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
break;
}
case ABIArg::GPR: {
ScratchF32 scratch(*this);
loadF32(scratch, arg);
masm.moveFloat32ToGPR(scratch, argLoc.gpr());
break;
}
case ABIArg::FPU: {
loadF32(argLoc.fpu(), arg);
break;
}
#if defined(JS_CODEGEN_REGISTER_PAIR)
case ABIArg::GPR_PAIR: {
MOZ_CRASH("Unexpected parameter passing discipline");
}
#endif
}
break;
}
default:
MOZ_CRASH("Function argument type");
}
}
void callDefinition(uint32_t funcIndex, const FunctionCall& call)
{
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Func);
masm.call(desc, funcIndex);
}
void callSymbolic(SymbolicAddress callee, const FunctionCall& call) {
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Symbolic);
masm.call(callee);
}
// Precondition: sync()
void callIndirect(uint32_t sigIndex, Stk& indexVal, const FunctionCall& call)
{
loadI32(WasmTableCallIndexReg, indexVal);
const SigWithId& sig = mg_.sigs[sigIndex];
CalleeDesc callee;
if (isCompilingAsmJS()) {
MOZ_ASSERT(sig.id.kind() == SigIdDesc::Kind::None);
const TableDesc& table = mg_.tables[mg_.asmJSSigToTableIndex[sigIndex]];
MOZ_ASSERT(IsPowerOfTwo(table.limits.initial));
masm.andPtr(Imm32((table.limits.initial - 1)), WasmTableCallIndexReg);
callee = CalleeDesc::asmJSTable(table);
} else {
MOZ_ASSERT(sig.id.kind() != SigIdDesc::Kind::None);
MOZ_ASSERT(mg_.tables.length() == 1);
const TableDesc& table = mg_.tables[0];
callee = CalleeDesc::wasmTable(table, sig.id);
}
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
masm.wasmCallIndirect(desc, callee);
}
// Precondition: sync()
void callImport(unsigned globalDataOffset, const FunctionCall& call)
{
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
CalleeDesc callee = CalleeDesc::import(globalDataOffset);
masm.wasmCallImport(desc, callee);
}
void builtinCall(SymbolicAddress builtin, const FunctionCall& call)
{
callSymbolic(builtin, call);
}
void builtinInstanceMethodCall(SymbolicAddress builtin, const ABIArg& instanceArg,
const FunctionCall& call)
{
// Builtin method calls assume the TLS register has been set.
loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Symbolic);
masm.wasmCallBuiltinInstanceMethod(instanceArg, builtin);
}
//////////////////////////////////////////////////////////////////////
//
// Sundry low-level code generators.
void addInterruptCheck()
{
// Always use signals for interrupts with Asm.JS/Wasm
MOZ_RELEASE_ASSERT(HaveSignalHandlers());
}
void jumpTable(LabelVector& labels) {
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
for (uint32_t i = 0; i < labels.length(); i++) {
CodeLabel cl;
masm.writeCodePointer(cl.patchAt());
cl.target()->bind(labels[i]->offset());
masm.addCodeLabel(cl);
}
#else
MOZ_CRASH("BaseCompiler platform hook: jumpTable");
#endif
}
void tableSwitch(Label* theTable, RegI32 switchValue) {
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
ScratchI32 scratch(*this);
CodeLabel tableCl;
masm.mov(tableCl.patchAt(), scratch);
tableCl.target()->bind(theTable->offset());
masm.addCodeLabel(tableCl);
masm.jmp(Operand(scratch, switchValue.reg, ScalePointer));
#elif defined(JS_CODEGEN_ARM)
ScratchI32 scratch(*this);
// Compute the offset from the next instruction to the jump table
Label here;
masm.bind(&here);
uint32_t offset = here.offset() - theTable->offset();
// Read PC+8
masm.ma_mov(pc, scratch);
// Required by ma_sub.
ScratchRegisterScope arm_scratch(*this);
// Compute the table base pointer
masm.ma_sub(Imm32(offset + 8), scratch, arm_scratch);
// Jump indirect via table element
masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue.reg, LSL, 2)), pc, Offset,
Assembler::Always);
#else
MOZ_CRASH("BaseCompiler platform hook: tableSwitch");
#endif
}
RegI32 captureReturnedI32() {
RegI32 rv = RegI32(ReturnReg);
MOZ_ASSERT(isAvailable(rv.reg));
needI32(rv);
return rv;
}
RegI64 captureReturnedI64() {
RegI64 rv = RegI64(ReturnReg64);
MOZ_ASSERT(isAvailable(rv.reg));
needI64(rv);
return rv;
}
RegF32 captureReturnedF32(const FunctionCall& call) {
RegF32 rv = RegF32(ReturnFloat32Reg);
MOZ_ASSERT(isAvailable(rv.reg));
needF32(rv);
#if defined(JS_CODEGEN_X86)
if (call.usesSystemAbi) {
masm.reserveStack(sizeof(float));
Operand op(esp, 0);
masm.fstp32(op);
masm.loadFloat32(op, rv.reg);
masm.freeStack(sizeof(float));
}
#elif defined(JS_CODEGEN_ARM)
if (call.usesSystemAbi && !call.hardFP)
masm.ma_vxfer(r0, rv.reg);
#endif
return rv;
}
RegF64 captureReturnedF64(const FunctionCall& call) {
RegF64 rv = RegF64(ReturnDoubleReg);
MOZ_ASSERT(isAvailable(rv.reg));
needF64(rv);
#if defined(JS_CODEGEN_X86)
if (call.usesSystemAbi) {
masm.reserveStack(sizeof(double));
Operand op(esp, 0);
masm.fstp(op);
masm.loadDouble(op, rv.reg);
masm.freeStack(sizeof(double));
}
#elif defined(JS_CODEGEN_ARM)
if (call.usesSystemAbi && !call.hardFP)
masm.ma_vxfer(r0, r1, rv.reg);
#endif
return rv;
}
void returnCleanup() {
popStackBeforeBranch(ctl_[0].framePushed);
masm.jump(&returnLabel_);
}
void pop2xI32ForIntMulDiv(RegI32* r0, RegI32* r1) {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
// srcDest must be eax, and edx will be clobbered.
need2xI32(specific_eax, specific_edx);
*r1 = popI32();
*r0 = popI32ToSpecific(specific_eax);
freeI32(specific_edx);
#else
pop2xI32(r0, r1);
#endif
}
void pop2xI64ForIntDiv(RegI64* r0, RegI64* r1) {
#ifdef JS_CODEGEN_X64
// srcDest must be rax, and rdx will be clobbered.
need2xI64(specific_rax, specific_rdx);
*r1 = popI64();
*r0 = popI64ToSpecific(specific_rax);
freeI64(specific_rdx);
#else
pop2xI64(r0, r1);
#endif
}
void checkDivideByZeroI32(RegI32 rhs, RegI32 srcDest, Label* done) {
if (isCompilingAsmJS()) {
// Truncated division by zero is zero (Infinity|0 == 0)
Label notDivByZero;
masm.branchTest32(Assembler::NonZero, rhs.reg, rhs.reg, ¬DivByZero);
masm.move32(Imm32(0), srcDest.reg);
masm.jump(done);
masm.bind(¬DivByZero);
} else {
masm.branchTest32(Assembler::Zero, rhs.reg, rhs.reg, trap(Trap::IntegerDivideByZero));
}
}
void checkDivideByZeroI64(RegI64 r) {
MOZ_ASSERT(!isCompilingAsmJS());
ScratchI32 scratch(*this);
masm.branchTest64(Assembler::Zero, r.reg, r.reg, scratch, trap(Trap::IntegerDivideByZero));
}
void checkDivideSignedOverflowI32(RegI32 rhs, RegI32 srcDest, Label* done, bool zeroOnOverflow) {
Label notMin;
masm.branch32(Assembler::NotEqual, srcDest.reg, Imm32(INT32_MIN), ¬Min);
if (zeroOnOverflow) {
masm.branch32(Assembler::NotEqual, rhs.reg, Imm32(-1), ¬Min);
masm.move32(Imm32(0), srcDest.reg);
masm.jump(done);
} else if (isCompilingAsmJS()) {
// (-INT32_MIN)|0 == INT32_MIN and INT32_MIN is already in srcDest.
masm.branch32(Assembler::Equal, rhs.reg, Imm32(-1), done);
} else {
masm.branch32(Assembler::Equal, rhs.reg, Imm32(-1), trap(Trap::IntegerOverflow));
}
masm.bind(¬Min);
}
void checkDivideSignedOverflowI64(RegI64 rhs, RegI64 srcDest, Label* done, bool zeroOnOverflow) {
MOZ_ASSERT(!isCompilingAsmJS());
Label notmin;
masm.branch64(Assembler::NotEqual, srcDest.reg, Imm64(INT64_MIN), ¬min);
masm.branch64(Assembler::NotEqual, rhs.reg, Imm64(-1), ¬min);
if (zeroOnOverflow) {
masm.xor64(srcDest.reg, srcDest.reg);
masm.jump(done);
} else {
masm.jump(trap(Trap::IntegerOverflow));
}
masm.bind(¬min);
}
#ifndef INT_DIV_I64_CALLOUT
void quotientI64(RegI64 rhs, RegI64 srcDest, IsUnsigned isUnsigned) {
Label done;
checkDivideByZeroI64(rhs);
if (!isUnsigned)
checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(false));
# if defined(JS_CODEGEN_X64)
// The caller must set up the following situation.
MOZ_ASSERT(srcDest.reg.reg == rax);
MOZ_ASSERT(isAvailable(rdx));
if (isUnsigned) {
masm.xorq(rdx, rdx);
masm.udivq(rhs.reg.reg);
} else {
masm.cqo();
masm.idivq(rhs.reg.reg);
}
# else
MOZ_CRASH("BaseCompiler platform hook: quotientI64");
# endif
masm.bind(&done);
}
void remainderI64(RegI64 rhs, RegI64 srcDest, IsUnsigned isUnsigned) {
Label done;
checkDivideByZeroI64(rhs);
if (!isUnsigned)
checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(true));
# if defined(JS_CODEGEN_X64)
// The caller must set up the following situation.
MOZ_ASSERT(srcDest.reg.reg == rax);
MOZ_ASSERT(isAvailable(rdx));
if (isUnsigned) {
masm.xorq(rdx, rdx);
masm.udivq(rhs.reg.reg);
} else {
masm.cqo();
masm.idivq(rhs.reg.reg);
}
masm.movq(rdx, rax);
# else
MOZ_CRASH("BaseCompiler platform hook: remainderI64");
# endif
masm.bind(&done);
}
#endif // INT_DIV_I64_CALLOUT
void pop2xI32ForShiftOrRotate(RegI32* r0, RegI32* r1) {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
*r1 = popI32(specific_ecx);
*r0 = popI32();
#else
pop2xI32(r0, r1);
#endif
}
void pop2xI64ForShiftOrRotate(RegI64* r0, RegI64* r1) {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
needI32(specific_ecx);
*r1 = widenI32(specific_ecx);
*r1 = popI64ToSpecific(*r1);
*r0 = popI64();
#else
pop2xI64(r0, r1);
#endif
}
void maskShiftCount32(RegI32 r) {
#if defined(JS_CODEGEN_ARM)
masm.and32(Imm32(31), r.reg);
#endif
}
bool popcnt32NeedsTemp() const {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
return !AssemblerX86Shared::HasPOPCNT();
#elif defined(JS_CODEGEN_ARM)
return true;
#else
MOZ_CRASH("BaseCompiler platform hook: popcnt32NeedsTemp");
#endif
}
bool popcnt64NeedsTemp() const {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
return !AssemblerX86Shared::HasPOPCNT();
#elif defined(JS_CODEGEN_ARM)
return true;
#else
MOZ_CRASH("BaseCompiler platform hook: popcnt64NeedsTemp");
#endif
}
void reinterpretI64AsF64(RegI64 src, RegF64 dest) {
#if defined(JS_CODEGEN_X64)
masm.vmovq(src.reg.reg, dest.reg);
#elif defined(JS_CODEGEN_X86)
masm.Push(src.reg.high);
masm.Push(src.reg.low);
masm.vmovq(Operand(esp, 0), dest.reg);
masm.freeStack(sizeof(uint64_t));
#elif defined(JS_CODEGEN_ARM)
masm.ma_vxfer(src.reg.low, src.reg.high, dest.reg);
#else
MOZ_CRASH("BaseCompiler platform hook: reinterpretI64AsF64");
#endif
}
void reinterpretF64AsI64(RegF64 src, RegI64 dest) {
#if defined(JS_CODEGEN_X64)
masm.vmovq(src.reg, dest.reg.reg);
#elif defined(JS_CODEGEN_X86)
masm.reserveStack(sizeof(uint64_t));
masm.vmovq(src.reg, Operand(esp, 0));
masm.Pop(dest.reg.low);
masm.Pop(dest.reg.high);
#elif defined(JS_CODEGEN_ARM)
masm.ma_vxfer(src.reg, dest.reg.low, dest.reg.high);
#else
MOZ_CRASH("BaseCompiler platform hook: reinterpretF64AsI64");
#endif
}
void wrapI64ToI32(RegI64 src, RegI32 dest) {
#if defined(JS_CODEGEN_X64)
// movl clears the high bits if the two registers are the same.
masm.movl(src.reg.reg, dest.reg);
#elif defined(JS_NUNBOX32)
masm.move32(src.reg.low, dest.reg);
#else
MOZ_CRASH("BaseCompiler platform hook: wrapI64ToI32");
#endif
}
RegI64 popI32ForSignExtendI64() {
#if defined(JS_CODEGEN_X86)
need2xI32(specific_edx, specific_eax);
RegI32 r0 = popI32ToSpecific(specific_eax);
RegI64 x0 = RegI64(Register64(specific_edx.reg, specific_eax.reg));
(void)r0; // x0 is the widening of r0
#else
RegI32 r0 = popI32();
RegI64 x0 = widenI32(r0);
#endif
return x0;
}
void signExtendI32ToI64(RegI32 src, RegI64 dest) {
#if defined(JS_CODEGEN_X64)
masm.movslq(src.reg, dest.reg.reg);
#elif defined(JS_CODEGEN_X86)
MOZ_ASSERT(dest.reg.low == src.reg);
MOZ_ASSERT(dest.reg.low == eax);
MOZ_ASSERT(dest.reg.high == edx);
masm.cdq();
#elif defined(JS_CODEGEN_ARM)
masm.ma_mov(src.reg, dest.reg.low);
masm.ma_asr(Imm32(31), src.reg, dest.reg.high);
#else
MOZ_CRASH("BaseCompiler platform hook: signExtendI32ToI64");
#endif
}
void extendU32ToI64(RegI32 src, RegI64 dest) {
#if defined(JS_CODEGEN_X64)
masm.movl(src.reg, dest.reg.reg);
#elif defined(JS_NUNBOX32)
masm.move32(src.reg, dest.reg.low);
masm.move32(Imm32(0), dest.reg.high);
#else
MOZ_CRASH("BaseCompiler platform hook: extendU32ToI64");
#endif
}
class OutOfLineTruncateF32OrF64ToI32 : public OutOfLineCode
{
AnyReg src;
RegI32 dest;
bool isAsmJS;
bool isUnsigned;
TrapOffset off;
public:
OutOfLineTruncateF32OrF64ToI32(AnyReg src, RegI32 dest, bool isAsmJS, bool isUnsigned,
TrapOffset off)
: src(src),
dest(dest),
isAsmJS(isAsmJS),
isUnsigned(isUnsigned),
off(off)
{
MOZ_ASSERT_IF(isAsmJS, !isUnsigned);
}
virtual void generate(MacroAssembler& masm) {
bool isFloat = src.tag == AnyReg::F32;
FloatRegister fsrc = isFloat ? src.f32().reg : src.f64().reg;
if (isAsmJS) {
saveVolatileReturnGPR(masm);
masm.outOfLineTruncateSlow(fsrc, dest.reg, isFloat, /* isAsmJS */ true);
restoreVolatileReturnGPR(masm);
masm.jump(rejoin());
} else {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
if (isFloat)
masm.outOfLineWasmTruncateFloat32ToInt32(fsrc, isUnsigned, off, rejoin());
else
masm.outOfLineWasmTruncateDoubleToInt32(fsrc, isUnsigned, off, rejoin());
#elif defined(JS_CODEGEN_ARM)
masm.outOfLineWasmTruncateToIntCheck(fsrc,
isFloat ? MIRType::Float32 : MIRType::Double,
MIRType::Int32, isUnsigned, rejoin(), off);
#else
(void)isUnsigned; // Suppress warnings
(void)off; // for unused private
MOZ_CRASH("BaseCompiler platform hook: OutOfLineTruncateF32OrF64ToI32 wasm");
#endif
}
}
};
MOZ_MUST_USE bool truncateF32ToI32(RegF32 src, RegI32 dest, bool isUnsigned) {
TrapOffset off = trapOffset();
OutOfLineCode* ool;
if (isCompilingAsmJS()) {
ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, true, false, off);
ool = addOutOfLineCode(ool);
if (!ool)
return false;
masm.branchTruncateFloat32ToInt32(src.reg, dest.reg, ool->entry());
} else {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM)
ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, false, isUnsigned,
off);
ool = addOutOfLineCode(ool);
if (!ool)
return false;
if (isUnsigned)
masm.wasmTruncateFloat32ToUInt32(src.reg, dest.reg, ool->entry());
else
masm.wasmTruncateFloat32ToInt32(src.reg, dest.reg, ool->entry());
#else
MOZ_CRASH("BaseCompiler platform hook: truncateF32ToI32 wasm");
#endif
}
masm.bind(ool->rejoin());
return true;
}
MOZ_MUST_USE bool truncateF64ToI32(RegF64 src, RegI32 dest, bool isUnsigned) {
TrapOffset off = trapOffset();
OutOfLineCode* ool;
if (isCompilingAsmJS()) {
ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, true, false, off);
ool = addOutOfLineCode(ool);
if (!ool)
return false;
masm.branchTruncateDoubleToInt32(src.reg, dest.reg, ool->entry());
} else {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM)
ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, false, isUnsigned,
off);
ool = addOutOfLineCode(ool);
if (!ool)
return false;
if (isUnsigned)
masm.wasmTruncateDoubleToUInt32(src.reg, dest.reg, ool->entry());
else
masm.wasmTruncateDoubleToInt32(src.reg, dest.reg, ool->entry());
#else
MOZ_CRASH("BaseCompiler platform hook: truncateF64ToI32 wasm");
#endif
}
masm.bind(ool->rejoin());
return true;
}
// This does not generate a value; if the truncation failed then it traps.
class OutOfLineTruncateCheckF32OrF64ToI64 : public OutOfLineCode
{
AnyReg src;
bool isUnsigned;
TrapOffset off;
public:
OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src, bool isUnsigned, TrapOffset off)
: src(src),
isUnsigned(isUnsigned),
off(off)
{}
virtual void generate(MacroAssembler& masm) {
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
if (src.tag == AnyReg::F32)
masm.outOfLineWasmTruncateFloat32ToInt64(src.f32().reg, isUnsigned, off, rejoin());
else if (src.tag == AnyReg::F64)
masm.outOfLineWasmTruncateDoubleToInt64(src.f64().reg, isUnsigned, off, rejoin());
else
MOZ_CRASH("unexpected type");
#elif defined(JS_CODEGEN_ARM)
if (src.tag == AnyReg::F32)
masm.outOfLineWasmTruncateToIntCheck(src.f32().reg, MIRType::Float32,
MIRType::Int64, isUnsigned, rejoin(), off);
else if (src.tag == AnyReg::F64)
masm.outOfLineWasmTruncateToIntCheck(src.f64().reg, MIRType::Double, MIRType::Int64,
isUnsigned, rejoin(), off);
else
MOZ_CRASH("unexpected type");
#else
(void)src;
(void)isUnsigned;
(void)off;
MOZ_CRASH("BaseCompiler platform hook: OutOfLineTruncateCheckF32OrF64ToI64");
#endif
}
};
#ifndef FLOAT_TO_I64_CALLOUT
MOZ_MUST_USE bool truncateF32ToI64(RegF32 src, RegI64 dest, bool isUnsigned, RegF64 temp) {
# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
OutOfLineCode* ool =
addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(src),
isUnsigned,
trapOffset()));
if (!ool)
return false;
if (isUnsigned)
masm.wasmTruncateFloat32ToUInt64(src.reg, dest.reg, ool->entry(),
ool->rejoin(), temp.reg);
else
masm.wasmTruncateFloat32ToInt64(src.reg, dest.reg, ool->entry(),
ool->rejoin(), temp.reg);
# else
MOZ_CRASH("BaseCompiler platform hook: truncateF32ToI64");
# endif
return true;
}
MOZ_MUST_USE bool truncateF64ToI64(RegF64 src, RegI64 dest, bool isUnsigned, RegF64 temp) {
# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
OutOfLineCode* ool =
addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(src),
isUnsigned,
trapOffset()));
if (!ool)
return false;
if (isUnsigned)
masm.wasmTruncateDoubleToUInt64(src.reg, dest.reg, ool->entry(),
ool->rejoin(), temp.reg);
else
masm.wasmTruncateDoubleToInt64(src.reg, dest.reg, ool->entry(),
ool->rejoin(), temp.reg);
# else
MOZ_CRASH("BaseCompiler platform hook: truncateF64ToI64");
# endif
return true;
}
#endif // FLOAT_TO_I64_CALLOUT
#ifndef I64_TO_FLOAT_CALLOUT
bool convertI64ToFloatNeedsTemp(bool isUnsigned) const {
# if defined(JS_CODEGEN_X86)
return isUnsigned && AssemblerX86Shared::HasSSE3();
# else
return false;
# endif
}
void convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, RegI32 temp) {
# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
if (isUnsigned)
masm.convertUInt64ToFloat32(src.reg, dest.reg, temp.reg);
else
masm.convertInt64ToFloat32(src.reg, dest.reg);
# else
MOZ_CRASH("BaseCompiler platform hook: convertI64ToF32");
# endif
}
void convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, RegI32 temp) {
# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
if (isUnsigned)
masm.convertUInt64ToDouble(src.reg, dest.reg, temp.reg);
else
masm.convertInt64ToDouble(src.reg, dest.reg);
# else
MOZ_CRASH("BaseCompiler platform hook: convertI64ToF64");
# endif
}
#endif // I64_TO_FLOAT_CALLOUT
void cmp64Set(Assembler::Condition cond, RegI64 lhs, RegI64 rhs, RegI32 dest) {
#if defined(JS_CODEGEN_X64)
masm.cmpq(rhs.reg.reg, lhs.reg.reg);
masm.emitSet(cond, dest.reg);
#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
// TODO / OPTIMIZE (Bug 1316822): This is pretty branchy, we should be
// able to do better.
Label done, condTrue;
masm.branch64(cond, lhs.reg, rhs.reg, &condTrue);
masm.move32(Imm32(0), dest.reg);
masm.jump(&done);
masm.bind(&condTrue);
masm.move32(Imm32(1), dest.reg);
masm.bind(&done);
#else
MOZ_CRASH("BaseCompiler platform hook: cmp64Set");
#endif
}
void unreachableTrap()
{
masm.jump(trap(Trap::Unreachable));
#ifdef DEBUG
masm.breakpoint();
#endif
}
//////////////////////////////////////////////////////////////////////
//
// Global variable access.
// CodeGenerator{X86,X64}::visitWasmLoadGlobal()
void loadGlobalVarI32(unsigned globalDataOffset, RegI32 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.loadRipRelativeInt32(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr), r.reg, scratch);
#else
MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI32");
#endif
}
void loadGlobalVarI64(unsigned globalDataOffset, RegI64 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.loadRipRelativeInt64(r.reg.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset labelLow = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.low);
masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET));
CodeOffset labelHigh = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.high);
masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET));
#elif defined(JS_CODEGEN_ARM)
ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch);
masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high,
scratch);
#else
MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI64");
#endif
}
void loadGlobalVarF32(unsigned globalDataOffset, RegF32 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.loadRipRelativeFloat32(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.vmovssWithPatch(PatchedAbsoluteAddress(), r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
unsigned addr = globalDataOffset - WasmGlobalRegBias;
VFPRegister vd(r.reg);
masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), vd.singleOverlay());
#else
MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF32");
#endif
}
void loadGlobalVarF64(unsigned globalDataOffset, RegF64 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.loadRipRelativeDouble(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.vmovsdWithPatch(PatchedAbsoluteAddress(), r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), r.reg);
#else
MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF64");
#endif
}
// CodeGeneratorX64::visitWasmStoreGlobal()
void storeGlobalVarI32(unsigned globalDataOffset, RegI32 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.storeRipRelativeInt32(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.movlWithPatch(r.reg, PatchedAbsoluteAddress());
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr), r.reg, scratch);
#else
MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI32");
#endif
}
void storeGlobalVarI64(unsigned globalDataOffset, RegI64 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.storeRipRelativeInt64(r.reg.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset labelLow = masm.movlWithPatch(r.reg.low, PatchedAbsoluteAddress());
masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET));
CodeOffset labelHigh = masm.movlWithPatch(r.reg.high, PatchedAbsoluteAddress());
masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET));
#elif defined(JS_CODEGEN_ARM)
ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch);
masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high,
scratch);
#else
MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI64");
#endif
}
void storeGlobalVarF32(unsigned globalDataOffset, RegF32 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.storeRipRelativeFloat32(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.vmovssWithPatch(r.reg, PatchedAbsoluteAddress());
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
unsigned addr = globalDataOffset - WasmGlobalRegBias;
VFPRegister vd(r.reg);
masm.ma_vstr(vd.singleOverlay(), VFPAddr(GlobalReg, VFPOffImm(addr)));
#else
MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF32");
#endif
}
void storeGlobalVarF64(unsigned globalDataOffset, RegF64 r)
{
#if defined(JS_CODEGEN_X64)
CodeOffset label = masm.storeRipRelativeDouble(r.reg);
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_X86)
CodeOffset label = masm.vmovsdWithPatch(r.reg, PatchedAbsoluteAddress());
masm.append(GlobalAccess(label, globalDataOffset));
#elif defined(JS_CODEGEN_ARM)
unsigned addr = globalDataOffset - WasmGlobalRegBias;
masm.ma_vstr(r.reg, VFPAddr(GlobalReg, VFPOffImm(addr)));
#else
MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF64");
#endif
}
//////////////////////////////////////////////////////////////////////
//
// Heap access.
#ifndef WASM_HUGE_MEMORY
class AsmJSLoadOOB : public OutOfLineCode
{
Scalar::Type viewType;
AnyRegister dest;
public:
AsmJSLoadOOB(Scalar::Type viewType, AnyRegister dest)
: viewType(viewType),
dest(dest)
{}
void generate(MacroAssembler& masm) {
# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
switch (viewType) {
case Scalar::Float32x4:
case Scalar::Int32x4:
case Scalar::Int8x16:
case Scalar::Int16x8:
case Scalar::MaxTypedArrayViewType:
MOZ_CRASH("unexpected array type");
case Scalar::Float32:
masm.loadConstantFloat32(float(GenericNaN()), dest.fpu());
break;
case Scalar::Float64:
masm.loadConstantDouble(GenericNaN(), dest.fpu());
break;
case Scalar::Int8:
case Scalar::Uint8:
case Scalar::Int16:
case Scalar::Uint16:
case Scalar::Int32:
case Scalar::Uint32:
case Scalar::Uint8Clamped:
masm.movePtr(ImmWord(0), dest.gpr());
break;
case Scalar::Int64:
MOZ_CRASH("unexpected array type");
}
masm.jump(rejoin());
# else
Unused << viewType;
Unused << dest;
MOZ_CRASH("Compiler bug: Unexpected platform.");
# endif
}
};
#endif
void checkOffset(MemoryAccessDesc* access, RegI32 ptr) {
if (access->offset() >= OffsetGuardLimit) {
masm.branchAdd32(Assembler::CarrySet, Imm32(access->offset()), ptr.reg,
trap(Trap::OutOfBounds));
access->clearOffset();
}
}
// This is the temp register passed as the last argument to load()
MOZ_MUST_USE size_t loadStoreTemps(MemoryAccessDesc& access) {
#if defined(JS_CODEGEN_ARM)
if (access.isUnaligned()) {
switch (access.type()) {
case Scalar::Float32:
return 1;
case Scalar::Float64:
return 2;
default:
break;
}
}
return 0;
#else
return 0;
#endif
}
// ptr and dest may be the same iff dest is I32.
// This may destroy ptr even if ptr and dest are not the same.
MOZ_MUST_USE bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest, RegI32 tmp1,
RegI32 tmp2)
{
checkOffset(&access, ptr);
OutOfLineCode* ool = nullptr;
#ifndef WASM_HUGE_MEMORY
if (access.isPlainAsmJS()) {
ool = new (alloc_) AsmJSLoadOOB(access.type(), dest.any());
if (!addOutOfLineCode(ool))
return false;
masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, ool->entry());
} else {
masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, trap(Trap::OutOfBounds));
}
#endif
#if defined(JS_CODEGEN_X64)
Operand srcAddr(HeapReg, ptr.reg, TimesOne, access.offset());
if (dest.tag == AnyReg::I64)
masm.wasmLoadI64(access, srcAddr, dest.i64().reg);
else
masm.wasmLoad(access, srcAddr, dest.any());
#elif defined(JS_CODEGEN_X86)
Operand srcAddr(ptr.reg, access.offset());
if (dest.tag == AnyReg::I64) {
masm.wasmLoadI64(access, srcAddr, dest.i64().reg);
} else {
bool byteRegConflict = access.byteSize() == 1 && !singleByteRegs_.has(dest.i32().reg);
AnyRegister out = byteRegConflict ? AnyRegister(ScratchRegX86) : dest.any();
masm.wasmLoad(access, srcAddr, out);
if (byteRegConflict)
masm.mov(ScratchRegX86, dest.i32().reg);
}
#elif defined(JS_CODEGEN_ARM)
if (access.offset() != 0)
masm.add32(Imm32(access.offset()), ptr.reg);
bool isSigned = true;
switch (access.type()) {
case Scalar::Uint8:
case Scalar::Uint16:
case Scalar::Uint32: {
isSigned = false;
MOZ_FALLTHROUGH;
case Scalar::Int8:
case Scalar::Int16:
case Scalar::Int32:
Register rt = dest.tag == AnyReg::I64 ? dest.i64().reg.low : dest.i32().reg;
loadI32(access, isSigned, ptr, rt);
if (dest.tag == AnyReg::I64) {
if (isSigned)
masm.ma_asr(Imm32(31), rt, dest.i64().reg.high);
else
masm.move32(Imm32(0), dest.i64().reg.high);
}
break;
}
case Scalar::Int64:
loadI64(access, ptr, dest.i64());
break;
case Scalar::Float32:
loadF32(access, ptr, dest.f32(), tmp1);
break;
case Scalar::Float64:
loadF64(access, ptr, dest.f64(), tmp1, tmp2);
break;
default:
MOZ_CRASH("Compiler bug: unexpected array type");
}
#else
MOZ_CRASH("BaseCompiler platform hook: load");
#endif
if (ool)
masm.bind(ool->rejoin());
return true;
}
// ptr and src must not be the same register.
// This may destroy ptr.
MOZ_MUST_USE bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src, RegI32 tmp1,
RegI32 tmp2)
{
checkOffset(&access, ptr);
Label rejoin;
#ifndef WASM_HUGE_MEMORY
if (access.isPlainAsmJS())
masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, &rejoin);
else
masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, trap(Trap::OutOfBounds));
#endif
// Emit the store
#if defined(JS_CODEGEN_X64)
Operand dstAddr(HeapReg, ptr.reg, TimesOne, access.offset());
masm.wasmStore(access, src.any(), dstAddr);
#elif defined(JS_CODEGEN_X86)
Operand dstAddr(ptr.reg, access.offset());
if (access.type() == Scalar::Int64) {
masm.wasmStoreI64(access, src.i64().reg, dstAddr);
} else {
AnyRegister value;
if (src.tag == AnyReg::I64) {
value = AnyRegister(src.i64().reg.low);
} else if (access.byteSize() == 1 && !singleByteRegs_.has(src.i32().reg)) {
masm.mov(src.i32().reg, ScratchRegX86);
value = AnyRegister(ScratchRegX86);
} else {
value = src.any();
}
masm.wasmStore(access, value, dstAddr);
}
#elif defined(JS_CODEGEN_ARM)
if (access.offset() != 0)
masm.add32(Imm32(access.offset()), ptr.reg);
switch (access.type()) {
case Scalar::Uint8:
MOZ_FALLTHROUGH;
case Scalar::Uint16:
MOZ_FALLTHROUGH;
case Scalar::Int8:
MOZ_FALLTHROUGH;
case Scalar::Int16:
MOZ_FALLTHROUGH;
case Scalar::Int32:
MOZ_FALLTHROUGH;
case Scalar::Uint32: {
Register rt = src.tag == AnyReg::I64 ? src.i64().reg.low : src.i32().reg;
storeI32(access, ptr, rt);
break;
}
case Scalar::Int64:
storeI64(access, ptr, src.i64());
break;
case Scalar::Float32:
storeF32(access, ptr, src.f32(), tmp1);
break;
case Scalar::Float64:
storeF64(access, ptr, src.f64(), tmp1, tmp2);
break;
default:
MOZ_CRASH("Compiler bug: unexpected array type");
}
#else
MOZ_CRASH("BaseCompiler platform hook: store");
#endif
if (rejoin.used())
masm.bind(&rejoin);
return true;
}
#ifdef JS_CODEGEN_ARM
void
loadI32(MemoryAccessDesc access, bool isSigned, RegI32 ptr, Register rt) {
if (access.byteSize() > 1 && access.isUnaligned()) {
masm.add32(HeapReg, ptr.reg);
SecondScratchRegisterScope scratch(*this);
masm.emitUnalignedLoad(isSigned, access.byteSize(), ptr.reg, scratch, rt, 0);
} else {
BufferOffset ld =
masm.ma_dataTransferN(js::jit::IsLoad, BitSize(access.byteSize()*8),
isSigned, HeapReg, ptr.reg, rt, Offset, Assembler::Always);
masm.append(access, ld.getOffset(), masm.framePushed());
}
}
void
storeI32(MemoryAccessDesc access, RegI32 ptr, Register rt) {
if (access.byteSize() > 1 && access.isUnaligned()) {
masm.add32(HeapReg, ptr.reg);
masm.emitUnalignedStore(access.byteSize(), ptr.reg, rt, 0);
} else {
BufferOffset st =
masm.ma_dataTransferN(js::jit::IsStore, BitSize(access.byteSize()*8),
IsSigned(false), ptr.reg, HeapReg, rt, Offset,
Assembler::Always);
masm.append(access, st.getOffset(), masm.framePushed());
}
}
void
loadI64(MemoryAccessDesc access, RegI32 ptr, RegI64 dest) {
if (access.isUnaligned()) {
masm.add32(HeapReg, ptr.reg);
SecondScratchRegisterScope scratch(*this);
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.low,
0);
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.high,
4);
} else {
BufferOffset ld;
ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
ptr.reg, dest.reg.low, Offset, Assembler::Always);
masm.append(access, ld.getOffset(), masm.framePushed());
masm.add32(Imm32(4), ptr.reg);
ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
ptr.reg, dest.reg.high, Offset, Assembler::Always);
masm.append(access, ld.getOffset(), masm.framePushed());
}
}
void
storeI64(MemoryAccessDesc access, RegI32 ptr, RegI64 src) {
if (access.isUnaligned()) {
masm.add32(HeapReg, ptr.reg);
masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.low, 0);
masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.high, 4);
} else {
BufferOffset st;
st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
ptr.reg, src.reg.low, Offset, Assembler::Always);
masm.append(access, st.getOffset(), masm.framePushed());
masm.add32(Imm32(4), ptr.reg);
st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
ptr.reg, src.reg.high, Offset, Assembler::Always);
masm.append(access, st.getOffset(), masm.framePushed());
}
}
void
loadF32(MemoryAccessDesc access, RegI32 ptr, RegF32 dest, RegI32 tmp1) {
masm.add32(HeapReg, ptr.reg);
if (access.isUnaligned()) {
SecondScratchRegisterScope scratch(*this);
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
masm.ma_vxfer(tmp1.reg, dest.reg);
} else {
BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
Assembler::Always);
masm.append(access, ld.getOffset(), masm.framePushed());
}
}
void
storeF32(MemoryAccessDesc access, RegI32 ptr, RegF32 src, RegI32 tmp1) {
masm.add32(HeapReg, ptr.reg);
if (access.isUnaligned()) {
masm.ma_vxfer(src.reg, tmp1.reg);
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
} else {
BufferOffset st =
masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
masm.append(access, st.getOffset(), masm.framePushed());
}
}
void
loadF64(MemoryAccessDesc access, RegI32 ptr, RegF64 dest, RegI32 tmp1, RegI32 tmp2) {
masm.add32(HeapReg, ptr.reg);
if (access.isUnaligned()) {
SecondScratchRegisterScope scratch(*this);
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp2.reg, 4);
masm.ma_vxfer(tmp1.reg, tmp2.reg, dest.reg);
} else {
BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
Assembler::Always);
masm.append(access, ld.getOffset(), masm.framePushed());
}
}
void
storeF64(MemoryAccessDesc access, RegI32 ptr, RegF64 src, RegI32 tmp1, RegI32 tmp2) {
masm.add32(HeapReg, ptr.reg);
if (access.isUnaligned()) {
masm.ma_vxfer(src.reg, tmp1.reg, tmp2.reg);
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp2.reg, 4);
} else {
BufferOffset st =
masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
masm.append(access, st.getOffset(), masm.framePushed());
}
}
#endif // JS_CODEGEN_ARM
////////////////////////////////////////////////////////////
// Generally speaking, ABOVE this point there should be no value
// stack manipulation (calls to popI32 etc).
// Generally speaking, BELOW this point there should be no
// platform dependencies. We make an exception for x86 register
// targeting, which is not too hard to keep clean.
////////////////////////////////////////////////////////////
//
// Sundry wrappers.
void pop2xI32(RegI32* r0, RegI32* r1) {
*r1 = popI32();
*r0 = popI32();
}
RegI32 popI32ToSpecific(RegI32 specific) {
freeI32(specific);
return popI32(specific);
}
void pop2xI64(RegI64* r0, RegI64* r1) {
*r1 = popI64();
*r0 = popI64();
}
RegI64 popI64ToSpecific(RegI64 specific) {
freeI64(specific);
return popI64(specific);
}
void pop2xF32(RegF32* r0, RegF32* r1) {
*r1 = popF32();
*r0 = popF32();
}
void pop2xF64(RegF64* r0, RegF64* r1) {
*r1 = popF64();
*r0 = popF64();
}
////////////////////////////////////////////////////////////
//
// Sundry helpers.
uint32_t readCallSiteLineOrBytecode() {
if (!func_.callSiteLineNums().empty())
return func_.callSiteLineNums()[lastReadCallSite_++];
return trapOffset().bytecodeOffset;
}
bool done() const {
return iter_.done();
}
bool isCompilingAsmJS() const {
return mg_.kind == ModuleKind::AsmJS;
}
TrapOffset trapOffset() const {
return iter_.trapOffset();
}
Maybe<TrapOffset> trapIfNotAsmJS() const {
return isCompilingAsmJS() ? Nothing() : Some(trapOffset());
}
TrapDesc trap(Trap t) const {
return TrapDesc(trapOffset(), t, masm.framePushed());
}
//////////////////////////////////////////////////////////////////////
MOZ_MUST_USE bool emitBody();
MOZ_MUST_USE bool emitBlock();
MOZ_MUST_USE bool emitLoop();
MOZ_MUST_USE bool emitIf();
MOZ_MUST_USE bool emitElse();
MOZ_MUST_USE bool emitEnd();
MOZ_MUST_USE bool emitBr();
MOZ_MUST_USE bool emitBrIf();
MOZ_MUST_USE bool emitBrTable();
MOZ_MUST_USE bool emitDrop();
MOZ_MUST_USE bool emitReturn();
MOZ_MUST_USE bool emitCallArgs(const ValTypeVector& args, FunctionCall& baselineCall);
MOZ_MUST_USE bool emitCall();
MOZ_MUST_USE bool emitCallIndirect(bool oldStyle);
MOZ_MUST_USE bool emitCommonMathCall(uint32_t lineOrBytecode, SymbolicAddress callee,
ValTypeVector& signature, ExprType retType);
MOZ_MUST_USE bool emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType);
MOZ_MUST_USE bool emitBinaryMathBuiltinCall(SymbolicAddress callee, ValType operandType);
#ifdef INT_DIV_I64_CALLOUT
MOZ_MUST_USE bool emitDivOrModI64BuiltinCall(SymbolicAddress callee, ValType operandType);
#endif
MOZ_MUST_USE bool emitGetLocal();
MOZ_MUST_USE bool emitSetLocal();
MOZ_MUST_USE bool emitTeeLocal();
MOZ_MUST_USE bool emitGetGlobal();
MOZ_MUST_USE bool emitSetGlobal();
MOZ_MUST_USE bool emitTeeGlobal();
MOZ_MUST_USE bool emitLoad(ValType type, Scalar::Type viewType);
MOZ_MUST_USE bool emitStore(ValType resultType, Scalar::Type viewType);
MOZ_MUST_USE bool emitTeeStore(ValType resultType, Scalar::Type viewType);
MOZ_MUST_USE bool emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType);
MOZ_MUST_USE bool emitSelect();
void endBlock(ExprType type, bool isFunctionBody);
void endLoop(ExprType type);
void endIfThen();
void endIfThenElse(ExprType type);
void doReturn(ExprType returnType);
void pushReturned(const FunctionCall& call, ExprType type);
void emitCompareI32(JSOp compareOp, MCompare::CompareType compareType);
void emitCompareI64(JSOp compareOp, MCompare::CompareType compareType);
void emitCompareF32(JSOp compareOp, MCompare::CompareType compareType);
void emitCompareF64(JSOp compareOp, MCompare::CompareType compareType);
void emitAddI32();
void emitAddI64();
void emitAddF64();
void emitAddF32();
void emitSubtractI32();
void emitSubtractI64();
void emitSubtractF32();
void emitSubtractF64();
void emitMultiplyI32();
void emitMultiplyI64();
void emitMultiplyF32();
void emitMultiplyF64();
void emitQuotientI32();
void emitQuotientU32();
void emitRemainderI32();
void emitRemainderU32();
#ifndef INT_DIV_I64_CALLOUT
void emitQuotientI64();
void emitQuotientU64();
void emitRemainderI64();
void emitRemainderU64();
#endif
void emitDivideF32();
void emitDivideF64();
void emitMinI32();
void emitMaxI32();
void emitMinMaxI32(Assembler::Condition cond);
void emitMinF32();
void emitMaxF32();
void emitMinF64();
void emitMaxF64();
void emitCopysignF32();
void emitCopysignF64();
void emitOrI32();
void emitOrI64();
void emitAndI32();
void emitAndI64();
void emitXorI32();
void emitXorI64();
void emitShlI32();
void emitShlI64();
void emitShrI32();
void emitShrI64();
void emitShrU32();
void emitShrU64();
void emitRotrI32();
void emitRotrI64();
void emitRotlI32();
void emitRotlI64();
void emitEqzI32();
void emitEqzI64();
void emitClzI32();
void emitClzI64();
void emitCtzI32();
void emitCtzI64();
void emitPopcntI32();
void emitPopcntI64();
void emitBitNotI32();
void emitAbsI32();
void emitAbsF32();
void emitAbsF64();
void emitNegateI32();
void emitNegateF32();
void emitNegateF64();
void emitSqrtF32();
void emitSqrtF64();
template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF32ToI32();
template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF64ToI32();
#ifdef FLOAT_TO_I64_CALLOUT
MOZ_MUST_USE bool emitConvertFloatingToInt64Callout(SymbolicAddress callee, ValType operandType,
ValType resultType);
#else
template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF32ToI64();
template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF64ToI64();
#endif
void emitWrapI64ToI32();
void emitExtendI32ToI64();
void emitExtendU32ToI64();
void emitReinterpretF32AsI32();
void emitReinterpretF64AsI64();
void emitConvertF64ToF32();
void emitConvertI32ToF32();
void emitConvertU32ToF32();
void emitConvertF32ToF64();
void emitConvertI32ToF64();
void emitConvertU32ToF64();
#ifdef I64_TO_FLOAT_CALLOUT
MOZ_MUST_USE bool emitConvertInt64ToFloatingCallout(SymbolicAddress callee, ValType operandType,
ValType resultType);
#else
void emitConvertI64ToF32();
void emitConvertU64ToF32();
void emitConvertI64ToF64();
void emitConvertU64ToF64();
#endif
void emitReinterpretI32AsF32();
void emitReinterpretI64AsF64();
MOZ_MUST_USE bool emitGrowMemory();
MOZ_MUST_USE bool emitCurrentMemory();
};
void
BaseCompiler::emitAddI32()
{
int32_t c;
if (popConstI32(c)) {
RegI32 r = popI32();
masm.add32(Imm32(c), r.reg);
pushI32(r);
} else {
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.add32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
}
void
BaseCompiler::emitAddI64()
{
// TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.add64(r1.reg, r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitAddF64()
{
// TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
RegF64 r0, r1;
pop2xF64(&r0, &r1);
masm.addDouble(r1.reg, r0.reg);
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitAddF32()
{
// TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
RegF32 r0, r1;
pop2xF32(&r0, &r1);
masm.addFloat32(r1.reg, r0.reg);
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitSubtractI32()
{
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.sub32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitSubtractI64()
{
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.sub64(r1.reg, r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitSubtractF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
masm.subFloat32(r1.reg, r0.reg);
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitSubtractF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
masm.subDouble(r1.reg, r0.reg);
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitMultiplyI32()
{
// TODO / OPTIMIZE: Multiplication by constant is common (Bug 1275442, 1316803)
RegI32 r0, r1;
pop2xI32ForIntMulDiv(&r0, &r1);
masm.mul32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitMultiplyI64()
{
// TODO / OPTIMIZE: Multiplication by constant is common (Bug 1275442, 1316803)
RegI64 r0, r1;
RegI32 temp;
#if defined(JS_CODEGEN_X64)
// srcDest must be rax, and rdx will be clobbered.
need2xI64(specific_rax, specific_rdx);
r1 = popI64();
r0 = popI64ToSpecific(specific_rax);
freeI64(specific_rdx);
#elif defined(JS_CODEGEN_X86)
need2xI32(specific_eax, specific_edx);
r1 = popI64();
r0 = popI64ToSpecific(RegI64(Register64(specific_edx.reg, specific_eax.reg)));
temp = needI32();
#else
pop2xI64(&r0, &r1);
temp = needI32();
#endif
masm.mul64(r1.reg, r0.reg, temp.reg);
if (temp.reg != Register::Invalid())
freeI32(temp);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitMultiplyF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
masm.mulFloat32(r1.reg, r0.reg);
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitMultiplyF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
masm.mulDouble(r1.reg, r0.reg);
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitQuotientI32()
{
// TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForIntMulDiv(&r0, &r1);
Label done;
checkDivideByZeroI32(r1, r0, &done);
checkDivideSignedOverflowI32(r1, r0, &done, ZeroOnOverflow(false));
masm.quotient32(r1.reg, r0.reg, IsUnsigned(false));
masm.bind(&done);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitQuotientU32()
{
// TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForIntMulDiv(&r0, &r1);
Label done;
checkDivideByZeroI32(r1, r0, &done);
masm.quotient32(r1.reg, r0.reg, IsUnsigned(true));
masm.bind(&done);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitRemainderI32()
{
// TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForIntMulDiv(&r0, &r1);
Label done;
checkDivideByZeroI32(r1, r0, &done);
checkDivideSignedOverflowI32(r1, r0, &done, ZeroOnOverflow(true));
masm.remainder32(r1.reg, r0.reg, IsUnsigned(false));
masm.bind(&done);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitRemainderU32()
{
// TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForIntMulDiv(&r0, &r1);
Label done;
checkDivideByZeroI32(r1, r0, &done);
masm.remainder32(r1.reg, r0.reg, IsUnsigned(true));
masm.bind(&done);
freeI32(r1);
pushI32(r0);
}
#ifndef INT_DIV_I64_CALLOUT
void
BaseCompiler::emitQuotientI64()
{
# ifdef JS_PUNBOX64
RegI64 r0, r1;
pop2xI64ForIntDiv(&r0, &r1);
quotientI64(r1, r0, IsUnsigned(false));
freeI64(r1);
pushI64(r0);
# else
MOZ_CRASH("BaseCompiler platform hook: emitQuotientI64");
# endif
}
void
BaseCompiler::emitQuotientU64()
{
# ifdef JS_PUNBOX64
RegI64 r0, r1;
pop2xI64ForIntDiv(&r0, &r1);
quotientI64(r1, r0, IsUnsigned(true));
freeI64(r1);
pushI64(r0);
# else
MOZ_CRASH("BaseCompiler platform hook: emitQuotientU64");
# endif
}
void
BaseCompiler::emitRemainderI64()
{
# ifdef JS_PUNBOX64
RegI64 r0, r1;
pop2xI64ForIntDiv(&r0, &r1);
remainderI64(r1, r0, IsUnsigned(false));
freeI64(r1);
pushI64(r0);
# else
MOZ_CRASH("BaseCompiler platform hook: emitRemainderI64");
# endif
}
void
BaseCompiler::emitRemainderU64()
{
# ifdef JS_PUNBOX64
RegI64 r0, r1;
pop2xI64ForIntDiv(&r0, &r1);
remainderI64(r1, r0, IsUnsigned(true));
freeI64(r1);
pushI64(r0);
# else
MOZ_CRASH("BaseCompiler platform hook: emitRemainderU64");
# endif
}
#endif // INT_DIV_I64_CALLOUT
void
BaseCompiler::emitDivideF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
masm.divFloat32(r1.reg, r0.reg);
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitDivideF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
masm.divDouble(r1.reg, r0.reg);
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitMinI32()
{
emitMinMaxI32(Assembler::LessThan);
}
void
BaseCompiler::emitMaxI32()
{
emitMinMaxI32(Assembler::GreaterThan);
}
void
BaseCompiler::emitMinMaxI32(Assembler::Condition cond)
{
Label done;
RegI32 r0, r1;
pop2xI32(&r0, &r1);
// TODO / OPTIMIZE (bug 1316823): Use conditional move on some platforms?
masm.branch32(cond, r0.reg, r1.reg, &done);
moveI32(r1, r0);
masm.bind(&done);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitMinF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
if (!isCompilingAsmJS()) {
// Convert signaling NaN to quiet NaNs.
//
// TODO / OPTIMIZE (bug 1316824): Don't do this if one of the operands
// is known to be a constant.
ScratchF32 zero(*this);
masm.loadConstantFloat32(0.f, zero);
masm.subFloat32(zero, r0.reg);
masm.subFloat32(zero, r1.reg);
}
masm.minFloat32(r1.reg, r0.reg, HandleNaNSpecially(true));
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitMaxF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
if (!isCompilingAsmJS()) {
// Convert signaling NaN to quiet NaNs.
//
// TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
ScratchF32 zero(*this);
masm.loadConstantFloat32(0.f, zero);
masm.subFloat32(zero, r0.reg);
masm.subFloat32(zero, r1.reg);
}
masm.maxFloat32(r1.reg, r0.reg, HandleNaNSpecially(true));
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitMinF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
if (!isCompilingAsmJS()) {
// Convert signaling NaN to quiet NaNs.
//
// TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
ScratchF64 zero(*this);
masm.loadConstantDouble(0, zero);
masm.subDouble(zero, r0.reg);
masm.subDouble(zero, r1.reg);
}
masm.minDouble(r1.reg, r0.reg, HandleNaNSpecially(true));
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitMaxF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
if (!isCompilingAsmJS()) {
// Convert signaling NaN to quiet NaNs.
//
// TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
ScratchF64 zero(*this);
masm.loadConstantDouble(0, zero);
masm.subDouble(zero, r0.reg);
masm.subDouble(zero, r1.reg);
}
masm.maxDouble(r1.reg, r0.reg, HandleNaNSpecially(true));
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitCopysignF32()
{
RegF32 r0, r1;
pop2xF32(&r0, &r1);
RegI32 i0 = needI32();
RegI32 i1 = needI32();
masm.moveFloat32ToGPR(r0.reg, i0.reg);
masm.moveFloat32ToGPR(r1.reg, i1.reg);
masm.and32(Imm32(INT32_MAX), i0.reg);
masm.and32(Imm32(INT32_MIN), i1.reg);
masm.or32(i1.reg, i0.reg);
masm.moveGPRToFloat32(i0.reg, r0.reg);
freeI32(i0);
freeI32(i1);
freeF32(r1);
pushF32(r0);
}
void
BaseCompiler::emitCopysignF64()
{
RegF64 r0, r1;
pop2xF64(&r0, &r1);
RegI64 x0 = needI64();
RegI64 x1 = needI64();
reinterpretF64AsI64(r0, x0);
reinterpretF64AsI64(r1, x1);
masm.and64(Imm64(INT64_MAX), x0.reg);
masm.and64(Imm64(INT64_MIN), x1.reg);
masm.or64(x1.reg, x0.reg);
reinterpretI64AsF64(x0, r0);
freeI64(x0);
freeI64(x1);
freeF64(r1);
pushF64(r0);
}
void
BaseCompiler::emitOrI32()
{
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.or32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitOrI64()
{
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.or64(r1.reg, r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitAndI32()
{
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.and32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitAndI64()
{
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.and64(r1.reg, r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitXorI32()
{
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.xor32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitXorI64()
{
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.xor64(r1.reg, r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitShlI32()
{
int32_t c;
if (popConstI32(c)) {
RegI32 r = popI32();
masm.lshift32(Imm32(c & 31), r.reg);
pushI32(r);
} else {
RegI32 r0, r1;
pop2xI32ForShiftOrRotate(&r0, &r1);
maskShiftCount32(r1);
masm.lshift32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
}
void
BaseCompiler::emitShlI64()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI64 r0, r1;
pop2xI64ForShiftOrRotate(&r0, &r1);
masm.lshift64(lowPart(r1), r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitShrI32()
{
int32_t c;
if (popConstI32(c)) {
RegI32 r = popI32();
masm.rshift32Arithmetic(Imm32(c & 31), r.reg);
pushI32(r);
} else {
RegI32 r0, r1;
pop2xI32ForShiftOrRotate(&r0, &r1);
maskShiftCount32(r1);
masm.rshift32Arithmetic(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
}
void
BaseCompiler::emitShrI64()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI64 r0, r1;
pop2xI64ForShiftOrRotate(&r0, &r1);
masm.rshift64Arithmetic(lowPart(r1), r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitShrU32()
{
int32_t c;
if (popConstI32(c)) {
RegI32 r = popI32();
masm.rshift32(Imm32(c & 31), r.reg);
pushI32(r);
} else {
RegI32 r0, r1;
pop2xI32ForShiftOrRotate(&r0, &r1);
maskShiftCount32(r1);
masm.rshift32(r1.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
}
void
BaseCompiler::emitShrU64()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI64 r0, r1;
pop2xI64ForShiftOrRotate(&r0, &r1);
masm.rshift64(lowPart(r1), r0.reg);
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitRotrI32()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForShiftOrRotate(&r0, &r1);
masm.rotateRight(r1.reg, r0.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitRotrI64()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI64 r0, r1;
pop2xI64ForShiftOrRotate(&r0, &r1);
masm.rotateRight64(lowPart(r1), r0.reg, r0.reg, maybeHighPart(r1));
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitRotlI32()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI32 r0, r1;
pop2xI32ForShiftOrRotate(&r0, &r1);
masm.rotateLeft(r1.reg, r0.reg, r0.reg);
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitRotlI64()
{
// TODO / OPTIMIZE: Constant rhs (Bug 1316803)
RegI64 r0, r1;
pop2xI64ForShiftOrRotate(&r0, &r1);
masm.rotateLeft64(lowPart(r1), r0.reg, r0.reg, maybeHighPart(r1));
freeI64(r1);
pushI64(r0);
}
void
BaseCompiler::emitEqzI32()
{
// TODO / OPTIMIZE: Boolean evaluation for control (Bug 1286816)
RegI32 r0 = popI32();
masm.cmp32Set(Assembler::Equal, r0.reg, Imm32(0), r0.reg);
pushI32(r0);
}
void
BaseCompiler::emitEqzI64()
{
// TODO / OPTIMIZE: Boolean evaluation for control (Bug 1286816)
// TODO / OPTIMIZE: Avoid the temp register (Bug 1316848)
RegI64 r0 = popI64();
RegI64 r1 = needI64();
setI64(0, r1);
RegI32 i0 = fromI64(r0);
cmp64Set(Assembler::Equal, r0, r1, i0);
freeI64(r1);
freeI64Except(r0, i0);
pushI32(i0);
}
void
BaseCompiler::emitClzI32()
{
RegI32 r0 = popI32();
masm.clz32(r0.reg, r0.reg, IsKnownNotZero(false));
pushI32(r0);
}
void
BaseCompiler::emitClzI64()
{
RegI64 r0 = popI64();
masm.clz64(r0.reg, lowPart(r0));
maybeClearHighPart(r0);
pushI64(r0);
}
void
BaseCompiler::emitCtzI32()
{
RegI32 r0 = popI32();
masm.ctz32(r0.reg, r0.reg, IsKnownNotZero(false));
pushI32(r0);
}
void
BaseCompiler::emitCtzI64()
{
RegI64 r0 = popI64();
masm.ctz64(r0.reg, lowPart(r0));
maybeClearHighPart(r0);
pushI64(r0);
}
void
BaseCompiler::emitPopcntI32()
{
RegI32 r0 = popI32();
if (popcnt32NeedsTemp()) {
RegI32 tmp = needI32();
masm.popcnt32(r0.reg, r0.reg, tmp.reg);
freeI32(tmp);
} else {
masm.popcnt32(r0.reg, r0.reg, invalidI32().reg);
}
pushI32(r0);
}
void
BaseCompiler::emitPopcntI64()
{
RegI64 r0 = popI64();
if (popcnt64NeedsTemp()) {
RegI32 tmp = needI32();
masm.popcnt64(r0.reg, r0.reg, tmp.reg);
freeI32(tmp);
} else {
masm.popcnt64(r0.reg, r0.reg, invalidI32().reg);
}
pushI64(r0);
}
void
BaseCompiler::emitBitNotI32()
{
RegI32 r0 = popI32();
masm.not32(r0.reg);
pushI32(r0);
}
void
BaseCompiler::emitAbsI32()
{
// TODO / OPTIMIZE (bug 1316823): Use conditional move on some platforms?
Label nonnegative;
RegI32 r0 = popI32();
masm.branch32(Assembler::GreaterThanOrEqual, r0.reg, Imm32(0), &nonnegative);
masm.neg32(r0.reg);
masm.bind(&nonnegative);
pushI32(r0);
}
void
BaseCompiler::emitAbsF32()
{
RegF32 r0 = popF32();
masm.absFloat32(r0.reg, r0.reg);
pushF32(r0);
}
void
BaseCompiler::emitAbsF64()
{
RegF64 r0 = popF64();
masm.absDouble(r0.reg, r0.reg);
pushF64(r0);
}
void
BaseCompiler::emitNegateI32()
{
RegI32 r0 = popI32();
masm.neg32(r0.reg);
pushI32(r0);
}
void
BaseCompiler::emitNegateF32()
{
RegF32 r0 = popF32();
masm.negateFloat(r0.reg);
pushF32(r0);
}
void
BaseCompiler::emitNegateF64()
{
RegF64 r0 = popF64();
masm.negateDouble(r0.reg);
pushF64(r0);
}
void
BaseCompiler::emitSqrtF32()
{
RegF32 r0 = popF32();
masm.sqrtFloat32(r0.reg, r0.reg);
pushF32(r0);
}
void
BaseCompiler::emitSqrtF64()
{
RegF64 r0 = popF64();
masm.sqrtDouble(r0.reg, r0.reg);
pushF64(r0);
}
template<bool isUnsigned>
bool
BaseCompiler::emitTruncateF32ToI32()
{
RegF32 r0 = popF32();
RegI32 i0 = needI32();
if (!truncateF32ToI32(r0, i0, isUnsigned))
return false;
freeF32(r0);
pushI32(i0);
return true;
}
template<bool isUnsigned>
bool
BaseCompiler::emitTruncateF64ToI32()
{
RegF64 r0 = popF64();
RegI32 i0 = needI32();
if (!truncateF64ToI32(r0, i0, isUnsigned))
return false;
freeF64(r0);
pushI32(i0);
return true;
}
#ifndef FLOAT_TO_I64_CALLOUT
template<bool isUnsigned>
bool
BaseCompiler::emitTruncateF32ToI64()
{
RegF32 r0 = popF32();
RegI64 x0 = needI64();
if (isUnsigned) {
RegF64 tmp = needF64();
if (!truncateF32ToI64(r0, x0, isUnsigned, tmp))
return false;
freeF64(tmp);
} else {
if (!truncateF32ToI64(r0, x0, isUnsigned, invalidF64()))
return false;
}
freeF32(r0);
pushI64(x0);
return true;
}
template<bool isUnsigned>
bool
BaseCompiler::emitTruncateF64ToI64()
{
RegF64 r0 = popF64();
RegI64 x0 = needI64();
if (isUnsigned) {
RegF64 tmp = needF64();
if (!truncateF64ToI64(r0, x0, isUnsigned, tmp))
return false;
freeF64(tmp);
} else {
if (!truncateF64ToI64(r0, x0, isUnsigned, invalidF64()))
return false;
}
freeF64(r0);
pushI64(x0);
return true;
}
#endif // FLOAT_TO_I64_CALLOUT
void
BaseCompiler::emitWrapI64ToI32()
{
RegI64 r0 = popI64();
RegI32 i0 = fromI64(r0);
wrapI64ToI32(r0, i0);
freeI64Except(r0, i0);
pushI32(i0);
}
void
BaseCompiler::emitExtendI32ToI64()
{
RegI64 x0 = popI32ForSignExtendI64();
RegI32 r0 = RegI32(lowPart(x0));
signExtendI32ToI64(r0, x0);
pushI64(x0);
// Note: no need to free r0, since it is part of x0
}
void
BaseCompiler::emitExtendU32ToI64()
{
RegI32 r0 = popI32();
RegI64 x0 = widenI32(r0);
extendU32ToI64(r0, x0);
pushI64(x0);
// Note: no need to free r0, since it is part of x0
}
void
BaseCompiler::emitReinterpretF32AsI32()
{
RegF32 r0 = popF32();
RegI32 i0 = needI32();
masm.moveFloat32ToGPR(r0.reg, i0.reg);
freeF32(r0);
pushI32(i0);
}
void
BaseCompiler::emitReinterpretF64AsI64()
{
RegF64 r0 = popF64();
RegI64 x0 = needI64();
reinterpretF64AsI64(r0, x0);
freeF64(r0);
pushI64(x0);
}
void
BaseCompiler::emitConvertF64ToF32()
{
RegF64 r0 = popF64();
RegF32 f0 = needF32();
masm.convertDoubleToFloat32(r0.reg, f0.reg);
freeF64(r0);
pushF32(f0);
}
void
BaseCompiler::emitConvertI32ToF32()
{
RegI32 r0 = popI32();
RegF32 f0 = needF32();
masm.convertInt32ToFloat32(r0.reg, f0.reg);
freeI32(r0);
pushF32(f0);
}
void
BaseCompiler::emitConvertU32ToF32()
{
RegI32 r0 = popI32();
RegF32 f0 = needF32();
masm.convertUInt32ToFloat32(r0.reg, f0.reg);
freeI32(r0);
pushF32(f0);
}
#ifndef I64_TO_FLOAT_CALLOUT
void
BaseCompiler::emitConvertI64ToF32()
{
RegI64 r0 = popI64();
RegF32 f0 = needF32();
convertI64ToF32(r0, IsUnsigned(false), f0, RegI32());
freeI64(r0);
pushF32(f0);
}
void
BaseCompiler::emitConvertU64ToF32()
{
RegI64 r0 = popI64();
RegF32 f0 = needF32();
RegI32 temp;
if (convertI64ToFloatNeedsTemp(IsUnsigned(true)))
temp = needI32();
convertI64ToF32(r0, IsUnsigned(true), f0, temp);
if (temp.reg != Register::Invalid())
freeI32(temp);
freeI64(r0);
pushF32(f0);
}
#endif
void
BaseCompiler::emitConvertF32ToF64()
{
RegF32 r0 = popF32();
RegF64 d0 = needF64();
masm.convertFloat32ToDouble(r0.reg, d0.reg);
freeF32(r0);
pushF64(d0);
}
void
BaseCompiler::emitConvertI32ToF64()
{
RegI32 r0 = popI32();
RegF64 d0 = needF64();
masm.convertInt32ToDouble(r0.reg, d0.reg);
freeI32(r0);
pushF64(d0);
}
void
BaseCompiler::emitConvertU32ToF64()
{
RegI32 r0 = popI32();
RegF64 d0 = needF64();
masm.convertUInt32ToDouble(r0.reg, d0.reg);
freeI32(r0);
pushF64(d0);
}
#ifndef I64_TO_FLOAT_CALLOUT
void
BaseCompiler::emitConvertI64ToF64()
{
RegI64 r0 = popI64();
RegF64 d0 = needF64();
convertI64ToF64(r0, IsUnsigned(false), d0, RegI32());
freeI64(r0);
pushF64(d0);
}
void
BaseCompiler::emitConvertU64ToF64()
{
RegI64 r0 = popI64();
RegF64 d0 = needF64();
RegI32 temp;
if (convertI64ToFloatNeedsTemp(IsUnsigned(true)))
temp = needI32();
convertI64ToF64(r0, IsUnsigned(true), d0, temp);
if (temp.reg != Register::Invalid())
freeI32(temp);
freeI64(r0);
pushF64(d0);
}
#endif // I64_TO_FLOAT_CALLOUT
void
BaseCompiler::emitReinterpretI32AsF32()
{
RegI32 r0 = popI32();
RegF32 f0 = needF32();
masm.moveGPRToFloat32(r0.reg, f0.reg);
freeI32(r0);
pushF32(f0);
}
void
BaseCompiler::emitReinterpretI64AsF64()
{
RegI64 r0 = popI64();
RegF64 d0 = needF64();
reinterpretI64AsF64(r0, d0);
freeI64(r0);
pushF64(d0);
}
// For blocks and loops and ifs:
//
// - Sync the value stack before going into the block in order to simplify exit
// from the block: all exits from the block can assume that there are no
// live registers except the one carrying the exit value.
// - The block can accumulate a number of dead values on the stacks, so when
// branching out of the block or falling out at the end be sure to
// pop the appropriate stacks back to where they were on entry, while
// preserving the exit value.
// - A continue branch in a loop is much like an exit branch, but the branch
// value must not be preserved.
// - The exit value is always in a designated join register (type dependent).
bool
BaseCompiler::emitBlock()
{
if (!iter_.readBlock())
return false;
UniquePooledLabel blockEnd(newLabel());
if (!blockEnd)
return false;
if (!deadCode_)
sync(); // Simplifies branching out from block
return pushControl(&blockEnd);
}
void
BaseCompiler::endBlock(ExprType type, bool isFunctionBody)
{
Control& block = controlItem(0);
// Save the value.
AnyReg r;
if (!deadCode_ && !IsVoid(type))
r = popJoinReg();
// Leave the block.
popStackOnBlockExit(block.framePushed);
// Bind after cleanup: branches out will have popped the stack.
if (block.label->used()) {
masm.bind(block.label);
if (deadCode_ && !IsVoid(type))
r = allocJoinReg(type);
deadCode_ = false;
}
MOZ_ASSERT(stk_.length() == block.stackSize);
// Retain the value stored in joinReg by all paths.
if (!deadCode_) {
if (!IsVoid(type))
pushJoinReg(r);
if (isFunctionBody)
doReturn(func_.sig().ret());
}
popControl();
}
bool
BaseCompiler::emitLoop()
{
if (!iter_.readLoop())
return false;
UniquePooledLabel blockCont(newLabel());
if (!blockCont)
return false;
if (!deadCode_)
sync(); // Simplifies branching out from block
if (!pushControl(&blockCont))
return false;
if (!deadCode_) {
masm.bind(controlItem(0).label);
addInterruptCheck();
}
return true;
}
void
BaseCompiler::endLoop(ExprType type)
{
Control& block = controlItem(0);
AnyReg r;
if (!deadCode_ && !IsVoid(type))
r = popJoinReg();
popStackOnBlockExit(block.framePushed);
MOZ_ASSERT(stk_.length() == block.stackSize);
popControl();
// Retain the value stored in joinReg by all paths.
if (!deadCode_ && !IsVoid(type))
pushJoinReg(r);
}
// The bodies of the "then" and "else" arms can be arbitrary sequences
// of expressions, they push control and increment the nesting and can
// even be targeted by jumps. A branch to the "if" block branches to
// the exit of the if, ie, it's like "break". Consider:
//
// (func (result i32)
// (if (i32.const 1)
// (begin (br 1) (unreachable))
// (begin (unreachable)))
// (i32.const 1))
//
// The branch causes neither of the unreachable expressions to be
// evaluated.
bool
BaseCompiler::emitIf()
{
Nothing unused_cond;
if (!iter_.readIf(&unused_cond))
return false;
UniquePooledLabel endLabel(newLabel());
if (!endLabel)
return false;
UniquePooledLabel elseLabel(newLabel());
if (!elseLabel)
return false;
RegI32 rc;
if (!deadCode_) {
rc = popI32();
sync(); // Simplifies branching out from the arms
}
if (!pushControl(&endLabel, &elseLabel))
return false;
if (!deadCode_) {
masm.branch32(Assembler::Equal, rc.reg, Imm32(0), controlItem(0).otherLabel);
freeI32(rc);
}
return true;
}
void
BaseCompiler::endIfThen()
{
Control& ifThen = controlItem(0);
popStackOnBlockExit(ifThen.framePushed);
if (ifThen.otherLabel->used())
masm.bind(ifThen.otherLabel);
if (ifThen.label->used())
masm.bind(ifThen.label);
deadCode_ = ifThen.deadOnArrival;
MOZ_ASSERT(stk_.length() == ifThen.stackSize);
popControl();
}
bool
BaseCompiler::emitElse()
{
ExprType thenType;
Nothing unused_thenValue;
if (!iter_.readElse(&thenType, &unused_thenValue))
return false;
Control& ifThenElse = controlItem(0);
// See comment in endIfThenElse, below.
// Exit the "then" branch.
ifThenElse.deadThenBranch = deadCode_;
AnyReg r;
if (!deadCode_ && !IsVoid(thenType))
r = popJoinReg();
popStackOnBlockExit(ifThenElse.framePushed);
if (!deadCode_)
masm.jump(ifThenElse.label);
if (ifThenElse.otherLabel->used())
masm.bind(ifThenElse.otherLabel);
// Reset to the "else" branch.
MOZ_ASSERT(stk_.length() == ifThenElse.stackSize);
if (!deadCode_ && !IsVoid(thenType))
freeJoinReg(r);
deadCode_ = ifThenElse.deadOnArrival;
return true;
}
void
BaseCompiler::endIfThenElse(ExprType type)
{
Control& ifThenElse = controlItem(0);
// The expression type is not a reliable guide to what we'll find
// on the stack, we could have (if E (i32.const 1) (unreachable))
// in which case the "else" arm is AnyType but the type of the
// full expression is I32. So restore whatever's there, not what
// we want to find there. The "then" arm has the same constraint.
AnyReg r;
if (!deadCode_ && !IsVoid(type))
r = popJoinReg();
popStackOnBlockExit(ifThenElse.framePushed);
if (ifThenElse.label->used())
masm.bind(ifThenElse.label);
if (!ifThenElse.deadOnArrival &&
(!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label->bound())) {
if (deadCode_ && !IsVoid(type))
r = allocJoinReg(type);
deadCode_ = false;
}
MOZ_ASSERT(stk_.length() == ifThenElse.stackSize);
popControl();
if (!deadCode_ && !IsVoid(type))
pushJoinReg(r);
}
bool
BaseCompiler::emitEnd()
{
LabelKind kind;
ExprType type;
Nothing unused_value;
if (!iter_.readEnd(&kind, &type, &unused_value))
return false;
switch (kind) {
case LabelKind::Block: endBlock(type, iter_.controlStackEmpty()); break;
case LabelKind::Loop: endLoop(type); break;
case LabelKind::UnreachableThen:
case LabelKind::Then: endIfThen(); break;
case LabelKind::Else: endIfThenElse(type); break;
}
return true;
}
bool
BaseCompiler::emitBr()
{
uint32_t relativeDepth;
ExprType type;
Nothing unused_value;
if (!iter_.readBr(&relativeDepth, &type, &unused_value))
return false;
if (deadCode_)
return true;
Control& target = controlItem(relativeDepth);
// Save any value in the designated join register, where the
// normal block exit code will also leave it.
AnyReg r;
if (!IsVoid(type))
r = popJoinReg();
popStackBeforeBranch(target.framePushed);
masm.jump(target.label);
// The register holding the join value is free for the remainder
// of this block.
if (!IsVoid(type))
freeJoinReg(r);
deadCode_ = true;
popValueStackTo(ctl_.back().stackSize);
return true;
}
bool
BaseCompiler::emitBrIf()
{
uint32_t relativeDepth;
ExprType type;
Nothing unused_value, unused_condition;
if (!iter_.readBrIf(&relativeDepth, &type, &unused_value, &unused_condition))
return false;
if (deadCode_)
return true;
Control& target = controlItem(relativeDepth);
// TODO / OPTIMIZE (Bug 1286816): Optimize boolean evaluation for control by
// allowing a conditional expression to be left on the stack and reified
// here as part of the branch instruction.
// Don't use joinReg for rc
maybeReserveJoinRegI(type);
// Condition value is on top, always I32.
RegI32 rc = popI32();
maybeUnreserveJoinRegI(type);
// Save any value in the designated join register, where the
// normal block exit code will also leave it.
AnyReg r;
if (!IsVoid(type))
r = popJoinReg();
Label notTaken;
masm.branch32(Assembler::Equal, rc.reg, Imm32(0), ¬Taken);
popStackBeforeBranch(target.framePushed);
masm.jump(target.label);
masm.bind(¬Taken);
// This register is free in the remainder of the block.
freeI32(rc);
// br_if returns its value(s).
if (!IsVoid(type))
pushJoinReg(r);
return true;
}
bool
BaseCompiler::emitBrTable()
{
uint32_t tableLength;
ExprType type;
Nothing unused_value, unused_index;
if (!iter_.readBrTable(&tableLength, &type, &unused_value, &unused_index))
return false;
LabelVector stubs;
if (!stubs.reserve(tableLength+1))
return false;
Uint32Vector depths;
if (!depths.reserve(tableLength))
return false;
for (size_t i = 0; i < tableLength; ++i) {
uint32_t depth;
if (!iter_.readBrTableEntry(&type, &unused_value, &depth))
return false;
depths.infallibleAppend(depth);
}
uint32_t defaultDepth;
if (!iter_.readBrTableDefault(&type, &unused_value, &defaultDepth))
return false;
if (deadCode_)
return true;
// Don't use joinReg for rc
maybeReserveJoinRegI(type);
// Table switch value always on top.
RegI32 rc = popI32();
maybeUnreserveJoinRegI(type);
AnyReg r;
if (!IsVoid(type))
r = popJoinReg();
Label dispatchCode;
masm.branch32(Assembler::Below, rc.reg, Imm32(tableLength), &dispatchCode);
// This is the out-of-range stub. rc is dead here but we don't need it.
popStackBeforeBranch(controlItem(defaultDepth).framePushed);
masm.jump(controlItem(defaultDepth).label);
// Emit stubs. rc is dead in all of these but we don't need it.
//
// TODO / OPTIMIZE (Bug 1316804): Branch directly to the case code if we
// can, don't emit an intermediate stub.
for (uint32_t i = 0; i < tableLength; i++) {
PooledLabel* stubLabel = newLabel();
// The labels in the vector are in the TempAllocator and will
// be freed by and by.
if (!stubLabel)
return false;
stubs.infallibleAppend(stubLabel);
masm.bind(stubLabel);
uint32_t k = depths[i];
popStackBeforeBranch(controlItem(k).framePushed);
masm.jump(controlItem(k).label);
}
// Emit table.
Label theTable;
masm.bind(&theTable);
jumpTable(stubs);
// Emit indirect jump. rc is live here.
masm.bind(&dispatchCode);
tableSwitch(&theTable, rc);
deadCode_ = true;
// Clean up.
freeI32(rc);
if (!IsVoid(type))
freeJoinReg(r);
for (uint32_t i = 0; i < tableLength; i++)
freeLabel(stubs[i]);
popValueStackTo(ctl_.back().stackSize);
return true;
}
bool
BaseCompiler::emitDrop()
{
if (!iter_.readDrop())
return false;
if (deadCode_)
return true;
popStackIfMemory();
popValueStackBy(1);
return true;
}
void
BaseCompiler::doReturn(ExprType type)
{
switch (type) {
case ExprType::Void: {
returnCleanup();
break;
}
case ExprType::I32: {
RegI32 rv = popI32(RegI32(ReturnReg));
returnCleanup();
freeI32(rv);
break;
}
case ExprType::I64: {
RegI64 rv = popI64(RegI64(ReturnReg64));
returnCleanup();
freeI64(rv);
break;
}
case ExprType::F64: {
RegF64 rv = popF64(RegF64(ReturnDoubleReg));
returnCleanup();
freeF64(rv);
break;
}
case ExprType::F32: {
RegF32 rv = popF32(RegF32(ReturnFloat32Reg));
returnCleanup();
freeF32(rv);
break;
}
default: {
MOZ_CRASH("Function return type");
}
}
}
bool
BaseCompiler::emitReturn()
{
Nothing unused_value;
if (!iter_.readReturn(&unused_value))
return false;
if (deadCode_)
return true;
doReturn(func_.sig().ret());
deadCode_ = true;
popValueStackTo(ctl_.back().stackSize);
return true;
}
bool
BaseCompiler::emitCallArgs(const ValTypeVector& args, FunctionCall& baselineCall)
{
MOZ_ASSERT(!deadCode_);
startCallArgs(baselineCall, stackArgAreaSize(args));
uint32_t numArgs = args.length();
for (size_t i = 0; i < numArgs; ++i) {
ValType argType = args[i];
Nothing arg_;
if (!iter_.readCallArg(argType, numArgs, i, &arg_))
return false;
Stk& arg = peek(numArgs - 1 - i);
passArg(baselineCall, argType, arg);
}
// Pass the TLS pointer as a hidden argument in WasmTlsReg. Load
// it directly out if its stack slot so we don't interfere with
// the stk_.
if (baselineCall.loadTlsBefore)
loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
if (!iter_.readCallArgsEnd(numArgs))
return false;
return true;
}
void
BaseCompiler::pushReturned(const FunctionCall& call, ExprType type)
{
switch (type) {
case ExprType::Void:
MOZ_CRASH("Compiler bug: attempt to push void return");
break;
case ExprType::I32: {
RegI32 rv = captureReturnedI32();
pushI32(rv);
break;
}
case ExprType::I64: {
RegI64 rv = captureReturnedI64();
pushI64(rv);
break;
}
case ExprType::F32: {
RegF32 rv = captureReturnedF32(call);
pushF32(rv);
break;
}
case ExprType::F64: {
RegF64 rv = captureReturnedF64(call);
pushF64(rv);
break;
}
default:
MOZ_CRASH("Function return type");
}
}
// For now, always sync() at the beginning of the call to easily save live
// values.
//
// TODO / OPTIMIZE (Bug 1316806): We may be able to avoid a full sync(), since
// all we want is to save live registers that won't be saved by the callee or
// that we need for outgoing args - we don't need to sync the locals. We can
// just push the necessary registers, it'll be like a lightweight sync.
//
// Even some of the pushing may be unnecessary if the registers will be consumed
// by the call, because then what we want is parallel assignment to the argument
// registers or onto the stack for outgoing arguments. A sync() is just
// simpler.
bool
BaseCompiler::emitCall()
{
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t funcIndex;
if (!iter_.readCall(&funcIndex))
return false;
if (deadCode_)
return true;
sync();
const Sig& sig = *mg_.funcSigs[funcIndex];
bool import = mg_.funcIsImport(funcIndex);
uint32_t numArgs = sig.args().length();
size_t stackSpace = stackConsumed(numArgs);
FunctionCall baselineCall(lineOrBytecode);
beginCall(baselineCall, UseABI::Wasm, import ? InterModule::True : InterModule::False);
if (!emitCallArgs(sig.args(), baselineCall))
return false;
if (!iter_.readCallReturn(sig.ret()))
return false;
if (import)
callImport(mg_.funcImportGlobalDataOffsets[funcIndex], baselineCall);
else
callDefinition(funcIndex, baselineCall);
endCall(baselineCall);
// TODO / OPTIMIZE (bug 1316827): It would be better to merge this
// freeStack() into the one in endCall, if we can.
popValueStackBy(numArgs);
masm.freeStack(stackSpace);
if (!IsVoid(sig.ret()))
pushReturned(baselineCall, sig.ret());
return true;
}
bool
BaseCompiler::emitCallIndirect(bool oldStyle)
{
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
uint32_t sigIndex;
Nothing callee_;
if (oldStyle) {
if (!iter_.readOldCallIndirect(&sigIndex))
return false;
} else {
if (!iter_.readCallIndirect(&sigIndex, &callee_))
return false;
}
if (deadCode_)
return true;
sync();
const SigWithId& sig = mg_.sigs[sigIndex];
// new style: Stack: ... arg1 .. argn callee
// old style: Stack: ... callee arg1 .. argn
uint32_t numArgs = sig.args().length();
size_t stackSpace = stackConsumed(numArgs + 1);
// The arguments must be at the stack top for emitCallArgs, so pop the
// callee if it is on top. Note this only pops the compiler's stack,
// not the CPU stack.
Stk callee = oldStyle ? peek(numArgs) : stk_.popCopy();
FunctionCall baselineCall(lineOrBytecode);
beginCall(baselineCall, UseABI::Wasm, InterModule::True);
if (!emitCallArgs(sig.args(), baselineCall))
return false;
if (oldStyle) {
if (!iter_.readOldCallIndirectCallee(&callee_))
return false;
}
if (!iter_.readCallReturn(sig.ret()))
return false;
callIndirect(sigIndex, callee, baselineCall);
endCall(baselineCall);
// For new style calls, the callee was popped off the compiler's
// stack above.
popValueStackBy(oldStyle ? numArgs + 1 : numArgs);
// TODO / OPTIMIZE (bug 1316827): It would be better to merge this
// freeStack() into the one in endCall, if we can.
masm.freeStack(stackSpace);
if (!IsVoid(sig.ret()))
pushReturned(baselineCall, sig.ret());
return true;
}
bool
BaseCompiler::emitCommonMathCall(uint32_t lineOrBytecode, SymbolicAddress callee,
ValTypeVector& signature, ExprType retType)
{
sync();
uint32_t numArgs = signature.length();
size_t stackSpace = stackConsumed(numArgs);
FunctionCall baselineCall(lineOrBytecode);
beginCall(baselineCall, UseABI::System, InterModule::False);
if (!emitCallArgs(signature, baselineCall))
return false;
if (!iter_.readCallReturn(retType))
return false;
builtinCall(callee, baselineCall);
endCall(baselineCall);
// TODO / OPTIMIZE (bug 1316827): It would be better to merge this
// freeStack() into the one in endCall, if we can.
popValueStackBy(numArgs);
masm.freeStack(stackSpace);
pushReturned(baselineCall, retType);
return true;
}
bool
BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType)
{
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
if (deadCode_)
return true;
return emitCommonMathCall(lineOrBytecode, callee,
operandType == ValType::F32 ? SigF_ : SigD_,
operandType == ValType::F32 ? ExprType::F32 : ExprType::F64);
}
bool
BaseCompiler::emitBinaryMathBuiltinCall(SymbolicAddress callee, ValType operandType)
{
MOZ_ASSERT(operandType == ValType::F64);
uint32_t lineOrBytecode = 0;
if (callee == SymbolicAddress::ModD) {
// Not actually a call in the binary representation
} else {
lineOrBytecode = readCallSiteLineOrBytecode();
}
if (deadCode_)
return true;
return emitCommonMathCall(lineOrBytecode, callee, SigDD_, ExprType::F64);
}
#ifdef INT_DIV_I64_CALLOUT
bool
BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee, ValType operandType)
{
MOZ_ASSERT(operandType == ValType::I64);
if (deadCode_)
return true;
sync();
needI64(abiReturnRegI64);
RegI32 temp = needI32();
RegI64 rhs = popI64();
RegI64 srcDest = popI64ToSpecific(abiReturnRegI64);
Label done;
checkDivideByZeroI64(rhs);
if (callee == SymbolicAddress::DivI64)
checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(false));
else if (callee == SymbolicAddress::ModI64)
checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(true));
masm.setupUnalignedABICall(temp.reg);
masm.passABIArg(srcDest.reg.high);
masm.passABIArg(srcDest.reg.low);
masm.passABIArg(rhs.reg.high);
masm.passABIArg(rhs.reg.low);
masm.callWithABI(callee);
masm.bind(&done);
freeI32(temp);
freeI64(rhs);
pushI64(srcDest);
return true;
}
#endif // INT_DIV_I64_CALLOUT
#ifdef I64_TO_FLOAT_CALLOUT
bool
BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee, ValType operandType,
ValType resultType)
{
sync();
RegI32 temp = needI32();
RegI64 input = popI64();
FunctionCall call(0);
masm.setupUnalignedABICall(temp.reg);
# ifdef JS_NUNBOX32
masm.passABIArg(input.reg.high);
masm.passABIArg(input.reg.low);
# else
MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout");
# endif
masm.callWithABI(callee, MoveOp::DOUBLE);
freeI32(temp);
freeI64(input);
RegF64 rv = captureReturnedF64(call);
if (resultType == ValType::F32) {
RegF32 rv2 = needF32();
masm.convertDoubleToFloat32(rv.reg, rv2.reg);
freeF64(rv);
pushF32(rv2);
} else {
pushF64(rv);
}
return true;
}
#endif // I64_TO_FLOAT_CALLOUT
#ifdef FLOAT_TO_I64_CALLOUT
// `Callee` always takes a double, so a float32 input must be converted.
bool
BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee, ValType operandType,
ValType resultType)
{
RegF64 doubleInput;
if (operandType == ValType::F32) {
doubleInput = needF64();
RegF32 input = popF32();
masm.convertFloat32ToDouble(input.reg, doubleInput.reg);
freeF32(input);
} else {
doubleInput = popF64();
}
// We may need the value after the call for the ool check.
RegF64 otherReg = needF64();
moveF64(doubleInput, otherReg);
pushF64(otherReg);
sync();
RegI32 temp = needI32();
FunctionCall call(0);
masm.setupUnalignedABICall(temp.reg);
masm.passABIArg(doubleInput.reg, MoveOp::DOUBLE);
masm.callWithABI(callee);
freeI32(temp);
freeF64(doubleInput);
RegI64 rv = captureReturnedI64();
RegF64 inputVal = popF64();
bool isUnsigned = callee == SymbolicAddress::TruncateDoubleToUint64;
// The OOL check just succeeds or fails, it does not generate a value.
OutOfLineCode* ool = new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(inputVal),
isUnsigned,
trapOffset());
ool = addOutOfLineCode(ool);
if (!ool)
return false;
masm.branch64(Assembler::Equal, rv.reg, Imm64(0x8000000000000000), ool->entry());
masm.bind(ool->rejoin());
pushI64(rv);
freeF64(inputVal);
return true;
}
#endif // FLOAT_TO_I64_CALLOUT
bool
BaseCompiler::emitGetLocal()
{
uint32_t slot;
if (!iter_.readGetLocal(locals_, &slot))
return false;
if (deadCode_)
return true;
// Local loads are pushed unresolved, ie, they may be deferred
// until needed, until they may be affected by a store, or until a
// sync. This is intended to reduce register pressure.
switch (locals_[slot]) {
case ValType::I32:
pushLocalI32(slot);
break;
case ValType::I64:
pushLocalI64(slot);
break;
case ValType::F64:
pushLocalF64(slot);
break;
case ValType::F32:
pushLocalF32(slot);
break;
default:
MOZ_CRASH("Local variable type");
}
return true;
}
bool
BaseCompiler::emitSetLocal()
{
uint32_t slot;
Nothing unused_value;
if (!iter_.readSetLocal(locals_, &slot, &unused_value))
return false;
if (deadCode_)
return true;
switch (locals_[slot]) {
case ValType::I32: {
RegI32 rv = popI32();
syncLocal(slot);
storeToFrameI32(rv.reg, frameOffsetFromSlot(slot, MIRType::Int32));
freeI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
syncLocal(slot);
storeToFrameI64(rv.reg, frameOffsetFromSlot(slot, MIRType::Int64));
freeI64(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
syncLocal(slot);
storeToFrameF64(rv.reg, frameOffsetFromSlot(slot, MIRType::Double));
freeF64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
syncLocal(slot);
storeToFrameF32(rv.reg, frameOffsetFromSlot(slot, MIRType::Float32));
freeF32(rv);
break;
}
default:
MOZ_CRASH("Local variable type");
}
return true;
}
bool
BaseCompiler::emitTeeLocal()
{
uint32_t slot;
Nothing unused_value;
if (!iter_.readTeeLocal(locals_, &slot, &unused_value))
return false;
if (deadCode_)
return true;
switch (locals_[slot]) {
case ValType::I32: {
RegI32 rv = popI32();
syncLocal(slot);
storeToFrameI32(rv.reg, frameOffsetFromSlot(slot, MIRType::Int32));
pushI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
syncLocal(slot);
storeToFrameI64(rv.reg, frameOffsetFromSlot(slot, MIRType::Int64));
pushI64(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
syncLocal(slot);
storeToFrameF64(rv.reg, frameOffsetFromSlot(slot, MIRType::Double));
pushF64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
syncLocal(slot);
storeToFrameF32(rv.reg, frameOffsetFromSlot(slot, MIRType::Float32));
pushF32(rv);
break;
}
default:
MOZ_CRASH("Local variable type");
}
return true;
}
bool
BaseCompiler::emitGetGlobal()
{
uint32_t id;
if (!iter_.readGetGlobal(mg_.globals, &id))
return false;
if (deadCode_)
return true;
const GlobalDesc& global = mg_.globals[id];
if (global.isConstant()) {
Val value = global.constantValue();
switch (value.type()) {
case ValType::I32:
pushI32(value.i32());
break;
case ValType::I64:
pushI64(value.i64());
break;
case ValType::F32:
pushF32(value.f32());
break;
case ValType::F64:
pushF64(value.f64());
break;
default:
MOZ_CRASH("Global constant type");
}
return true;
}
switch (global.type()) {
case ValType::I32: {
RegI32 rv = needI32();
loadGlobalVarI32(global.offset(), rv);
pushI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = needI64();
loadGlobalVarI64(global.offset(), rv);
pushI64(rv);
break;
}
case ValType::F32: {
RegF32 rv = needF32();
loadGlobalVarF32(global.offset(), rv);
pushF32(rv);
break;
}
case ValType::F64: {
RegF64 rv = needF64();
loadGlobalVarF64(global.offset(), rv);
pushF64(rv);
break;
}
default:
MOZ_CRASH("Global variable type");
break;
}
return true;
}
bool
BaseCompiler::emitSetGlobal()
{
uint32_t id;
Nothing unused_value;
if (!iter_.readSetGlobal(mg_.globals, &id, &unused_value))
return false;
if (deadCode_)
return true;
const GlobalDesc& global = mg_.globals[id];
switch (global.type()) {
case ValType::I32: {
RegI32 rv = popI32();
storeGlobalVarI32(global.offset(), rv);
freeI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
storeGlobalVarI64(global.offset(), rv);
freeI64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
storeGlobalVarF32(global.offset(), rv);
freeF32(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
storeGlobalVarF64(global.offset(), rv);
freeF64(rv);
break;
}
default:
MOZ_CRASH("Global variable type");
break;
}
return true;
}
bool
BaseCompiler::emitTeeGlobal()
{
uint32_t id;
Nothing unused_value;
if (!iter_.readTeeGlobal(mg_.globals, &id, &unused_value))
return false;
if (deadCode_)
return true;
const GlobalDesc& global = mg_.globals[id];
switch (global.type()) {
case ValType::I32: {
RegI32 rv = popI32();
storeGlobalVarI32(global.offset(), rv);
pushI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
storeGlobalVarI64(global.offset(), rv);
pushI64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
storeGlobalVarF32(global.offset(), rv);
pushF32(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
storeGlobalVarF64(global.offset(), rv);
pushF64(rv);
break;
}
default:
MOZ_CRASH("Global variable type");
break;
}
return true;
}
bool
BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
{
LinearMemoryAddress<Nothing> addr;
if (!iter_.readLoad(type, Scalar::byteSize(viewType), &addr))
return false;
if (deadCode_)
return true;
// TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
// accesses below the minimum heap length.
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
size_t temps = loadStoreTemps(access);
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
switch (type) {
case ValType::I32: {
RegI32 rp = popI32();
#ifdef JS_CODEGEN_ARM
RegI32 rv = access.isUnaligned() ? needI32() : rp;
#else
RegI32 rv = rp;
#endif
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
pushI32(rv);
if (rp != rv)
freeI32(rp);
break;
}
case ValType::I64: {
RegI64 rv;
RegI32 rp;
#ifdef JS_CODEGEN_X86
rv = abiReturnRegI64;
needI64(rv);
rp = popI32();
#else
rp = popI32();
rv = needI64();
#endif
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
pushI64(rv);
freeI32(rp);
break;
}
case ValType::F32: {
RegI32 rp = popI32();
RegF32 rv = needF32();
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
pushF32(rv);
freeI32(rp);
break;
}
case ValType::F64: {
RegI32 rp = popI32();
RegF64 rv = needF64();
if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
pushF64(rv);
freeI32(rp);
break;
}
default:
MOZ_CRASH("load type");
break;
}
if (temps >= 1)
freeI32(tmp1);
if (temps >= 2)
freeI32(tmp2);
return true;
}
bool
BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
{
LinearMemoryAddress<Nothing> addr;
Nothing unused_value;
if (!iter_.readStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
return false;
if (deadCode_)
return true;
// TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
// accesses below the minimum heap length.
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
size_t temps = loadStoreTemps(access);
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
switch (resultType) {
case ValType::I32: {
RegI32 rp, rv;
pop2xI32(&rp, &rv);
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
freeI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
freeI64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
freeF32(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
freeF64(rv);
break;
}
default:
MOZ_CRASH("store type");
break;
}
if (temps >= 1)
freeI32(tmp1);
if (temps >= 2)
freeI32(tmp2);
return true;
}
bool
BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType)
{
LinearMemoryAddress<Nothing> addr;
Nothing unused_value;
if (!iter_.readTeeStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
return false;
if (deadCode_)
return true;
// TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
// accesses below the minimum heap length.
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
size_t temps = loadStoreTemps(access);
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
switch (resultType) {
case ValType::I32: {
RegI32 rp, rv;
pop2xI32(&rp, &rv);
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
pushI32(rv);
break;
}
case ValType::I64: {
RegI64 rv = popI64();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
pushI64(rv);
break;
}
case ValType::F32: {
RegF32 rv = popF32();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
pushF32(rv);
break;
}
case ValType::F64: {
RegF64 rv = popF64();
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
return false;
freeI32(rp);
pushF64(rv);
break;
}
default:
MOZ_CRASH("store type");
break;
}
if (temps >= 1)
freeI32(tmp1);
if (temps >= 2)
freeI32(tmp2);
return true;
}
bool
BaseCompiler::emitSelect()
{
ValType type;
Nothing unused_trueValue;
Nothing unused_falseValue;
Nothing unused_condition;
if (!iter_.readSelect(&type, &unused_trueValue, &unused_falseValue, &unused_condition))
return false;
if (deadCode_)
return true;
// I32 condition on top, then false, then true.
RegI32 rc = popI32();
switch (type) {
case ValType::I32: {
Label done;
RegI32 r0, r1;
pop2xI32(&r0, &r1);
masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
moveI32(r1, r0);
masm.bind(&done);
freeI32(r1);
pushI32(r0);
break;
}
case ValType::I64: {
Label done;
RegI64 r0, r1;
pop2xI64(&r0, &r1);
masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
moveI64(r1, r0);
masm.bind(&done);
freeI64(r1);
pushI64(r0);
break;
}
case ValType::F32: {
Label done;
RegF32 r0, r1;
pop2xF32(&r0, &r1);
masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
moveF32(r1, r0);
masm.bind(&done);
freeF32(r1);
pushF32(r0);
break;
}
case ValType::F64: {
Label done;
RegF64 r0, r1;
pop2xF64(&r0, &r1);
masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
moveF64(r1, r0);
masm.bind(&done);
freeF64(r1);
pushF64(r0);
break;
}
default: {
MOZ_CRASH("select type");
}
}
freeI32(rc);
return true;
}
void
BaseCompiler::emitCompareI32(JSOp compareOp, MCompare::CompareType compareType)
{
// TODO / OPTIMIZE (bug 1286816): if we want to generate good code for
// boolean operators for control it is possible to delay generating code
// here by pushing a compare operation on the stack, after all it is
// side-effect free. The popping code for br_if will handle it differently,
// but other popI32() will just force code generation.
//
// TODO / OPTIMIZE (bug 1286816): Comparisons against constants using the
// same popConstant pattern as for add().
MOZ_ASSERT(compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32);
RegI32 r0, r1;
pop2xI32(&r0, &r1);
bool u = compareType == MCompare::Compare_UInt32;
switch (compareOp) {
case JSOP_EQ:
masm.cmp32Set(Assembler::Equal, r0.reg, r1.reg, r0.reg);
break;
case JSOP_NE:
masm.cmp32Set(Assembler::NotEqual, r0.reg, r1.reg, r0.reg);
break;
case JSOP_LE:
masm.cmp32Set(u ? Assembler::BelowOrEqual : Assembler::LessThanOrEqual, r0.reg, r1.reg, r0.reg);
break;
case JSOP_LT:
masm.cmp32Set(u ? Assembler::Below : Assembler::LessThan, r0.reg, r1.reg, r0.reg);
break;
case JSOP_GE:
masm.cmp32Set(u ? Assembler::AboveOrEqual : Assembler::GreaterThanOrEqual, r0.reg, r1.reg, r0.reg);
break;
case JSOP_GT:
masm.cmp32Set(u ? Assembler::Above : Assembler::GreaterThan, r0.reg, r1.reg, r0.reg);
break;
default:
MOZ_CRASH("Compiler bug: Unexpected compare opcode");
}
freeI32(r1);
pushI32(r0);
}
void
BaseCompiler::emitCompareI64(JSOp compareOp, MCompare::CompareType compareType)
{
MOZ_ASSERT(compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64);
RegI64 r0, r1;
pop2xI64(&r0, &r1);
RegI32 i0(fromI64(r0));
bool u = compareType == MCompare::Compare_UInt64;
switch (compareOp) {
case JSOP_EQ:
cmp64Set(Assembler::Equal, r0, r1, i0);
break;
case JSOP_NE:
cmp64Set(Assembler::NotEqual, r0, r1, i0);
break;
case JSOP_LE:
cmp64Set(u ? Assembler::BelowOrEqual : Assembler::LessThanOrEqual, r0, r1, i0);
break;
case JSOP_LT:
cmp64Set(u ? Assembler::Below : Assembler::LessThan, r0, r1, i0);
break;
case JSOP_GE:
cmp64Set(u ? Assembler::AboveOrEqual : Assembler::GreaterThanOrEqual, r0, r1, i0);
break;
case JSOP_GT:
cmp64Set(u ? Assembler::Above : Assembler::GreaterThan, r0, r1, i0);
break;
default:
MOZ_CRASH("Compiler bug: Unexpected compare opcode");
}
freeI64(r1);
freeI64Except(r0, i0);
pushI32(i0);
}
void
BaseCompiler::emitCompareF32(JSOp compareOp, MCompare::CompareType compareType)
{
MOZ_ASSERT(compareType == MCompare::Compare_Float32);
Label across;
RegF32 r0, r1;
pop2xF32(&r0, &r1);
RegI32 i0 = needI32();
masm.mov(ImmWord(1), i0.reg);
switch (compareOp) {
case JSOP_EQ:
masm.branchFloat(Assembler::DoubleEqual, r0.reg, r1.reg, &across);
break;
case JSOP_NE:
masm.branchFloat(Assembler::DoubleNotEqualOrUnordered, r0.reg, r1.reg, &across);
break;
case JSOP_LE:
masm.branchFloat(Assembler::DoubleLessThanOrEqual, r0.reg, r1.reg, &across);
break;
case JSOP_LT:
masm.branchFloat(Assembler::DoubleLessThan, r0.reg, r1.reg, &across);
break;
case JSOP_GE:
masm.branchFloat(Assembler::DoubleGreaterThanOrEqual, r0.reg, r1.reg, &across);
break;
case JSOP_GT:
masm.branchFloat(Assembler::DoubleGreaterThan, r0.reg, r1.reg, &across);
break;
default:
MOZ_CRASH("Compiler bug: Unexpected compare opcode");
}
masm.mov(ImmWord(0), i0.reg);
masm.bind(&across);
freeF32(r0);
freeF32(r1);
pushI32(i0);
}
void
BaseCompiler::emitCompareF64(JSOp compareOp, MCompare::CompareType compareType)
{
MOZ_ASSERT(compareType == MCompare::Compare_Double);
Label across;
RegF64 r0, r1;
pop2xF64(&r0, &r1);
RegI32 i0 = needI32();
masm.mov(ImmWord(1), i0.reg);
switch (compareOp) {
case JSOP_EQ:
masm.branchDouble(Assembler::DoubleEqual, r0.reg, r1.reg, &across);
break;
case JSOP_NE:
masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, r0.reg, r1.reg, &across);
break;
case JSOP_LE:
masm.branchDouble(Assembler::DoubleLessThanOrEqual, r0.reg, r1.reg, &across);
break;
case JSOP_LT:
masm.branchDouble(Assembler::DoubleLessThan, r0.reg, r1.reg, &across);
break;
case JSOP_GE:
masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, r0.reg, r1.reg, &across);
break;
case JSOP_GT:
masm.branchDouble(Assembler::DoubleGreaterThan, r0.reg, r1.reg, &across);
break;
default:
MOZ_CRASH("Compiler bug: Unexpected compare opcode");
}
masm.mov(ImmWord(0), i0.reg);
masm.bind(&across);
freeF64(r0);
freeF64(r1);
pushI32(i0);
}
bool
BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType)
{
LinearMemoryAddress<Nothing> addr;
Nothing unused_value;
if (!iter_.readTeeStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
return false;
if (deadCode_)
return true;
// TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
// accesses below the minimum heap length.
MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
size_t temps = loadStoreTemps(access);
RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
if (resultType == ValType::F32 && viewType == Scalar::Float64) {
RegF32 rv = popF32();
RegF64 rw = needF64();
masm.convertFloat32ToDouble(rv.reg, rw.reg);
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
return false;
pushF32(rv);
freeI32(rp);
freeF64(rw);
}
else if (resultType == ValType::F64 && viewType == Scalar::Float32) {
RegF64 rv = popF64();
RegF32 rw = needF32();
masm.convertDoubleToFloat32(rv.reg, rw.reg);
RegI32 rp = popI32();
if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
return false;
pushF64(rv);
freeI32(rp);
freeF32(rw);
}
else
MOZ_CRASH("unexpected coerced store");
if (temps >= 1)
freeI32(tmp1);
if (temps >= 2)
freeI32(tmp2);
return true;
}
bool
BaseCompiler::emitGrowMemory()
{
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
Nothing arg;
if (!iter_.readGrowMemory(&arg))
return false;
if (deadCode_)
return true;
sync();
uint32_t numArgs = 1;
size_t stackSpace = stackConsumed(numArgs);
FunctionCall baselineCall(lineOrBytecode);
beginCall(baselineCall, UseABI::System, InterModule::True);
ABIArg instanceArg = reservePointerArgument(baselineCall);
startCallArgs(baselineCall, stackArgAreaSize(SigI_));
passArg(baselineCall, ValType::I32, peek(0));
builtinInstanceMethodCall(SymbolicAddress::GrowMemory, instanceArg, baselineCall);
endCall(baselineCall);
popValueStackBy(numArgs);
masm.freeStack(stackSpace);
pushReturned(baselineCall, ExprType::I32);
return true;
}
bool
BaseCompiler::emitCurrentMemory()
{
uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
if (!iter_.readCurrentMemory())
return false;
if (deadCode_)
return true;
sync();
FunctionCall baselineCall(lineOrBytecode);
beginCall(baselineCall, UseABI::System, InterModule::False);
ABIArg instanceArg = reservePointerArgument(baselineCall);
startCallArgs(baselineCall, stackArgAreaSize(Sig_));
builtinInstanceMethodCall(SymbolicAddress::CurrentMemory, instanceArg, baselineCall);
endCall(baselineCall);
pushReturned(baselineCall, ExprType::I32);
return true;
}
bool
BaseCompiler::emitBody()
{
uint32_t overhead = 0;
for (;;) {
Nothing unused_a, unused_b;
#define emitBinary(doEmit, type) \
iter_.readBinary(type, &unused_a, &unused_b) && (deadCode_ || (doEmit(), true))
#define emitUnary(doEmit, type) \
iter_.readUnary(type, &unused_a) && (deadCode_ || (doEmit(), true))
#define emitComparison(doEmit, operandType, compareOp, compareType) \
iter_.readComparison(operandType, &unused_a, &unused_b) && \
(deadCode_ || (doEmit(compareOp, compareType), true))
#define emitConversion(doEmit, inType, outType) \
iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || (doEmit(), true))
#define emitConversionOOM(doEmit, inType, outType) \
iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || doEmit())
#define emitCalloutConversionOOM(doEmit, symbol, inType, outType) \
iter_.readConversion(inType, outType, &unused_a) && \
(deadCode_ || doEmit(symbol, inType, outType))
#define CHECK(E) if (!(E)) goto done
#define NEXT() continue
#define CHECK_NEXT(E) if (!(E)) goto done; continue
// TODO / EVALUATE (bug 1316845): Not obvious that this attempt at
// reducing overhead is really paying off relative to making the check
// every iteration.
if (overhead == 0) {
// Check every 50 expressions -- a happy medium between
// memory usage and checking overhead.
overhead = 50;
// Checking every 50 expressions should be safe, as the
// baseline JIT does very little allocation per expression.
CHECK(alloc_.ensureBallast());
// The pushiest opcode is LOOP, which pushes two values
// per instance.
CHECK(stk_.reserve(stk_.length() + overhead * 2));
}
overhead--;
if (done())
return true;
uint16_t op;
CHECK(iter_.readOp(&op));
switch (op) {
// Control opcodes
case uint16_t(Op::Nop):
CHECK(iter_.readNop());
NEXT();
case uint16_t(Op::Drop):
CHECK_NEXT(emitDrop());
case uint16_t(Op::Block):
CHECK_NEXT(emitBlock());
case uint16_t(Op::Loop):
CHECK_NEXT(emitLoop());
case uint16_t(Op::If):
CHECK_NEXT(emitIf());
case uint16_t(Op::Else):
CHECK_NEXT(emitElse());
case uint16_t(Op::End):
CHECK_NEXT(emitEnd());
case uint16_t(Op::Br):
CHECK_NEXT(emitBr());
case uint16_t(Op::BrIf):
CHECK_NEXT(emitBrIf());
case uint16_t(Op::BrTable):
CHECK_NEXT(emitBrTable());
case uint16_t(Op::Return):
CHECK_NEXT(emitReturn());
case uint16_t(Op::Unreachable):
CHECK(iter_.readUnreachable());
if (!deadCode_) {
unreachableTrap();
deadCode_ = true;
popValueStackTo(ctl_.back().stackSize);
}
NEXT();
// Calls
case uint16_t(Op::Call):
CHECK_NEXT(emitCall());
case uint16_t(Op::CallIndirect):
CHECK_NEXT(emitCallIndirect(/* oldStyle = */ false));
case uint16_t(Op::OldCallIndirect):
CHECK_NEXT(emitCallIndirect(/* oldStyle = */ true));
// Locals and globals
case uint16_t(Op::GetLocal):
CHECK_NEXT(emitGetLocal());
case uint16_t(Op::SetLocal):
CHECK_NEXT(emitSetLocal());
case uint16_t(Op::TeeLocal):
CHECK_NEXT(emitTeeLocal());
case uint16_t(Op::GetGlobal):
CHECK_NEXT(emitGetGlobal());
case uint16_t(Op::SetGlobal):
CHECK_NEXT(emitSetGlobal());
case uint16_t(Op::TeeGlobal):
CHECK_NEXT(emitTeeGlobal());
// Select
case uint16_t(Op::Select):
CHECK_NEXT(emitSelect());
// I32
case uint16_t(Op::I32Const): {
int32_t i32;
CHECK(iter_.readI32Const(&i32));
if (!deadCode_)
pushI32(i32);
NEXT();
}
case uint16_t(Op::I32Add):
CHECK_NEXT(emitBinary(emitAddI32, ValType::I32));
case uint16_t(Op::I32Sub):
CHECK_NEXT(emitBinary(emitSubtractI32, ValType::I32));
case uint16_t(Op::I32Mul):
CHECK_NEXT(emitBinary(emitMultiplyI32, ValType::I32));
case uint16_t(Op::I32DivS):
CHECK_NEXT(emitBinary(emitQuotientI32, ValType::I32));
case uint16_t(Op::I32DivU):
CHECK_NEXT(emitBinary(emitQuotientU32, ValType::I32));
case uint16_t(Op::I32RemS):
CHECK_NEXT(emitBinary(emitRemainderI32, ValType::I32));
case uint16_t(Op::I32RemU):
CHECK_NEXT(emitBinary(emitRemainderU32, ValType::I32));
case uint16_t(Op::I32Min):
CHECK_NEXT(emitBinary(emitMinI32, ValType::I32));
case uint16_t(Op::I32Max):
CHECK_NEXT(emitBinary(emitMaxI32, ValType::I32));
case uint16_t(Op::I32Eqz):
CHECK_NEXT(emitConversion(emitEqzI32, ValType::I32, ValType::I32));
case uint16_t(Op::I32TruncSF32):
CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI32<false>, ValType::F32, ValType::I32));
case uint16_t(Op::I32TruncUF32):
CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI32<true>, ValType::F32, ValType::I32));
case uint16_t(Op::I32TruncSF64):
CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI32<false>, ValType::F64, ValType::I32));
case uint16_t(Op::I32TruncUF64):
CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI32<true>, ValType::F64, ValType::I32));
case uint16_t(Op::I32WrapI64):
CHECK_NEXT(emitConversion(emitWrapI64ToI32, ValType::I64, ValType::I32));
case uint16_t(Op::I32ReinterpretF32):
CHECK_NEXT(emitConversion(emitReinterpretF32AsI32, ValType::F32, ValType::I32));
case uint16_t(Op::I32Clz):
CHECK_NEXT(emitUnary(emitClzI32, ValType::I32));
case uint16_t(Op::I32Ctz):
CHECK_NEXT(emitUnary(emitCtzI32, ValType::I32));
case uint16_t(Op::I32Popcnt):
CHECK_NEXT(emitUnary(emitPopcntI32, ValType::I32));
case uint16_t(Op::I32Abs):
CHECK_NEXT(emitUnary(emitAbsI32, ValType::I32));
case uint16_t(Op::I32Neg):
CHECK_NEXT(emitUnary(emitNegateI32, ValType::I32));
case uint16_t(Op::I32Or):
CHECK_NEXT(emitBinary(emitOrI32, ValType::I32));
case uint16_t(Op::I32And):
CHECK_NEXT(emitBinary(emitAndI32, ValType::I32));
case uint16_t(Op::I32Xor):
CHECK_NEXT(emitBinary(emitXorI32, ValType::I32));
case uint16_t(Op::I32Shl):
CHECK_NEXT(emitBinary(emitShlI32, ValType::I32));
case uint16_t(Op::I32ShrS):
CHECK_NEXT(emitBinary(emitShrI32, ValType::I32));
case uint16_t(Op::I32ShrU):
CHECK_NEXT(emitBinary(emitShrU32, ValType::I32));
case uint16_t(Op::I32BitNot):
CHECK_NEXT(emitUnary(emitBitNotI32, ValType::I32));
case uint16_t(Op::I32Load8S):
CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Load8U):
CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint8));
case uint16_t(Op::I32Load16S):
CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int16));
case uint16_t(Op::I32Load16U):
CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint16));
case uint16_t(Op::I32Load):
CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int32));
case uint16_t(Op::I32Store8):
CHECK_NEXT(emitStore(ValType::I32, Scalar::Int8));
case uint16_t(Op::I32TeeStore8):
CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int8));
case uint16_t(Op::I32Store16):
CHECK_NEXT(emitStore(ValType::I32, Scalar::Int16));
case uint16_t(Op::I32TeeStore16):
CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int16));
case uint16_t(Op::I32Store):
CHECK_NEXT(emitStore(ValType::I32, Scalar::Int32));
case uint16_t(Op::I32TeeStore):
CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int32));
case uint16_t(Op::I32Rotr):
CHECK_NEXT(emitBinary(emitRotrI32, ValType::I32));
case uint16_t(Op::I32Rotl):
CHECK_NEXT(emitBinary(emitRotlI32, ValType::I32));
// I64
case uint16_t(Op::I64Const): {
int64_t i64;
CHECK(iter_.readI64Const(&i64));
if (!deadCode_)
pushI64(i64);
NEXT();
}
case uint16_t(Op::I64Add):
CHECK_NEXT(emitBinary(emitAddI64, ValType::I64));
case uint16_t(Op::I64Sub):
CHECK_NEXT(emitBinary(emitSubtractI64, ValType::I64));
case uint16_t(Op::I64Mul):
CHECK_NEXT(emitBinary(emitMultiplyI64, ValType::I64));
case uint16_t(Op::I64DivS):
#ifdef INT_DIV_I64_CALLOUT
CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::DivI64, ValType::I64));
#else
CHECK_NEXT(emitBinary(emitQuotientI64, ValType::I64));
#endif
case uint16_t(Op::I64DivU):
#ifdef INT_DIV_I64_CALLOUT
CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::UDivI64, ValType::I64));
#else
CHECK_NEXT(emitBinary(emitQuotientU64, ValType::I64));
#endif
case uint16_t(Op::I64RemS):
#ifdef INT_DIV_I64_CALLOUT
CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::ModI64, ValType::I64));
#else
CHECK_NEXT(emitBinary(emitRemainderI64, ValType::I64));
#endif
case uint16_t(Op::I64RemU):
#ifdef INT_DIV_I64_CALLOUT
CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::UModI64, ValType::I64));
#else
CHECK_NEXT(emitBinary(emitRemainderU64, ValType::I64));
#endif
case uint16_t(Op::I64TruncSF32):
#ifdef FLOAT_TO_I64_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
SymbolicAddress::TruncateDoubleToInt64,
ValType::F32, ValType::I64));
#else
CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI64<false>, ValType::F32, ValType::I64));
#endif
case uint16_t(Op::I64TruncUF32):
#ifdef FLOAT_TO_I64_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
SymbolicAddress::TruncateDoubleToUint64,
ValType::F32, ValType::I64));
#else
CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI64<true>, ValType::F32, ValType::I64));
#endif
case uint16_t(Op::I64TruncSF64):
#ifdef FLOAT_TO_I64_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
SymbolicAddress::TruncateDoubleToInt64,
ValType::F64, ValType::I64));
#else
CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI64<false>, ValType::F64, ValType::I64));
#endif
case uint16_t(Op::I64TruncUF64):
#ifdef FLOAT_TO_I64_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
SymbolicAddress::TruncateDoubleToUint64,
ValType::F64, ValType::I64));
#else
CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI64<true>, ValType::F64, ValType::I64));
#endif
case uint16_t(Op::I64ExtendSI32):
CHECK_NEXT(emitConversion(emitExtendI32ToI64, ValType::I32, ValType::I64));
case uint16_t(Op::I64ExtendUI32):
CHECK_NEXT(emitConversion(emitExtendU32ToI64, ValType::I32, ValType::I64));
case uint16_t(Op::I64ReinterpretF64):
CHECK_NEXT(emitConversion(emitReinterpretF64AsI64, ValType::F64, ValType::I64));
case uint16_t(Op::I64Or):
CHECK_NEXT(emitBinary(emitOrI64, ValType::I64));
case uint16_t(Op::I64And):
CHECK_NEXT(emitBinary(emitAndI64, ValType::I64));
case uint16_t(Op::I64Xor):
CHECK_NEXT(emitBinary(emitXorI64, ValType::I64));
case uint16_t(Op::I64Shl):
CHECK_NEXT(emitBinary(emitShlI64, ValType::I64));
case uint16_t(Op::I64ShrS):
CHECK_NEXT(emitBinary(emitShrI64, ValType::I64));
case uint16_t(Op::I64ShrU):
CHECK_NEXT(emitBinary(emitShrU64, ValType::I64));
case uint16_t(Op::I64Rotr):
CHECK_NEXT(emitBinary(emitRotrI64, ValType::I64));
case uint16_t(Op::I64Rotl):
CHECK_NEXT(emitBinary(emitRotlI64, ValType::I64));
case uint16_t(Op::I64Clz):
CHECK_NEXT(emitUnary(emitClzI64, ValType::I64));
case uint16_t(Op::I64Ctz):
CHECK_NEXT(emitUnary(emitCtzI64, ValType::I64));
case uint16_t(Op::I64Popcnt):
CHECK_NEXT(emitUnary(emitPopcntI64, ValType::I64));
case uint16_t(Op::I64Eqz):
CHECK_NEXT(emitConversion(emitEqzI64, ValType::I64, ValType::I32));
case uint16_t(Op::I64Load8S):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Load16S):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Load32S):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int32));
case uint16_t(Op::I64Load8U):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint8));
case uint16_t(Op::I64Load16U):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint16));
case uint16_t(Op::I64Load32U):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint32));
case uint16_t(Op::I64Load):
CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int64));
case uint16_t(Op::I64Store8):
CHECK_NEXT(emitStore(ValType::I64, Scalar::Int8));
case uint16_t(Op::I64TeeStore8):
CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int8));
case uint16_t(Op::I64Store16):
CHECK_NEXT(emitStore(ValType::I64, Scalar::Int16));
case uint16_t(Op::I64TeeStore16):
CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int16));
case uint16_t(Op::I64Store32):
CHECK_NEXT(emitStore(ValType::I64, Scalar::Int32));
case uint16_t(Op::I64TeeStore32):
CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int32));
case uint16_t(Op::I64Store):
CHECK_NEXT(emitStore(ValType::I64, Scalar::Int64));
case uint16_t(Op::I64TeeStore):
CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int64));
// F32
case uint16_t(Op::F32Const): {
RawF32 f32;
CHECK(iter_.readF32Const(&f32));
if (!deadCode_)
pushF32(f32);
NEXT();
}
case uint16_t(Op::F32Add):
CHECK_NEXT(emitBinary(emitAddF32, ValType::F32));
case uint16_t(Op::F32Sub):
CHECK_NEXT(emitBinary(emitSubtractF32, ValType::F32));
case uint16_t(Op::F32Mul):
CHECK_NEXT(emitBinary(emitMultiplyF32, ValType::F32));
case uint16_t(Op::F32Div):
CHECK_NEXT(emitBinary(emitDivideF32, ValType::F32));
case uint16_t(Op::F32Min):
CHECK_NEXT(emitBinary(emitMinF32, ValType::F32));
case uint16_t(Op::F32Max):
CHECK_NEXT(emitBinary(emitMaxF32, ValType::F32));
case uint16_t(Op::F32Neg):
CHECK_NEXT(emitUnary(emitNegateF32, ValType::F32));
case uint16_t(Op::F32Abs):
CHECK_NEXT(emitUnary(emitAbsF32, ValType::F32));
case uint16_t(Op::F32Sqrt):
CHECK_NEXT(emitUnary(emitSqrtF32, ValType::F32));
case uint16_t(Op::F32Ceil):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CeilF, ValType::F32));
case uint16_t(Op::F32Floor):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::FloorF, ValType::F32));
case uint16_t(Op::F32DemoteF64):
CHECK_NEXT(emitConversion(emitConvertF64ToF32, ValType::F64, ValType::F32));
case uint16_t(Op::F32ConvertSI32):
CHECK_NEXT(emitConversion(emitConvertI32ToF32, ValType::I32, ValType::F32));
case uint16_t(Op::F32ConvertUI32):
CHECK_NEXT(emitConversion(emitConvertU32ToF32, ValType::I32, ValType::F32));
case uint16_t(Op::F32ConvertSI64):
#ifdef I64_TO_FLOAT_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
SymbolicAddress::Int64ToFloatingPoint,
ValType::I64, ValType::F32));
#else
CHECK_NEXT(emitConversion(emitConvertI64ToF32, ValType::I64, ValType::F32));
#endif
case uint16_t(Op::F32ConvertUI64):
#ifdef I64_TO_FLOAT_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
SymbolicAddress::Uint64ToFloatingPoint,
ValType::I64, ValType::F32));
#else
CHECK_NEXT(emitConversion(emitConvertU64ToF32, ValType::I64, ValType::F32));
#endif
case uint16_t(Op::F32ReinterpretI32):
CHECK_NEXT(emitConversion(emitReinterpretI32AsF32, ValType::I32, ValType::F32));
case uint16_t(Op::F32Load):
CHECK_NEXT(emitLoad(ValType::F32, Scalar::Float32));
case uint16_t(Op::F32Store):
CHECK_NEXT(emitStore(ValType::F32, Scalar::Float32));
case uint16_t(Op::F32TeeStore):
CHECK_NEXT(emitTeeStore(ValType::F32, Scalar::Float32));
case uint16_t(Op::F32TeeStoreF64):
CHECK_NEXT(emitTeeStoreWithCoercion(ValType::F32, Scalar::Float64));
case uint16_t(Op::F32CopySign):
CHECK_NEXT(emitBinary(emitCopysignF32, ValType::F32));
case uint16_t(Op::F32Nearest):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntF, ValType::F32));
case uint16_t(Op::F32Trunc):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TruncF, ValType::F32));
// F64
case uint16_t(Op::F64Const): {
RawF64 f64;
CHECK(iter_.readF64Const(&f64));
if (!deadCode_)
pushF64(f64);
NEXT();
}
case uint16_t(Op::F64Add):
CHECK_NEXT(emitBinary(emitAddF64, ValType::F64));
case uint16_t(Op::F64Sub):
CHECK_NEXT(emitBinary(emitSubtractF64, ValType::F64));
case uint16_t(Op::F64Mul):
CHECK_NEXT(emitBinary(emitMultiplyF64, ValType::F64));
case uint16_t(Op::F64Div):
CHECK_NEXT(emitBinary(emitDivideF64, ValType::F64));
case uint16_t(Op::F64Mod):
CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::ModD, ValType::F64));
case uint16_t(Op::F64Min):
CHECK_NEXT(emitBinary(emitMinF64, ValType::F64));
case uint16_t(Op::F64Max):
CHECK_NEXT(emitBinary(emitMaxF64, ValType::F64));
case uint16_t(Op::F64Neg):
CHECK_NEXT(emitUnary(emitNegateF64, ValType::F64));
case uint16_t(Op::F64Abs):
CHECK_NEXT(emitUnary(emitAbsF64, ValType::F64));
case uint16_t(Op::F64Sqrt):
CHECK_NEXT(emitUnary(emitSqrtF64, ValType::F64));
case uint16_t(Op::F64Ceil):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CeilD, ValType::F64));
case uint16_t(Op::F64Floor):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::FloorD, ValType::F64));
case uint16_t(Op::F64Sin):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::SinD, ValType::F64));
case uint16_t(Op::F64Cos):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CosD, ValType::F64));
case uint16_t(Op::F64Tan):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TanD, ValType::F64));
case uint16_t(Op::F64Asin):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ASinD, ValType::F64));
case uint16_t(Op::F64Acos):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ACosD, ValType::F64));
case uint16_t(Op::F64Atan):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ATanD, ValType::F64));
case uint16_t(Op::F64Exp):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ExpD, ValType::F64));
case uint16_t(Op::F64Log):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::LogD, ValType::F64));
case uint16_t(Op::F64Pow):
CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::PowD, ValType::F64));
case uint16_t(Op::F64Atan2):
CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::ATan2D, ValType::F64));
case uint16_t(Op::F64PromoteF32):
CHECK_NEXT(emitConversion(emitConvertF32ToF64, ValType::F32, ValType::F64));
case uint16_t(Op::F64ConvertSI32):
CHECK_NEXT(emitConversion(emitConvertI32ToF64, ValType::I32, ValType::F64));
case uint16_t(Op::F64ConvertUI32):
CHECK_NEXT(emitConversion(emitConvertU32ToF64, ValType::I32, ValType::F64));
case uint16_t(Op::F64ConvertSI64):
#ifdef I64_TO_FLOAT_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
SymbolicAddress::Int64ToFloatingPoint,
ValType::I64, ValType::F64));
#else
CHECK_NEXT(emitConversion(emitConvertI64ToF64, ValType::I64, ValType::F64));
#endif
case uint16_t(Op::F64ConvertUI64):
#ifdef I64_TO_FLOAT_CALLOUT
CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
SymbolicAddress::Uint64ToFloatingPoint,
ValType::I64, ValType::F64));
#else
CHECK_NEXT(emitConversion(emitConvertU64ToF64, ValType::I64, ValType::F64));
#endif
case uint16_t(Op::F64Load):
CHECK_NEXT(emitLoad(ValType::F64, Scalar::Float64));
case uint16_t(Op::F64Store):
CHECK_NEXT(emitStore(ValType::F64, Scalar::Float64));
case uint16_t(Op::F64TeeStore):
CHECK_NEXT(emitTeeStore(ValType::F64, Scalar::Float64));
case uint16_t(Op::F64TeeStoreF32):
CHECK_NEXT(emitTeeStoreWithCoercion(ValType::F64, Scalar::Float32));
case uint16_t(Op::F64ReinterpretI64):
CHECK_NEXT(emitConversion(emitReinterpretI64AsF64, ValType::I64, ValType::F64));
case uint16_t(Op::F64CopySign):
CHECK_NEXT(emitBinary(emitCopysignF64, ValType::F64));
case uint16_t(Op::F64Nearest):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntD, ValType::F64));
case uint16_t(Op::F64Trunc):
CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TruncD, ValType::F64));
// Comparisons
case uint16_t(Op::I32Eq):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_EQ, MCompare::Compare_Int32));
case uint16_t(Op::I32Ne):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_NE, MCompare::Compare_Int32));
case uint16_t(Op::I32LtS):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LT, MCompare::Compare_Int32));
case uint16_t(Op::I32LeS):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LE, MCompare::Compare_Int32));
case uint16_t(Op::I32GtS):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GT, MCompare::Compare_Int32));
case uint16_t(Op::I32GeS):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GE, MCompare::Compare_Int32));
case uint16_t(Op::I32LtU):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LT, MCompare::Compare_UInt32));
case uint16_t(Op::I32LeU):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LE, MCompare::Compare_UInt32));
case uint16_t(Op::I32GtU):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GT, MCompare::Compare_UInt32));
case uint16_t(Op::I32GeU):
CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GE, MCompare::Compare_UInt32));
case uint16_t(Op::I64Eq):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_EQ, MCompare::Compare_Int64));
case uint16_t(Op::I64Ne):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_NE, MCompare::Compare_Int64));
case uint16_t(Op::I64LtS):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LT, MCompare::Compare_Int64));
case uint16_t(Op::I64LeS):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LE, MCompare::Compare_Int64));
case uint16_t(Op::I64GtS):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GT, MCompare::Compare_Int64));
case uint16_t(Op::I64GeS):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GE, MCompare::Compare_Int64));
case uint16_t(Op::I64LtU):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LT, MCompare::Compare_UInt64));
case uint16_t(Op::I64LeU):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LE, MCompare::Compare_UInt64));
case uint16_t(Op::I64GtU):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GT, MCompare::Compare_UInt64));
case uint16_t(Op::I64GeU):
CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GE, MCompare::Compare_UInt64));
case uint16_t(Op::F32Eq):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_EQ, MCompare::Compare_Float32));
case uint16_t(Op::F32Ne):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_NE, MCompare::Compare_Float32));
case uint16_t(Op::F32Lt):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_LT, MCompare::Compare_Float32));
case uint16_t(Op::F32Le):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_LE, MCompare::Compare_Float32));
case uint16_t(Op::F32Gt):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_GT, MCompare::Compare_Float32));
case uint16_t(Op::F32Ge):
CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_GE, MCompare::Compare_Float32));
case uint16_t(Op::F64Eq):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_EQ, MCompare::Compare_Double));
case uint16_t(Op::F64Ne):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_NE, MCompare::Compare_Double));
case uint16_t(Op::F64Lt):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_LT, MCompare::Compare_Double));
case uint16_t(Op::F64Le):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_LE, MCompare::Compare_Double));
case uint16_t(Op::F64Gt):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_GT, MCompare::Compare_Double));
case uint16_t(Op::F64Ge):
CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_GE, MCompare::Compare_Double));
// SIMD
#define CASE(TYPE, OP, SIGN) \
case uint16_t(Op::TYPE##OP): \
MOZ_CRASH("Unimplemented SIMD");
#define I8x16CASE(OP) CASE(I8x16, OP, SimdSign::Signed)
#define I16x8CASE(OP) CASE(I16x8, OP, SimdSign::Signed)
#define I32x4CASE(OP) CASE(I32x4, OP, SimdSign::Signed)
#define F32x4CASE(OP) CASE(F32x4, OP, SimdSign::NotApplicable)
#define B8x16CASE(OP) CASE(B8x16, OP, SimdSign::NotApplicable)
#define B16x8CASE(OP) CASE(B16x8, OP, SimdSign::NotApplicable)
#define B32x4CASE(OP) CASE(B32x4, OP, SimdSign::NotApplicable)
#define ENUMERATE(TYPE, FORALL, DO) \
case uint16_t(Op::TYPE##Constructor): \
FORALL(DO)
ENUMERATE(I8x16, FORALL_INT8X16_ASMJS_OP, I8x16CASE)
ENUMERATE(I16x8, FORALL_INT16X8_ASMJS_OP, I16x8CASE)
ENUMERATE(I32x4, FORALL_INT32X4_ASMJS_OP, I32x4CASE)
ENUMERATE(F32x4, FORALL_FLOAT32X4_ASMJS_OP, F32x4CASE)
ENUMERATE(B8x16, FORALL_BOOL_SIMD_OP, B8x16CASE)
ENUMERATE(B16x8, FORALL_BOOL_SIMD_OP, B16x8CASE)
ENUMERATE(B32x4, FORALL_BOOL_SIMD_OP, B32x4CASE)
#undef CASE
#undef I8x16CASE
#undef I16x8CASE
#undef I32x4CASE
#undef F32x4CASE
#undef B8x16CASE
#undef B16x8CASE
#undef B32x4CASE
#undef ENUMERATE
case uint16_t(Op::I8x16Const):
case uint16_t(Op::I16x8Const):
case uint16_t(Op::I32x4Const):
case uint16_t(Op::F32x4Const):
case uint16_t(Op::B8x16Const):
case uint16_t(Op::B16x8Const):
case uint16_t(Op::B32x4Const):
case uint16_t(Op::I32x4shiftRightByScalarU):
case uint16_t(Op::I8x16addSaturateU):
case uint16_t(Op::I8x16subSaturateU):
case uint16_t(Op::I8x16shiftRightByScalarU):
case uint16_t(Op::I8x16lessThanU):
case uint16_t(Op::I8x16lessThanOrEqualU):
case uint16_t(Op::I8x16greaterThanU):
case uint16_t(Op::I8x16greaterThanOrEqualU):
case uint16_t(Op::I8x16extractLaneU):
case uint16_t(Op::I16x8addSaturateU):
case uint16_t(Op::I16x8subSaturateU):
case uint16_t(Op::I16x8shiftRightByScalarU):
case uint16_t(Op::I16x8lessThanU):
case uint16_t(Op::I16x8lessThanOrEqualU):
case uint16_t(Op::I16x8greaterThanU):
case uint16_t(Op::I16x8greaterThanOrEqualU):
case uint16_t(Op::I16x8extractLaneU):
case uint16_t(Op::I32x4lessThanU):
case uint16_t(Op::I32x4lessThanOrEqualU):
case uint16_t(Op::I32x4greaterThanU):
case uint16_t(Op::I32x4greaterThanOrEqualU):
case uint16_t(Op::I32x4fromFloat32x4U):
MOZ_CRASH("Unimplemented SIMD");
// Atomics
case uint16_t(Op::I32AtomicsLoad):
case uint16_t(Op::I32AtomicsStore):
case uint16_t(Op::I32AtomicsBinOp):
case uint16_t(Op::I32AtomicsCompareExchange):
case uint16_t(Op::I32AtomicsExchange):
MOZ_CRASH("Unimplemented Atomics");
// Memory Related
case uint16_t(Op::GrowMemory):
CHECK_NEXT(emitGrowMemory());
case uint16_t(Op::CurrentMemory):
CHECK_NEXT(emitCurrentMemory());
}
MOZ_CRASH("unexpected wasm opcode");
#undef CHECK
#undef NEXT
#undef CHECK_NEXT
#undef emitBinary
#undef emitUnary
#undef emitComparison
#undef emitConversion
#undef emitConversionOOM
#undef emitCalloutConversionOOM
}
done:
return false;
}
bool
BaseCompiler::emitFunction()
{
// emitBody() will ensure that there is enough memory reserved in the
// vector for infallible allocation to succeed within the compiler, but we
// need a little headroom for the initial pushControl(), which pushes a
// void value onto the value stack.
if (!stk_.reserve(8))
return false;
const Sig& sig = func_.sig();
if (!iter_.readFunctionStart(sig.ret()))
return false;
beginFunction();
UniquePooledLabel functionEnd(newLabel());
if (!pushControl(&functionEnd))
return false;
if (!emitBody())
return false;
if (!iter_.readFunctionEnd())
return false;
if (!endFunction())
return false;
return true;
}
BaseCompiler::BaseCompiler(const ModuleGeneratorData& mg,
Decoder& decoder,
const FuncBytes& func,
const ValTypeVector& locals,
FuncCompileResults& compileResults)
: mg_(mg),
iter_(decoder, func.lineOrBytecode()),
func_(func),
lastReadCallSite_(0),
alloc_(compileResults.alloc()),
locals_(locals),
localSize_(0),
varLow_(0),
varHigh_(0),
maxFramePushed_(0),
deadCode_(false),
prologueTrapOffset_(trapOffset()),
compileResults_(compileResults),
masm(compileResults_.masm()),
availGPR_(GeneralRegisterSet::All()),
availFPU_(FloatRegisterSet::All()),
#ifdef DEBUG
scratchRegisterTaken_(false),
#endif
tlsSlot_(0),
#ifdef JS_CODEGEN_X64
specific_rax(RegI64(Register64(rax))),
specific_rcx(RegI64(Register64(rcx))),
specific_rdx(RegI64(Register64(rdx))),
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
specific_eax(RegI32(eax)),
specific_ecx(RegI32(ecx)),
specific_edx(RegI32(edx)),
#endif
#ifdef JS_CODEGEN_X86
singleByteRegs_(GeneralRegisterSet(Registers::SingleByteRegs)),
abiReturnRegI64(RegI64(Register64(edx, eax))),
#endif
#ifdef JS_CODEGEN_ARM
abiReturnRegI64(ReturnReg64),
#endif
joinRegI32(RegI32(ReturnReg)),
joinRegI64(RegI64(ReturnReg64)),
joinRegF32(RegF32(ReturnFloat32Reg)),
joinRegF64(RegF64(ReturnDoubleReg))
{
// jit/RegisterAllocator.h: RegisterAllocator::RegisterAllocator()
#if defined(JS_CODEGEN_X64)
availGPR_.take(HeapReg);
#elif defined(JS_CODEGEN_ARM)
availGPR_.take(HeapReg);
availGPR_.take(GlobalReg);
availGPR_.take(ScratchRegARM);
#elif defined(JS_CODEGEN_ARM64)
availGPR_.take(HeapReg);
availGPR_.take(HeapLenReg);
availGPR_.take(GlobalReg);
#elif defined(JS_CODEGEN_X86)
availGPR_.take(ScratchRegX86);
#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
availGPR_.take(HeapReg);
availGPR_.take(GlobalReg);
#endif
labelPool_.setAllocator(alloc_);
}
bool
BaseCompiler::init()
{
if (!SigDD_.append(ValType::F64) || !SigDD_.append(ValType::F64))
return false;
if (!SigD_.append(ValType::F64))
return false;
if (!SigF_.append(ValType::F32))
return false;
if (!SigI_.append(ValType::I32))
return false;
if (!SigI64I64_.append(ValType::I64) || !SigI64I64_.append(ValType::I64))
return false;
const ValTypeVector& args = func_.sig().args();
// localInfo_ contains an entry for every local in locals_, followed by
// entries for special locals. Currently the only special local is the TLS
// pointer.
tlsSlot_ = locals_.length();
if (!localInfo_.resize(locals_.length() + 1))
return false;
localSize_ = 0;
for (ABIArgIter<const ValTypeVector> i(args); !i.done(); i++) {
Local& l = localInfo_[i.index()];
switch (i.mirType()) {
case MIRType::Int32:
if (i->argInRegister())
l.init(MIRType::Int32, pushLocal(4));
else
l.init(MIRType::Int32, -(i->offsetFromArgBase() + sizeof(Frame)));
break;
case MIRType::Int64:
if (i->argInRegister())
l.init(MIRType::Int64, pushLocal(8));
else
l.init(MIRType::Int64, -(i->offsetFromArgBase() + sizeof(Frame)));
break;
case MIRType::Double:
if (i->argInRegister())
l.init(MIRType::Double, pushLocal(8));
else
l.init(MIRType::Double, -(i->offsetFromArgBase() + sizeof(Frame)));
break;
case MIRType::Float32:
if (i->argInRegister())
l.init(MIRType::Float32, pushLocal(4));
else
l.init(MIRType::Float32, -(i->offsetFromArgBase() + sizeof(Frame)));
break;
default:
MOZ_CRASH("Argument type");
}
}
// Reserve a stack slot for the TLS pointer outside the varLow..varHigh
// range so it isn't zero-filled like the normal locals.
localInfo_[tlsSlot_].init(MIRType::Pointer, pushLocal(sizeof(void*)));
varLow_ = localSize_;
for (size_t i = args.length(); i < locals_.length(); i++) {
Local& l = localInfo_[i];
switch (locals_[i]) {
case ValType::I32:
l.init(MIRType::Int32, pushLocal(4));
break;
case ValType::F32:
l.init(MIRType::Float32, pushLocal(4));
break;
case ValType::F64:
l.init(MIRType::Double, pushLocal(8));
break;
case ValType::I64:
l.init(MIRType::Int64, pushLocal(8));
break;
default:
MOZ_CRASH("Compiler bug: Unexpected local type");
}
}
varHigh_ = localSize_;
localSize_ = AlignBytes(localSize_, 16u);
addInterruptCheck();
return true;
}
void
BaseCompiler::finish()
{
MOZ_ASSERT(done(), "all bytes must be consumed");
MOZ_ASSERT(func_.callSiteLineNums().length() == lastReadCallSite_);
masm.flushBuffer();
}
static LiveRegisterSet
volatileReturnGPR()
{
GeneralRegisterSet rtn;
rtn.addAllocatable(ReturnReg);
return LiveRegisterSet(RegisterSet::VolatileNot(RegisterSet(rtn, FloatRegisterSet())));
}
LiveRegisterSet BaseCompiler::VolatileReturnGPR = volatileReturnGPR();
} // wasm
} // js
bool
js::wasm::BaselineCanCompile(const FunctionGenerator* fg)
{
// On all platforms we require signals for AsmJS/Wasm.
// If we made it this far we must have signals.
MOZ_RELEASE_ASSERT(wasm::HaveSignalHandlers());
#if defined(JS_CODEGEN_ARM)
// Simplifying assumption: require SDIV and UDIV.
//
// I have no good data on ARM populations allowing me to say that
// X% of devices in the market implement SDIV and UDIV. However,
// they are definitely implemented on the Cortex-A7 and Cortex-A15
// and on all ARMv8 systems.
if (!HasIDIV())
return false;
#endif
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
if (fg->usesAtomics())
return false;
if (fg->usesSimd())
return false;
return true;
#else
return false;
#endif
}
bool
js::wasm::BaselineCompileFunction(IonCompileTask* task)
{
MOZ_ASSERT(task->mode() == IonCompileTask::CompileMode::Baseline);
const FuncBytes& func = task->func();
FuncCompileResults& results = task->results();
Decoder d(func.bytes());
// Build the local types vector.
ValTypeVector locals;
if (!locals.appendAll(func.sig().args()))
return false;
if (!DecodeLocalEntries(d, task->mg().kind, &locals))
return false;
// The MacroAssembler will sometimes access the jitContext.
JitContext jitContext(&results.alloc());
// One-pass baseline compilation.
BaseCompiler f(task->mg(), d, func, locals, results);
if (!f.init())
return false;
if (!f.emitFunction())
return false;
f.finish();
return true;
}
#undef INT_DIV_I64_CALLOUT
#undef I64_TO_FLOAT_CALLOUT
#undef FLOAT_TO_I64_CALLOUT