/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "jit/CodeGenerator.h" #include "mozilla/Assertions.h" #include "mozilla/Attributes.h" #include "mozilla/Casting.h" #include "mozilla/DebugOnly.h" #include "mozilla/EnumeratedArray.h" #include "mozilla/EnumeratedRange.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/ScopeExit.h" #include "mozilla/SizePrintfMacros.h" #include "jslibmath.h" #include "jsmath.h" #include "jsnum.h" #include "jsprf.h" #include "jsstr.h" #include "builtin/Eval.h" #include "builtin/TypedObject.h" #include "gc/Nursery.h" #include "irregexp/NativeRegExpMacroAssembler.h" #include "jit/AtomicOperations.h" #include "jit/BaselineCompiler.h" #include "jit/IonBuilder.h" #include "jit/IonCaches.h" #include "jit/IonOptimizationLevels.h" #include "jit/JitcodeMap.h" #include "jit/JitSpewer.h" #include "jit/Linker.h" #include "jit/Lowering.h" #include "jit/MIRGenerator.h" #include "jit/MoveEmitter.h" #include "jit/RangeAnalysis.h" #include "jit/SharedICHelpers.h" #include "vm/AsyncFunction.h" #include "vm/MatchPairs.h" #include "vm/RegExpObject.h" #include "vm/RegExpStatics.h" #include "vm/TraceLogging.h" #include "vm/Unicode.h" #include "jsboolinlines.h" #include "jit/MacroAssembler-inl.h" #include "jit/shared/CodeGenerator-shared-inl.h" #include "jit/shared/Lowering-shared-inl.h" #include "vm/Interpreter-inl.h" using namespace js; using namespace js::jit; using mozilla::AssertedCast; using mozilla::DebugOnly; using mozilla::FloatingPoint; using mozilla::Maybe; using mozilla::NegativeInfinity; using mozilla::PositiveInfinity; using JS::GenericNaN; namespace js { namespace jit { // This out-of-line cache is used to do a double dispatch including it-self and // the wrapped IonCache. class OutOfLineUpdateCache : public OutOfLineCodeBase<CodeGenerator>, public IonCacheVisitor { private: LInstruction* lir_; size_t cacheIndex_; RepatchLabel entry_; public: OutOfLineUpdateCache(LInstruction* lir, size_t cacheIndex) : lir_(lir), cacheIndex_(cacheIndex) { } void bind(MacroAssembler* masm) { // The binding of the initial jump is done in // CodeGenerator::visitOutOfLineCache. } size_t getCacheIndex() const { return cacheIndex_; } LInstruction* lir() const { return lir_; } RepatchLabel& entry() { return entry_; } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineCache(this); } // ICs' visit functions delegating the work to the CodeGen visit funtions. #define VISIT_CACHE_FUNCTION(op) \ void visit##op##IC(CodeGenerator* codegen) { \ CodeGenerator::DataPtr<op##IC> ic(codegen, getCacheIndex()); \ codegen->visit##op##IC(this, ic); \ } IONCACHE_KIND_LIST(VISIT_CACHE_FUNCTION) #undef VISIT_CACHE_FUNCTION }; // This function is declared here because it needs to instantiate an // OutOfLineUpdateCache, but we want to keep it visible inside the // CodeGeneratorShared such as we can specialize inline caches in function of // the architecture. void CodeGeneratorShared::addCache(LInstruction* lir, size_t cacheIndex) { if (cacheIndex == SIZE_MAX) { masm.setOOM(); return; } DataPtr<IonCache> cache(this, cacheIndex); MInstruction* mir = lir->mirRaw()->toInstruction(); if (mir->resumePoint()) cache->setScriptedLocation(mir->block()->info().script(), mir->resumePoint()->pc()); else cache->setIdempotent(); OutOfLineUpdateCache* ool = new(alloc()) OutOfLineUpdateCache(lir, cacheIndex); addOutOfLineCode(ool, mir); cache->emitInitialJump(masm, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineCache(OutOfLineUpdateCache* ool) { DataPtr<IonCache> cache(this, ool->getCacheIndex()); // Register the location of the OOL path in the IC. cache->setFallbackLabel(masm.labelForPatch()); masm.bind(&ool->entry()); // Dispatch to ICs' accept functions. cache->accept(this, ool); } StringObject* MNewStringObject::templateObj() const { return &templateObj_->as<StringObject>(); } CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm) : CodeGeneratorSpecific(gen, graph, masm) , ionScriptLabels_(gen->alloc()) , scriptCounts_(nullptr) , simdRefreshTemplatesDuringLink_(0) { } CodeGenerator::~CodeGenerator() { MOZ_ASSERT_IF(!gen->compilingWasm(), masm.numSymbolicAccesses() == 0); js_delete(scriptCounts_); } typedef bool (*StringToNumberFn)(ExclusiveContext*, JSString*, double*); static const VMFunction StringToNumberInfo = FunctionInfo<StringToNumberFn>(StringToNumber, "StringToNumber"); void CodeGenerator::visitValueToInt32(LValueToInt32* lir) { ValueOperand operand = ToValue(lir, LValueToInt32::Input); Register output = ToRegister(lir->output()); FloatRegister temp = ToFloatRegister(lir->tempFloat()); MDefinition* input; if (lir->mode() == LValueToInt32::NORMAL) input = lir->mirNormal()->input(); else input = lir->mirTruncate()->input(); Label fails; if (lir->mode() == LValueToInt32::TRUNCATE) { OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir()); // We can only handle strings in truncation contexts, like bitwise // operations. Label* stringEntry; Label* stringRejoin; Register stringReg; if (input->mightBeType(MIRType::String)) { stringReg = ToRegister(lir->temp()); OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(stringReg), StoreFloatRegisterTo(temp)); stringEntry = oolString->entry(); stringRejoin = oolString->rejoin(); } else { stringReg = InvalidReg; stringEntry = nullptr; stringRejoin = nullptr; } masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin, oolDouble->entry(), stringReg, temp, output, &fails); masm.bind(oolDouble->rejoin()); } else { masm.convertValueToInt32(operand, input, temp, output, &fails, lir->mirNormal()->canBeNegativeZero(), lir->mirNormal()->conversion()); } bailoutFrom(&fails, lir->snapshot()); } void CodeGenerator::visitValueToDouble(LValueToDouble* lir) { MToDouble* mir = lir->mir(); ValueOperand operand = ToValue(lir, LValueToDouble::Input); FloatRegister output = ToFloatRegister(lir->output()); Register tag = masm.splitTagForTest(operand); Label isDouble, isInt32, isBool, isNull, isUndefined, done; bool hasBoolean = false, hasNull = false, hasUndefined = false; masm.branchTestDouble(Assembler::Equal, tag, &isDouble); masm.branchTestInt32(Assembler::Equal, tag, &isInt32); if (mir->conversion() != MToFPInstruction::NumbersOnly) { masm.branchTestBoolean(Assembler::Equal, tag, &isBool); masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined); hasBoolean = true; hasUndefined = true; if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) { masm.branchTestNull(Assembler::Equal, tag, &isNull); hasNull = true; } } bailout(lir->snapshot()); if (hasNull) { masm.bind(&isNull); masm.loadConstantDouble(0.0, output); masm.jump(&done); } if (hasUndefined) { masm.bind(&isUndefined); masm.loadConstantDouble(GenericNaN(), output); masm.jump(&done); } if (hasBoolean) { masm.bind(&isBool); masm.boolValueToDouble(operand, output); masm.jump(&done); } masm.bind(&isInt32); masm.int32ValueToDouble(operand, output); masm.jump(&done); masm.bind(&isDouble); masm.unboxDouble(operand, output); masm.bind(&done); } void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) { MToFloat32* mir = lir->mir(); ValueOperand operand = ToValue(lir, LValueToFloat32::Input); FloatRegister output = ToFloatRegister(lir->output()); Register tag = masm.splitTagForTest(operand); Label isDouble, isInt32, isBool, isNull, isUndefined, done; bool hasBoolean = false, hasNull = false, hasUndefined = false; masm.branchTestDouble(Assembler::Equal, tag, &isDouble); masm.branchTestInt32(Assembler::Equal, tag, &isInt32); if (mir->conversion() != MToFPInstruction::NumbersOnly) { masm.branchTestBoolean(Assembler::Equal, tag, &isBool); masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined); hasBoolean = true; hasUndefined = true; if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) { masm.branchTestNull(Assembler::Equal, tag, &isNull); hasNull = true; } } bailout(lir->snapshot()); if (hasNull) { masm.bind(&isNull); masm.loadConstantFloat32(0.0f, output); masm.jump(&done); } if (hasUndefined) { masm.bind(&isUndefined); masm.loadConstantFloat32(float(GenericNaN()), output); masm.jump(&done); } if (hasBoolean) { masm.bind(&isBool); masm.boolValueToFloat32(operand, output); masm.jump(&done); } masm.bind(&isInt32); masm.int32ValueToFloat32(operand, output); masm.jump(&done); masm.bind(&isDouble); // ARM and MIPS may not have a double register available if we've // allocated output as a float32. #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) masm.unboxDouble(operand, ScratchDoubleReg); masm.convertDoubleToFloat32(ScratchDoubleReg, output); #else masm.unboxDouble(operand, output); masm.convertDoubleToFloat32(output, output); #endif masm.bind(&done); } void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) { masm.convertInt32ToDouble(ToRegister(lir->input()), ToFloatRegister(lir->output())); } void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) { masm.convertFloat32ToDouble(ToFloatRegister(lir->input()), ToFloatRegister(lir->output())); } void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) { masm.convertDoubleToFloat32(ToFloatRegister(lir->input()), ToFloatRegister(lir->output())); } void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) { masm.convertInt32ToFloat32(ToRegister(lir->input()), ToFloatRegister(lir->output())); } void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) { Label fail; FloatRegister input = ToFloatRegister(lir->input()); Register output = ToRegister(lir->output()); masm.convertDoubleToInt32(input, output, &fail, lir->mir()->canBeNegativeZero()); bailoutFrom(&fail, lir->snapshot()); } void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) { Label fail; FloatRegister input = ToFloatRegister(lir->input()); Register output = ToRegister(lir->output()); masm.convertFloat32ToInt32(input, output, &fail, lir->mir()->canBeNegativeZero()); bailoutFrom(&fail, lir->snapshot()); } void CodeGenerator::emitOOLTestObject(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined, Register scratch) { saveVolatile(scratch); masm.setupUnalignedABICall(scratch); masm.passABIArg(objreg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::EmulatesUndefined)); masm.storeCallBoolResult(scratch); restoreVolatile(scratch); masm.branchIfTrueBool(scratch, ifEmulatesUndefined); masm.jump(ifDoesntEmulateUndefined); } // Base out-of-line code generator for all tests of the truthiness of an // object, where the object might not be truthy. (Recall that per spec all // objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class // flag to permit objects to look like |undefined| in certain contexts, // including in object truthiness testing.) We check truthiness inline except // when we're testing it on a proxy (or if TI guarantees us that the specified // object will never emulate |undefined|), in which case out-of-line code will // call EmulatesUndefined for a conclusive answer. class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator> { Register objreg_; Register scratch_; Label* ifEmulatesUndefined_; Label* ifDoesntEmulateUndefined_; #ifdef DEBUG bool initialized() { return ifEmulatesUndefined_ != nullptr; } #endif public: OutOfLineTestObject() #ifdef DEBUG : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) #endif { } void accept(CodeGenerator* codegen) final override { MOZ_ASSERT(initialized()); codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_, ifDoesntEmulateUndefined_, scratch_); } // Specify the register where the object to be tested is found, labels to // jump to if the object is truthy or falsy, and a scratch register for // use in the out-of-line path. void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined, Register scratch) { MOZ_ASSERT(!initialized()); MOZ_ASSERT(ifEmulatesUndefined); objreg_ = objreg; scratch_ = scratch; ifEmulatesUndefined_ = ifEmulatesUndefined; ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined; } }; // A subclass of OutOfLineTestObject containing two extra labels, for use when // the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line // code. The user should bind these labels in inline code, and specify them as // targets via setInputAndTargets, as appropriate. class OutOfLineTestObjectWithLabels : public OutOfLineTestObject { Label label1_; Label label2_; public: OutOfLineTestObjectWithLabels() { } Label* label1() { return &label1_; } Label* label2() { return &label2_; } }; void CodeGenerator::testObjectEmulatesUndefinedKernel(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined, Register scratch, OutOfLineTestObject* ool) { ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch); // Perform a fast-path check of the object's class flags if the object's // not a proxy. Let out-of-line code handle the slow cases that require // saving registers, making a function call, and restoring registers. masm.branchTestObjectTruthy(false, objreg, scratch, ool->entry(), ifEmulatesUndefined); } void CodeGenerator::branchTestObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined, Register scratch, OutOfLineTestObject* ool) { MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(), "ifDoesntEmulateUndefined will be bound to the fallthrough path"); testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch, ool); masm.bind(ifDoesntEmulateUndefined); } void CodeGenerator::testObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined, Register scratch, OutOfLineTestObject* ool) { testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch, ool); masm.jump(ifDoesntEmulateUndefined); } void CodeGenerator::testValueTruthyKernel(const ValueOperand& value, const LDefinition* scratch1, const LDefinition* scratch2, FloatRegister fr, Label* ifTruthy, Label* ifFalsy, OutOfLineTestObject* ool, MDefinition* valueMIR) { // Count the number of possible type tags we might have, so we'll know when // we've checked them all and hence can avoid emitting a tag check for the // last one. In particular, whenever tagCount is 1 that means we've tried // all but one of them already so we know exactly what's left based on the // mightBe* booleans. bool mightBeUndefined = valueMIR->mightBeType(MIRType::Undefined); bool mightBeNull = valueMIR->mightBeType(MIRType::Null); bool mightBeBoolean = valueMIR->mightBeType(MIRType::Boolean); bool mightBeInt32 = valueMIR->mightBeType(MIRType::Int32); bool mightBeObject = valueMIR->mightBeType(MIRType::Object); bool mightBeString = valueMIR->mightBeType(MIRType::String); bool mightBeSymbol = valueMIR->mightBeType(MIRType::Symbol); bool mightBeDouble = valueMIR->mightBeType(MIRType::Double); int tagCount = int(mightBeUndefined) + int(mightBeNull) + int(mightBeBoolean) + int(mightBeInt32) + int(mightBeObject) + int(mightBeString) + int(mightBeSymbol) + int(mightBeDouble); MOZ_ASSERT_IF(!valueMIR->emptyResultTypeSet(), tagCount > 0); // If we know we're null or undefined, we're definitely falsy, no // need to even check the tag. if (int(mightBeNull) + int(mightBeUndefined) == tagCount) { masm.jump(ifFalsy); return; } Register tag = masm.splitTagForTest(value); if (mightBeUndefined) { MOZ_ASSERT(tagCount > 1); masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy); --tagCount; } if (mightBeNull) { MOZ_ASSERT(tagCount > 1); masm.branchTestNull(Assembler::Equal, tag, ifFalsy); --tagCount; } if (mightBeBoolean) { MOZ_ASSERT(tagCount != 0); Label notBoolean; if (tagCount != 1) masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean); masm.branchTestBooleanTruthy(false, value, ifFalsy); if (tagCount != 1) masm.jump(ifTruthy); // Else just fall through to truthiness. masm.bind(¬Boolean); --tagCount; } if (mightBeInt32) { MOZ_ASSERT(tagCount != 0); Label notInt32; if (tagCount != 1) masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32); masm.branchTestInt32Truthy(false, value, ifFalsy); if (tagCount != 1) masm.jump(ifTruthy); // Else just fall through to truthiness. masm.bind(¬Int32); --tagCount; } if (mightBeObject) { MOZ_ASSERT(tagCount != 0); if (ool) { Label notObject; if (tagCount != 1) masm.branchTestObject(Assembler::NotEqual, tag, ¬Object); Register objreg = masm.extractObject(value, ToRegister(scratch1)); testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, ToRegister(scratch2), ool); masm.bind(¬Object); } else { if (tagCount != 1) masm.branchTestObject(Assembler::Equal, tag, ifTruthy); // Else just fall through to truthiness. } --tagCount; } else { MOZ_ASSERT(!ool, "We better not have an unused OOL path, since the code generator will try to " "generate code for it but we never set up its labels, which will cause null " "derefs of those labels."); } if (mightBeString) { // Test if a string is non-empty. MOZ_ASSERT(tagCount != 0); Label notString; if (tagCount != 1) masm.branchTestString(Assembler::NotEqual, tag, ¬String); masm.branchTestStringTruthy(false, value, ifFalsy); if (tagCount != 1) masm.jump(ifTruthy); // Else just fall through to truthiness. masm.bind(¬String); --tagCount; } if (mightBeSymbol) { // All symbols are truthy. MOZ_ASSERT(tagCount != 0); if (tagCount != 1) masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy); // Else fall through to ifTruthy. --tagCount; } if (mightBeDouble) { MOZ_ASSERT(tagCount == 1); // If we reach here the value is a double. masm.unboxDouble(value, fr); masm.branchTestDoubleTruthy(false, fr, ifFalsy); --tagCount; } MOZ_ASSERT(tagCount == 0); // Fall through for truthy. } void CodeGenerator::testValueTruthy(const ValueOperand& value, const LDefinition* scratch1, const LDefinition* scratch2, FloatRegister fr, Label* ifTruthy, Label* ifFalsy, OutOfLineTestObject* ool, MDefinition* valueMIR) { testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool, valueMIR); masm.jump(ifTruthy); } void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) { MIRType inputType = lir->mir()->input()->type(); MOZ_ASSERT(inputType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(), "If the object couldn't emulate undefined, this should have been folded."); Label* truthy = getJumpLabelForBranch(lir->ifTruthy()); Label* falsy = getJumpLabelForBranch(lir->ifFalsy()); Register input = ToRegister(lir->input()); if (lir->mir()->operandMightEmulateUndefined()) { if (inputType == MIRType::ObjectOrNull) masm.branchTestPtr(Assembler::Zero, input, input, falsy); OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject(); addOutOfLineCode(ool, lir->mir()); testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()), ool); } else { MOZ_ASSERT(inputType == MIRType::ObjectOrNull); testZeroEmitBranch(Assembler::NotEqual, input, lir->ifTruthy(), lir->ifFalsy()); } } void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) { OutOfLineTestObject* ool = nullptr; MDefinition* input = lir->mir()->input(); // Unfortunately, it's possible that someone (e.g. phi elimination) switched // out our input after we did cacheOperandMightEmulateUndefined. So we // might think it can emulate undefined _and_ know that it can't be an // object. if (lir->mir()->operandMightEmulateUndefined() && input->mightBeType(MIRType::Object)) { ool = new(alloc()) OutOfLineTestObject(); addOutOfLineCode(ool, lir->mir()); } Label* truthy = getJumpLabelForBranch(lir->ifTruthy()); Label* falsy = getJumpLabelForBranch(lir->ifFalsy()); testValueTruthy(ToValue(lir, LTestVAndBranch::Input), lir->temp1(), lir->temp2(), ToFloatRegister(lir->tempFloat()), truthy, falsy, ool, input); } void CodeGenerator::visitFunctionDispatch(LFunctionDispatch* lir) { MFunctionDispatch* mir = lir->mir(); Register input = ToRegister(lir->input()); Label* lastLabel; size_t casesWithFallback; // Determine if the last case is fallback or an ordinary case. if (!mir->hasFallback()) { MOZ_ASSERT(mir->numCases() > 0); casesWithFallback = mir->numCases(); lastLabel = skipTrivialBlocks(mir->getCaseBlock(mir->numCases() - 1))->lir()->label(); } else { casesWithFallback = mir->numCases() + 1; lastLabel = skipTrivialBlocks(mir->getFallback())->lir()->label(); } // Compare function pointers, except for the last case. for (size_t i = 0; i < casesWithFallback - 1; i++) { MOZ_ASSERT(i < mir->numCases()); LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir(); if (ObjectGroup* funcGroup = mir->getCaseObjectGroup(i)) { masm.branchPtr(Assembler::Equal, Address(input, JSObject::offsetOfGroup()), ImmGCPtr(funcGroup), target->label()); } else { JSFunction* func = mir->getCase(i); masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label()); } } // Jump to the last case. masm.jump(lastLabel); } void CodeGenerator::visitObjectGroupDispatch(LObjectGroupDispatch* lir) { MObjectGroupDispatch* mir = lir->mir(); Register input = ToRegister(lir->input()); Register temp = ToRegister(lir->temp()); // Load the incoming ObjectGroup in temp. masm.loadPtr(Address(input, JSObject::offsetOfGroup()), temp); // Compare ObjectGroups. MacroAssembler::BranchGCPtr lastBranch; LBlock* lastBlock = nullptr; InlinePropertyTable* propTable = mir->propTable(); for (size_t i = 0; i < mir->numCases(); i++) { JSFunction* func = mir->getCase(i); LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir(); DebugOnly<bool> found = false; for (size_t j = 0; j < propTable->numEntries(); j++) { if (propTable->getFunction(j) != func) continue; if (lastBranch.isInitialized()) lastBranch.emit(masm); ObjectGroup* group = propTable->getObjectGroup(j); lastBranch = MacroAssembler::BranchGCPtr(Assembler::Equal, temp, ImmGCPtr(group), target->label()); lastBlock = target; found = true; } MOZ_ASSERT(found); } // Jump to fallback block if we have an unknown ObjectGroup. If there's no // fallback block, we should have handled all cases. if (!mir->hasFallback()) { MOZ_ASSERT(lastBranch.isInitialized()); #ifdef DEBUG Label ok; lastBranch.relink(&ok); lastBranch.emit(masm); masm.assumeUnreachable("Unexpected ObjectGroup"); masm.bind(&ok); #endif if (!isNextBlock(lastBlock)) masm.jump(lastBlock->label()); return; } LBlock* fallback = skipTrivialBlocks(mir->getFallback())->lir(); if (!lastBranch.isInitialized()) { if (!isNextBlock(fallback)) masm.jump(fallback->label()); return; } lastBranch.invertCondition(); lastBranch.relink(fallback->label()); lastBranch.emit(masm); if (!isNextBlock(lastBlock)) masm.jump(lastBlock->label()); } void CodeGenerator::visitBooleanToString(LBooleanToString* lir) { Register input = ToRegister(lir->input()); Register output = ToRegister(lir->output()); const JSAtomState& names = GetJitContext()->runtime->names(); Label true_, done; masm.branchTest32(Assembler::NonZero, input, input, &true_); masm.movePtr(ImmGCPtr(names.false_), output); masm.jump(&done); masm.bind(&true_); masm.movePtr(ImmGCPtr(names.true_), output); masm.bind(&done); } void CodeGenerator::emitIntToString(Register input, Register output, Label* ool) { masm.branch32(Assembler::AboveOrEqual, input, Imm32(StaticStrings::INT_STATIC_LIMIT), ool); // Fast path for small integers. masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().intStaticTable), output); masm.loadPtr(BaseIndex(output, input, ScalePointer), output); } typedef JSFlatString* (*IntToStringFn)(ExclusiveContext*, int); static const VMFunction IntToStringInfo = FunctionInfo<IntToStringFn>(Int32ToString<CanGC>, "Int32ToString"); void CodeGenerator::visitIntToString(LIntToString* lir) { Register input = ToRegister(lir->input()); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(IntToStringInfo, lir, ArgList(input), StoreRegisterTo(output)); emitIntToString(input, output, ool->entry()); masm.bind(ool->rejoin()); } typedef JSString* (*DoubleToStringFn)(ExclusiveContext*, double); static const VMFunction DoubleToStringInfo = FunctionInfo<DoubleToStringFn>(NumberToString<CanGC>, "NumberToString"); void CodeGenerator::visitDoubleToString(LDoubleToString* lir) { FloatRegister input = ToFloatRegister(lir->input()); Register temp = ToRegister(lir->tempInt()); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(DoubleToStringInfo, lir, ArgList(input), StoreRegisterTo(output)); // Try double to integer conversion and run integer to string code. masm.convertDoubleToInt32(input, temp, ool->entry(), true); emitIntToString(temp, output, ool->entry()); masm.bind(ool->rejoin()); } typedef JSString* (*PrimitiveToStringFn)(JSContext*, HandleValue); static const VMFunction PrimitiveToStringInfo = FunctionInfo<PrimitiveToStringFn>(ToStringSlow, "ToStringSlow"); void CodeGenerator::visitValueToString(LValueToString* lir) { ValueOperand input = ToValue(lir, LValueToString::Input); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(PrimitiveToStringInfo, lir, ArgList(input), StoreRegisterTo(output)); Label done; Register tag = masm.splitTagForTest(input); const JSAtomState& names = GetJitContext()->runtime->names(); // String if (lir->mir()->input()->mightBeType(MIRType::String)) { Label notString; masm.branchTestString(Assembler::NotEqual, tag, ¬String); masm.unboxString(input, output); masm.jump(&done); masm.bind(¬String); } // Integer if (lir->mir()->input()->mightBeType(MIRType::Int32)) { Label notInteger; masm.branchTestInt32(Assembler::NotEqual, tag, ¬Integer); Register unboxed = ToTempUnboxRegister(lir->tempToUnbox()); unboxed = masm.extractInt32(input, unboxed); emitIntToString(unboxed, output, ool->entry()); masm.jump(&done); masm.bind(¬Integer); } // Double if (lir->mir()->input()->mightBeType(MIRType::Double)) { // Note: no fastpath. Need two extra registers and can only convert doubles // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT. masm.branchTestDouble(Assembler::Equal, tag, ool->entry()); } // Undefined if (lir->mir()->input()->mightBeType(MIRType::Undefined)) { Label notUndefined; masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined); masm.movePtr(ImmGCPtr(names.undefined), output); masm.jump(&done); masm.bind(¬Undefined); } // Null if (lir->mir()->input()->mightBeType(MIRType::Null)) { Label notNull; masm.branchTestNull(Assembler::NotEqual, tag, ¬Null); masm.movePtr(ImmGCPtr(names.null), output); masm.jump(&done); masm.bind(¬Null); } // Boolean if (lir->mir()->input()->mightBeType(MIRType::Boolean)) { Label notBoolean, true_; masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean); masm.branchTestBooleanTruthy(true, input, &true_); masm.movePtr(ImmGCPtr(names.false_), output); masm.jump(&done); masm.bind(&true_); masm.movePtr(ImmGCPtr(names.true_), output); masm.jump(&done); masm.bind(¬Boolean); } // Object if (lir->mir()->input()->mightBeType(MIRType::Object)) { // Bail. MOZ_ASSERT(lir->mir()->fallible()); Label bail; masm.branchTestObject(Assembler::Equal, tag, &bail); bailoutFrom(&bail, lir->snapshot()); } // Symbol if (lir->mir()->input()->mightBeType(MIRType::Symbol)) masm.branchTestSymbol(Assembler::Equal, tag, ool->entry()); #ifdef DEBUG masm.assumeUnreachable("Unexpected type for MValueToString."); #endif masm.bind(&done); masm.bind(ool->rejoin()); } typedef JSObject* (*ToObjectFn)(JSContext*, HandleValue, bool); static const VMFunction ToObjectInfo = FunctionInfo<ToObjectFn>(ToObjectSlow, "ToObjectSlow"); void CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir) { ValueOperand input = ToValue(lir, LValueToObjectOrNull::Input); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(ToObjectInfo, lir, ArgList(input, Imm32(0)), StoreRegisterTo(output)); Label done; masm.branchTestObject(Assembler::Equal, input, &done); masm.branchTestNull(Assembler::NotEqual, input, ool->entry()); masm.bind(&done); masm.unboxNonDouble(input, output); masm.bind(ool->rejoin()); } typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, JSObject*); static const VMFunction CloneRegExpObjectInfo = FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject, "CloneRegExpObject"); void CodeGenerator::visitRegExp(LRegExp* lir) { pushArg(ImmGCPtr(lir->mir()->source())); callVM(CloneRegExpObjectInfo, lir); } // Amount of space to reserve on the stack when executing RegExps inline. static const size_t RegExpReservedStack = sizeof(irregexp::InputOutputData) + sizeof(MatchPairs) + RegExpObject::MaxPairCount * sizeof(MatchPair); static size_t RegExpPairsVectorStartOffset(size_t inputOutputDataStartOffset) { return inputOutputDataStartOffset + sizeof(irregexp::InputOutputData) + sizeof(MatchPairs); } static Address RegExpPairCountAddress(MacroAssembler& masm, size_t inputOutputDataStartOffset) { return Address(masm.getStackPointer(), inputOutputDataStartOffset + sizeof(irregexp::InputOutputData) + MatchPairs::offsetOfPairCount()); } // Prepare an InputOutputData and optional MatchPairs which space has been // allocated for on the stack, and try to execute a RegExp on a string input. // If the RegExp was successfully executed and matched the input, fallthrough, // otherwise jump to notFound or failure. static bool PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Register input, Register lastIndex, Register temp1, Register temp2, Register temp3, size_t inputOutputDataStartOffset, RegExpShared::CompilationMode mode, Label* notFound, Label* failure) { size_t matchPairsStartOffset = inputOutputDataStartOffset + sizeof(irregexp::InputOutputData); size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset); Address inputStartAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputStart)); Address inputEndAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputEnd)); Address matchesPointerAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, matches)); Address startIndexAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, startIndex)); Address endIndexAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, endIndex)); Address matchResultAddress(masm.getStackPointer(), inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, result)); Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset); Address pairsPointerAddress(masm.getStackPointer(), matchPairsStartOffset + MatchPairs::offsetOfPairs()); Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset); RegExpStatics* res = cx->global()->getRegExpStatics(cx); if (!res) return false; #ifdef JS_USE_LINK_REGISTER if (mode != RegExpShared::MatchOnly) masm.pushReturnAddress(); #endif if (mode == RegExpShared::Normal) { // First, fill in a skeletal MatchPairs instance on the stack. This will be // passed to the OOL stub in the caller if we aren't able to execute the // RegExp inline, and that stub needs to be able to determine whether the // execution finished successfully. masm.store32(Imm32(1), pairCountAddress); masm.store32(Imm32(-1), pairsVectorAddress); masm.computeEffectiveAddress(pairsVectorAddress, temp1); masm.storePtr(temp1, pairsPointerAddress); } // Check for a linear input string. masm.branchIfRopeOrExternal(input, temp1, failure); // Get the RegExpShared for the RegExp. masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp1); masm.branchPtr(Assembler::Equal, temp1, ImmWord(0), failure); // ES6 21.2.2.2 step 2. // See RegExp.cpp ExecuteRegExp for more detail. { Label done; masm.branchTest32(Assembler::Zero, Address(temp1, RegExpShared::offsetOfFlags()), Imm32(UnicodeFlag), &done); // If input is latin1, there should not be surrogate pair. masm.branchLatin1String(input, &done); // Check if |lastIndex > 0 && lastIndex < input->length()|. // lastIndex should already have no sign here. masm.branchTest32(Assembler::Zero, lastIndex, lastIndex, &done); masm.loadStringLength(input, temp2); masm.branch32(Assembler::AboveOrEqual, lastIndex, temp2, &done); // Check if input[lastIndex] is trail surrogate. masm.loadStringChars(input, temp2); masm.computeEffectiveAddress(BaseIndex(temp2, lastIndex, TimesTwo), temp3); masm.load16ZeroExtend(Address(temp3, 0), temp3); masm.branch32(Assembler::Below, temp3, Imm32(unicode::TrailSurrogateMin), &done); masm.branch32(Assembler::Above, temp3, Imm32(unicode::TrailSurrogateMax), &done); // Check if input[lastIndex-1] is lead surrogate. masm.move32(lastIndex, temp3); masm.sub32(Imm32(1), temp3); masm.computeEffectiveAddress(BaseIndex(temp2, temp3, TimesTwo), temp3); masm.load16ZeroExtend(Address(temp3, 0), temp3); masm.branch32(Assembler::Below, temp3, Imm32(unicode::LeadSurrogateMin), &done); masm.branch32(Assembler::Above, temp3, Imm32(unicode::LeadSurrogateMax), &done); // Move lastIndex to lead surrogate. masm.subPtr(Imm32(1), lastIndex); masm.bind(&done); } if (mode == RegExpShared::Normal) { // Don't handle RegExps with excessive parens. masm.load32(Address(temp1, RegExpShared::offsetOfParenCount()), temp2); masm.branch32(Assembler::AboveOrEqual, temp2, Imm32(RegExpObject::MaxPairCount), failure); // Fill in the paren count in the MatchPairs on the stack. masm.add32(Imm32(1), temp2); masm.store32(temp2, pairCountAddress); } // Load the code pointer for the type of input string we have, and compute // the input start/end pointers in the InputOutputData. Register codePointer = temp1; { masm.loadStringChars(input, temp2); masm.storePtr(temp2, inputStartAddress); masm.loadStringLength(input, temp3); Label isLatin1, done; masm.branchLatin1String(input, &isLatin1); { masm.lshiftPtr(Imm32(1), temp3); masm.loadPtr(Address(temp1, RegExpShared::offsetOfTwoByteJitCode(mode)), codePointer); } masm.jump(&done); { masm.bind(&isLatin1); masm.loadPtr(Address(temp1, RegExpShared::offsetOfLatin1JitCode(mode)), codePointer); } masm.bind(&done); masm.addPtr(temp3, temp2); masm.storePtr(temp2, inputEndAddress); } // Check the RegExpShared has been compiled for this type of input. masm.branchPtr(Assembler::Equal, codePointer, ImmWord(0), failure); masm.loadPtr(Address(codePointer, JitCode::offsetOfCode()), codePointer); // Finish filling in the InputOutputData instance on the stack. if (mode == RegExpShared::Normal) { masm.computeEffectiveAddress(Address(masm.getStackPointer(), matchPairsStartOffset), temp2); masm.storePtr(temp2, matchesPointerAddress); } else { // Use InputOutputData.endIndex itself for output. masm.computeEffectiveAddress(endIndexAddress, temp2); masm.storePtr(temp2, endIndexAddress); } masm.storePtr(lastIndex, startIndexAddress); masm.store32(Imm32(0), matchResultAddress); // Save any volatile inputs. LiveGeneralRegisterSet volatileRegs; if (lastIndex.volatile_()) volatileRegs.add(lastIndex); if (input.volatile_()) volatileRegs.add(input); if (regexp.volatile_()) volatileRegs.add(regexp); // Execute the RegExp. masm.computeEffectiveAddress(Address(masm.getStackPointer(), inputOutputDataStartOffset), temp2); masm.PushRegsInMask(volatileRegs); masm.setupUnalignedABICall(temp3); masm.passABIArg(temp2); masm.callWithABI(codePointer); masm.PopRegsInMask(volatileRegs); Label success; masm.branch32(Assembler::Equal, matchResultAddress, Imm32(RegExpRunStatus_Success_NotFound), notFound); masm.branch32(Assembler::Equal, matchResultAddress, Imm32(RegExpRunStatus_Error), failure); // Lazily update the RegExpStatics. masm.movePtr(ImmPtr(res), temp1); Address pendingInputAddress(temp1, RegExpStatics::offsetOfPendingInput()); Address matchesInputAddress(temp1, RegExpStatics::offsetOfMatchesInput()); Address lazySourceAddress(temp1, RegExpStatics::offsetOfLazySource()); Address lazyIndexAddress(temp1, RegExpStatics::offsetOfLazyIndex()); masm.patchableCallPreBarrier(pendingInputAddress, MIRType::String); masm.patchableCallPreBarrier(matchesInputAddress, MIRType::String); masm.patchableCallPreBarrier(lazySourceAddress, MIRType::String); masm.storePtr(input, pendingInputAddress); masm.storePtr(input, matchesInputAddress); masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex())); masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation())); masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp2); masm.loadPtr(Address(temp2, RegExpShared::offsetOfSource()), temp3); masm.storePtr(temp3, lazySourceAddress); masm.load32(Address(temp2, RegExpShared::offsetOfFlags()), temp3); masm.store32(temp3, Address(temp1, RegExpStatics::offsetOfLazyFlags())); if (mode == RegExpShared::MatchOnly) { // endIndex is passed via temp3. masm.load32(endIndexAddress, temp3); } return true; } static void CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len, Register byteOpScratch, size_t fromWidth, size_t toWidth); class CreateDependentString { Register string_; Register temp_; Label* failure_; enum class FallbackKind : uint8_t { InlineString, FatInlineString, NotInlineString, Count }; mozilla::EnumeratedArray<FallbackKind, FallbackKind::Count, Label> fallbacks_, joins_; public: // Generate code that creates DependentString. // Caller should call generateFallback after masm.ret(), to generate // fallback path. void generate(MacroAssembler& masm, const JSAtomState& names, bool latin1, Register string, Register base, Register temp1, Register temp2, BaseIndex startIndexAddress, BaseIndex limitIndexAddress, Label* failure); // Generate fallback path for creating DependentString. void generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave); }; void CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names, bool latin1, Register string, Register base, Register temp1, Register temp2, BaseIndex startIndexAddress, BaseIndex limitIndexAddress, Label* failure) { string_ = string; temp_ = temp2; failure_ = failure; // Compute the string length. masm.load32(startIndexAddress, temp2); masm.load32(limitIndexAddress, temp1); masm.sub32(temp2, temp1); Label done, nonEmpty; // Zero length matches use the empty string. masm.branchTest32(Assembler::NonZero, temp1, temp1, &nonEmpty); masm.movePtr(ImmGCPtr(names.empty), string); masm.jump(&done); masm.bind(&nonEmpty); Label notInline; int32_t maxInlineLength = latin1 ? (int32_t) JSFatInlineString::MAX_LENGTH_LATIN1 : (int32_t) JSFatInlineString::MAX_LENGTH_TWO_BYTE; masm.branch32(Assembler::Above, temp1, Imm32(maxInlineLength), ¬Inline); { // Make a thin or fat inline string. Label stringAllocated, fatInline; int32_t maxThinInlineLength = latin1 ? (int32_t) JSThinInlineString::MAX_LENGTH_LATIN1 : (int32_t) JSThinInlineString::MAX_LENGTH_TWO_BYTE; masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline); int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS; masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString]); masm.bind(&joins_[FallbackKind::InlineString]); masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags())); masm.jump(&stringAllocated); masm.bind(&fatInline); int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS; masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString]); masm.bind(&joins_[FallbackKind::FatInlineString]); masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags())); masm.bind(&stringAllocated); masm.store32(temp1, Address(string, JSString::offsetOfLength())); masm.push(string); masm.push(base); // Adjust the start index address for the above pushes. MOZ_ASSERT(startIndexAddress.base == masm.getStackPointer()); BaseIndex newStartIndexAddress = startIndexAddress; newStartIndexAddress.offset += 2 * sizeof(void*); // Load chars pointer for the new string. masm.addPtr(ImmWord(JSInlineString::offsetOfInlineStorage()), string); // Load the source characters pointer. masm.loadStringChars(base, base); masm.load32(newStartIndexAddress, temp2); if (latin1) masm.addPtr(temp2, base); else masm.computeEffectiveAddress(BaseIndex(base, temp2, TimesTwo), base); CopyStringChars(masm, string, base, temp1, temp2, latin1 ? 1 : 2, latin1 ? 1 : 2); // Null-terminate. if (latin1) masm.store8(Imm32(0), Address(string, 0)); else masm.store16(Imm32(0), Address(string, 0)); masm.pop(base); masm.pop(string); } masm.jump(&done); masm.bind(¬Inline); { // Make a dependent string. int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS; masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString]); masm.bind(&joins_[FallbackKind::NotInlineString]); masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags())); masm.store32(temp1, Address(string, JSString::offsetOfLength())); masm.loadPtr(Address(base, JSString::offsetOfNonInlineChars()), temp1); masm.load32(startIndexAddress, temp2); if (latin1) masm.addPtr(temp2, temp1); else masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1); masm.storePtr(temp1, Address(string, JSString::offsetOfNonInlineChars())); masm.storePtr(base, Address(string, JSDependentString::offsetOfBase())); // Follow any base pointer if the input is itself a dependent string. // Watch for undepended strings, which have a base pointer but don't // actually share their characters with it. Label noBase; masm.branchTest32(Assembler::Zero, Address(base, JSString::offsetOfFlags()), Imm32(JSString::HAS_BASE_BIT), &noBase); masm.branchTest32(Assembler::NonZero, Address(base, JSString::offsetOfFlags()), Imm32(JSString::FLAT_BIT), &noBase); masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp1); masm.storePtr(temp1, Address(string, JSDependentString::offsetOfBase())); masm.bind(&noBase); } masm.bind(&done); } static void* AllocateString(JSContext* cx) { return js::Allocate<JSString, NoGC>(cx); } static void* AllocateFatInlineString(JSContext* cx) { return js::Allocate<JSFatInlineString, NoGC>(cx); } void CreateDependentString::generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave) { regsToSave.take(string_); regsToSave.take(temp_); for (FallbackKind kind : mozilla::MakeEnumeratedRange(FallbackKind::Count)) { masm.bind(&fallbacks_[kind]); masm.PushRegsInMask(regsToSave); masm.setupUnalignedABICall(string_); masm.loadJSContext(string_); masm.passABIArg(string_); masm.callWithABI(kind == FallbackKind::FatInlineString ? JS_FUNC_TO_DATA_PTR(void*, AllocateFatInlineString) : JS_FUNC_TO_DATA_PTR(void*, AllocateString)); masm.storeCallPointerResult(string_); masm.PopRegsInMask(regsToSave); masm.branchPtr(Assembler::Equal, string_, ImmWord(0), failure_); masm.jump(&joins_[kind]); } } static void* CreateMatchResultFallbackFunc(JSContext* cx, gc::AllocKind kind, size_t nDynamicSlots) { return js::Allocate<JSObject, NoGC>(cx, kind, nDynamicSlots, gc::DefaultHeap, &ArrayObject::class_); } static void CreateMatchResultFallback(MacroAssembler& masm, LiveRegisterSet regsToSave, Register object, Register temp2, Register temp5, ArrayObject* templateObj, Label* fail) { MOZ_ASSERT(templateObj->group()->clasp() == &ArrayObject::class_); regsToSave.take(object); regsToSave.take(temp2); regsToSave.take(temp5); masm.PushRegsInMask(regsToSave); masm.setupUnalignedABICall(object); masm.loadJSContext(object); masm.passABIArg(object); masm.move32(Imm32(int32_t(templateObj->asTenured().getAllocKind())), temp2); masm.passABIArg(temp2); masm.move32(Imm32(int32_t(templateObj->as<NativeObject>().numDynamicSlots())), temp5); masm.passABIArg(temp5); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, CreateMatchResultFallbackFunc)); masm.storeCallPointerResult(object); masm.PopRegsInMask(regsToSave); masm.branchPtr(Assembler::Equal, object, ImmWord(0), fail); masm.initGCThing(object, temp2, templateObj, true, false); } JitCode* JitCompartment::generateRegExpMatcherStub(JSContext* cx) { Register regexp = RegExpMatcherRegExpReg; Register input = RegExpMatcherStringReg; Register lastIndex = RegExpMatcherLastIndexReg; ValueOperand result = JSReturnOperand; // We are free to clobber all registers, as LRegExpMatcher is a call instruction. AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(input); regs.take(regexp); regs.take(lastIndex); // temp5 is used in single byte instructions when creating dependent // strings, and has restrictions on which register it can be on some // platforms. Register temp5; { AllocatableGeneralRegisterSet oregs = regs; do { temp5 = oregs.takeAny(); } while (!MacroAssembler::canUseInSingleByteInstruction(temp5)); regs.take(temp5); } Register temp1 = regs.takeAny(); Register temp2 = regs.takeAny(); Register temp3 = regs.takeAny(); Register maybeTemp4 = InvalidReg; if (!regs.empty()) { // There are not enough registers on x86. maybeTemp4 = regs.takeAny(); } ArrayObject* templateObject = cx->compartment()->regExps.getOrCreateMatchResultTemplateObject(cx); if (!templateObject) return nullptr; // The template object should have enough space for the maximum number of // pairs this stub can handle. MOZ_ASSERT(ObjectElements::VALUES_PER_HEADER + RegExpObject::MaxPairCount == gc::GetGCKindSlots(templateObject->asTenured().getAllocKind())); MacroAssembler masm(cx); // The InputOutputData is placed above the return address on the stack. size_t inputOutputDataStartOffset = sizeof(void*); Label notFound, oolEntry; if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2, temp5, inputOutputDataStartOffset, RegExpShared::Normal, ¬Found, &oolEntry)) { return nullptr; } // Construct the result. Register object = temp1; Label matchResultFallback, matchResultJoin; masm.createGCObject(object, temp2, templateObject, gc::DefaultHeap, &matchResultFallback); masm.bind(&matchResultJoin); // Initialize slots of result object. masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2); masm.storeValue(templateObject->getSlot(0), Address(temp2, 0)); masm.storeValue(templateObject->getSlot(1), Address(temp2, sizeof(Value))); size_t elementsOffset = NativeObject::offsetOfFixedElements(); #ifdef DEBUG // Assert the initial value of initializedLength and length to make sure // restoration on failure case works. { Label initLengthOK, lengthOK; masm.branch32(Assembler::Equal, Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()), Imm32(templateObject->getDenseInitializedLength()), &initLengthOK); masm.assumeUnreachable("Initial value of the match object's initializedLength does not match to restoration."); masm.bind(&initLengthOK); masm.branch32(Assembler::Equal, Address(object, elementsOffset + ObjectElements::offsetOfLength()), Imm32(templateObject->length()), &lengthOK); masm.assumeUnreachable("Initial value of The match object's length does not match to restoration."); masm.bind(&lengthOK); } #endif Register matchIndex = temp2; masm.move32(Imm32(0), matchIndex); size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset); Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset); Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset); BaseIndex stringAddress(object, matchIndex, TimesEight, elementsOffset); JS_STATIC_ASSERT(sizeof(MatchPair) == 8); BaseIndex stringIndexAddress(masm.getStackPointer(), matchIndex, TimesEight, pairsVectorStartOffset + offsetof(MatchPair, start)); BaseIndex stringLimitAddress(masm.getStackPointer(), matchIndex, TimesEight, pairsVectorStartOffset + offsetof(MatchPair, limit)); // Loop to construct the match strings. There are two different loops, // depending on whether the input is latin1. CreateDependentString depStr[2]; { Label isLatin1, done; masm.branchLatin1String(input, &isLatin1); Label* failure = &oolEntry; Register temp4 = (maybeTemp4 == InvalidReg) ? lastIndex : maybeTemp4; Label failureRestore; if (maybeTemp4 == InvalidReg) { failure = &failureRestore; // Save lastIndex value to temporary space. masm.store32(lastIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength())); } for (int isLatin = 0; isLatin <= 1; isLatin++) { if (isLatin) masm.bind(&isLatin1); Label matchLoop; masm.bind(&matchLoop); Label isUndefined, storeDone; masm.branch32(Assembler::LessThan, stringIndexAddress, Imm32(0), &isUndefined); depStr[isLatin].generate(masm, cx->names(), isLatin, temp3, input, temp4, temp5, stringIndexAddress, stringLimitAddress, failure); masm.storeValue(JSVAL_TYPE_STRING, temp3, stringAddress); masm.jump(&storeDone); masm.bind(&isUndefined); masm.storeValue(UndefinedValue(), stringAddress); masm.bind(&storeDone); masm.add32(Imm32(1), matchIndex); masm.branch32(Assembler::LessThanOrEqual, pairCountAddress, matchIndex, &done); masm.jump(&matchLoop); } if (maybeTemp4 == InvalidReg) { // Restore lastIndex value from temporary space, both for success // and failure cases. masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex); masm.jump(&done); masm.bind(&failureRestore); masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex); // Restore the match object for failure case. masm.store32(Imm32(templateObject->getDenseInitializedLength()), Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength())); masm.store32(Imm32(templateObject->length()), Address(object, elementsOffset + ObjectElements::offsetOfLength())); masm.jump(&oolEntry); } masm.bind(&done); } // Fill in the rest of the output object. masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength())); masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength())); masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2); MOZ_ASSERT(templateObject->numFixedSlots() == 0); MOZ_ASSERT(templateObject->lookupPure(cx->names().index)->slot() == 0); MOZ_ASSERT(templateObject->lookupPure(cx->names().input)->slot() == 1); masm.load32(pairsVectorAddress, temp3); masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0)); masm.storeValue(JSVAL_TYPE_STRING, input, Address(temp2, sizeof(Value))); // All done! masm.tagValue(JSVAL_TYPE_OBJECT, object, result); masm.ret(); masm.bind(¬Found); masm.moveValue(NullValue(), result); masm.ret(); // Fallback paths for CreateDependentString and createGCObject. // Need to save all registers in use when they were called. LiveRegisterSet regsToSave(RegisterSet::Volatile()); regsToSave.addUnchecked(regexp); regsToSave.addUnchecked(input); regsToSave.addUnchecked(lastIndex); regsToSave.addUnchecked(temp1); regsToSave.addUnchecked(temp2); regsToSave.addUnchecked(temp3); if (maybeTemp4 != InvalidReg) regsToSave.addUnchecked(maybeTemp4); regsToSave.addUnchecked(temp5); for (int isLatin = 0; isLatin <= 1; isLatin++) depStr[isLatin].generateFallback(masm, regsToSave); masm.bind(&matchResultFallback); CreateMatchResultFallback(masm, regsToSave, object, temp2, temp5, templateObject, &oolEntry); masm.jump(&matchResultJoin); // Use an undefined value to signal to the caller that the OOL stub needs to be called. masm.bind(&oolEntry); masm.moveValue(UndefinedValue(), result); masm.ret(); Linker linker(masm); AutoFlushICache afc("RegExpMatcherStub"); JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE); if (!code) return nullptr; #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "RegExpMatcherStub"); #endif if (cx->zone()->needsIncrementalBarrier()) code->togglePreBarriers(true, DontReprotect); return code; } class OutOfLineRegExpMatcher : public OutOfLineCodeBase<CodeGenerator> { LRegExpMatcher* lir_; public: explicit OutOfLineRegExpMatcher(LRegExpMatcher* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineRegExpMatcher(this); } LRegExpMatcher* lir() const { return lir_; } }; typedef bool (*RegExpMatcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input, int32_t lastIndex, MatchPairs* pairs, MutableHandleValue output); static const VMFunction RegExpMatcherRawInfo = FunctionInfo<RegExpMatcherRawFn>(RegExpMatcherRaw, "RegExpMatcherRaw"); void CodeGenerator::visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool) { LRegExpMatcher* lir = ool->lir(); Register lastIndex = ToRegister(lir->lastIndex()); Register input = ToRegister(lir->string()); Register regexp = ToRegister(lir->regexp()); AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(lastIndex); regs.take(input); regs.take(regexp); Register temp = regs.takeAny(); masm.computeEffectiveAddress(Address(masm.getStackPointer(), sizeof(irregexp::InputOutputData)), temp); pushArg(temp); pushArg(lastIndex); pushArg(input); pushArg(regexp); // We are not using oolCallVM because we are in a Call, and that live // registers are already saved by the the register allocator. callVM(RegExpMatcherRawInfo, lir); masm.jump(ool->rejoin()); } void CodeGenerator::visitRegExpMatcher(LRegExpMatcher* lir) { MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg); MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg); MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg); MOZ_ASSERT(GetValueOutput(lir) == JSReturnOperand); #if defined(JS_NUNBOX32) MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Type); MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Data); MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Type); MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Data); MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Type); MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Data); #elif defined(JS_PUNBOX64) MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg); MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg); MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg); #endif masm.reserveStack(RegExpReservedStack); OutOfLineRegExpMatcher* ool = new(alloc()) OutOfLineRegExpMatcher(lir); addOutOfLineCode(ool, lir->mir()); JitCode* regExpMatcherStub = gen->compartment->jitCompartment()->regExpMatcherStubNoBarrier(); masm.call(regExpMatcherStub); masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry()); masm.bind(ool->rejoin()); masm.freeStack(RegExpReservedStack); } static const int32_t RegExpSearcherResultNotFound = -1; static const int32_t RegExpSearcherResultFailed = -2; JitCode* JitCompartment::generateRegExpSearcherStub(JSContext* cx) { Register regexp = RegExpTesterRegExpReg; Register input = RegExpTesterStringReg; Register lastIndex = RegExpTesterLastIndexReg; Register result = ReturnReg; // We are free to clobber all registers, as LRegExpSearcher is a call instruction. AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(input); regs.take(regexp); regs.take(lastIndex); Register temp1 = regs.takeAny(); Register temp2 = regs.takeAny(); Register temp3 = regs.takeAny(); MacroAssembler masm(cx); // The InputOutputData is placed above the return address on the stack. size_t inputOutputDataStartOffset = sizeof(void*); Label notFound, oolEntry; if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2, temp3, inputOutputDataStartOffset, RegExpShared::Normal, ¬Found, &oolEntry)) { return nullptr; } size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset); Address stringIndexAddress(masm.getStackPointer(), pairsVectorStartOffset + offsetof(MatchPair, start)); Address stringLimitAddress(masm.getStackPointer(), pairsVectorStartOffset + offsetof(MatchPair, limit)); masm.load32(stringIndexAddress, result); masm.load32(stringLimitAddress, input); masm.lshiftPtr(Imm32(15), input); masm.or32(input, result); masm.ret(); masm.bind(¬Found); masm.move32(Imm32(RegExpSearcherResultNotFound), result); masm.ret(); masm.bind(&oolEntry); masm.move32(Imm32(RegExpSearcherResultFailed), result); masm.ret(); Linker linker(masm); AutoFlushICache afc("RegExpSearcherStub"); JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE); if (!code) return nullptr; #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "RegExpSearcherStub"); #endif if (cx->zone()->needsIncrementalBarrier()) code->togglePreBarriers(true, DontReprotect); return code; } class OutOfLineRegExpSearcher : public OutOfLineCodeBase<CodeGenerator> { LRegExpSearcher* lir_; public: explicit OutOfLineRegExpSearcher(LRegExpSearcher* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineRegExpSearcher(this); } LRegExpSearcher* lir() const { return lir_; } }; typedef bool (*RegExpSearcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input, int32_t lastIndex, MatchPairs* pairs, int32_t* result); static const VMFunction RegExpSearcherRawInfo = FunctionInfo<RegExpSearcherRawFn>(RegExpSearcherRaw, "RegExpSearcherRaw"); void CodeGenerator::visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool) { LRegExpSearcher* lir = ool->lir(); Register lastIndex = ToRegister(lir->lastIndex()); Register input = ToRegister(lir->string()); Register regexp = ToRegister(lir->regexp()); AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(lastIndex); regs.take(input); regs.take(regexp); Register temp = regs.takeAny(); masm.computeEffectiveAddress(Address(masm.getStackPointer(), sizeof(irregexp::InputOutputData)), temp); pushArg(temp); pushArg(lastIndex); pushArg(input); pushArg(regexp); // We are not using oolCallVM because we are in a Call, and that live // registers are already saved by the the register allocator. callVM(RegExpSearcherRawInfo, lir); masm.jump(ool->rejoin()); } void CodeGenerator::visitRegExpSearcher(LRegExpSearcher* lir) { MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg); MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg); MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg); MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg); MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg); MOZ_ASSERT(RegExpTesterStringReg != ReturnReg); MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg); masm.reserveStack(RegExpReservedStack); OutOfLineRegExpSearcher* ool = new(alloc()) OutOfLineRegExpSearcher(lir); addOutOfLineCode(ool, lir->mir()); JitCode* regExpSearcherStub = gen->compartment->jitCompartment()->regExpSearcherStubNoBarrier(); masm.call(regExpSearcherStub); masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed), ool->entry()); masm.bind(ool->rejoin()); masm.freeStack(RegExpReservedStack); } static const int32_t RegExpTesterResultNotFound = -1; static const int32_t RegExpTesterResultFailed = -2; JitCode* JitCompartment::generateRegExpTesterStub(JSContext* cx) { Register regexp = RegExpTesterRegExpReg; Register input = RegExpTesterStringReg; Register lastIndex = RegExpTesterLastIndexReg; Register result = ReturnReg; MacroAssembler masm(cx); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif // We are free to clobber all registers, as LRegExpTester is a call instruction. AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(input); regs.take(regexp); regs.take(lastIndex); Register temp1 = regs.takeAny(); Register temp2 = regs.takeAny(); Register temp3 = regs.takeAny(); masm.reserveStack(sizeof(irregexp::InputOutputData)); Label notFound, oolEntry; if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2, temp3, 0, RegExpShared::MatchOnly, ¬Found, &oolEntry)) { return nullptr; } Label done; // temp3 contains endIndex. masm.move32(temp3, result); masm.jump(&done); masm.bind(¬Found); masm.move32(Imm32(RegExpTesterResultNotFound), result); masm.jump(&done); masm.bind(&oolEntry); masm.move32(Imm32(RegExpTesterResultFailed), result); masm.bind(&done); masm.freeStack(sizeof(irregexp::InputOutputData)); masm.ret(); Linker linker(masm); AutoFlushICache afc("RegExpTesterStub"); JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE); if (!code) return nullptr; #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "RegExpTesterStub"); #endif if (cx->zone()->needsIncrementalBarrier()) code->togglePreBarriers(true, DontReprotect); return code; } class OutOfLineRegExpTester : public OutOfLineCodeBase<CodeGenerator> { LRegExpTester* lir_; public: explicit OutOfLineRegExpTester(LRegExpTester* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineRegExpTester(this); } LRegExpTester* lir() const { return lir_; } }; typedef bool (*RegExpTesterRawFn)(JSContext* cx, HandleObject regexp, HandleString input, int32_t lastIndex, int32_t* result); static const VMFunction RegExpTesterRawInfo = FunctionInfo<RegExpTesterRawFn>(RegExpTesterRaw, "RegExpTesterRaw"); void CodeGenerator::visitOutOfLineRegExpTester(OutOfLineRegExpTester* ool) { LRegExpTester* lir = ool->lir(); Register lastIndex = ToRegister(lir->lastIndex()); Register input = ToRegister(lir->string()); Register regexp = ToRegister(lir->regexp()); pushArg(lastIndex); pushArg(input); pushArg(regexp); // We are not using oolCallVM because we are in a Call, and that live // registers are already saved by the the register allocator. callVM(RegExpTesterRawInfo, lir); masm.jump(ool->rejoin()); } void CodeGenerator::visitRegExpTester(LRegExpTester* lir) { MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg); MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg); MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg); MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg); MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg); MOZ_ASSERT(RegExpTesterStringReg != ReturnReg); MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg); OutOfLineRegExpTester* ool = new(alloc()) OutOfLineRegExpTester(lir); addOutOfLineCode(ool, lir->mir()); JitCode* regExpTesterStub = gen->compartment->jitCompartment()->regExpTesterStubNoBarrier(); masm.call(regExpTesterStub); masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpTesterResultFailed), ool->entry()); masm.bind(ool->rejoin()); } class OutOfLineRegExpPrototypeOptimizable : public OutOfLineCodeBase<CodeGenerator> { LRegExpPrototypeOptimizable* ins_; public: explicit OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins) : ins_(ins) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineRegExpPrototypeOptimizable(this); } LRegExpPrototypeOptimizable* ins() const { return ins_; } }; void CodeGenerator::visitRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins) { Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); Register temp = ToRegister(ins->temp()); OutOfLineRegExpPrototypeOptimizable* ool = new(alloc()) OutOfLineRegExpPrototypeOptimizable(ins); addOutOfLineCode(ool, ins->mir()); masm.loadJSContext(temp); masm.loadPtr(Address(temp, JSContext::offsetOfCompartment()), temp); size_t offset = JSCompartment::offsetOfRegExps() + RegExpCompartment::offsetOfOptimizableRegExpPrototypeShape(); masm.loadPtr(Address(temp, offset), temp); masm.loadPtr(Address(object, ShapedObject::offsetOfShape()), output); masm.branchPtr(Assembler::NotEqual, output, temp, ool->entry()); masm.move32(Imm32(0x1), output); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineRegExpPrototypeOptimizable(OutOfLineRegExpPrototypeOptimizable* ool) { LRegExpPrototypeOptimizable* ins = ool->ins(); Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); saveVolatile(output); masm.setupUnalignedABICall(output); masm.loadJSContext(output); masm.passABIArg(output); masm.passABIArg(object); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpPrototypeOptimizableRaw)); masm.storeCallBoolResult(output); restoreVolatile(output); masm.jump(ool->rejoin()); } class OutOfLineRegExpInstanceOptimizable : public OutOfLineCodeBase<CodeGenerator> { LRegExpInstanceOptimizable* ins_; public: explicit OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins) : ins_(ins) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineRegExpInstanceOptimizable(this); } LRegExpInstanceOptimizable* ins() const { return ins_; } }; void CodeGenerator::visitRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins) { Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); Register temp = ToRegister(ins->temp()); OutOfLineRegExpInstanceOptimizable* ool = new(alloc()) OutOfLineRegExpInstanceOptimizable(ins); addOutOfLineCode(ool, ins->mir()); masm.loadJSContext(temp); masm.loadPtr(Address(temp, JSContext::offsetOfCompartment()), temp); size_t offset = JSCompartment::offsetOfRegExps() + RegExpCompartment::offsetOfOptimizableRegExpInstanceShape(); masm.loadPtr(Address(temp, offset), temp); masm.loadPtr(Address(object, ShapedObject::offsetOfShape()), output); masm.branchPtr(Assembler::NotEqual, output, temp, ool->entry()); masm.move32(Imm32(0x1), output); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineRegExpInstanceOptimizable(OutOfLineRegExpInstanceOptimizable* ool) { LRegExpInstanceOptimizable* ins = ool->ins(); Register object = ToRegister(ins->object()); Register proto = ToRegister(ins->proto()); Register output = ToRegister(ins->output()); saveVolatile(output); masm.setupUnalignedABICall(output); masm.loadJSContext(output); masm.passABIArg(output); masm.passABIArg(object); masm.passABIArg(proto); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpInstanceOptimizableRaw)); masm.storeCallBoolResult(output); restoreVolatile(output); masm.jump(ool->rejoin()); } static void FindFirstDollarIndex(MacroAssembler& masm, Register str, Register len, Register chars, Register temp, Register output, bool isLatin1) { masm.loadStringChars(str, chars); masm.move32(Imm32(0), output); Label start, done; masm.bind(&start); if (isLatin1) masm.load8ZeroExtend(BaseIndex(chars, output, TimesOne), temp); else masm.load16ZeroExtend(BaseIndex(chars, output, TimesTwo), temp); masm.branch32(Assembler::Equal, temp, Imm32('$'), &done); masm.add32(Imm32(1), output); masm.branch32(Assembler::NotEqual, output, len, &start); masm.move32(Imm32(-1), output); masm.bind(&done); } typedef bool (*GetFirstDollarIndexRawFn)(JSContext*, HandleString, int32_t*); static const VMFunction GetFirstDollarIndexRawInfo = FunctionInfo<GetFirstDollarIndexRawFn>(GetFirstDollarIndexRaw, "GetFirstDollarIndexRaw"); void CodeGenerator::visitGetFirstDollarIndex(LGetFirstDollarIndex* ins) { Register str = ToRegister(ins->str()); Register output = ToRegister(ins->output()); Register temp0 = ToRegister(ins->temp0()); Register temp1 = ToRegister(ins->temp1()); Register len = ToRegister(ins->temp2()); OutOfLineCode* ool = oolCallVM(GetFirstDollarIndexRawInfo, ins, ArgList(str), StoreRegisterTo(output)); masm.branchIfRope(str, ool->entry()); masm.loadStringLength(str, len); Label isLatin1, done; masm.branchLatin1String(str, &isLatin1); { FindFirstDollarIndex(masm, str, len, temp0, temp1, output, /* isLatin1 = */ false); } masm.jump(&done); { masm.bind(&isLatin1); FindFirstDollarIndex(masm, str, len, temp0, temp1, output, /* isLatin1 = */ true); } masm.bind(&done); masm.bind(ool->rejoin()); } typedef JSString* (*StringReplaceFn)(JSContext*, HandleString, HandleString, HandleString); static const VMFunction StringFlatReplaceInfo = FunctionInfo<StringReplaceFn>(js::str_flat_replace_string, "str_flat_replace_string"); static const VMFunction StringReplaceInfo = FunctionInfo<StringReplaceFn>(StringReplace, "StringReplace"); void CodeGenerator::visitStringReplace(LStringReplace* lir) { if (lir->replacement()->isConstant()) pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString())); else pushArg(ToRegister(lir->replacement())); if (lir->pattern()->isConstant()) pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString())); else pushArg(ToRegister(lir->pattern())); if (lir->string()->isConstant()) pushArg(ImmGCPtr(lir->string()->toConstant()->toString())); else pushArg(ToRegister(lir->string())); if (lir->mir()->isFlatReplacement()) callVM(StringFlatReplaceInfo, lir); else callVM(StringReplaceInfo, lir); } void CodeGenerator::emitSharedStub(ICStub::Kind kind, LInstruction* lir) { JSScript* script = lir->mirRaw()->block()->info().script(); jsbytecode* pc = lir->mirRaw()->toInstruction()->resumePoint()->pc(); #ifdef JS_USE_LINK_REGISTER // Some architectures don't push the return address on the stack but // use the link register. In that case the stack isn't aligned. Push // to make sure we are aligned. masm.Push(Imm32(0)); #endif // Create descriptor signifying end of Ion frame. uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, JitStubFrameLayout::Size()); masm.Push(Imm32(descriptor)); // Call into the stubcode. CodeOffset patchOffset; IonICEntry entry(script->pcToOffset(pc), ICEntry::Kind_Op, script); EmitCallIC(&patchOffset, masm); entry.setReturnOffset(CodeOffset(masm.currentOffset())); SharedStub sharedStub(kind, entry, patchOffset); masm.propagateOOM(sharedStubs_.append(sharedStub)); // Fix up upon return. uint32_t callOffset = masm.currentOffset(); #ifdef JS_USE_LINK_REGISTER masm.freeStack(sizeof(intptr_t) * 2); #else masm.freeStack(sizeof(intptr_t)); #endif markSafepointAt(callOffset, lir); } void CodeGenerator::visitBinarySharedStub(LBinarySharedStub* lir) { JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc()); switch (jsop) { case JSOP_ADD: case JSOP_SUB: case JSOP_MUL: case JSOP_DIV: case JSOP_MOD: case JSOP_POW: emitSharedStub(ICStub::Kind::BinaryArith_Fallback, lir); break; case JSOP_LT: case JSOP_LE: case JSOP_GT: case JSOP_GE: case JSOP_EQ: case JSOP_NE: case JSOP_STRICTEQ: case JSOP_STRICTNE: emitSharedStub(ICStub::Kind::Compare_Fallback, lir); break; default: MOZ_CRASH("Unsupported jsop in shared stubs."); } } void CodeGenerator::visitUnarySharedStub(LUnarySharedStub* lir) { JSOp jsop = JSOp(*lir->mir()->resumePoint()->pc()); switch (jsop) { case JSOP_BITNOT: case JSOP_NEG: emitSharedStub(ICStub::Kind::UnaryArith_Fallback, lir); break; case JSOP_CALLPROP: case JSOP_GETPROP: case JSOP_LENGTH: emitSharedStub(ICStub::Kind::GetProp_Fallback, lir); break; default: MOZ_CRASH("Unsupported jsop in shared stubs."); } } void CodeGenerator::visitNullarySharedStub(LNullarySharedStub* lir) { jsbytecode* pc = lir->mir()->resumePoint()->pc(); JSOp jsop = JSOp(*pc); switch (jsop) { case JSOP_NEWARRAY: { uint32_t length = GET_UINT32(pc); MOZ_ASSERT(length <= INT32_MAX, "the bytecode emitter must fail to compile code that would " "produce JSOP_NEWARRAY with a length exceeding int32_t range"); // Pass length in R0. masm.move32(Imm32(AssertedCast<int32_t>(length)), R0.scratchReg()); emitSharedStub(ICStub::Kind::NewArray_Fallback, lir); break; } case JSOP_NEWOBJECT: emitSharedStub(ICStub::Kind::NewObject_Fallback, lir); break; case JSOP_NEWINIT: { JSProtoKey key = JSProtoKey(GET_UINT8(pc)); if (key == JSProto_Array) { masm.move32(Imm32(0), R0.scratchReg()); emitSharedStub(ICStub::Kind::NewArray_Fallback, lir); } else { emitSharedStub(ICStub::Kind::NewObject_Fallback, lir); } break; } default: MOZ_CRASH("Unsupported jsop in shared stubs."); } } typedef JSObject* (*LambdaFn)(JSContext*, HandleFunction, HandleObject); static const VMFunction LambdaInfo = FunctionInfo<LambdaFn>(js::Lambda, "Lambda"); void CodeGenerator::visitLambdaForSingleton(LLambdaForSingleton* lir) { pushArg(ToRegister(lir->environmentChain())); pushArg(ImmGCPtr(lir->mir()->info().fun)); callVM(LambdaInfo, lir); } void CodeGenerator::visitLambda(LLambda* lir) { Register envChain = ToRegister(lir->environmentChain()); Register output = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); const LambdaFunctionInfo& info = lir->mir()->info(); OutOfLineCode* ool = oolCallVM(LambdaInfo, lir, ArgList(ImmGCPtr(info.fun), envChain), StoreRegisterTo(output)); MOZ_ASSERT(!info.singletonType); masm.createGCObject(output, tempReg, info.fun, gc::DefaultHeap, ool->entry()); emitLambdaInit(output, envChain, info); if (info.flags & JSFunction::EXTENDED) { MOZ_ASSERT(info.fun->allowSuperProperty() || info.fun->isSelfHostedBuiltin() || info.fun->isAsync()); static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized"); masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(0))); masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1))); } masm.bind(ool->rejoin()); } class OutOfLineLambdaArrow : public OutOfLineCodeBase<CodeGenerator> { public: LLambdaArrow* lir; Label entryNoPop_; explicit OutOfLineLambdaArrow(LLambdaArrow* lir) : lir(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineLambdaArrow(this); } Label* entryNoPop() { return &entryNoPop_; } }; typedef JSObject* (*LambdaArrowFn)(JSContext*, HandleFunction, HandleObject, HandleValue); static const VMFunction LambdaArrowInfo = FunctionInfo<LambdaArrowFn>(js::LambdaArrow, "LambdaArrow"); void CodeGenerator::visitOutOfLineLambdaArrow(OutOfLineLambdaArrow* ool) { Register envChain = ToRegister(ool->lir->environmentChain()); ValueOperand newTarget = ToValue(ool->lir, LLambdaArrow::NewTargetValue); Register output = ToRegister(ool->lir->output()); const LambdaFunctionInfo& info = ool->lir->mir()->info(); // When we get here, we may need to restore part of the newTarget, // which has been conscripted into service as a temp register. masm.pop(newTarget.scratchReg()); masm.bind(ool->entryNoPop()); saveLive(ool->lir); pushArg(newTarget); pushArg(envChain); pushArg(ImmGCPtr(info.fun)); callVM(LambdaArrowInfo, ool->lir); StoreRegisterTo(output).generate(this); restoreLiveIgnore(ool->lir, StoreRegisterTo(output).clobbered()); masm.jump(ool->rejoin()); } void CodeGenerator::visitLambdaArrow(LLambdaArrow* lir) { Register envChain = ToRegister(lir->environmentChain()); ValueOperand newTarget = ToValue(lir, LLambdaArrow::NewTargetValue); Register output = ToRegister(lir->output()); const LambdaFunctionInfo& info = lir->mir()->info(); OutOfLineLambdaArrow* ool = new (alloc()) OutOfLineLambdaArrow(lir); addOutOfLineCode(ool, lir->mir()); MOZ_ASSERT(!info.useSingletonForClone); if (info.singletonType) { // If the function has a singleton type, this instruction will only be // executed once so we don't bother inlining it. masm.jump(ool->entryNoPop()); masm.bind(ool->rejoin()); return; } // There's not enough registers on x86 with the profiler enabled to request // a temp. Instead, spill part of one of the values, being prepared to // restore it if necessary on the out of line path. Register tempReg = newTarget.scratchReg(); masm.push(newTarget.scratchReg()); masm.createGCObject(output, tempReg, info.fun, gc::DefaultHeap, ool->entry()); masm.pop(newTarget.scratchReg()); emitLambdaInit(output, envChain, info); // Initialize extended slots. Lexical |this| is stored in the first one. MOZ_ASSERT(info.flags & JSFunction::EXTENDED); static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized"); static_assert(FunctionExtended::ARROW_NEWTARGET_SLOT == 0, "|new.target| must be stored in first slot"); masm.storeValue(newTarget, Address(output, FunctionExtended::offsetOfExtendedSlot(0))); masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1))); masm.bind(ool->rejoin()); } void CodeGenerator::emitLambdaInit(Register output, Register envChain, const LambdaFunctionInfo& info) { // Initialize nargs and flags. We do this with a single uint32 to avoid // 16-bit writes. union { struct S { uint16_t nargs; uint16_t flags; } s; uint32_t word; } u; u.s.nargs = info.nargs; u.s.flags = info.flags; MOZ_ASSERT(JSFunction::offsetOfFlags() == JSFunction::offsetOfNargs() + 2); masm.store32(Imm32(u.word), Address(output, JSFunction::offsetOfNargs())); masm.storePtr(ImmGCPtr(info.scriptOrLazyScript), Address(output, JSFunction::offsetOfNativeOrScript())); masm.storePtr(envChain, Address(output, JSFunction::offsetOfEnvironment())); masm.storePtr(ImmGCPtr(info.fun->displayAtom()), Address(output, JSFunction::offsetOfAtom())); } void CodeGenerator::visitOsiPoint(LOsiPoint* lir) { // Note: markOsiPoint ensures enough space exists between the last // LOsiPoint and this one to patch adjacent call instructions. MOZ_ASSERT(masm.framePushed() == frameSize()); uint32_t osiCallPointOffset = markOsiPoint(lir); LSafepoint* safepoint = lir->associatedSafepoint(); MOZ_ASSERT(!safepoint->osiCallPointOffset()); safepoint->setOsiCallPointOffset(osiCallPointOffset); #ifdef DEBUG // There should be no movegroups or other instructions between // an instruction and its OsiPoint. This is necessary because // we use the OsiPoint's snapshot from within VM calls. for (LInstructionReverseIterator iter(current->rbegin(lir)); iter != current->rend(); iter++) { if (*iter == lir) continue; MOZ_ASSERT(!iter->isMoveGroup()); MOZ_ASSERT(iter->safepoint() == safepoint); break; } #endif #ifdef CHECK_OSIPOINT_REGISTERS if (shouldVerifyOsiPointRegs(safepoint)) verifyOsiPointRegs(safepoint); #endif } void CodeGenerator::visitGoto(LGoto* lir) { jumpToBlock(lir->target()); } // Out-of-line path to execute any move groups between the start of a loop // header and its interrupt check, then invoke the interrupt handler. class OutOfLineInterruptCheckImplicit : public OutOfLineCodeBase<CodeGenerator> { public: LBlock* block; LInterruptCheck* lir; OutOfLineInterruptCheckImplicit(LBlock* block, LInterruptCheck* lir) : block(block), lir(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineInterruptCheckImplicit(this); } }; typedef bool (*InterruptCheckFn)(JSContext*); static const VMFunction InterruptCheckInfo = FunctionInfo<InterruptCheckFn>(InterruptCheck, "InterruptCheck"); void CodeGenerator::visitOutOfLineInterruptCheckImplicit(OutOfLineInterruptCheckImplicit* ool) { #ifdef CHECK_OSIPOINT_REGISTERS // This is path is entered from the patched back-edge of the loop. This // means that the JitAtivation flags used for checking the validity of the // OSI points are not reseted by the path generated by generateBody, so we // have to reset it here. resetOsiPointRegs(ool->lir->safepoint()); #endif LInstructionIterator iter = ool->block->begin(); for (; iter != ool->block->end(); iter++) { if (iter->isMoveGroup()) { // Replay this move group that preceds the interrupt check at the // start of the loop header. Any incoming jumps here will be from // the backedge and will skip over the move group emitted inline. visitMoveGroup(iter->toMoveGroup()); } else { break; } } MOZ_ASSERT(*iter == ool->lir); saveLive(ool->lir); callVM(InterruptCheckInfo, ool->lir); restoreLive(ool->lir); masm.jump(ool->rejoin()); } void CodeGenerator::visitTableSwitch(LTableSwitch* ins) { MTableSwitch* mir = ins->mir(); Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label(); const LAllocation* temp; if (mir->getOperand(0)->type() != MIRType::Int32) { temp = ins->tempInt()->output(); // The input is a double, so try and convert it to an integer. // If it does not fit in an integer, take the default case. masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp), defaultcase, false); } else { temp = ins->index(); } emitTableSwitchDispatch(mir, ToRegister(temp), ToRegisterOrInvalid(ins->tempPointer())); } void CodeGenerator::visitTableSwitchV(LTableSwitchV* ins) { MTableSwitch* mir = ins->mir(); Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label(); Register index = ToRegister(ins->tempInt()); ValueOperand value = ToValue(ins, LTableSwitchV::InputValue); Register tag = masm.extractTag(value, index); masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase); Label unboxInt, isInt; masm.branchTestInt32(Assembler::Equal, tag, &unboxInt); { FloatRegister floatIndex = ToFloatRegister(ins->tempFloat()); masm.unboxDouble(value, floatIndex); masm.convertDoubleToInt32(floatIndex, index, defaultcase, false); masm.jump(&isInt); } masm.bind(&unboxInt); masm.unboxInt32(value, index); masm.bind(&isInt); emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer())); } typedef JSObject* (*DeepCloneObjectLiteralFn)(JSContext*, HandleObject, NewObjectKind); static const VMFunction DeepCloneObjectLiteralInfo = FunctionInfo<DeepCloneObjectLiteralFn>(DeepCloneObjectLiteral, "DeepCloneObjectLiteral"); void CodeGenerator::visitCloneLiteral(LCloneLiteral* lir) { pushArg(ImmWord(TenuredObject)); pushArg(ToRegister(lir->getObjectLiteral())); callVM(DeepCloneObjectLiteralInfo, lir); } void CodeGenerator::visitParameter(LParameter* lir) { } void CodeGenerator::visitCallee(LCallee* lir) { Register callee = ToRegister(lir->output()); Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken()); masm.loadFunctionFromCalleeToken(ptr, callee); } void CodeGenerator::visitIsConstructing(LIsConstructing* lir) { Register output = ToRegister(lir->output()); Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken()); masm.loadPtr(calleeToken, output); // We must be inside a function. MOZ_ASSERT(current->mir()->info().script()->functionNonDelazifying()); // The low bit indicates whether this call is constructing, just clear the // other bits. static_assert(CalleeToken_Function == 0x0, "CalleeTokenTag value should match"); static_assert(CalleeToken_FunctionConstructing == 0x1, "CalleeTokenTag value should match"); masm.andPtr(Imm32(0x1), output); } void CodeGenerator::visitStart(LStart* lir) { } void CodeGenerator::visitReturn(LReturn* lir) { #if defined(JS_NUNBOX32) DebugOnly<LAllocation*> type = lir->getOperand(TYPE_INDEX); DebugOnly<LAllocation*> payload = lir->getOperand(PAYLOAD_INDEX); MOZ_ASSERT(ToRegister(type) == JSReturnReg_Type); MOZ_ASSERT(ToRegister(payload) == JSReturnReg_Data); #elif defined(JS_PUNBOX64) DebugOnly<LAllocation*> result = lir->getOperand(0); MOZ_ASSERT(ToRegister(result) == JSReturnReg); #endif // Don't emit a jump to the return label if this is the last block. if (current->mir() != *gen->graph().poBegin()) masm.jump(&returnLabel_); } void CodeGenerator::visitOsrEntry(LOsrEntry* lir) { Register temp = ToRegister(lir->temp()); // Remember the OSR entry offset into the code buffer. masm.flushBuffer(); setOsrEntryOffset(masm.size()); #ifdef JS_TRACE_LOGGING emitTracelogStopEvent(TraceLogger_Baseline); emitTracelogStartEvent(TraceLogger_IonMonkey); #endif // If profiling, save the current frame pointer to a per-thread global field. if (isProfilerInstrumentationEnabled()) masm.profilerEnterFrame(masm.getStackPointer(), temp); // Allocate the full frame for this function // Note we have a new entry here. So we reset MacroAssembler::framePushed() // to 0, before reserving the stack. MOZ_ASSERT(masm.framePushed() == frameSize()); masm.setFramePushed(0); // Ensure that the Ion frames is properly aligned. masm.assertStackAlignment(JitStackAlignment, 0); masm.reserveStack(frameSize()); } void CodeGenerator::visitOsrEnvironmentChain(LOsrEnvironmentChain* lir) { const LAllocation* frame = lir->getOperand(0); const LDefinition* object = lir->getDef(0); const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfEnvironmentChain(); masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object)); } void CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject* lir) { const LAllocation* frame = lir->getOperand(0); const LDefinition* object = lir->getDef(0); const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj(); masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object)); } void CodeGenerator::visitOsrValue(LOsrValue* value) { const LAllocation* frame = value->getOperand(0); const ValueOperand out = ToOutValue(value); const ptrdiff_t frameOffset = value->mir()->frameOffset(); masm.loadValue(Address(ToRegister(frame), frameOffset), out); } void CodeGenerator::visitOsrReturnValue(LOsrReturnValue* lir) { const LAllocation* frame = lir->getOperand(0); const ValueOperand out = ToOutValue(lir); Address flags = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags()); Address retval = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue()); masm.moveValue(UndefinedValue(), out); Label done; masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL), &done); masm.loadValue(retval, out); masm.bind(&done); } void CodeGenerator::visitStackArgT(LStackArgT* lir) { const LAllocation* arg = lir->getArgument(); MIRType argType = lir->type(); uint32_t argslot = lir->argslot(); MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount()); int32_t stack_offset = StackOffsetOfPassedArg(argslot); Address dest(masm.getStackPointer(), stack_offset); if (arg->isFloatReg()) masm.storeDouble(ToFloatRegister(arg), dest); else if (arg->isRegister()) masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest); else masm.storeValue(arg->toConstant()->toJSValue(), dest); } void CodeGenerator::visitStackArgV(LStackArgV* lir) { ValueOperand val = ToValue(lir, 0); uint32_t argslot = lir->argslot(); MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount()); int32_t stack_offset = StackOffsetOfPassedArg(argslot); masm.storeValue(val, Address(masm.getStackPointer(), stack_offset)); } void CodeGenerator::visitMoveGroup(LMoveGroup* group) { if (!group->numMoves()) return; MoveResolver& resolver = masm.moveResolver(); for (size_t i = 0; i < group->numMoves(); i++) { const LMove& move = group->getMove(i); LAllocation from = move.from(); LAllocation to = move.to(); LDefinition::Type type = move.type(); // No bogus moves. MOZ_ASSERT(from != to); MOZ_ASSERT(!from.isConstant()); MoveOp::Type moveType; switch (type) { case LDefinition::OBJECT: case LDefinition::SLOTS: #ifdef JS_NUNBOX32 case LDefinition::TYPE: case LDefinition::PAYLOAD: #else case LDefinition::BOX: #endif case LDefinition::GENERAL: moveType = MoveOp::GENERAL; break; case LDefinition::INT32: moveType = MoveOp::INT32; break; case LDefinition::FLOAT32: moveType = MoveOp::FLOAT32; break; case LDefinition::DOUBLE: moveType = MoveOp::DOUBLE; break; case LDefinition::SIMD128INT: moveType = MoveOp::SIMD128INT; break; case LDefinition::SIMD128FLOAT: moveType = MoveOp::SIMD128FLOAT; break; default: MOZ_CRASH("Unexpected move type"); } masm.propagateOOM(resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType)); } masm.propagateOOM(resolver.resolve()); if (masm.oom()) return; MoveEmitter emitter(masm); #ifdef JS_CODEGEN_X86 if (group->maybeScratchRegister().isGeneralReg()) emitter.setScratchRegister(group->maybeScratchRegister().toGeneralReg()->reg()); else resolver.sortMemoryToMemoryMoves(); #endif emitter.emit(resolver); emitter.finish(); } void CodeGenerator::visitInteger(LInteger* lir) { masm.move32(Imm32(lir->getValue()), ToRegister(lir->output())); } void CodeGenerator::visitInteger64(LInteger64* lir) { masm.move64(Imm64(lir->getValue()), ToOutRegister64(lir)); } void CodeGenerator::visitPointer(LPointer* lir) { if (lir->kind() == LPointer::GC_THING) masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output())); else masm.movePtr(ImmPtr(lir->ptr()), ToRegister(lir->output())); } void CodeGenerator::visitKeepAliveObject(LKeepAliveObject* lir) { // No-op. } void CodeGenerator::visitSlots(LSlots* lir) { Address slots(ToRegister(lir->object()), NativeObject::offsetOfSlots()); masm.loadPtr(slots, ToRegister(lir->output())); } void CodeGenerator::visitLoadSlotT(LLoadSlotT* lir) { Register base = ToRegister(lir->slots()); int32_t offset = lir->mir()->slot() * sizeof(js::Value); AnyRegister result = ToAnyRegister(lir->output()); masm.loadUnboxedValue(Address(base, offset), lir->mir()->type(), result); } void CodeGenerator::visitLoadSlotV(LLoadSlotV* lir) { ValueOperand dest = ToOutValue(lir); Register base = ToRegister(lir->input()); int32_t offset = lir->mir()->slot() * sizeof(js::Value); masm.loadValue(Address(base, offset), dest); } void CodeGenerator::visitStoreSlotT(LStoreSlotT* lir) { Register base = ToRegister(lir->slots()); int32_t offset = lir->mir()->slot() * sizeof(js::Value); Address dest(base, offset); if (lir->mir()->needsBarrier()) emitPreBarrier(dest); MIRType valueType = lir->mir()->value()->type(); if (valueType == MIRType::ObjectOrNull) { masm.storeObjectOrNull(ToRegister(lir->value()), dest); } else { ConstantOrRegister value; if (lir->value()->isConstant()) value = ConstantOrRegister(lir->value()->toConstant()->toJSValue()); else value = TypedOrValueRegister(valueType, ToAnyRegister(lir->value())); masm.storeUnboxedValue(value, valueType, dest, lir->mir()->slotType()); } } void CodeGenerator::visitStoreSlotV(LStoreSlotV* lir) { Register base = ToRegister(lir->slots()); int32_t offset = lir->mir()->slot() * sizeof(Value); const ValueOperand value = ToValue(lir, LStoreSlotV::Value); if (lir->mir()->needsBarrier()) emitPreBarrier(Address(base, offset)); masm.storeValue(value, Address(base, offset)); } static void GuardReceiver(MacroAssembler& masm, const ReceiverGuard& guard, Register obj, Register scratch, Label* miss, bool checkNullExpando) { if (guard.group) { masm.branchTestObjGroup(Assembler::NotEqual, obj, guard.group, miss); Address expandoAddress(obj, UnboxedPlainObject::offsetOfExpando()); if (guard.shape) { masm.loadPtr(expandoAddress, scratch); masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), miss); masm.branchTestObjShape(Assembler::NotEqual, scratch, guard.shape, miss); } else if (checkNullExpando) { masm.branchPtr(Assembler::NotEqual, expandoAddress, ImmWord(0), miss); } } else { masm.branchTestObjShape(Assembler::NotEqual, obj, guard.shape, miss); } } void CodeGenerator::emitGetPropertyPolymorphic(LInstruction* ins, Register obj, Register scratch, const TypedOrValueRegister& output) { MGetPropertyPolymorphic* mir = ins->mirRaw()->toGetPropertyPolymorphic(); Label done; for (size_t i = 0; i < mir->numReceivers(); i++) { ReceiverGuard receiver = mir->receiver(i); Label next; masm.comment("GuardReceiver"); GuardReceiver(masm, receiver, obj, scratch, &next, /* checkNullExpando = */ false); if (receiver.shape) { masm.comment("loadTypedOrValue"); // If this is an unboxed expando access, GuardReceiver loaded the // expando object into scratch. Register target = receiver.group ? scratch : obj; Shape* shape = mir->shape(i); if (shape->slot() < shape->numFixedSlots()) { // Fixed slot. masm.loadTypedOrValue(Address(target, NativeObject::getFixedSlotOffset(shape->slot())), output); } else { // Dynamic slot. uint32_t offset = (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value); masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch); masm.loadTypedOrValue(Address(scratch, offset), output); } } else { masm.comment("loadUnboxedProperty"); const UnboxedLayout::Property* property = receiver.group->unboxedLayout().lookup(mir->name()); Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset); masm.loadUnboxedProperty(propertyAddr, property->type, output); } if (i == mir->numReceivers() - 1) { bailoutFrom(&next, ins->snapshot()); } else { masm.jump(&done); masm.bind(&next); } } masm.bind(&done); } void CodeGenerator::visitGetPropertyPolymorphicV(LGetPropertyPolymorphicV* ins) { Register obj = ToRegister(ins->obj()); ValueOperand output = GetValueOutput(ins); emitGetPropertyPolymorphic(ins, obj, output.scratchReg(), output); } void CodeGenerator::visitGetPropertyPolymorphicT(LGetPropertyPolymorphicT* ins) { Register obj = ToRegister(ins->obj()); TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->output())); Register temp = (output.type() == MIRType::Double) ? ToRegister(ins->temp()) : output.typedReg().gpr(); emitGetPropertyPolymorphic(ins, obj, temp, output); } template <typename T> static void EmitUnboxedPreBarrier(MacroAssembler &masm, T address, JSValueType type) { if (type == JSVAL_TYPE_OBJECT) masm.patchableCallPreBarrier(address, MIRType::Object); else if (type == JSVAL_TYPE_STRING) masm.patchableCallPreBarrier(address, MIRType::String); else MOZ_ASSERT(!UnboxedTypeNeedsPreBarrier(type)); } void CodeGenerator::emitSetPropertyPolymorphic(LInstruction* ins, Register obj, Register scratch, const ConstantOrRegister& value) { MSetPropertyPolymorphic* mir = ins->mirRaw()->toSetPropertyPolymorphic(); Label done; for (size_t i = 0; i < mir->numReceivers(); i++) { ReceiverGuard receiver = mir->receiver(i); Label next; GuardReceiver(masm, receiver, obj, scratch, &next, /* checkNullExpando = */ false); if (receiver.shape) { // If this is an unboxed expando access, GuardReceiver loaded the // expando object into scratch. Register target = receiver.group ? scratch : obj; Shape* shape = mir->shape(i); if (shape->slot() < shape->numFixedSlots()) { // Fixed slot. Address addr(target, NativeObject::getFixedSlotOffset(shape->slot())); if (mir->needsBarrier()) emitPreBarrier(addr); masm.storeConstantOrRegister(value, addr); } else { // Dynamic slot. masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch); Address addr(scratch, (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value)); if (mir->needsBarrier()) emitPreBarrier(addr); masm.storeConstantOrRegister(value, addr); } } else { const UnboxedLayout::Property* property = receiver.group->unboxedLayout().lookup(mir->name()); Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset); EmitUnboxedPreBarrier(masm, propertyAddr, property->type); masm.storeUnboxedProperty(propertyAddr, property->type, value, nullptr); } if (i == mir->numReceivers() - 1) { bailoutFrom(&next, ins->snapshot()); } else { masm.jump(&done); masm.bind(&next); } } masm.bind(&done); } void CodeGenerator::visitSetPropertyPolymorphicV(LSetPropertyPolymorphicV* ins) { Register obj = ToRegister(ins->obj()); Register temp = ToRegister(ins->temp()); ValueOperand value = ToValue(ins, LSetPropertyPolymorphicV::Value); emitSetPropertyPolymorphic(ins, obj, temp, TypedOrValueRegister(value)); } void CodeGenerator::visitSetPropertyPolymorphicT(LSetPropertyPolymorphicT* ins) { Register obj = ToRegister(ins->obj()); Register temp = ToRegister(ins->temp()); ConstantOrRegister value; if (ins->mir()->value()->isConstant()) value = ConstantOrRegister(ins->mir()->value()->toConstant()->toJSValue()); else value = TypedOrValueRegister(ins->mir()->value()->type(), ToAnyRegister(ins->value())); emitSetPropertyPolymorphic(ins, obj, temp, value); } void CodeGenerator::visitElements(LElements* lir) { Address elements(ToRegister(lir->object()), lir->mir()->unboxed() ? UnboxedArrayObject::offsetOfElements() : NativeObject::offsetOfElements()); masm.loadPtr(elements, ToRegister(lir->output())); } typedef bool (*ConvertElementsToDoublesFn)(JSContext*, uintptr_t); static const VMFunction ConvertElementsToDoublesInfo = FunctionInfo<ConvertElementsToDoublesFn>(ObjectElements::ConvertElementsToDoubles, "ObjectElements::ConvertElementsToDoubles"); void CodeGenerator::visitConvertElementsToDoubles(LConvertElementsToDoubles* lir) { Register elements = ToRegister(lir->elements()); OutOfLineCode* ool = oolCallVM(ConvertElementsToDoublesInfo, lir, ArgList(elements), StoreNothing()); Address convertedAddress(elements, ObjectElements::offsetOfFlags()); Imm32 bit(ObjectElements::CONVERT_DOUBLE_ELEMENTS); masm.branchTest32(Assembler::Zero, convertedAddress, bit, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitMaybeToDoubleElement(LMaybeToDoubleElement* lir) { Register elements = ToRegister(lir->elements()); Register value = ToRegister(lir->value()); ValueOperand out = ToOutValue(lir); FloatRegister temp = ToFloatRegister(lir->tempFloat()); Label convert, done; // If the CONVERT_DOUBLE_ELEMENTS flag is set, convert the int32 // value to double. Else, just box it. masm.branchTest32(Assembler::NonZero, Address(elements, ObjectElements::offsetOfFlags()), Imm32(ObjectElements::CONVERT_DOUBLE_ELEMENTS), &convert); masm.tagValue(JSVAL_TYPE_INT32, value, out); masm.jump(&done); masm.bind(&convert); masm.convertInt32ToDouble(value, temp); masm.boxDouble(temp, out); masm.bind(&done); } typedef bool (*CopyElementsForWriteFn)(ExclusiveContext*, NativeObject*); static const VMFunction CopyElementsForWriteInfo = FunctionInfo<CopyElementsForWriteFn>(NativeObject::CopyElementsForWrite, "NativeObject::CopyElementsForWrite"); void CodeGenerator::visitMaybeCopyElementsForWrite(LMaybeCopyElementsForWrite* lir) { Register object = ToRegister(lir->object()); Register temp = ToRegister(lir->temp()); OutOfLineCode* ool = oolCallVM(CopyElementsForWriteInfo, lir, ArgList(object), StoreNothing()); if (lir->mir()->checkNative()) { masm.loadObjClass(object, temp); masm.branchTest32(Assembler::NonZero, Address(temp, Class::offsetOfFlags()), Imm32(Class::NON_NATIVE), ool->rejoin()); } masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp); masm.branchTest32(Assembler::NonZero, Address(temp, ObjectElements::offsetOfFlags()), Imm32(ObjectElements::COPY_ON_WRITE), ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment* lir) { Address environment(ToRegister(lir->function()), JSFunction::offsetOfEnvironment()); masm.loadPtr(environment, ToRegister(lir->output())); } void CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity* guard) { Register input = ToRegister(guard->input()); Register expected = ToRegister(guard->expected()); Assembler::Condition cond = guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual; bailoutCmpPtr(cond, input, expected, guard->snapshot()); } void CodeGenerator::visitGuardReceiverPolymorphic(LGuardReceiverPolymorphic* lir) { const MGuardReceiverPolymorphic* mir = lir->mir(); Register obj = ToRegister(lir->object()); Register temp = ToRegister(lir->temp()); Label done; for (size_t i = 0; i < mir->numReceivers(); i++) { const ReceiverGuard& receiver = mir->receiver(i); Label next; GuardReceiver(masm, receiver, obj, temp, &next, /* checkNullExpando = */ true); if (i == mir->numReceivers() - 1) { bailoutFrom(&next, lir->snapshot()); } else { masm.jump(&done); masm.bind(&next); } } masm.bind(&done); } void CodeGenerator::visitGuardUnboxedExpando(LGuardUnboxedExpando* lir) { Label miss; Register obj = ToRegister(lir->object()); masm.branchPtr(lir->mir()->requireExpando() ? Assembler::Equal : Assembler::NotEqual, Address(obj, UnboxedPlainObject::offsetOfExpando()), ImmWord(0), &miss); bailoutFrom(&miss, lir->snapshot()); } void CodeGenerator::visitLoadUnboxedExpando(LLoadUnboxedExpando* lir) { Register obj = ToRegister(lir->object()); Register result = ToRegister(lir->getDef(0)); masm.loadPtr(Address(obj, UnboxedPlainObject::offsetOfExpando()), result); } void CodeGenerator::visitTypeBarrierV(LTypeBarrierV* lir) { ValueOperand operand = ToValue(lir, LTypeBarrierV::Input); Register scratch = ToTempRegisterOrInvalid(lir->temp()); Label miss; masm.guardTypeSet(operand, lir->mir()->resultTypeSet(), lir->mir()->barrierKind(), scratch, &miss); bailoutFrom(&miss, lir->snapshot()); } void CodeGenerator::visitTypeBarrierO(LTypeBarrierO* lir) { Register obj = ToRegister(lir->object()); Register scratch = ToTempRegisterOrInvalid(lir->temp()); Label miss, ok; if (lir->mir()->type() == MIRType::ObjectOrNull) { masm.comment("Object or Null"); Label* nullTarget = lir->mir()->resultTypeSet()->mightBeMIRType(MIRType::Null) ? &ok : &miss; masm.branchTestPtr(Assembler::Zero, obj, obj, nullTarget); } else { MOZ_ASSERT(lir->mir()->type() == MIRType::Object); MOZ_ASSERT(lir->mir()->barrierKind() != BarrierKind::TypeTagOnly); } if (lir->mir()->barrierKind() != BarrierKind::TypeTagOnly) { masm.comment("Type tag only"); masm.guardObjectType(obj, lir->mir()->resultTypeSet(), scratch, &miss); } bailoutFrom(&miss, lir->snapshot()); masm.bind(&ok); } void CodeGenerator::visitMonitorTypes(LMonitorTypes* lir) { ValueOperand operand = ToValue(lir, LMonitorTypes::Input); Register scratch = ToTempUnboxRegister(lir->temp()); Label matched, miss; masm.guardTypeSet(operand, lir->mir()->typeSet(), lir->mir()->barrierKind(), scratch, &miss); bailoutFrom(&miss, lir->snapshot()); } // Out-of-line path to update the store buffer. class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator> { LInstruction* lir_; const LAllocation* object_; public: OutOfLineCallPostWriteBarrier(LInstruction* lir, const LAllocation* object) : lir_(lir), object_(object) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineCallPostWriteBarrier(this); } LInstruction* lir() const { return lir_; } const LAllocation* object() const { return object_; } }; static void EmitStoreBufferCheckForConstant(MacroAssembler& masm, JSObject* object, AllocatableGeneralRegisterSet& regs, Label* exit, Label* callVM) { Register temp = regs.takeAny(); const gc::TenuredCell* cell = &object->asTenured(); gc::Arena* arena = cell->arena(); Register cells = temp; masm.loadPtr(AbsoluteAddress(&arena->bufferedCells), cells); size_t index = gc::ArenaCellSet::getCellIndex(cell); size_t word; uint32_t mask; gc::ArenaCellSet::getWordIndexAndMask(index, &word, &mask); size_t offset = gc::ArenaCellSet::offsetOfBits() + word * sizeof(uint32_t); masm.branchTest32(Assembler::NonZero, Address(cells, offset), Imm32(mask), exit); // Check whether this is the sentinel set and if so call the VM to allocate // one for this arena. masm.branchPtr(Assembler::Equal, Address(cells, gc::ArenaCellSet::offsetOfArena()), ImmPtr(nullptr), callVM); // Add the cell to the set. masm.or32(Imm32(mask), Address(cells, offset)); masm.jump(exit); regs.add(temp); } static void EmitPostWriteBarrier(MacroAssembler& masm, Register objreg, JSObject* maybeConstant, bool isGlobal, AllocatableGeneralRegisterSet& regs) { MOZ_ASSERT_IF(isGlobal, maybeConstant); Label callVM; Label exit; // We already have a fast path to check whether a global is in the store // buffer. if (!isGlobal && maybeConstant) EmitStoreBufferCheckForConstant(masm, maybeConstant, regs, &exit, &callVM); // Call into the VM to barrier the write. masm.bind(&callVM); Register runtimereg = regs.takeAny(); masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg); void (*fun)(JSRuntime*, JSObject*) = isGlobal ? PostGlobalWriteBarrier : PostWriteBarrier; masm.setupUnalignedABICall(regs.takeAny()); masm.passABIArg(runtimereg); masm.passABIArg(objreg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun)); masm.bind(&exit); } void CodeGenerator::emitPostWriteBarrier(const LAllocation* obj) { AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); Register objreg; JSObject* object = nullptr; bool isGlobal = false; if (obj->isConstant()) { object = &obj->toConstant()->toObject(); isGlobal = isGlobalObject(object); objreg = regs.takeAny(); masm.movePtr(ImmGCPtr(object), objreg); } else { objreg = ToRegister(obj); regs.takeUnchecked(objreg); } EmitPostWriteBarrier(masm, objreg, object, isGlobal, regs); } void CodeGenerator::emitPostWriteBarrier(Register objreg) { AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); regs.takeUnchecked(objreg); EmitPostWriteBarrier(masm, objreg, nullptr, false, regs); } void CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool) { saveLiveVolatile(ool->lir()); const LAllocation* obj = ool->object(); emitPostWriteBarrier(obj); restoreLiveVolatile(ool->lir()); masm.jump(ool->rejoin()); } void CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal, OutOfLineCode* ool) { // Check whether an object is a global that we have already barriered before // calling into the VM. if (!maybeGlobal->isConstant()) return; JSObject* obj = &maybeGlobal->toConstant()->toObject(); if (!isGlobalObject(obj)) return; JSCompartment* comp = obj->compartment(); auto addr = AbsoluteAddress(&comp->globalWriteBarriered); masm.branch32(Assembler::NotEqual, addr, Imm32(0), ool->rejoin()); } template <class LPostBarrierType> void CodeGenerator::visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode* ool) { addOutOfLineCode(ool, lir->mir()); Register temp = ToTempRegisterOrInvalid(lir->temp()); if (lir->object()->isConstant()) { // Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier. MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject())); } else { masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp, ool->rejoin()); } maybeEmitGlobalBarrierCheck(lir->object(), ool); Register valueObj = ToRegister(lir->value()); masm.branchTestPtr(Assembler::Zero, valueObj, valueObj, ool->rejoin()); masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->value()), temp, ool->entry()); masm.bind(ool->rejoin()); } template <class LPostBarrierType> void CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool) { addOutOfLineCode(ool, lir->mir()); Register temp = ToTempRegisterOrInvalid(lir->temp()); if (lir->object()->isConstant()) { // Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier. MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject())); } else { masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp, ool->rejoin()); } maybeEmitGlobalBarrierCheck(lir->object(), ool); ValueOperand value = ToValue(lir, LPostBarrierType::Input); masm.branchValueIsNurseryObject(Assembler::Equal, value, temp, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir) { auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object()); visitPostWriteBarrierCommonO(lir, ool); } void CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV* lir) { auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object()); visitPostWriteBarrierCommonV(lir, ool); } // Out-of-line path to update the store buffer. class OutOfLineCallPostWriteElementBarrier : public OutOfLineCodeBase<CodeGenerator> { LInstruction* lir_; const LAllocation* object_; const LAllocation* index_; public: OutOfLineCallPostWriteElementBarrier(LInstruction* lir, const LAllocation* object, const LAllocation* index) : lir_(lir), object_(object), index_(index) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineCallPostWriteElementBarrier(this); } LInstruction* lir() const { return lir_; } const LAllocation* object() const { return object_; } const LAllocation* index() const { return index_; } }; void CodeGenerator::visitOutOfLineCallPostWriteElementBarrier(OutOfLineCallPostWriteElementBarrier* ool) { saveLiveVolatile(ool->lir()); const LAllocation* obj = ool->object(); const LAllocation* index = ool->index(); Register objreg = obj->isConstant() ? InvalidReg : ToRegister(obj); Register indexreg = ToRegister(index); AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); regs.takeUnchecked(indexreg); if (obj->isConstant()) { objreg = regs.takeAny(); masm.movePtr(ImmGCPtr(&obj->toConstant()->toObject()), objreg); } else { regs.takeUnchecked(objreg); } Register runtimereg = regs.takeAny(); masm.setupUnalignedABICall(runtimereg); masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg); masm.passABIArg(runtimereg); masm.passABIArg(objreg); masm.passABIArg(indexreg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostWriteElementBarrier)); restoreLiveVolatile(ool->lir()); masm.jump(ool->rejoin()); } void CodeGenerator::visitPostWriteElementBarrierO(LPostWriteElementBarrierO* lir) { auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index()); visitPostWriteBarrierCommonO(lir, ool); } void CodeGenerator::visitPostWriteElementBarrierV(LPostWriteElementBarrierV* lir) { auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index()); visitPostWriteBarrierCommonV(lir, ool); } void CodeGenerator::visitCallNative(LCallNative* call) { WrappedFunction* target = call->getSingleTarget(); MOZ_ASSERT(target); MOZ_ASSERT(target->isNative()); int callargslot = call->argslot(); int unusedStack = StackOffsetOfPassedArg(callargslot); // Registers used for callWithABI() argument-passing. const Register argContextReg = ToRegister(call->getArgContextReg()); const Register argUintNReg = ToRegister(call->getArgUintNReg()); const Register argVpReg = ToRegister(call->getArgVpReg()); // Misc. temporary registers. const Register tempReg = ToRegister(call->getTempReg()); DebugOnly<uint32_t> initialStack = masm.framePushed(); masm.checkStackAlignment(); // Native functions have the signature: // bool (*)(JSContext*, unsigned, Value* vp) // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward // are the function arguments. // Allocate space for the outparam, moving the StackPointer to what will be &vp[1]. masm.adjustStack(unusedStack); // Push a Value containing the callee object: natives are allowed to access their callee before // setitng the return value. The StackPointer is moved to &vp[0]. masm.Push(ObjectValue(*target->rawJSFunction())); // Preload arguments into registers. masm.loadJSContext(argContextReg); masm.move32(Imm32(call->numActualArgs()), argUintNReg); masm.moveStackPtrTo(argVpReg); masm.Push(argUintNReg); // Construct native exit frame. uint32_t safepointOffset = masm.buildFakeExitFrame(tempReg); masm.enterFakeExitFrameForNative(call->mir()->isConstructing()); markSafepointAt(safepointOffset, call); emitTracelogStartEvent(TraceLogger_Call); // Construct and execute call. masm.setupUnalignedABICall(tempReg); masm.passABIArg(argContextReg); masm.passABIArg(argUintNReg); masm.passABIArg(argVpReg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->native())); emitTracelogStopEvent(TraceLogger_Call); // Test for failure. masm.branchIfFalseBool(ReturnReg, masm.failureLabel()); // Load the outparam vp[0] into output register(s). masm.loadValue(Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()), JSReturnOperand); // The next instruction is removing the footer of the exit frame, so there // is no need for leaveFakeExitFrame. // Move the StackPointer back to its original location, unwinding the native exit frame. masm.adjustStack(NativeExitFrameLayout::Size() - unusedStack); MOZ_ASSERT(masm.framePushed() == initialStack); } static void LoadDOMPrivate(MacroAssembler& masm, Register obj, Register priv) { // Load the value in DOM_OBJECT_SLOT for a native or proxy DOM object. This // will be in the first slot but may be fixed or non-fixed. MOZ_ASSERT(obj != priv); // Check shape->numFixedSlots != 0. masm.loadPtr(Address(obj, ShapedObject::offsetOfShape()), priv); Label hasFixedSlots, done; masm.branchTest32(Assembler::NonZero, Address(priv, Shape::offsetOfSlotInfo()), Imm32(Shape::fixedSlotsMask()), &hasFixedSlots); masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), priv); masm.loadPrivate(Address(priv, 0), priv); masm.jump(&done); masm.bind(&hasFixedSlots); masm.loadPrivate(Address(obj, NativeObject::getFixedSlotOffset(0)), priv); masm.bind(&done); } void CodeGenerator::visitCallDOMNative(LCallDOMNative* call) { WrappedFunction* target = call->getSingleTarget(); MOZ_ASSERT(target); MOZ_ASSERT(target->isNative()); MOZ_ASSERT(target->jitInfo()); MOZ_ASSERT(call->mir()->isCallDOMNative()); int callargslot = call->argslot(); int unusedStack = StackOffsetOfPassedArg(callargslot); // Registers used for callWithABI() argument-passing. const Register argJSContext = ToRegister(call->getArgJSContext()); const Register argObj = ToRegister(call->getArgObj()); const Register argPrivate = ToRegister(call->getArgPrivate()); const Register argArgs = ToRegister(call->getArgArgs()); DebugOnly<uint32_t> initialStack = masm.framePushed(); masm.checkStackAlignment(); // DOM methods have the signature: // bool (*)(JSContext*, HandleObject, void* private, const JSJitMethodCallArgs& args) // Where args is initialized from an argc and a vp, vp[0] is space for an // outparam and the callee, vp[1] is |this|, and vp[2] onward are the // function arguments. Note that args stores the argv, not the vp, and // argv == vp + 2. // Nestle the stack up against the pushed arguments, leaving StackPointer at // &vp[1] masm.adjustStack(unusedStack); // argObj is filled with the extracted object, then returned. Register obj = masm.extractObject(Address(masm.getStackPointer(), 0), argObj); MOZ_ASSERT(obj == argObj); // Push a Value containing the callee object: natives are allowed to access their callee before // setitng the return value. After this the StackPointer points to &vp[0]. masm.Push(ObjectValue(*target->rawJSFunction())); // Now compute the argv value. Since StackPointer is pointing to &vp[0] and // argv is &vp[2] we just need to add 2*sizeof(Value) to the current // StackPointer. JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgv == 0); JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgc == IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv); masm.computeEffectiveAddress(Address(masm.getStackPointer(), 2 * sizeof(Value)), argArgs); LoadDOMPrivate(masm, obj, argPrivate); // Push argc from the call instruction into what will become the IonExitFrame masm.Push(Imm32(call->numActualArgs())); // Push our argv onto the stack masm.Push(argArgs); // And store our JSJitMethodCallArgs* in argArgs. masm.moveStackPtrTo(argArgs); // Push |this| object for passing HandleObject. We push after argc to // maintain the same sp-relative location of the object pointer with other // DOMExitFrames. masm.Push(argObj); masm.moveStackPtrTo(argObj); // Construct native exit frame. uint32_t safepointOffset = masm.buildFakeExitFrame(argJSContext); masm.enterFakeExitFrame(IonDOMMethodExitFrameLayoutToken); markSafepointAt(safepointOffset, call); // Construct and execute call. masm.setupUnalignedABICall(argJSContext); masm.loadJSContext(argJSContext); masm.passABIArg(argJSContext); masm.passABIArg(argObj); masm.passABIArg(argPrivate); masm.passABIArg(argArgs); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->jitInfo()->method)); if (target->jitInfo()->isInfallible) { masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()), JSReturnOperand); } else { // Test for failure. masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); // Load the outparam vp[0] into output register(s). masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()), JSReturnOperand); } // The next instruction is removing the footer of the exit frame, so there // is no need for leaveFakeExitFrame. // Move the StackPointer back to its original location, unwinding the native exit frame. masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack); MOZ_ASSERT(masm.framePushed() == initialStack); } typedef bool (*GetIntrinsicValueFn)(JSContext* cx, HandlePropertyName, MutableHandleValue); static const VMFunction GetIntrinsicValueInfo = FunctionInfo<GetIntrinsicValueFn>(GetIntrinsicValue, "GetIntrinsicValue"); void CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue* lir) { pushArg(ImmGCPtr(lir->mir()->name())); callVM(GetIntrinsicValueInfo, lir); } typedef bool (*InvokeFunctionFn)(JSContext*, HandleObject, bool, uint32_t, Value*, MutableHandleValue); static const VMFunction InvokeFunctionInfo = FunctionInfo<InvokeFunctionFn>(InvokeFunction, "InvokeFunction"); void CodeGenerator::emitCallInvokeFunction(LInstruction* call, Register calleereg, bool constructing, uint32_t argc, uint32_t unusedStack) { // Nestle %esp up to the argument vector. // Each path must account for framePushed_ separately, for callVM to be valid. masm.freeStack(unusedStack); pushArg(masm.getStackPointer()); // argv. pushArg(Imm32(argc)); // argc. pushArg(Imm32(constructing)); // constructing. pushArg(calleereg); // JSFunction*. callVM(InvokeFunctionInfo, call); // Un-nestle %esp from the argument vector. No prefix was pushed. masm.reserveStack(unusedStack); } void CodeGenerator::visitCallGeneric(LCallGeneric* call) { Register calleereg = ToRegister(call->getFunction()); Register objreg = ToRegister(call->getTempObject()); Register nargsreg = ToRegister(call->getNargsReg()); uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot()); Label invoke, thunk, makeCall, end; // Known-target case is handled by LCallKnown. MOZ_ASSERT(!call->hasSingleTarget()); // Generate an ArgumentsRectifier. JitCode* argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier(); masm.checkStackAlignment(); // Guard that calleereg is actually a function object. masm.loadObjClass(calleereg, nargsreg); masm.branchPtr(Assembler::NotEqual, nargsreg, ImmPtr(&JSFunction::class_), &invoke); // Guard that calleereg is an interpreted function with a JSScript. // If we are constructing, also ensure the callee is a constructor. if (call->mir()->isConstructing()) { masm.branchIfNotInterpretedConstructor(calleereg, nargsreg, &invoke); } else { masm.branchIfFunctionHasNoScript(calleereg, &invoke); masm.branchFunctionKind(Assembler::Equal, JSFunction::ClassConstructor, calleereg, objreg, &invoke); } // Knowing that calleereg is a non-native function, load the JSScript. masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg); // Load script jitcode. masm.loadBaselineOrIonRaw(objreg, objreg, &invoke); // Nestle the StackPointer up to the argument vector. masm.freeStack(unusedStack); // Construct the IonFramePrefix. uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, JitFrameLayout::Size()); masm.Push(Imm32(call->numActualArgs())); masm.PushCalleeToken(calleereg, call->mir()->isConstructing()); masm.Push(Imm32(descriptor)); // Check whether the provided arguments satisfy target argc. // We cannot have lowered to LCallGeneric with a known target. Assert that we didn't // add any undefineds in IonBuilder. NB: MCall::numStackArgs includes |this|. DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing(); MOZ_ASSERT(call->numActualArgs() == call->mir()->numStackArgs() - numNonArgsOnStack); masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nargsreg); masm.branch32(Assembler::Above, nargsreg, Imm32(call->numActualArgs()), &thunk); masm.jump(&makeCall); // Argument fixed needed. Load the ArgumentsRectifier. masm.bind(&thunk); { MOZ_ASSERT(ArgumentsRectifierReg != objreg); masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking. masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg); masm.move32(Imm32(call->numActualArgs()), ArgumentsRectifierReg); } // Finally call the function in objreg. masm.bind(&makeCall); uint32_t callOffset = masm.callJit(objreg); markSafepointAt(callOffset, call); // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass. // The return address has already been removed from the Ion frame. int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*); masm.adjustStack(prefixGarbage - unusedStack); masm.jump(&end); // Handle uncompiled or native functions. masm.bind(&invoke); emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(), unusedStack); masm.bind(&end); // If the return value of the constructing function is Primitive, // replace the return value with the Object from CreateThis. if (call->mir()->isConstructing()) { Label notPrimitive; masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive); masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand); masm.bind(¬Primitive); } } typedef bool (*InvokeFunctionShuffleFn)(JSContext*, HandleObject, uint32_t, uint32_t, Value*, MutableHandleValue); static const VMFunction InvokeFunctionShuffleInfo = FunctionInfo<InvokeFunctionShuffleFn>(InvokeFunctionShuffleNewTarget, "InvokeFunctionShuffleNewTarget"); void CodeGenerator::emitCallInvokeFunctionShuffleNewTarget(LCallKnown* call, Register calleeReg, uint32_t numFormals, uint32_t unusedStack) { masm.freeStack(unusedStack); pushArg(masm.getStackPointer()); pushArg(Imm32(numFormals)); pushArg(Imm32(call->numActualArgs())); pushArg(calleeReg); callVM(InvokeFunctionShuffleInfo, call); masm.reserveStack(unusedStack); } void CodeGenerator::visitCallKnown(LCallKnown* call) { Register calleereg = ToRegister(call->getFunction()); Register objreg = ToRegister(call->getTempObject()); uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot()); WrappedFunction* target = call->getSingleTarget(); Label end, uncompiled; // Native single targets are handled by LCallNative. MOZ_ASSERT(!target->isNative()); // Missing arguments must have been explicitly appended by the IonBuilder. DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing(); MOZ_ASSERT(target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack); MOZ_ASSERT_IF(call->isConstructing(), target->isConstructor()); masm.checkStackAlignment(); if (target->isClassConstructor() && !call->isConstructing()) { emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(), unusedStack); return; } MOZ_ASSERT_IF(target->isClassConstructor(), call->isConstructing()); // The calleereg is known to be a non-native function, but might point to // a LazyScript instead of a JSScript. masm.branchIfFunctionHasNoScript(calleereg, &uncompiled); // Knowing that calleereg is a non-native function, load the JSScript. masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg); // Load script jitcode. if (call->mir()->needsArgCheck()) masm.loadBaselineOrIonRaw(objreg, objreg, &uncompiled); else masm.loadBaselineOrIonNoArgCheck(objreg, objreg, &uncompiled); // Nestle the StackPointer up to the argument vector. masm.freeStack(unusedStack); // Construct the IonFramePrefix. uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS, JitFrameLayout::Size()); masm.Push(Imm32(call->numActualArgs())); masm.PushCalleeToken(calleereg, call->mir()->isConstructing()); masm.Push(Imm32(descriptor)); // Finally call the function in objreg. uint32_t callOffset = masm.callJit(objreg); markSafepointAt(callOffset, call); // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass. // The return address has already been removed from the Ion frame. int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*); masm.adjustStack(prefixGarbage - unusedStack); masm.jump(&end); // Handle uncompiled functions. masm.bind(&uncompiled); if (call->isConstructing() && target->nargs() > call->numActualArgs()) emitCallInvokeFunctionShuffleNewTarget(call, calleereg, target->nargs(), unusedStack); else emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(), unusedStack); masm.bind(&end); // If the return value of the constructing function is Primitive, // replace the return value with the Object from CreateThis. if (call->mir()->isConstructing()) { Label notPrimitive; masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive); masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand); masm.bind(¬Primitive); } } template<typename T> void CodeGenerator::emitCallInvokeFunction(T* apply, Register extraStackSize) { Register objreg = ToRegister(apply->getTempObject()); MOZ_ASSERT(objreg != extraStackSize); // Push the space used by the arguments. masm.moveStackPtrTo(objreg); masm.Push(extraStackSize); pushArg(objreg); // argv. pushArg(ToRegister(apply->getArgc())); // argc. pushArg(Imm32(false)); // isConstrucing. pushArg(ToRegister(apply->getFunction())); // JSFunction*. // This specialization og callVM restore the extraStackSize after the call. callVM(InvokeFunctionInfo, apply, &extraStackSize); masm.Pop(extraStackSize); } // Do not bailout after the execution of this function since the stack no longer // correspond to what is expected by the snapshots. void CodeGenerator::emitAllocateSpaceForApply(Register argcreg, Register extraStackSpace, Label* end) { // Initialize the loop counter AND Compute the stack usage (if == 0) masm.movePtr(argcreg, extraStackSpace); // Align the JitFrameLayout on the JitStackAlignment. if (JitStackValueAlignment > 1) { MOZ_ASSERT(frameSize() % JitStackAlignment == 0, "Stack padding assumes that the frameSize is correct"); MOZ_ASSERT(JitStackValueAlignment == 2); Label noPaddingNeeded; // if the number of arguments is odd, then we do not need any padding. masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded); masm.addPtr(Imm32(1), extraStackSpace); masm.bind(&noPaddingNeeded); } // Reserve space for copying the arguments. NativeObject::elementsSizeMustNotOverflow(); masm.lshiftPtr(Imm32(ValueShift), extraStackSpace); masm.subFromStackPtr(extraStackSpace); #ifdef DEBUG // Put a magic value in the space reserved for padding. Note, this code // cannot be merged with the previous test, as not all architectures can // write below their stack pointers. if (JitStackValueAlignment > 1) { MOZ_ASSERT(JitStackValueAlignment == 2); Label noPaddingNeeded; // if the number of arguments is odd, then we do not need any padding. masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded); BaseValueIndex dstPtr(masm.getStackPointer(), argcreg); masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr); masm.bind(&noPaddingNeeded); } #endif // Skip the copy of arguments if there are none. masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, end); } // Destroys argvIndex and copyreg. void CodeGenerator::emitCopyValuesForApply(Register argvSrcBase, Register argvIndex, Register copyreg, size_t argvSrcOffset, size_t argvDstOffset) { Label loop; masm.bind(&loop); // As argvIndex is off by 1, and we use the decBranchPtr instruction // to loop back, we have to substract the size of the word which are // copied. BaseValueIndex srcPtr(argvSrcBase, argvIndex, argvSrcOffset - sizeof(void*)); BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex, argvDstOffset - sizeof(void*)); masm.loadPtr(srcPtr, copyreg); masm.storePtr(copyreg, dstPtr); // Handle 32 bits architectures. if (sizeof(Value) == 2 * sizeof(void*)) { BaseValueIndex srcPtrLow(argvSrcBase, argvIndex, argvSrcOffset - 2 * sizeof(void*)); BaseValueIndex dstPtrLow(masm.getStackPointer(), argvIndex, argvDstOffset - 2 * sizeof(void*)); masm.loadPtr(srcPtrLow, copyreg); masm.storePtr(copyreg, dstPtrLow); } masm.decBranchPtr(Assembler::NonZero, argvIndex, Imm32(1), &loop); } void CodeGenerator::emitPopArguments(Register extraStackSpace) { // Pop |this| and Arguments. masm.freeStack(extraStackSpace); } void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply, Register extraStackSpace) { // Holds the function nargs. Initially the number of args to the caller. Register argcreg = ToRegister(apply->getArgc()); Register copyreg = ToRegister(apply->getTempObject()); Label end; emitAllocateSpaceForApply(argcreg, extraStackSpace, &end); // We are making a copy of the arguments which are above the JitFrameLayout // of the current Ion frame. // // [arg1] [arg0] <- src [this] [JitFrameLayout] [.. frameSize ..] [pad] [arg1] [arg0] <- dst // Compute the source and destination offsets into the stack. size_t argvSrcOffset = frameSize() + JitFrameLayout::offsetOfActualArgs(); size_t argvDstOffset = 0; // Save the extra stack space, and re-use the register as a base. masm.push(extraStackSpace); Register argvSrcBase = extraStackSpace; argvSrcOffset += sizeof(void*); argvDstOffset += sizeof(void*); // Save the actual number of register, and re-use the register as an index register. masm.push(argcreg); Register argvIndex = argcreg; argvSrcOffset += sizeof(void*); argvDstOffset += sizeof(void*); // srcPtr = (StackPointer + extraStackSpace) + argvSrcOffset // dstPtr = (StackPointer ) + argvDstOffset masm.addStackPtrTo(argvSrcBase); // Copy arguments. emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset, argvDstOffset); // Restore argcreg and the extra stack space counter. masm.pop(argcreg); masm.pop(extraStackSpace); // Join with all arguments copied and the extra stack usage computed. masm.bind(&end); // Push |this|. masm.addPtr(Imm32(sizeof(Value)), extraStackSpace); masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex)); } void CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply, Register extraStackSpace) { Label noCopy, epilogue; Register tmpArgc = ToRegister(apply->getTempObject()); Register elementsAndArgc = ToRegister(apply->getElements()); // Invariants guarded in the caller: // - the array is not too long // - the array length equals its initialized length // The array length is our argc for the purposes of allocating space. Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength()); masm.load32(length, tmpArgc); // Allocate space for the values. emitAllocateSpaceForApply(tmpArgc, extraStackSpace, &noCopy); // Copy the values. This code is skipped entirely if there are // no values. size_t argvDstOffset = 0; Register argvSrcBase = elementsAndArgc; // Elements value masm.push(extraStackSpace); Register copyreg = extraStackSpace; argvDstOffset += sizeof(void*); masm.push(tmpArgc); Register argvIndex = tmpArgc; argvDstOffset += sizeof(void*); // Copy emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, 0, argvDstOffset); // Restore. masm.pop(elementsAndArgc); masm.pop(extraStackSpace); masm.jump(&epilogue); // Clear argc if we skipped the copy step. masm.bind(&noCopy); masm.movePtr(ImmPtr(0), elementsAndArgc); // Join with all arguments copied and the extra stack usage computed. // Note, "elements" has become "argc". masm.bind(&epilogue); // Push |this|. masm.addPtr(Imm32(sizeof(Value)), extraStackSpace); masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex)); } template<typename T> void CodeGenerator::emitApplyGeneric(T* apply) { // Holds the function object. Register calleereg = ToRegister(apply->getFunction()); // Temporary register for modifying the function object. Register objreg = ToRegister(apply->getTempObject()); Register extraStackSpace = ToRegister(apply->getTempStackCounter()); // Holds the function nargs, computed in the invoker or (for // ApplyArray) in the argument pusher. Register argcreg = ToRegister(apply->getArgc()); // Unless already known, guard that calleereg is actually a function object. if (!apply->hasSingleTarget()) { masm.loadObjClass(calleereg, objreg); ImmPtr ptr = ImmPtr(&JSFunction::class_); bailoutCmpPtr(Assembler::NotEqual, objreg, ptr, apply->snapshot()); } // Copy the arguments of the current function. // // In the case of ApplyArray, also compute argc: the argc register // and the elements register are the same; argc must not be // referenced before the call to emitPushArguments() and elements // must not be referenced after it returns. // // objreg is dead across this call. // // extraStackSpace is garbage on entry and defined on exit. emitPushArguments(apply, extraStackSpace); masm.checkStackAlignment(); // If the function is native, only emit the call to InvokeFunction. if (apply->hasSingleTarget() && apply->getSingleTarget()->isNative()) { emitCallInvokeFunction(apply, extraStackSpace); emitPopArguments(extraStackSpace); return; } Label end, invoke; // Guard that calleereg is an interpreted function with a JSScript. masm.branchIfFunctionHasNoScript(calleereg, &invoke); // Knowing that calleereg is a non-native function, load the JSScript. masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg); // Load script jitcode. masm.loadBaselineOrIonRaw(objreg, objreg, &invoke); // Call with an Ion frame or a rectifier frame. { // Create the frame descriptor. unsigned pushed = masm.framePushed(); Register stackSpace = extraStackSpace; masm.addPtr(Imm32(pushed), stackSpace); masm.makeFrameDescriptor(stackSpace, JitFrame_IonJS, JitFrameLayout::Size()); masm.Push(argcreg); masm.Push(calleereg); masm.Push(stackSpace); // descriptor Label underflow, rejoin; // Check whether the provided arguments satisfy target argc. if (!apply->hasSingleTarget()) { Register nformals = extraStackSpace; masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nformals); masm.branch32(Assembler::Below, argcreg, nformals, &underflow); } else { masm.branch32(Assembler::Below, argcreg, Imm32(apply->getSingleTarget()->nargs()), &underflow); } // Skip the construction of the rectifier frame because we have no // underflow. masm.jump(&rejoin); // Argument fixup needed. Get ready to call the argumentsRectifier. { masm.bind(&underflow); // Hardcode the address of the argumentsRectifier code. JitCode* argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier(); MOZ_ASSERT(ArgumentsRectifierReg != objreg); masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking. masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg); masm.movePtr(argcreg, ArgumentsRectifierReg); } masm.bind(&rejoin); // Finally call the function in objreg, as assigned by one of the paths above. uint32_t callOffset = masm.callJit(objreg); markSafepointAt(callOffset, apply); // Recover the number of arguments from the frame descriptor. masm.loadPtr(Address(masm.getStackPointer(), 0), stackSpace); masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), stackSpace); masm.subPtr(Imm32(pushed), stackSpace); // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass. // The return address has already been removed from the Ion frame. int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*); masm.adjustStack(prefixGarbage); masm.jump(&end); } // Handle uncompiled or native functions. { masm.bind(&invoke); emitCallInvokeFunction(apply, extraStackSpace); } // Pop arguments and continue. masm.bind(&end); emitPopArguments(extraStackSpace); } void CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply) { // Limit the number of parameters we can handle to a number that does not risk // us allocating too much stack, notably on Windows where there is a 4K guard page // that has to be touched to extend the stack. See bug 1351278. The value "3000" // is the size of the guard page minus an arbitrary, but large, safety margin. LSnapshot* snapshot = apply->snapshot(); Register argcreg = ToRegister(apply->getArgc()); uint32_t limit = 3000 / sizeof(Value); bailoutCmp32(Assembler::Above, argcreg, Imm32(limit), snapshot); emitApplyGeneric(apply); } void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) { LSnapshot* snapshot = apply->snapshot(); Register tmp = ToRegister(apply->getTempObject()); Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength()); masm.load32(length, tmp); // See comment in visitApplyArgsGeneric, above. uint32_t limit = 3000 / sizeof(Value); bailoutCmp32(Assembler::Above, tmp, Imm32(limit), snapshot); // Ensure that the array does not contain an uninitialized tail. Address initializedLength(ToRegister(apply->getElements()), ObjectElements::offsetOfInitializedLength()); masm.sub32(initializedLength, tmp); bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot); emitApplyGeneric(apply); } typedef bool (*ArraySpliceDenseFn)(JSContext*, HandleObject, uint32_t, uint32_t); static const VMFunction ArraySpliceDenseInfo = FunctionInfo<ArraySpliceDenseFn>(ArraySpliceDense, "ArraySpliceDense"); void CodeGenerator::visitArraySplice(LArraySplice* lir) { pushArg(ToRegister(lir->getDeleteCount())); pushArg(ToRegister(lir->getStart())); pushArg(ToRegister(lir->getObject())); callVM(ArraySpliceDenseInfo, lir); } void CodeGenerator::visitBail(LBail* lir) { bailout(lir->snapshot()); } void CodeGenerator::visitUnreachable(LUnreachable* lir) { masm.assumeUnreachable("end-of-block assumed unreachable"); } void CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir) { encode(lir->snapshot()); } void CodeGenerator::visitGetDynamicName(LGetDynamicName* lir) { Register envChain = ToRegister(lir->getEnvironmentChain()); Register name = ToRegister(lir->getName()); Register temp1 = ToRegister(lir->temp1()); Register temp2 = ToRegister(lir->temp2()); Register temp3 = ToRegister(lir->temp3()); masm.loadJSContext(temp3); /* Make space for the outparam. */ masm.adjustStack(-int32_t(sizeof(Value))); masm.moveStackPtrTo(temp2); masm.setupUnalignedABICall(temp1); masm.passABIArg(temp3); masm.passABIArg(envChain); masm.passABIArg(name); masm.passABIArg(temp2); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GetDynamicName)); const ValueOperand out = ToOutValue(lir); masm.loadValue(Address(masm.getStackPointer(), 0), out); masm.adjustStack(sizeof(Value)); Label undefined; masm.branchTestUndefined(Assembler::Equal, out, &undefined); bailoutFrom(&undefined, lir->snapshot()); } typedef bool (*DirectEvalSFn)(JSContext*, HandleObject, HandleScript, HandleValue, HandleString, jsbytecode*, MutableHandleValue); static const VMFunction DirectEvalStringInfo = FunctionInfo<DirectEvalSFn>(DirectEvalStringFromIon, "DirectEvalStringFromIon"); void CodeGenerator::visitCallDirectEval(LCallDirectEval* lir) { Register envChain = ToRegister(lir->getEnvironmentChain()); Register string = ToRegister(lir->getString()); pushArg(ImmPtr(lir->mir()->pc())); pushArg(string); pushArg(ToValue(lir, LCallDirectEval::NewTarget)); pushArg(ImmGCPtr(current->mir()->info().script())); pushArg(envChain); callVM(DirectEvalStringInfo, lir); } void CodeGenerator::generateArgumentsChecks(bool bailout) { // Registers safe for use before generatePrologue(). static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code()); // This function can be used the normal way to check the argument types, // before entering the function and bailout when arguments don't match. // For debug purpose, this is can also be used to force/check that the // arguments are correct. Upon fail it will hit a breakpoint. MIRGraph& mir = gen->graph(); MResumePoint* rp = mir.entryResumePoint(); // No registers are allocated yet, so it's safe to grab anything. Register temp = GeneralRegisterSet(EntryTempMask).getAny(); const CompileInfo& info = gen->info(); Label miss; for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) { // All initial parameters are guaranteed to be MParameters. MParameter* param = rp->getOperand(i)->toParameter(); const TypeSet* types = param->resultTypeSet(); if (!types || types->unknown()) continue; // Calculate the offset on the stack of the argument. // (i - info.startArgSlot()) - Compute index of arg within arg vector. // ... * sizeof(Value) - Scale by value size. // ArgToStackOffset(...) - Compute displacement within arg vector. int32_t offset = ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value)); masm.guardTypeSet(Address(masm.getStackPointer(), offset), types, BarrierKind::TypeSet, temp, &miss); } if (miss.used()) { if (bailout) { bailoutFrom(&miss, graph.entrySnapshot()); } else { Label success; masm.jump(&success); masm.bind(&miss); // Check for cases where the type set guard might have missed due to // changing object groups. for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) { MParameter* param = rp->getOperand(i)->toParameter(); const TemporaryTypeSet* types = param->resultTypeSet(); if (!types || types->unknown()) continue; Label skip; Address addr(masm.getStackPointer(), ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value))); masm.branchTestObject(Assembler::NotEqual, addr, &skip); Register obj = masm.extractObject(addr, temp); masm.guardTypeSetMightBeIncomplete(types, obj, temp, &success); masm.bind(&skip); } masm.assumeUnreachable("Argument check fail."); masm.bind(&success); } } } // Out-of-line path to report over-recursed error and fail. class CheckOverRecursedFailure : public OutOfLineCodeBase<CodeGenerator> { LInstruction* lir_; public: explicit CheckOverRecursedFailure(LInstruction* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitCheckOverRecursedFailure(this); } LInstruction* lir() const { return lir_; } }; void CodeGenerator::visitCheckOverRecursed(LCheckOverRecursed* lir) { // If we don't push anything on the stack, skip the check. if (omitOverRecursedCheck()) return; // Ensure that this frame will not cross the stack limit. // This is a weak check, justified by Ion using the C stack: we must always // be some distance away from the actual limit, since if the limit is // crossed, an error must be thrown, which requires more frames. // // It must always be possible to trespass past the stack limit. // Ion may legally place frames very close to the limit. Calling additional // C functions may then violate the limit without any checking. // Since Ion frames exist on the C stack, the stack limit may be // dynamically set by JS_SetThreadStackLimit() and JS_SetNativeStackQuota(). const void* limitAddr = GetJitContext()->runtime->addressOfJitStackLimit(); CheckOverRecursedFailure* ool = new(alloc()) CheckOverRecursedFailure(lir); addOutOfLineCode(ool, lir->mir()); // Conditional forward (unlikely) branch to failure. masm.branchStackPtrRhs(Assembler::AboveOrEqual, AbsoluteAddress(limitAddr), ool->entry()); masm.bind(ool->rejoin()); } typedef bool (*DefVarFn)(JSContext*, HandlePropertyName, unsigned, HandleObject); static const VMFunction DefVarInfo = FunctionInfo<DefVarFn>(DefVar, "DefVar"); void CodeGenerator::visitDefVar(LDefVar* lir) { Register envChain = ToRegister(lir->environmentChain()); pushArg(envChain); // JSObject* pushArg(Imm32(lir->mir()->attrs())); // unsigned pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName* callVM(DefVarInfo, lir); } typedef bool (*DefLexicalFn)(JSContext*, HandlePropertyName, unsigned); static const VMFunction DefLexicalInfo = FunctionInfo<DefLexicalFn>(DefGlobalLexical, "DefGlobalLexical"); void CodeGenerator::visitDefLexical(LDefLexical* lir) { pushArg(Imm32(lir->mir()->attrs())); // unsigned pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName* callVM(DefLexicalInfo, lir); } typedef bool (*DefFunOperationFn)(JSContext*, HandleScript, HandleObject, HandleFunction); static const VMFunction DefFunOperationInfo = FunctionInfo<DefFunOperationFn>(DefFunOperation, "DefFunOperation"); void CodeGenerator::visitDefFun(LDefFun* lir) { Register envChain = ToRegister(lir->environmentChain()); Register fun = ToRegister(lir->fun()); pushArg(fun); pushArg(envChain); pushArg(ImmGCPtr(current->mir()->info().script())); callVM(DefFunOperationInfo, lir); } typedef bool (*CheckOverRecursedFn)(JSContext*); static const VMFunction CheckOverRecursedInfo = FunctionInfo<CheckOverRecursedFn>(CheckOverRecursed, "CheckOverRecursed"); void CodeGenerator::visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool) { // The OOL path is hit if the recursion depth has been exceeded. // Throw an InternalError for over-recursion. // LFunctionEnvironment can appear before LCheckOverRecursed, so we have // to save all live registers to avoid crashes if CheckOverRecursed triggers // a GC. saveLive(ool->lir()); callVM(CheckOverRecursedInfo, ool->lir()); restoreLive(ool->lir()); masm.jump(ool->rejoin()); } IonScriptCounts* CodeGenerator::maybeCreateScriptCounts() { // If scripts are being profiled, create a new IonScriptCounts for the // profiling data, which will be attached to the associated JSScript or // wasm module after code generation finishes. if (!GetJitContext()->hasProfilingScripts()) return nullptr; // This test inhibits IonScriptCount creation for wasm code which is // currently incompatible with wasm codegen for two reasons: (1) wasm code // must be serializable and script count codegen bakes in absolute // addresses, (2) wasm code does not have a JSScript with which to associate // code coverage data. JSScript* script = gen->info().script(); if (!script) return nullptr; UniquePtr<IonScriptCounts> counts(js_new<IonScriptCounts>()); if (!counts || !counts->init(graph.numBlocks())) return nullptr; for (size_t i = 0; i < graph.numBlocks(); i++) { MBasicBlock* block = graph.getBlock(i)->mir(); uint32_t offset = 0; char* description = nullptr; if (MResumePoint* resume = block->entryResumePoint()) { // Find a PC offset in the outermost script to use. If this // block is from an inlined script, find a location in the // outer script to associate information about the inlining // with. while (resume->caller()) resume = resume->caller(); offset = script->pcToOffset(resume->pc()); if (block->entryResumePoint()->caller()) { // Get the filename and line number of the inner script. JSScript* innerScript = block->info().script(); description = (char*) js_calloc(200); if (description) { snprintf(description, 200, "%s:%" PRIuSIZE, innerScript->filename(), innerScript->lineno()); } } } if (!counts->block(i).init(block->id(), offset, description, block->numSuccessors())) return nullptr; for (size_t j = 0; j < block->numSuccessors(); j++) counts->block(i).setSuccessor(j, skipTrivialBlocks(block->getSuccessor(j))->id()); } scriptCounts_ = counts.release(); return scriptCounts_; } // Structure for managing the state tracked for a block by script counters. struct ScriptCountBlockState { IonBlockCounts& block; MacroAssembler& masm; Sprinter printer; public: ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm) : block(*block), masm(*masm), printer(GetJitContext()->cx, false) { } bool init() { if (!printer.init()) return false; // Bump the hit count for the block at the start. This code is not // included in either the text for the block or the instruction byte // counts. masm.inc64(AbsoluteAddress(block.addressOfHitCount())); // Collect human readable assembly for the code generated in the block. masm.setPrinter(&printer); return true; } void visitInstruction(LInstruction* ins) { // Prefix stream of assembly instructions with their LIR instruction // name and any associated high level info. if (const char* extra = ins->extraName()) printer.printf("[%s:%s]\n", ins->opName(), extra); else printer.printf("[%s]\n", ins->opName()); } ~ScriptCountBlockState() { masm.setPrinter(nullptr); if (!printer.hadOutOfMemory()) block.setCode(printer.string()); } }; void CodeGenerator::branchIfInvalidated(Register temp, Label* invalidated) { CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), temp); masm.propagateOOM(ionScriptLabels_.append(label)); // If IonScript::invalidationCount_ != 0, the script has been invalidated. masm.branch32(Assembler::NotEqual, Address(temp, IonScript::offsetOfInvalidationCount()), Imm32(0), invalidated); } void CodeGenerator::emitAssertObjectOrStringResult(Register input, MIRType type, const TemporaryTypeSet* typeset) { MOZ_ASSERT(type == MIRType::Object || type == MIRType::ObjectOrNull || type == MIRType::String || type == MIRType::Symbol); AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(input); Register temp = regs.takeAny(); masm.push(temp); // Don't check if the script has been invalidated. In that case invalid // types are expected (until we reach the OsiPoint and bailout). Label done; branchIfInvalidated(temp, &done); if ((type == MIRType::Object || type == MIRType::ObjectOrNull) && typeset && !typeset->unknownObject()) { // We have a result TypeSet, assert this object is in it. Label miss, ok; if (type == MIRType::ObjectOrNull) masm.branchPtr(Assembler::Equal, input, ImmWord(0), &ok); if (typeset->getObjectCount() > 0) masm.guardObjectType(input, typeset, temp, &miss); else masm.jump(&miss); masm.jump(&ok); masm.bind(&miss); masm.guardTypeSetMightBeIncomplete(typeset, input, temp, &ok); masm.assumeUnreachable("MIR instruction returned object with unexpected type"); masm.bind(&ok); } // Check that we have a valid GC pointer. saveVolatile(); masm.setupUnalignedABICall(temp); masm.loadJSContext(temp); masm.passABIArg(temp); masm.passABIArg(input); void* callee; switch (type) { case MIRType::Object: callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectPtr); break; case MIRType::ObjectOrNull: callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectOrNullPtr); break; case MIRType::String: callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidStringPtr); break; case MIRType::Symbol: callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidSymbolPtr); break; default: MOZ_CRASH(); } masm.callWithABI(callee); restoreVolatile(); masm.bind(&done); masm.pop(temp); } void CodeGenerator::emitAssertResultV(const ValueOperand input, const TemporaryTypeSet* typeset) { AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); regs.take(input); Register temp1 = regs.takeAny(); Register temp2 = regs.takeAny(); masm.push(temp1); masm.push(temp2); // Don't check if the script has been invalidated. In that case invalid // types are expected (until we reach the OsiPoint and bailout). Label done; branchIfInvalidated(temp1, &done); if (typeset && !typeset->unknown()) { // We have a result TypeSet, assert this value is in it. Label miss, ok; masm.guardTypeSet(input, typeset, BarrierKind::TypeSet, temp1, &miss); masm.jump(&ok); masm.bind(&miss); // Check for cases where the type set guard might have missed due to // changing object groups. Label realMiss; masm.branchTestObject(Assembler::NotEqual, input, &realMiss); Register payload = masm.extractObject(input, temp1); masm.guardTypeSetMightBeIncomplete(typeset, payload, temp1, &ok); masm.bind(&realMiss); masm.assumeUnreachable("MIR instruction returned value with unexpected type"); masm.bind(&ok); } // Check that we have a valid GC pointer. saveVolatile(); masm.pushValue(input); masm.moveStackPtrTo(temp1); masm.setupUnalignedABICall(temp2); masm.loadJSContext(temp2); masm.passABIArg(temp2); masm.passABIArg(temp1); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, AssertValidValue)); masm.popValue(input); restoreVolatile(); masm.bind(&done); masm.pop(temp2); masm.pop(temp1); } #ifdef DEBUG void CodeGenerator::emitObjectOrStringResultChecks(LInstruction* lir, MDefinition* mir) { if (lir->numDefs() == 0) return; MOZ_ASSERT(lir->numDefs() == 1); Register output = ToRegister(lir->getDef(0)); emitAssertObjectOrStringResult(output, mir->type(), mir->resultTypeSet()); } void CodeGenerator::emitValueResultChecks(LInstruction* lir, MDefinition* mir) { if (lir->numDefs() == 0) return; MOZ_ASSERT(lir->numDefs() == BOX_PIECES); if (!lir->getDef(0)->output()->isRegister()) return; ValueOperand output = ToOutValue(lir); emitAssertResultV(output, mir->resultTypeSet()); } void CodeGenerator::emitDebugResultChecks(LInstruction* ins) { // In debug builds, check that LIR instructions return valid values. MDefinition* mir = ins->mirRaw(); if (!mir) return; switch (mir->type()) { case MIRType::Object: case MIRType::ObjectOrNull: case MIRType::String: case MIRType::Symbol: emitObjectOrStringResultChecks(ins, mir); break; case MIRType::Value: emitValueResultChecks(ins, mir); break; default: break; } } void CodeGenerator::emitDebugForceBailing(LInstruction* lir) { if (!lir->snapshot()) return; if (lir->isStart()) return; if (lir->isOsiPoint()) return; masm.comment("emitDebugForceBailing"); const void* bailAfterAddr = GetJitContext()->runtime->addressOfIonBailAfter(); AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); Label done, notBail, bail; masm.branch32(Assembler::Equal, AbsoluteAddress(bailAfterAddr), Imm32(0), &done); { Register temp = regs.takeAny(); masm.push(temp); masm.load32(AbsoluteAddress(bailAfterAddr), temp); masm.sub32(Imm32(1), temp); masm.store32(temp, AbsoluteAddress(bailAfterAddr)); masm.branch32(Assembler::NotEqual, temp, Imm32(0), ¬Bail); { masm.pop(temp); masm.jump(&bail); bailoutFrom(&bail, lir->snapshot()); } masm.bind(¬Bail); masm.pop(temp); } masm.bind(&done); } #endif bool CodeGenerator::generateBody() { IonScriptCounts* counts = maybeCreateScriptCounts(); #if defined(JS_ION_PERF) PerfSpewer* perfSpewer = &perfSpewer_; if (gen->compilingWasm()) perfSpewer = &gen->perfSpewer(); #endif for (size_t i = 0; i < graph.numBlocks(); i++) { current = graph.getBlock(i); // Don't emit any code for trivial blocks, containing just a goto. Such // blocks are created to split critical edges, and if we didn't end up // putting any instructions in them, we can skip them. if (current->isTrivial()) continue; #ifdef JS_JITSPEW const char* filename = nullptr; size_t lineNumber = 0; unsigned columnNumber = 0; if (current->mir()->info().script()) { filename = current->mir()->info().script()->filename(); if (current->mir()->pc()) lineNumber = PCToLineNumber(current->mir()->info().script(), current->mir()->pc(), &columnNumber); } else { #ifdef DEBUG lineNumber = current->mir()->lineno(); columnNumber = current->mir()->columnIndex(); #endif } JitSpew(JitSpew_Codegen, "# block%" PRIuSIZE " %s:%" PRIuSIZE ":%u%s:", i, filename ? filename : "?", lineNumber, columnNumber, current->mir()->isLoopHeader() ? " (loop header)" : ""); #endif masm.bind(current->label()); mozilla::Maybe<ScriptCountBlockState> blockCounts; if (counts) { blockCounts.emplace(&counts->block(i), &masm); if (!blockCounts->init()) return false; } #if defined(JS_ION_PERF) perfSpewer->startBasicBlock(current->mir(), masm); #endif for (LInstructionIterator iter = current->begin(); iter != current->end(); iter++) { if (!alloc().ensureBallast()) return false; #ifdef JS_JITSPEW JitSpewStart(JitSpew_Codegen, "instruction %s", iter->opName()); if (const char* extra = iter->extraName()) JitSpewCont(JitSpew_Codegen, ":%s", extra); JitSpewFin(JitSpew_Codegen); #endif if (counts) blockCounts->visitInstruction(*iter); #ifdef CHECK_OSIPOINT_REGISTERS if (iter->safepoint()) resetOsiPointRegs(iter->safepoint()); #endif if (iter->mirRaw()) { // Only add instructions that have a tracked inline script tree. if (iter->mirRaw()->trackedTree()) { if (!addNativeToBytecodeEntry(iter->mirRaw()->trackedSite())) return false; } // Track the start native offset of optimizations. if (iter->mirRaw()->trackedOptimizations()) { if (!addTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations())) return false; } } #ifdef DEBUG setElement(*iter); // needed to encode correct snapshot location. emitDebugForceBailing(*iter); #endif iter->accept(this); // Track the end native offset of optimizations. if (iter->mirRaw() && iter->mirRaw()->trackedOptimizations()) extendTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations()); #ifdef DEBUG if (!counts) emitDebugResultChecks(*iter); #endif } if (masm.oom()) return false; #if defined(JS_ION_PERF) perfSpewer->endBasicBlock(masm); #endif } return true; } // Out-of-line object allocation for LNewArray. class OutOfLineNewArray : public OutOfLineCodeBase<CodeGenerator> { LNewArray* lir_; public: explicit OutOfLineNewArray(LNewArray* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineNewArray(this); } LNewArray* lir() const { return lir_; } }; typedef JSObject* (*NewArrayOperationFn)(JSContext*, HandleScript, jsbytecode*, uint32_t, NewObjectKind); static const VMFunction NewArrayOperationInfo = FunctionInfo<NewArrayOperationFn>(NewArrayOperation, "NewArrayOperation"); static JSObject* NewArrayWithGroup(JSContext* cx, uint32_t length, HandleObjectGroup group, bool convertDoubleElements) { JSObject* res = NewFullyAllocatedArrayTryUseGroup(cx, group, length); if (!res) return nullptr; if (convertDoubleElements) res->as<ArrayObject>().setShouldConvertDoubleElements(); return res; } typedef JSObject* (*NewArrayWithGroupFn)(JSContext*, uint32_t, HandleObjectGroup, bool); static const VMFunction NewArrayWithGroupInfo = FunctionInfo<NewArrayWithGroupFn>(NewArrayWithGroup, "NewArrayWithGroup"); void CodeGenerator::visitNewArrayCallVM(LNewArray* lir) { Register objReg = ToRegister(lir->output()); MOZ_ASSERT(!lir->isCall()); saveLive(lir); JSObject* templateObject = lir->mir()->templateObject(); if (templateObject) { pushArg(Imm32(lir->mir()->convertDoubleElements())); pushArg(ImmGCPtr(templateObject->group())); pushArg(Imm32(lir->mir()->length())); callVM(NewArrayWithGroupInfo, lir); } else { pushArg(Imm32(GenericObject)); pushArg(Imm32(lir->mir()->length())); pushArg(ImmPtr(lir->mir()->pc())); pushArg(ImmGCPtr(lir->mir()->block()->info().script())); callVM(NewArrayOperationInfo, lir); } if (ReturnReg != objReg) masm.movePtr(ReturnReg, objReg); restoreLive(lir); } typedef JSObject* (*NewDerivedTypedObjectFn)(JSContext*, HandleObject type, HandleObject owner, int32_t offset); static const VMFunction CreateDerivedTypedObjInfo = FunctionInfo<NewDerivedTypedObjectFn>(CreateDerivedTypedObj, "CreateDerivedTypedObj"); void CodeGenerator::visitNewDerivedTypedObject(LNewDerivedTypedObject* lir) { pushArg(ToRegister(lir->offset())); pushArg(ToRegister(lir->owner())); pushArg(ToRegister(lir->type())); callVM(CreateDerivedTypedObjInfo, lir); } void CodeGenerator::visitAtan2D(LAtan2D* lir) { Register temp = ToRegister(lir->temp()); FloatRegister y = ToFloatRegister(lir->y()); FloatRegister x = ToFloatRegister(lir->x()); masm.setupUnalignedABICall(temp); masm.passABIArg(y, MoveOp::DOUBLE); masm.passABIArg(x, MoveOp::DOUBLE); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaAtan2), MoveOp::DOUBLE); MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg); } void CodeGenerator::visitHypot(LHypot* lir) { Register temp = ToRegister(lir->temp()); uint32_t numArgs = lir->numArgs(); masm.setupUnalignedABICall(temp); for (uint32_t i = 0 ; i < numArgs; ++i) masm.passABIArg(ToFloatRegister(lir->getOperand(i)), MoveOp::DOUBLE); switch(numArgs) { case 2: masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaHypot), MoveOp::DOUBLE); break; case 3: masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot3), MoveOp::DOUBLE); break; case 4: masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot4), MoveOp::DOUBLE); break; default: MOZ_CRASH("Unexpected number of arguments to hypot function."); } MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg); } void CodeGenerator::visitNewArray(LNewArray* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); JSObject* templateObject = lir->mir()->templateObject(); DebugOnly<uint32_t> length = lir->mir()->length(); MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT); if (lir->mir()->isVMCall()) { visitNewArrayCallVM(lir); return; } OutOfLineNewArray* ool = new(alloc()) OutOfLineNewArray(lir); addOutOfLineCode(ool, lir->mir()); masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(), /* initContents = */ true, lir->mir()->convertDoubleElements()); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineNewArray(OutOfLineNewArray* ool) { visitNewArrayCallVM(ool->lir()); masm.jump(ool->rejoin()); } void CodeGenerator::visitNewArrayCopyOnWrite(LNewArrayCopyOnWrite* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); ArrayObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); // If we have a template object, we can inline call object creation. OutOfLineCode* ool = oolCallVM(NewArrayCopyOnWriteInfo, lir, ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)), StoreRegisterTo(objReg)); masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry()); masm.bind(ool->rejoin()); } typedef JSObject* (*ArrayConstructorOneArgFn)(JSContext*, HandleObjectGroup, int32_t length); static const VMFunction ArrayConstructorOneArgInfo = FunctionInfo<ArrayConstructorOneArgFn>(ArrayConstructorOneArg, "ArrayConstructorOneArg"); void CodeGenerator::visitNewArrayDynamicLength(LNewArrayDynamicLength* lir) { Register lengthReg = ToRegister(lir->length()); Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); JSObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); OutOfLineCode* ool = oolCallVM(ArrayConstructorOneArgInfo, lir, ArgList(ImmGCPtr(templateObject->group()), lengthReg), StoreRegisterTo(objReg)); bool canInline = true; size_t inlineLength = 0; if (templateObject->is<ArrayObject>()) { if (templateObject->as<ArrayObject>().hasFixedElements()) { size_t numSlots = gc::GetGCKindSlots(templateObject->asTenured().getAllocKind()); inlineLength = numSlots - ObjectElements::VALUES_PER_HEADER; } else { canInline = false; } } else { if (templateObject->as<UnboxedArrayObject>().hasInlineElements()) { size_t nbytes = templateObject->tenuredSizeOfThis() - UnboxedArrayObject::offsetOfInlineElements(); inlineLength = nbytes / templateObject->as<UnboxedArrayObject>().elementSize(); } else { canInline = false; } } if (canInline) { // Try to do the allocation inline if the template object is big enough // for the length in lengthReg. If the length is bigger we could still // use the template object and not allocate the elements, but it's more // efficient to do a single big allocation than (repeatedly) reallocating // the array later on when filling it. masm.branch32(Assembler::Above, lengthReg, Imm32(inlineLength), ool->entry()); masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry()); size_t lengthOffset = NativeObject::offsetOfFixedElements() + ObjectElements::offsetOfLength(); masm.store32(lengthReg, Address(objReg, lengthOffset)); } else { masm.jump(ool->entry()); } masm.bind(ool->rejoin()); } typedef TypedArrayObject* (*TypedArrayConstructorOneArgFn)(JSContext*, HandleObject, int32_t length); static const VMFunction TypedArrayConstructorOneArgInfo = FunctionInfo<TypedArrayConstructorOneArgFn>(TypedArrayCreateWithTemplate, "TypedArrayCreateWithTemplate"); void CodeGenerator::visitNewTypedArray(LNewTypedArray* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp1()); Register lengthReg = ToRegister(lir->temp2()); LiveRegisterSet liveRegs = lir->safepoint()->liveRegs(); JSObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>(); uint32_t n = ttemplate->length(); OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir, ArgList(ImmGCPtr(templateObject), Imm32(n)), StoreRegisterTo(objReg)); masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry(), /*initContents*/true, /*convertDoubleElements*/false); masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(), ttemplate, TypedArrayLength::Fixed); masm.bind(ool->rejoin()); } void CodeGenerator::visitNewTypedArrayDynamicLength(LNewTypedArrayDynamicLength* lir) { Register lengthReg = ToRegister(lir->length()); Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); LiveRegisterSet liveRegs = lir->safepoint()->liveRegs(); JSObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>(); OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir, ArgList(ImmGCPtr(templateObject), lengthReg), StoreRegisterTo(objReg)); masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry(), /*initContents*/true, /*convertDoubleElements*/false); masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(), ttemplate, TypedArrayLength::Dynamic); masm.bind(ool->rejoin()); } // Out-of-line object allocation for JSOP_NEWOBJECT. class OutOfLineNewObject : public OutOfLineCodeBase<CodeGenerator> { LNewObject* lir_; public: explicit OutOfLineNewObject(LNewObject* lir) : lir_(lir) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineNewObject(this); } LNewObject* lir() const { return lir_; } }; typedef JSObject* (*NewInitObjectWithTemplateFn)(JSContext*, HandleObject); static const VMFunction NewInitObjectWithTemplateInfo = FunctionInfo<NewInitObjectWithTemplateFn>(NewObjectOperationWithTemplate, "NewObjectOperationWithTemplate"); typedef JSObject* (*NewInitObjectFn)(JSContext*, HandleScript, jsbytecode* pc, NewObjectKind); static const VMFunction NewInitObjectInfo = FunctionInfo<NewInitObjectFn>(NewObjectOperation, "NewObjectOperation"); typedef PlainObject* (*ObjectCreateWithTemplateFn)(JSContext*, HandlePlainObject); static const VMFunction ObjectCreateWithTemplateInfo = FunctionInfo<ObjectCreateWithTemplateFn>(ObjectCreateWithTemplate, "ObjectCreateWithTemplate"); void CodeGenerator::visitNewObjectVMCall(LNewObject* lir) { Register objReg = ToRegister(lir->output()); MOZ_ASSERT(!lir->isCall()); saveLive(lir); JSObject* templateObject = lir->mir()->templateObject(); // If we're making a new object with a class prototype (that is, an object // that derives its class from its prototype instead of being // PlainObject::class_'d) from self-hosted code, we need a different init // function. switch (lir->mir()->mode()) { case MNewObject::ObjectLiteral: if (templateObject) { pushArg(ImmGCPtr(templateObject)); callVM(NewInitObjectWithTemplateInfo, lir); } else { pushArg(Imm32(GenericObject)); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); pushArg(ImmGCPtr(lir->mir()->block()->info().script())); callVM(NewInitObjectInfo, lir); } break; case MNewObject::ObjectCreate: pushArg(ImmGCPtr(templateObject)); callVM(ObjectCreateWithTemplateInfo, lir); break; } if (ReturnReg != objReg) masm.movePtr(ReturnReg, objReg); restoreLive(lir); } static bool ShouldInitFixedSlots(LInstruction* lir, JSObject* obj) { if (!obj->isNative()) return true; NativeObject* templateObj = &obj->as<NativeObject>(); // Look for StoreFixedSlot instructions following an object allocation // that write to this object before a GC is triggered or this object is // passed to a VM call. If all fixed slots will be initialized, the // allocation code doesn't need to set the slots to |undefined|. uint32_t nfixed = templateObj->numUsedFixedSlots(); if (nfixed == 0) return false; // Only optimize if all fixed slots are initially |undefined|, so that we // can assume incremental pre-barriers are not necessary. See also the // comment below. for (uint32_t slot = 0; slot < nfixed; slot++) { if (!templateObj->getSlot(slot).isUndefined()) return true; } // Keep track of the fixed slots that are initialized. initializedSlots is // a bit mask with a bit for each slot. MOZ_ASSERT(nfixed <= NativeObject::MAX_FIXED_SLOTS); static_assert(NativeObject::MAX_FIXED_SLOTS <= 32, "Slot bits must fit in 32 bits"); uint32_t initializedSlots = 0; uint32_t numInitialized = 0; MInstruction* allocMir = lir->mirRaw()->toInstruction(); MBasicBlock* block = allocMir->block(); // Skip the allocation instruction. MInstructionIterator iter = block->begin(allocMir); MOZ_ASSERT(*iter == allocMir); iter++; while (true) { for (; iter != block->end(); iter++) { if (iter->isNop() || iter->isConstant() || iter->isPostWriteBarrier()) { // These instructions won't trigger a GC or read object slots. continue; } if (iter->isStoreFixedSlot()) { MStoreFixedSlot* store = iter->toStoreFixedSlot(); if (store->object() != allocMir) return true; // We may not initialize this object slot on allocation, so the // pre-barrier could read uninitialized memory. Simply disable // the barrier for this store: the object was just initialized // so the barrier is not necessary. store->setNeedsBarrier(false); uint32_t slot = store->slot(); MOZ_ASSERT(slot < nfixed); if ((initializedSlots & (1 << slot)) == 0) { numInitialized++; initializedSlots |= (1 << slot); if (numInitialized == nfixed) { // All fixed slots will be initialized. MOZ_ASSERT(mozilla::CountPopulation32(initializedSlots) == nfixed); return false; } } continue; } if (iter->isGoto()) { block = iter->toGoto()->target(); if (block->numPredecessors() != 1) return true; break; } // Unhandled instruction, assume it bails or reads object slots. return true; } iter = block->begin(); } MOZ_CRASH("Shouldn't get here"); } void CodeGenerator::visitNewObject(LNewObject* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); JSObject* templateObject = lir->mir()->templateObject(); if (lir->mir()->isVMCall()) { visitNewObjectVMCall(lir); return; } OutOfLineNewObject* ool = new(alloc()) OutOfLineNewObject(lir); addOutOfLineCode(ool, lir->mir()); bool initContents = ShouldInitFixedSlots(lir, templateObject); masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(), initContents); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineNewObject(OutOfLineNewObject* ool) { visitNewObjectVMCall(ool->lir()); masm.jump(ool->rejoin()); } typedef InlineTypedObject* (*NewTypedObjectFn)(JSContext*, Handle<InlineTypedObject*>, gc::InitialHeap); static const VMFunction NewTypedObjectInfo = FunctionInfo<NewTypedObjectFn>(InlineTypedObject::createCopy, "InlineTypedObject::createCopy"); void CodeGenerator::visitNewTypedObject(LNewTypedObject* lir) { Register object = ToRegister(lir->output()); Register temp = ToRegister(lir->temp()); InlineTypedObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir, ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)), StoreRegisterTo(object)); masm.createGCObject(object, temp, templateObject, initialHeap, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitSimdBox(LSimdBox* lir) { FloatRegister in = ToFloatRegister(lir->input()); Register object = ToRegister(lir->output()); Register temp = ToRegister(lir->temp()); InlineTypedObject* templateObject = lir->mir()->templateObject(); gc::InitialHeap initialHeap = lir->mir()->initialHeap(); MIRType type = lir->mir()->input()->type(); registerSimdTemplate(lir->mir()->simdType()); MOZ_ASSERT(lir->safepoint()->liveRegs().has(in), "Save the input register across oolCallVM"); OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir, ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)), StoreRegisterTo(object)); masm.createGCObject(object, temp, templateObject, initialHeap, ool->entry()); masm.bind(ool->rejoin()); Address objectData(object, InlineTypedObject::offsetOfDataStart()); switch (type) { case MIRType::Int8x16: case MIRType::Int16x8: case MIRType::Int32x4: case MIRType::Bool8x16: case MIRType::Bool16x8: case MIRType::Bool32x4: masm.storeUnalignedSimd128Int(in, objectData); break; case MIRType::Float32x4: masm.storeUnalignedSimd128Float(in, objectData); break; default: MOZ_CRASH("Unknown SIMD kind when generating code for SimdBox."); } } void CodeGenerator::registerSimdTemplate(SimdType simdType) { simdRefreshTemplatesDuringLink_ |= 1 << uint32_t(simdType); } void CodeGenerator::captureSimdTemplate(JSContext* cx) { JitCompartment* jitCompartment = cx->compartment()->jitCompartment(); while (simdRefreshTemplatesDuringLink_) { uint32_t typeIndex = mozilla::CountTrailingZeroes32(simdRefreshTemplatesDuringLink_); simdRefreshTemplatesDuringLink_ ^= 1 << typeIndex; SimdType type = SimdType(typeIndex); // Note: the weak-reference on the template object should not have been // garbage collected. It is either registered by IonBuilder, or verified // before using it in the EagerSimdUnbox phase. jitCompartment->registerSimdTemplateObjectFor(type); } } void CodeGenerator::visitSimdUnbox(LSimdUnbox* lir) { Register object = ToRegister(lir->input()); FloatRegister simd = ToFloatRegister(lir->output()); Register temp = ToRegister(lir->temp()); Label bail; // obj->group() masm.loadPtr(Address(object, JSObject::offsetOfGroup()), temp); // Guard that the object has the same representation as the one produced for // SIMD value-type. Address clasp(temp, ObjectGroup::offsetOfClasp()); static_assert(!SimdTypeDescr::Opaque, "SIMD objects are transparent"); masm.branchPtr(Assembler::NotEqual, clasp, ImmPtr(&InlineTransparentTypedObject::class_), &bail); // obj->type()->typeDescr() // The previous class pointer comparison implies that the addendumKind is // Addendum_TypeDescr. masm.loadPtr(Address(temp, ObjectGroup::offsetOfAddendum()), temp); // Check for the /Kind/ reserved slot of the TypeDescr. This is an Int32 // Value which is equivalent to the object class check. static_assert(JS_DESCR_SLOT_KIND < NativeObject::MAX_FIXED_SLOTS, "Load from fixed slots"); Address typeDescrKind(temp, NativeObject::getFixedSlotOffset(JS_DESCR_SLOT_KIND)); masm.assertTestInt32(Assembler::Equal, typeDescrKind, "MOZ_ASSERT(obj->type()->typeDescr()->getReservedSlot(JS_DESCR_SLOT_KIND).isInt32())"); masm.branch32(Assembler::NotEqual, masm.ToPayload(typeDescrKind), Imm32(js::type::Simd), &bail); SimdType type = lir->mir()->simdType(); // Check if the SimdTypeDescr /Type/ match the specialization of this // MSimdUnbox instruction. static_assert(JS_DESCR_SLOT_TYPE < NativeObject::MAX_FIXED_SLOTS, "Load from fixed slots"); Address typeDescrType(temp, NativeObject::getFixedSlotOffset(JS_DESCR_SLOT_TYPE)); masm.assertTestInt32(Assembler::Equal, typeDescrType, "MOZ_ASSERT(obj->type()->typeDescr()->getReservedSlot(JS_DESCR_SLOT_TYPE).isInt32())"); masm.branch32(Assembler::NotEqual, masm.ToPayload(typeDescrType), Imm32(int32_t(type)), &bail); // Load the value from the data of the InlineTypedObject. Address objectData(object, InlineTypedObject::offsetOfDataStart()); switch (lir->mir()->type()) { case MIRType::Int8x16: case MIRType::Int16x8: case MIRType::Int32x4: case MIRType::Bool8x16: case MIRType::Bool16x8: case MIRType::Bool32x4: masm.loadUnalignedSimd128Int(objectData, simd); break; case MIRType::Float32x4: masm.loadUnalignedSimd128Float(objectData, simd); break; default: MOZ_CRASH("The impossible happened!"); } bailoutFrom(&bail, lir->snapshot()); } typedef js::NamedLambdaObject* (*NewNamedLambdaObjectFn)(JSContext*, HandleFunction, gc::InitialHeap); static const VMFunction NewNamedLambdaObjectInfo = FunctionInfo<NewNamedLambdaObjectFn>(NamedLambdaObject::createTemplateObject, "NamedLambdaObject::createTemplateObject"); void CodeGenerator::visitNewNamedLambdaObject(LNewNamedLambdaObject* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); EnvironmentObject* templateObj = lir->mir()->templateObj(); const CompileInfo& info = lir->mir()->block()->info(); // If we have a template object, we can inline call object creation. OutOfLineCode* ool = oolCallVM(NewNamedLambdaObjectInfo, lir, ArgList(ImmGCPtr(info.funMaybeLazy()), Imm32(gc::DefaultHeap)), StoreRegisterTo(objReg)); bool initContents = ShouldInitFixedSlots(lir, templateObj); masm.createGCObject(objReg, tempReg, templateObj, gc::DefaultHeap, ool->entry(), initContents); masm.bind(ool->rejoin()); } typedef JSObject* (*NewCallObjectFn)(JSContext*, HandleShape, HandleObjectGroup); static const VMFunction NewCallObjectInfo = FunctionInfo<NewCallObjectFn>(NewCallObject, "NewCallObject"); void CodeGenerator::visitNewCallObject(LNewCallObject* lir) { Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); CallObject* templateObj = lir->mir()->templateObject(); OutOfLineCode* ool = oolCallVM(NewCallObjectInfo, lir, ArgList(ImmGCPtr(templateObj->lastProperty()), ImmGCPtr(templateObj->group())), StoreRegisterTo(objReg)); // Inline call object creation, using the OOL path only for tricky cases. bool initContents = ShouldInitFixedSlots(lir, templateObj); masm.createGCObject(objReg, tempReg, templateObj, gc::DefaultHeap, ool->entry(), initContents); masm.bind(ool->rejoin()); } typedef JSObject* (*NewSingletonCallObjectFn)(JSContext*, HandleShape); static const VMFunction NewSingletonCallObjectInfo = FunctionInfo<NewSingletonCallObjectFn>(NewSingletonCallObject, "NewSingletonCallObject"); void CodeGenerator::visitNewSingletonCallObject(LNewSingletonCallObject* lir) { Register objReg = ToRegister(lir->output()); JSObject* templateObj = lir->mir()->templateObject(); OutOfLineCode* ool; ool = oolCallVM(NewSingletonCallObjectInfo, lir, ArgList(ImmGCPtr(templateObj->as<CallObject>().lastProperty())), StoreRegisterTo(objReg)); // Objects can only be given singleton types in VM calls. We make the call // out of line to not bloat inline code, even if (naively) this seems like // extra work. masm.jump(ool->entry()); masm.bind(ool->rejoin()); } typedef JSObject* (*NewStringObjectFn)(JSContext*, HandleString); static const VMFunction NewStringObjectInfo = FunctionInfo<NewStringObjectFn>(NewStringObject, "NewStringObject"); void CodeGenerator::visitNewStringObject(LNewStringObject* lir) { Register input = ToRegister(lir->input()); Register output = ToRegister(lir->output()); Register temp = ToRegister(lir->temp()); StringObject* templateObj = lir->mir()->templateObj(); OutOfLineCode* ool = oolCallVM(NewStringObjectInfo, lir, ArgList(input), StoreRegisterTo(output)); masm.createGCObject(output, temp, templateObj, gc::DefaultHeap, ool->entry()); masm.loadStringLength(input, temp); masm.storeValue(JSVAL_TYPE_STRING, input, Address(output, StringObject::offsetOfPrimitiveValue())); masm.storeValue(JSVAL_TYPE_INT32, temp, Address(output, StringObject::offsetOfLength())); masm.bind(ool->rejoin()); } typedef bool(*InitElemFn)(JSContext* cx, jsbytecode* pc, HandleObject obj, HandleValue id, HandleValue value); static const VMFunction InitElemInfo = FunctionInfo<InitElemFn>(InitElemOperation, "InitElemOperation"); void CodeGenerator::visitInitElem(LInitElem* lir) { Register objReg = ToRegister(lir->getObject()); pushArg(ToValue(lir, LInitElem::ValueIndex)); pushArg(ToValue(lir, LInitElem::IdIndex)); pushArg(objReg); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); callVM(InitElemInfo, lir); } typedef bool (*InitElemGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandleValue, HandleObject); static const VMFunction InitElemGetterSetterInfo = FunctionInfo<InitElemGetterSetterFn>(InitGetterSetterOperation, "InitGetterSetterOperation"); void CodeGenerator::visitInitElemGetterSetter(LInitElemGetterSetter* lir) { Register obj = ToRegister(lir->object()); Register value = ToRegister(lir->value()); pushArg(value); pushArg(ToValue(lir, LInitElemGetterSetter::IdIndex)); pushArg(obj); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); callVM(InitElemGetterSetterInfo, lir); } typedef bool(*MutatePrototypeFn)(JSContext* cx, HandlePlainObject obj, HandleValue value); static const VMFunction MutatePrototypeInfo = FunctionInfo<MutatePrototypeFn>(MutatePrototype, "MutatePrototype"); void CodeGenerator::visitMutateProto(LMutateProto* lir) { Register objReg = ToRegister(lir->getObject()); pushArg(ToValue(lir, LMutateProto::ValueIndex)); pushArg(objReg); callVM(MutatePrototypeInfo, lir); } typedef bool(*InitPropFn)(JSContext*, HandleObject, HandlePropertyName, HandleValue, jsbytecode* pc); static const VMFunction InitPropInfo = FunctionInfo<InitPropFn>(InitProp, "InitProp"); void CodeGenerator::visitInitProp(LInitProp* lir) { Register objReg = ToRegister(lir->getObject()); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); pushArg(ToValue(lir, LInitProp::ValueIndex)); pushArg(ImmGCPtr(lir->mir()->propertyName())); pushArg(objReg); callVM(InitPropInfo, lir); } typedef bool(*InitPropGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandlePropertyName, HandleObject); static const VMFunction InitPropGetterSetterInfo = FunctionInfo<InitPropGetterSetterFn>(InitGetterSetterOperation, "InitGetterSetterOperation"); void CodeGenerator::visitInitPropGetterSetter(LInitPropGetterSetter* lir) { Register obj = ToRegister(lir->object()); Register value = ToRegister(lir->value()); pushArg(value); pushArg(ImmGCPtr(lir->mir()->name())); pushArg(obj); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); callVM(InitPropGetterSetterInfo, lir); } typedef bool (*CreateThisFn)(JSContext* cx, HandleObject callee, HandleObject newTarget, MutableHandleValue rval); static const VMFunction CreateThisInfoCodeGen = FunctionInfo<CreateThisFn>(CreateThis, "CreateThis"); void CodeGenerator::visitCreateThis(LCreateThis* lir) { const LAllocation* callee = lir->getCallee(); const LAllocation* newTarget = lir->getNewTarget(); if (newTarget->isConstant()) pushArg(ImmGCPtr(&newTarget->toConstant()->toObject())); else pushArg(ToRegister(newTarget)); if (callee->isConstant()) pushArg(ImmGCPtr(&callee->toConstant()->toObject())); else pushArg(ToRegister(callee)); callVM(CreateThisInfoCodeGen, lir); } static JSObject* CreateThisForFunctionWithProtoWrapper(JSContext* cx, HandleObject callee, HandleObject newTarget, HandleObject proto) { return CreateThisForFunctionWithProto(cx, callee, newTarget, proto); } typedef JSObject* (*CreateThisWithProtoFn)(JSContext* cx, HandleObject callee, HandleObject newTarget, HandleObject proto); static const VMFunction CreateThisWithProtoInfo = FunctionInfo<CreateThisWithProtoFn>(CreateThisForFunctionWithProtoWrapper, "CreateThisForFunctionWithProtoWrapper"); void CodeGenerator::visitCreateThisWithProto(LCreateThisWithProto* lir) { const LAllocation* callee = lir->getCallee(); const LAllocation* newTarget = lir->getNewTarget(); const LAllocation* proto = lir->getPrototype(); if (proto->isConstant()) pushArg(ImmGCPtr(&proto->toConstant()->toObject())); else pushArg(ToRegister(proto)); if (newTarget->isConstant()) pushArg(ImmGCPtr(&newTarget->toConstant()->toObject())); else pushArg(ToRegister(newTarget)); if (callee->isConstant()) pushArg(ImmGCPtr(&callee->toConstant()->toObject())); else pushArg(ToRegister(callee)); callVM(CreateThisWithProtoInfo, lir); } void CodeGenerator::visitCreateThisWithTemplate(LCreateThisWithTemplate* lir) { JSObject* templateObject = lir->mir()->templateObject(); Register objReg = ToRegister(lir->output()); Register tempReg = ToRegister(lir->temp()); OutOfLineCode* ool = oolCallVM(NewInitObjectWithTemplateInfo, lir, ArgList(ImmGCPtr(templateObject)), StoreRegisterTo(objReg)); // Allocate. If the FreeList is empty, call to VM, which may GC. bool initContents = !templateObject->is<PlainObject>() || ShouldInitFixedSlots(lir, &templateObject->as<PlainObject>()); masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(), initContents); masm.bind(ool->rejoin()); } typedef JSObject* (*NewIonArgumentsObjectFn)(JSContext* cx, JitFrameLayout* frame, HandleObject); static const VMFunction NewIonArgumentsObjectInfo = FunctionInfo<NewIonArgumentsObjectFn>((NewIonArgumentsObjectFn) ArgumentsObject::createForIon, "ArgumentsObject::createForIon"); void CodeGenerator::visitCreateArgumentsObject(LCreateArgumentsObject* lir) { // This should be getting constructed in the first block only, and not any OSR entry blocks. MOZ_ASSERT(lir->mir()->block()->id() == 0); Register callObj = ToRegister(lir->getCallObject()); Register temp = ToRegister(lir->temp0()); Label done; if (ArgumentsObject* templateObj = lir->mir()->templateObject()) { Register objTemp = ToRegister(lir->temp1()); Register cxTemp = ToRegister(lir->temp2()); masm.Push(callObj); // Try to allocate an arguments object. This will leave the reserved // slots uninitialized, so it's important we don't GC until we // initialize these slots in ArgumentsObject::finishForIon. Label failure; masm.createGCObject(objTemp, temp, templateObj, gc::DefaultHeap, &failure, /* initContents = */ false); masm.moveStackPtrTo(temp); masm.addPtr(Imm32(masm.framePushed()), temp); masm.setupUnalignedABICall(cxTemp); masm.loadJSContext(cxTemp); masm.passABIArg(cxTemp); masm.passABIArg(temp); masm.passABIArg(callObj); masm.passABIArg(objTemp); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ArgumentsObject::finishForIon)); masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure); // Discard saved callObj on the stack. masm.addToStackPtr(Imm32(sizeof(uintptr_t))); masm.jump(&done); masm.bind(&failure); masm.Pop(callObj); } masm.moveStackPtrTo(temp); masm.addPtr(Imm32(frameSize()), temp); pushArg(callObj); pushArg(temp); callVM(NewIonArgumentsObjectInfo, lir); masm.bind(&done); } void CodeGenerator::visitGetArgumentsObjectArg(LGetArgumentsObjectArg* lir) { Register temp = ToRegister(lir->getTemp(0)); Register argsObj = ToRegister(lir->getArgsObject()); ValueOperand out = ToOutValue(lir); masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp); Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value)); masm.loadValue(argAddr, out); #ifdef DEBUG Label success; masm.branchTestMagic(Assembler::NotEqual, out, &success); masm.assumeUnreachable("Result from ArgumentObject shouldn't be JSVAL_TYPE_MAGIC."); masm.bind(&success); #endif } void CodeGenerator::visitSetArgumentsObjectArg(LSetArgumentsObjectArg* lir) { Register temp = ToRegister(lir->getTemp(0)); Register argsObj = ToRegister(lir->getArgsObject()); ValueOperand value = ToValue(lir, LSetArgumentsObjectArg::ValueIndex); masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp); Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value)); emitPreBarrier(argAddr); #ifdef DEBUG Label success; masm.branchTestMagic(Assembler::NotEqual, argAddr, &success); masm.assumeUnreachable("Result in ArgumentObject shouldn't be JSVAL_TYPE_MAGIC."); masm.bind(&success); #endif masm.storeValue(value, argAddr); } void CodeGenerator::visitReturnFromCtor(LReturnFromCtor* lir) { ValueOperand value = ToValue(lir, LReturnFromCtor::ValueIndex); Register obj = ToRegister(lir->getObject()); Register output = ToRegister(lir->output()); Label valueIsObject, end; masm.branchTestObject(Assembler::Equal, value, &valueIsObject); // Value is not an object. Return that other object. masm.movePtr(obj, output); masm.jump(&end); // Value is an object. Return unbox(Value). masm.bind(&valueIsObject); Register payload = masm.extractObject(value, output); if (payload != output) masm.movePtr(payload, output); masm.bind(&end); } typedef bool (*BoxNonStrictThisFn)(JSContext*, HandleValue, MutableHandleValue); static const VMFunction BoxNonStrictThisInfo = FunctionInfo<BoxNonStrictThisFn>(BoxNonStrictThis, "BoxNonStrictThis"); void CodeGenerator::visitComputeThis(LComputeThis* lir) { ValueOperand value = ToValue(lir, LComputeThis::ValueIndex); ValueOperand output = ToOutValue(lir); OutOfLineCode* ool = oolCallVM(BoxNonStrictThisInfo, lir, ArgList(value), StoreValueTo(output)); masm.branchTestObject(Assembler::NotEqual, value, ool->entry()); masm.moveValue(value, output); masm.bind(ool->rejoin()); } void CodeGenerator::visitArrowNewTarget(LArrowNewTarget* lir) { Register callee = ToRegister(lir->callee()); ValueOperand output = ToOutValue(lir); masm.loadValue(Address(callee, FunctionExtended::offsetOfArrowNewTargetSlot()), output); } void CodeGenerator::visitArrayLength(LArrayLength* lir) { Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength()); masm.load32(length, ToRegister(lir->output())); } void CodeGenerator::visitSetArrayLength(LSetArrayLength* lir) { Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength()); RegisterOrInt32Constant newLength = ToRegisterOrInt32Constant(lir->index()); masm.inc32(&newLength); masm.store32(newLength, length); // Restore register value if it is used/captured after. masm.dec32(&newLength); } template <class OrderedHashTable> static void RangeFront(MacroAssembler&, Register, Register, Register); template <> void RangeFront<ValueMap>(MacroAssembler& masm, Register range, Register i, Register front) { masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front); masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front); static_assert(ValueMap::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0"); static_assert(ValueMap::sizeofImplData() == 24, "sizeof(Data) is 24"); masm.mulBy3(i, i); masm.lshiftPtr(Imm32(3), i); masm.addPtr(i, front); } template <> void RangeFront<ValueSet>(MacroAssembler& masm, Register range, Register i, Register front) { masm.loadPtr(Address(range, ValueSet::Range::offsetOfHashTable()), front); masm.loadPtr(Address(front, ValueSet::offsetOfImplData()), front); static_assert(ValueSet::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0"); static_assert(ValueSet::sizeofImplData() == 16, "sizeof(Data) is 16"); masm.lshiftPtr(Imm32(4), i); masm.addPtr(i, front); } template <class OrderedHashTable> static void RangePopFront(MacroAssembler& masm, Register range, Register front, Register dataLength, Register temp) { Register i = temp; masm.add32(Imm32(1), Address(range, OrderedHashTable::Range::offsetOfCount())); masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), i); masm.add32(Imm32(1), i); Label done, seek; masm.bind(&seek); masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done); // We can add sizeof(Data) to |front| to select the next element, because // |front| and |range.ht.data[i]| point to the same location. static_assert(OrderedHashTable::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0"); masm.addPtr(Imm32(OrderedHashTable::sizeofImplData()), front); masm.branchTestMagic(Assembler::NotEqual, Address(front, OrderedHashTable::offsetOfEntryKey()), JS_HASH_KEY_EMPTY, &done); masm.add32(Imm32(1), i); masm.jump(&seek); masm.bind(&done); masm.store32(i, Address(range, OrderedHashTable::Range::offsetOfI())); } template <class OrderedHashTable> static inline void RangeDestruct(MacroAssembler& masm, Register range, Register temp0, Register temp1) { Register next = temp0; Register prevp = temp1; masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfNext()), next); masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfPrevP()), prevp); masm.storePtr(next, Address(prevp, 0)); Label hasNoNext; masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext); masm.storePtr(prevp, Address(next, OrderedHashTable::Range::offsetOfPrevP())); masm.bind(&hasNoNext); masm.callFreeStub(range); } template <> void CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result, Register temp, Register front) { size_t elementsOffset = NativeObject::offsetOfFixedElements(); Address keyAddress(front, ValueMap::Entry::offsetOfKey()); Address valueAddress(front, ValueMap::Entry::offsetOfValue()); Address keyElemAddress(result, elementsOffset); Address valueElemAddress(result, elementsOffset + sizeof(Value)); masm.patchableCallPreBarrier(keyElemAddress, MIRType::Value); masm.patchableCallPreBarrier(valueElemAddress, MIRType::Value); masm.storeValue(keyAddress, keyElemAddress, temp); masm.storeValue(valueAddress, valueElemAddress, temp); Label keyIsNotObject, valueIsNotNurseryObject, emitBarrier; masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject); masm.branchValueIsNurseryObject(Assembler::Equal, keyAddress, temp, &emitBarrier); masm.bind(&keyIsNotObject); masm.branchTestObject(Assembler::NotEqual, valueAddress, &valueIsNotNurseryObject); masm.branchValueIsNurseryObject(Assembler::NotEqual, valueAddress, temp, &valueIsNotNurseryObject); { masm.bind(&emitBarrier); saveVolatile(temp); emitPostWriteBarrier(result); restoreVolatile(temp); } masm.bind(&valueIsNotNurseryObject); } template <> void CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result, Register temp, Register front) { size_t elementsOffset = NativeObject::offsetOfFixedElements(); Address keyAddress(front, ValueSet::offsetOfEntryKey()); Address keyElemAddress(result, elementsOffset); masm.patchableCallPreBarrier(keyElemAddress, MIRType::Value); masm.storeValue(keyAddress, keyElemAddress, temp); Label keyIsNotObject; masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject); masm.branchValueIsNurseryObject(Assembler::NotEqual, keyAddress, temp, &keyIsNotObject); { saveVolatile(temp); emitPostWriteBarrier(result); restoreVolatile(temp); } masm.bind(&keyIsNotObject); } template <class IteratorObject, class OrderedHashTable> void CodeGenerator::emitGetNextEntryForIterator(LGetNextEntryForIterator* lir) { Register iter = ToRegister(lir->iter()); Register result = ToRegister(lir->result()); Register temp = ToRegister(lir->temp0()); Register dataLength = ToRegister(lir->temp1()); Register range = ToRegister(lir->temp2()); Register output = ToRegister(lir->output()); masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot)), range); Label iterAlreadyDone, iterDone, done; masm.branchTestPtr(Assembler::Zero, range, range, &iterAlreadyDone); masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), temp); masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfHashTable()), dataLength); masm.load32(Address(dataLength, OrderedHashTable::offsetOfImplDataLength()), dataLength); masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone); { masm.push(iter); Register front = iter; RangeFront<OrderedHashTable>(masm, range, temp, front); emitLoadIteratorValues<OrderedHashTable>(result, temp, front); RangePopFront<OrderedHashTable>(masm, range, front, dataLength, temp); masm.pop(iter); masm.move32(Imm32(0), output); } masm.jump(&done); { masm.bind(&iterDone); RangeDestruct<OrderedHashTable>(masm, range, temp, dataLength); masm.storeValue(PrivateValue(nullptr), Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot))); masm.bind(&iterAlreadyDone); masm.move32(Imm32(1), output); } masm.bind(&done); } void CodeGenerator::visitGetNextEntryForIterator(LGetNextEntryForIterator* lir) { if (lir->mir()->mode() == MGetNextEntryForIterator::Map) { emitGetNextEntryForIterator<MapIteratorObject, ValueMap>(lir); } else { MOZ_ASSERT(lir->mir()->mode() == MGetNextEntryForIterator::Set); emitGetNextEntryForIterator<SetIteratorObject, ValueSet>(lir); } } void CodeGenerator::visitTypedArrayLength(LTypedArrayLength* lir) { Register obj = ToRegister(lir->object()); Register out = ToRegister(lir->output()); masm.unboxInt32(Address(obj, TypedArrayObject::lengthOffset()), out); } void CodeGenerator::visitTypedArrayElements(LTypedArrayElements* lir) { Register obj = ToRegister(lir->object()); Register out = ToRegister(lir->output()); masm.loadPtr(Address(obj, TypedArrayObject::dataOffset()), out); } void CodeGenerator::visitSetDisjointTypedElements(LSetDisjointTypedElements* lir) { Register target = ToRegister(lir->target()); Register targetOffset = ToRegister(lir->targetOffset()); Register source = ToRegister(lir->source()); Register temp = ToRegister(lir->temp()); masm.setupUnalignedABICall(temp); masm.passABIArg(target); masm.passABIArg(targetOffset); masm.passABIArg(source); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::SetDisjointTypedElements)); } void CodeGenerator::visitTypedObjectDescr(LTypedObjectDescr* lir) { Register obj = ToRegister(lir->object()); Register out = ToRegister(lir->output()); masm.loadPtr(Address(obj, JSObject::offsetOfGroup()), out); masm.loadPtr(Address(out, ObjectGroup::offsetOfAddendum()), out); } void CodeGenerator::visitTypedObjectElements(LTypedObjectElements* lir) { Register obj = ToRegister(lir->object()); Register out = ToRegister(lir->output()); if (lir->mir()->definitelyOutline()) { masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out); } else { Label inlineObject, done; masm.loadObjClass(obj, out); masm.branchPtr(Assembler::Equal, out, ImmPtr(&InlineOpaqueTypedObject::class_), &inlineObject); masm.branchPtr(Assembler::Equal, out, ImmPtr(&InlineTransparentTypedObject::class_), &inlineObject); masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out); masm.jump(&done); masm.bind(&inlineObject); masm.computeEffectiveAddress(Address(obj, InlineTypedObject::offsetOfDataStart()), out); masm.bind(&done); } } void CodeGenerator::visitSetTypedObjectOffset(LSetTypedObjectOffset* lir) { Register object = ToRegister(lir->object()); Register offset = ToRegister(lir->offset()); Register temp0 = ToRegister(lir->temp0()); Register temp1 = ToRegister(lir->temp1()); // Compute the base pointer for the typed object's owner. masm.loadPtr(Address(object, OutlineTypedObject::offsetOfOwner()), temp0); Label inlineObject, done; masm.loadObjClass(temp0, temp1); masm.branchPtr(Assembler::Equal, temp1, ImmPtr(&InlineOpaqueTypedObject::class_), &inlineObject); masm.branchPtr(Assembler::Equal, temp1, ImmPtr(&InlineTransparentTypedObject::class_), &inlineObject); masm.loadPrivate(Address(temp0, ArrayBufferObject::offsetOfDataSlot()), temp0); masm.jump(&done); masm.bind(&inlineObject); masm.addPtr(ImmWord(InlineTypedObject::offsetOfDataStart()), temp0); masm.bind(&done); // Compute the new data pointer and set it in the object. masm.addPtr(offset, temp0); masm.storePtr(temp0, Address(object, OutlineTypedObject::offsetOfData())); } void CodeGenerator::visitStringLength(LStringLength* lir) { Register input = ToRegister(lir->string()); Register output = ToRegister(lir->output()); masm.loadStringLength(input, output); } void CodeGenerator::visitMinMaxI(LMinMaxI* ins) { Register first = ToRegister(ins->first()); Register output = ToRegister(ins->output()); MOZ_ASSERT(first == output); Label done; Assembler::Condition cond = ins->mir()->isMax() ? Assembler::GreaterThan : Assembler::LessThan; if (ins->second()->isConstant()) { masm.branch32(cond, first, Imm32(ToInt32(ins->second())), &done); masm.move32(Imm32(ToInt32(ins->second())), output); } else { masm.branch32(cond, first, ToRegister(ins->second()), &done); masm.move32(ToRegister(ins->second()), output); } masm.bind(&done); } void CodeGenerator::visitAbsI(LAbsI* ins) { Register input = ToRegister(ins->input()); Label positive; MOZ_ASSERT(input == ToRegister(ins->output())); masm.branchTest32(Assembler::NotSigned, input, input, &positive); masm.neg32(input); LSnapshot* snapshot = ins->snapshot(); #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) if (snapshot) bailoutCmp32(Assembler::Equal, input, Imm32(INT32_MIN), snapshot); #else if (snapshot) bailoutIf(Assembler::Overflow, snapshot); #endif masm.bind(&positive); } void CodeGenerator::visitPowI(LPowI* ins) { FloatRegister value = ToFloatRegister(ins->value()); Register power = ToRegister(ins->power()); Register temp = ToRegister(ins->temp()); MOZ_ASSERT(power != temp); masm.setupUnalignedABICall(temp); masm.passABIArg(value, MoveOp::DOUBLE); masm.passABIArg(power); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::powi), MoveOp::DOUBLE); MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg); } void CodeGenerator::visitPowD(LPowD* ins) { FloatRegister value = ToFloatRegister(ins->value()); FloatRegister power = ToFloatRegister(ins->power()); Register temp = ToRegister(ins->temp()); masm.setupUnalignedABICall(temp); masm.passABIArg(value, MoveOp::DOUBLE); masm.passABIArg(power, MoveOp::DOUBLE); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaPow), MoveOp::DOUBLE); MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg); } void CodeGenerator::visitMathFunctionD(LMathFunctionD* ins) { Register temp = ToRegister(ins->temp()); FloatRegister input = ToFloatRegister(ins->input()); MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg); masm.setupUnalignedABICall(temp); const MathCache* mathCache = ins->mir()->cache(); if (mathCache) { masm.movePtr(ImmPtr(mathCache), temp); masm.passABIArg(temp); } masm.passABIArg(input, MoveOp::DOUBLE); # define MAYBE_CACHED(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached) void* funptr = nullptr; switch (ins->mir()->function()) { case MMathFunction::Log: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log)); break; case MMathFunction::Sin: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sin)); break; case MMathFunction::Cos: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cos)); break; case MMathFunction::Exp: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_exp)); break; case MMathFunction::Tan: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tan)); break; case MMathFunction::ATan: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atan)); break; case MMathFunction::ASin: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asin)); break; case MMathFunction::ACos: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acos)); break; case MMathFunction::Log10: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log10)); break; case MMathFunction::Log2: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log2)); break; case MMathFunction::Log1P: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log1p)); break; case MMathFunction::ExpM1: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_expm1)); break; case MMathFunction::CosH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cosh)); break; case MMathFunction::SinH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sinh)); break; case MMathFunction::TanH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tanh)); break; case MMathFunction::ACosH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acosh)); break; case MMathFunction::ASinH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asinh)); break; case MMathFunction::ATanH: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atanh)); break; case MMathFunction::Sign: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sign)); break; case MMathFunction::Trunc: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_trunc)); break; case MMathFunction::Cbrt: funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cbrt)); break; case MMathFunction::Floor: funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_floor_impl); break; case MMathFunction::Ceil: funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_ceil_impl); break; case MMathFunction::Round: funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_round_impl); break; default: MOZ_CRASH("Unknown math function"); } # undef MAYBE_CACHED masm.callWithABI(funptr, MoveOp::DOUBLE); } void CodeGenerator::visitMathFunctionF(LMathFunctionF* ins) { Register temp = ToRegister(ins->temp()); FloatRegister input = ToFloatRegister(ins->input()); MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnFloat32Reg); masm.setupUnalignedABICall(temp); masm.passABIArg(input, MoveOp::FLOAT32); void* funptr = nullptr; switch (ins->mir()->function()) { case MMathFunction::Floor: funptr = JS_FUNC_TO_DATA_PTR(void*, floorf); break; case MMathFunction::Round: funptr = JS_FUNC_TO_DATA_PTR(void*, math_roundf_impl); break; case MMathFunction::Ceil: funptr = JS_FUNC_TO_DATA_PTR(void*, ceilf); break; default: MOZ_CRASH("Unknown or unsupported float32 math function"); } masm.callWithABI(funptr, MoveOp::FLOAT32); } void CodeGenerator::visitModD(LModD* ins) { FloatRegister lhs = ToFloatRegister(ins->lhs()); FloatRegister rhs = ToFloatRegister(ins->rhs()); Register temp = ToRegister(ins->temp()); MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg); masm.setupUnalignedABICall(temp); masm.passABIArg(lhs, MoveOp::DOUBLE); masm.passABIArg(rhs, MoveOp::DOUBLE); if (gen->compilingWasm()) masm.callWithABI(wasm::SymbolicAddress::ModD, MoveOp::DOUBLE); else masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, NumberMod), MoveOp::DOUBLE); } typedef bool (*BinaryFn)(JSContext*, MutableHandleValue, MutableHandleValue, MutableHandleValue); static const VMFunction AddInfo = FunctionInfo<BinaryFn>(js::AddValues, "AddValues"); static const VMFunction SubInfo = FunctionInfo<BinaryFn>(js::SubValues, "SubValues"); static const VMFunction MulInfo = FunctionInfo<BinaryFn>(js::MulValues, "MulValues"); static const VMFunction DivInfo = FunctionInfo<BinaryFn>(js::DivValues, "DivValues"); static const VMFunction ModInfo = FunctionInfo<BinaryFn>(js::ModValues, "ModValues"); static const VMFunction UrshInfo = FunctionInfo<BinaryFn>(js::UrshValues, "UrshValues"); void CodeGenerator::visitBinaryV(LBinaryV* lir) { pushArg(ToValue(lir, LBinaryV::RhsInput)); pushArg(ToValue(lir, LBinaryV::LhsInput)); switch (lir->jsop()) { case JSOP_ADD: callVM(AddInfo, lir); break; case JSOP_SUB: callVM(SubInfo, lir); break; case JSOP_MUL: callVM(MulInfo, lir); break; case JSOP_DIV: callVM(DivInfo, lir); break; case JSOP_MOD: callVM(ModInfo, lir); break; case JSOP_URSH: callVM(UrshInfo, lir); break; default: MOZ_CRASH("Unexpected binary op"); } } typedef bool (*StringCompareFn)(JSContext*, HandleString, HandleString, bool*); static const VMFunction StringsEqualInfo = FunctionInfo<StringCompareFn>(jit::StringsEqual<true>, "StringsEqual"); static const VMFunction StringsNotEqualInfo = FunctionInfo<StringCompareFn>(jit::StringsEqual<false>, "StringsEqual"); void CodeGenerator::emitCompareS(LInstruction* lir, JSOp op, Register left, Register right, Register output) { MOZ_ASSERT(lir->isCompareS() || lir->isCompareStrictS()); OutOfLineCode* ool = nullptr; if (op == JSOP_EQ || op == JSOP_STRICTEQ) { ool = oolCallVM(StringsEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output)); } else { MOZ_ASSERT(op == JSOP_NE || op == JSOP_STRICTNE); ool = oolCallVM(StringsNotEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output)); } masm.compareStrings(op, left, right, output, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitCompareStrictS(LCompareStrictS* lir) { JSOp op = lir->mir()->jsop(); MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE); const ValueOperand leftV = ToValue(lir, LCompareStrictS::Lhs); Register right = ToRegister(lir->right()); Register output = ToRegister(lir->output()); Register tempToUnbox = ToTempUnboxRegister(lir->tempToUnbox()); Label string, done; masm.branchTestString(Assembler::Equal, leftV, &string); masm.move32(Imm32(op == JSOP_STRICTNE), output); masm.jump(&done); masm.bind(&string); Register left = masm.extractString(leftV, tempToUnbox); emitCompareS(lir, op, left, right, output); masm.bind(&done); } void CodeGenerator::visitCompareS(LCompareS* lir) { JSOp op = lir->mir()->jsop(); Register left = ToRegister(lir->left()); Register right = ToRegister(lir->right()); Register output = ToRegister(lir->output()); emitCompareS(lir, op, left, right, output); } typedef bool (*CompareFn)(JSContext*, MutableHandleValue, MutableHandleValue, bool*); static const VMFunction EqInfo = FunctionInfo<CompareFn>(jit::LooselyEqual<true>, "LooselyEqual"); static const VMFunction NeInfo = FunctionInfo<CompareFn>(jit::LooselyEqual<false>, "LooselyEqual"); static const VMFunction StrictEqInfo = FunctionInfo<CompareFn>(jit::StrictlyEqual<true>, "StrictlyEqual"); static const VMFunction StrictNeInfo = FunctionInfo<CompareFn>(jit::StrictlyEqual<false>, "StrictlyEqual"); static const VMFunction LtInfo = FunctionInfo<CompareFn>(jit::LessThan, "LessThan"); static const VMFunction LeInfo = FunctionInfo<CompareFn>(jit::LessThanOrEqual, "LessThanOrEqual"); static const VMFunction GtInfo = FunctionInfo<CompareFn>(jit::GreaterThan, "GreaterThan"); static const VMFunction GeInfo = FunctionInfo<CompareFn>(jit::GreaterThanOrEqual, "GreaterThanOrEqual"); void CodeGenerator::visitCompareVM(LCompareVM* lir) { pushArg(ToValue(lir, LBinaryV::RhsInput)); pushArg(ToValue(lir, LBinaryV::LhsInput)); switch (lir->mir()->jsop()) { case JSOP_EQ: callVM(EqInfo, lir); break; case JSOP_NE: callVM(NeInfo, lir); break; case JSOP_STRICTEQ: callVM(StrictEqInfo, lir); break; case JSOP_STRICTNE: callVM(StrictNeInfo, lir); break; case JSOP_LT: callVM(LtInfo, lir); break; case JSOP_LE: callVM(LeInfo, lir); break; case JSOP_GT: callVM(GtInfo, lir); break; case JSOP_GE: callVM(GeInfo, lir); break; default: MOZ_CRASH("Unexpected compare op"); } } void CodeGenerator::visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV* lir) { JSOp op = lir->mir()->jsop(); MCompare::CompareType compareType = lir->mir()->compareType(); MOZ_ASSERT(compareType == MCompare::Compare_Undefined || compareType == MCompare::Compare_Null); const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedV::Value); Register output = ToRegister(lir->output()); if (op == JSOP_EQ || op == JSOP_NE) { MOZ_ASSERT(lir->mir()->lhs()->type() != MIRType::Object || lir->mir()->operandMightEmulateUndefined(), "Operands which can't emulate undefined should have been folded"); OutOfLineTestObjectWithLabels* ool = nullptr; Maybe<Label> label1, label2; Label* nullOrLikeUndefined; Label* notNullOrLikeUndefined; if (lir->mir()->operandMightEmulateUndefined()) { ool = new(alloc()) OutOfLineTestObjectWithLabels(); addOutOfLineCode(ool, lir->mir()); nullOrLikeUndefined = ool->label1(); notNullOrLikeUndefined = ool->label2(); } else { label1.emplace(); label2.emplace(); nullOrLikeUndefined = label1.ptr(); notNullOrLikeUndefined = label2.ptr(); } Register tag = masm.splitTagForTest(value); MDefinition* input = lir->mir()->lhs(); if (input->mightBeType(MIRType::Null)) masm.branchTestNull(Assembler::Equal, tag, nullOrLikeUndefined); if (input->mightBeType(MIRType::Undefined)) masm.branchTestUndefined(Assembler::Equal, tag, nullOrLikeUndefined); if (ool) { // Check whether it's a truthy object or a falsy object that emulates // undefined. masm.branchTestObject(Assembler::NotEqual, tag, notNullOrLikeUndefined); Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox())); branchTestObjectEmulatesUndefined(objreg, nullOrLikeUndefined, notNullOrLikeUndefined, ToRegister(lir->temp()), ool); // fall through } Label done; // It's not null or undefined, and if it's an object it doesn't // emulate undefined, so it's not like undefined. masm.move32(Imm32(op == JSOP_NE), output); masm.jump(&done); masm.bind(nullOrLikeUndefined); masm.move32(Imm32(op == JSOP_EQ), output); // Both branches meet here. masm.bind(&done); return; } MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE); Assembler::Condition cond = JSOpToCondition(compareType, op); if (compareType == MCompare::Compare_Null) masm.testNullSet(cond, value, output); else masm.testUndefinedSet(cond, value, output); } void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchV(LIsNullOrLikeUndefinedAndBranchV* lir) { JSOp op = lir->cmpMir()->jsop(); MCompare::CompareType compareType = lir->cmpMir()->compareType(); MOZ_ASSERT(compareType == MCompare::Compare_Undefined || compareType == MCompare::Compare_Null); const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedAndBranchV::Value); if (op == JSOP_EQ || op == JSOP_NE) { MBasicBlock* ifTrue; MBasicBlock* ifFalse; if (op == JSOP_EQ) { ifTrue = lir->ifTrue(); ifFalse = lir->ifFalse(); } else { // Swap branches. ifTrue = lir->ifFalse(); ifFalse = lir->ifTrue(); op = JSOP_EQ; } MOZ_ASSERT(lir->cmpMir()->lhs()->type() != MIRType::Object || lir->cmpMir()->operandMightEmulateUndefined(), "Operands which can't emulate undefined should have been folded"); OutOfLineTestObject* ool = nullptr; if (lir->cmpMir()->operandMightEmulateUndefined()) { ool = new(alloc()) OutOfLineTestObject(); addOutOfLineCode(ool, lir->cmpMir()); } Register tag = masm.splitTagForTest(value); Label* ifTrueLabel = getJumpLabelForBranch(ifTrue); Label* ifFalseLabel = getJumpLabelForBranch(ifFalse); MDefinition* input = lir->cmpMir()->lhs(); if (input->mightBeType(MIRType::Null)) masm.branchTestNull(Assembler::Equal, tag, ifTrueLabel); if (input->mightBeType(MIRType::Undefined)) masm.branchTestUndefined(Assembler::Equal, tag, ifTrueLabel); if (ool) { masm.branchTestObject(Assembler::NotEqual, tag, ifFalseLabel); // Objects that emulate undefined are loosely equal to null/undefined. Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox())); Register scratch = ToRegister(lir->temp()); testObjectEmulatesUndefined(objreg, ifTrueLabel, ifFalseLabel, scratch, ool); } else { masm.jump(ifFalseLabel); } return; } MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE); Assembler::Condition cond = JSOpToCondition(compareType, op); if (compareType == MCompare::Compare_Null) testNullEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse()); else testUndefinedEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse()); } void CodeGenerator::visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT * lir) { MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null); MIRType lhsType = lir->mir()->lhs()->type(); MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull); JSOp op = lir->mir()->jsop(); MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE, "Strict equality should have been folded"); MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(), "If the object couldn't emulate undefined, this should have been folded."); Register objreg = ToRegister(lir->input()); Register output = ToRegister(lir->output()); if ((op == JSOP_EQ || op == JSOP_NE) && lir->mir()->operandMightEmulateUndefined()) { OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels(); addOutOfLineCode(ool, lir->mir()); Label* emulatesUndefined = ool->label1(); Label* doesntEmulateUndefined = ool->label2(); if (lhsType == MIRType::ObjectOrNull) masm.branchTestPtr(Assembler::Zero, objreg, objreg, emulatesUndefined); branchTestObjectEmulatesUndefined(objreg, emulatesUndefined, doesntEmulateUndefined, output, ool); Label done; masm.move32(Imm32(op == JSOP_NE), output); masm.jump(&done); masm.bind(emulatesUndefined); masm.move32(Imm32(op == JSOP_EQ), output); masm.bind(&done); } else { MOZ_ASSERT(lhsType == MIRType::ObjectOrNull); Label isNull, done; masm.branchTestPtr(Assembler::Zero, objreg, objreg, &isNull); masm.move32(Imm32(op == JSOP_NE || op == JSOP_STRICTNE), output); masm.jump(&done); masm.bind(&isNull); masm.move32(Imm32(op == JSOP_EQ || op == JSOP_STRICTEQ), output); masm.bind(&done); } } void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchT(LIsNullOrLikeUndefinedAndBranchT* lir) { DebugOnly<MCompare::CompareType> compareType = lir->cmpMir()->compareType(); MOZ_ASSERT(compareType == MCompare::Compare_Undefined || compareType == MCompare::Compare_Null); MIRType lhsType = lir->cmpMir()->lhs()->type(); MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull); JSOp op = lir->cmpMir()->jsop(); MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE, "Strict equality should have been folded"); MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->cmpMir()->operandMightEmulateUndefined(), "If the object couldn't emulate undefined, this should have been folded."); MBasicBlock* ifTrue; MBasicBlock* ifFalse; if (op == JSOP_EQ || op == JSOP_STRICTEQ) { ifTrue = lir->ifTrue(); ifFalse = lir->ifFalse(); } else { // Swap branches. ifTrue = lir->ifFalse(); ifFalse = lir->ifTrue(); } Register input = ToRegister(lir->getOperand(0)); if ((op == JSOP_EQ || op == JSOP_NE) && lir->cmpMir()->operandMightEmulateUndefined()) { OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject(); addOutOfLineCode(ool, lir->cmpMir()); Label* ifTrueLabel = getJumpLabelForBranch(ifTrue); Label* ifFalseLabel = getJumpLabelForBranch(ifFalse); if (lhsType == MIRType::ObjectOrNull) masm.branchTestPtr(Assembler::Zero, input, input, ifTrueLabel); // Objects that emulate undefined are loosely equal to null/undefined. Register scratch = ToRegister(lir->temp()); testObjectEmulatesUndefined(input, ifTrueLabel, ifFalseLabel, scratch, ool); } else { MOZ_ASSERT(lhsType == MIRType::ObjectOrNull); testZeroEmitBranch(Assembler::Equal, input, ifTrue, ifFalse); } } typedef JSString* (*ConcatStringsFn)(ExclusiveContext*, HandleString, HandleString); static const VMFunction ConcatStringsInfo = FunctionInfo<ConcatStringsFn>(ConcatStrings<CanGC>, "ConcatStrings"); void CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs, Register output) { OutOfLineCode* ool = oolCallVM(ConcatStringsInfo, lir, ArgList(lhs, rhs), StoreRegisterTo(output)); JitCode* stringConcatStub = gen->compartment->jitCompartment()->stringConcatStubNoBarrier(); masm.call(stringConcatStub); masm.branchTestPtr(Assembler::Zero, output, output, ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitConcat(LConcat* lir) { Register lhs = ToRegister(lir->lhs()); Register rhs = ToRegister(lir->rhs()); Register output = ToRegister(lir->output()); MOZ_ASSERT(lhs == CallTempReg0); MOZ_ASSERT(rhs == CallTempReg1); MOZ_ASSERT(ToRegister(lir->temp1()) == CallTempReg0); MOZ_ASSERT(ToRegister(lir->temp2()) == CallTempReg1); MOZ_ASSERT(ToRegister(lir->temp3()) == CallTempReg2); MOZ_ASSERT(ToRegister(lir->temp4()) == CallTempReg3); MOZ_ASSERT(ToRegister(lir->temp5()) == CallTempReg4); MOZ_ASSERT(output == CallTempReg5); emitConcat(lir, lhs, rhs, output); } static void CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len, Register byteOpScratch, size_t fromWidth, size_t toWidth) { // Copy |len| char16_t code units from |from| to |to|. Assumes len > 0 // (checked below in debug builds), and when done |to| must point to the // next available char. #ifdef DEBUG Label ok; masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok); masm.assumeUnreachable("Length should be greater than 0."); masm.bind(&ok); #endif MOZ_ASSERT(fromWidth == 1 || fromWidth == 2); MOZ_ASSERT(toWidth == 1 || toWidth == 2); MOZ_ASSERT_IF(toWidth == 1, fromWidth == 1); Label start; masm.bind(&start); if (fromWidth == 2) masm.load16ZeroExtend(Address(from, 0), byteOpScratch); else masm.load8ZeroExtend(Address(from, 0), byteOpScratch); if (toWidth == 2) masm.store16(byteOpScratch, Address(to, 0)); else masm.store8(byteOpScratch, Address(to, 0)); masm.addPtr(Imm32(fromWidth), from); masm.addPtr(Imm32(toWidth), to); masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start); } static void CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input, Register destChars, Register temp1, Register temp2) { // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may // have to inflate. Label isLatin1, done; masm.loadStringLength(input, temp1); masm.branchLatin1String(input, &isLatin1); { masm.loadStringChars(input, input); CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char16_t), sizeof(char16_t)); masm.jump(&done); } masm.bind(&isLatin1); { masm.loadStringChars(input, input); CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char), sizeof(char16_t)); } masm.bind(&done); } static void ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register output, Register temp1, Register temp2, Register temp3, Label* failure, Label* failurePopTemps, bool isTwoByte) { // State: result length in temp2. // Ensure both strings are linear. masm.branchIfRope(lhs, failure); masm.branchIfRope(rhs, failure); // Allocate a JSThinInlineString or JSFatInlineString. size_t maxThinInlineLength; if (isTwoByte) maxThinInlineLength = JSThinInlineString::MAX_LENGTH_TWO_BYTE; else maxThinInlineLength = JSThinInlineString::MAX_LENGTH_LATIN1; Label isFat, allocDone; masm.branch32(Assembler::Above, temp2, Imm32(maxThinInlineLength), &isFat); { uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS; if (!isTwoByte) flags |= JSString::LATIN1_CHARS_BIT; masm.newGCString(output, temp1, failure); masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags())); masm.jump(&allocDone); } masm.bind(&isFat); { uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS; if (!isTwoByte) flags |= JSString::LATIN1_CHARS_BIT; masm.newGCFatInlineString(output, temp1, failure); masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags())); } masm.bind(&allocDone); // Store length. masm.store32(temp2, Address(output, JSString::offsetOfLength())); // Load chars pointer in temp2. masm.computeEffectiveAddress(Address(output, JSInlineString::offsetOfInlineStorage()), temp2); { // Copy lhs chars. Note that this advances temp2 to point to the next // char. This also clobbers the lhs register. if (isTwoByte) { CopyStringCharsMaybeInflate(masm, lhs, temp2, temp1, temp3); } else { masm.loadStringLength(lhs, temp3); masm.loadStringChars(lhs, lhs); CopyStringChars(masm, temp2, lhs, temp3, temp1, sizeof(char), sizeof(char)); } // Copy rhs chars. Clobbers the rhs register. if (isTwoByte) { CopyStringCharsMaybeInflate(masm, rhs, temp2, temp1, temp3); } else { masm.loadStringLength(rhs, temp3); masm.loadStringChars(rhs, rhs); CopyStringChars(masm, temp2, rhs, temp3, temp1, sizeof(char), sizeof(char)); } // Null-terminate. if (isTwoByte) masm.store16(Imm32(0), Address(temp2, 0)); else masm.store8(Imm32(0), Address(temp2, 0)); } masm.ret(); } typedef JSString* (*SubstringKernelFn)(JSContext* cx, HandleString str, int32_t begin, int32_t len); static const VMFunction SubstringKernelInfo = FunctionInfo<SubstringKernelFn>(SubstringKernel, "SubstringKernel"); void CodeGenerator::visitSubstr(LSubstr* lir) { Register string = ToRegister(lir->string()); Register begin = ToRegister(lir->begin()); Register length = ToRegister(lir->length()); Register output = ToRegister(lir->output()); Register temp = ToRegister(lir->temp()); Register temp3 = ToRegister(lir->temp3()); // On x86 there are not enough registers. In that case reuse the string // register as temporary. Register temp2 = lir->temp2()->isBogusTemp() ? string : ToRegister(lir->temp2()); Address stringFlags(string, JSString::offsetOfFlags()); Label isLatin1, notInline, nonZero, isInlinedLatin1; // For every edge case use the C++ variant. // Note: we also use this upon allocation failure in newGCString and // newGCFatInlineString. To squeeze out even more performance those failures // can be handled by allocate in ool code and returning to jit code to fill // in all data. OutOfLineCode* ool = oolCallVM(SubstringKernelInfo, lir, ArgList(string, begin, length), StoreRegisterTo(output)); Label* slowPath = ool->entry(); Label* done = ool->rejoin(); // Zero length, return emptystring. masm.branchTest32(Assembler::NonZero, length, length, &nonZero); const JSAtomState& names = GetJitContext()->runtime->names(); masm.movePtr(ImmGCPtr(names.empty), output); masm.jump(done); // Use slow path for ropes. masm.bind(&nonZero); masm.branchIfRopeOrExternal(string, temp, slowPath); // Handle inlined strings by creating a FatInlineString. masm.branchTest32(Assembler::Zero, stringFlags, Imm32(JSString::INLINE_CHARS_BIT), ¬Inline); masm.newGCFatInlineString(output, temp, slowPath); masm.store32(length, Address(output, JSString::offsetOfLength())); Address stringStorage(string, JSInlineString::offsetOfInlineStorage()); Address outputStorage(output, JSInlineString::offsetOfInlineStorage()); masm.branchLatin1String(string, &isInlinedLatin1); { masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS), Address(output, JSString::offsetOfFlags())); masm.computeEffectiveAddress(stringStorage, temp); if (temp2 == string) masm.push(string); BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t))); masm.computeEffectiveAddress(chars, temp2); masm.computeEffectiveAddress(outputStorage, temp); CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char16_t), sizeof(char16_t)); masm.load32(Address(output, JSString::offsetOfLength()), length); masm.store16(Imm32(0), Address(temp, 0)); if (temp2 == string) masm.pop(string); masm.jump(done); } masm.bind(&isInlinedLatin1); { masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS | JSString::LATIN1_CHARS_BIT), Address(output, JSString::offsetOfFlags())); if (temp2 == string) masm.push(string); masm.computeEffectiveAddress(stringStorage, temp2); static_assert(sizeof(char) == 1, "begin index shouldn't need scaling"); masm.addPtr(begin, temp2); masm.computeEffectiveAddress(outputStorage, temp); CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char), sizeof(char)); masm.load32(Address(output, JSString::offsetOfLength()), length); masm.store8(Imm32(0), Address(temp, 0)); if (temp2 == string) masm.pop(string); masm.jump(done); } // Handle other cases with a DependentString. masm.bind(¬Inline); masm.newGCString(output, temp, slowPath); masm.store32(length, Address(output, JSString::offsetOfLength())); masm.storePtr(string, Address(output, JSDependentString::offsetOfBase())); masm.branchLatin1String(string, &isLatin1); { masm.store32(Imm32(JSString::DEPENDENT_FLAGS), Address(output, JSString::offsetOfFlags())); masm.loadPtr(Address(string, JSString::offsetOfNonInlineChars()), temp); BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t))); masm.computeEffectiveAddress(chars, temp); masm.storePtr(temp, Address(output, JSString::offsetOfNonInlineChars())); masm.jump(done); } masm.bind(&isLatin1); { masm.store32(Imm32(JSString::DEPENDENT_FLAGS | JSString::LATIN1_CHARS_BIT), Address(output, JSString::offsetOfFlags())); masm.loadPtr(Address(string, JSString::offsetOfNonInlineChars()), temp); static_assert(sizeof(char) == 1, "begin index shouldn't need scaling"); masm.addPtr(begin, temp); masm.storePtr(temp, Address(output, JSString::offsetOfNonInlineChars())); masm.jump(done); } masm.bind(done); } JitCode* JitCompartment::generateStringConcatStub(JSContext* cx) { MacroAssembler masm(cx); Register lhs = CallTempReg0; Register rhs = CallTempReg1; Register temp1 = CallTempReg2; Register temp2 = CallTempReg3; Register temp3 = CallTempReg4; Register output = CallTempReg5; Label failure, failurePopTemps; #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif // If lhs is empty, return rhs. Label leftEmpty; masm.loadStringLength(lhs, temp1); masm.branchTest32(Assembler::Zero, temp1, temp1, &leftEmpty); // If rhs is empty, return lhs. Label rightEmpty; masm.loadStringLength(rhs, temp2); masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty); masm.add32(temp1, temp2); // Check if we can use a JSFatInlineString. The result is a Latin1 string if // lhs and rhs are both Latin1, so we AND the flags. Label isFatInlineTwoByte, isFatInlineLatin1; masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1); masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1); Label isLatin1, notInline; masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1); { masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE), &isFatInlineTwoByte); masm.jump(¬Inline); } masm.bind(&isLatin1); { masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), &isFatInlineLatin1); } masm.bind(¬Inline); // Keep AND'ed flags in temp1. // Ensure result length <= JSString::MAX_LENGTH. masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure); // Allocate a new rope. masm.newGCString(output, temp3, &failure); // Store rope length and flags. temp1 still holds the result of AND'ing the // lhs and rhs flags, so we just have to clear the other flags to get our // rope flags (Latin1 if both lhs and rhs are Latin1). static_assert(JSString::ROPE_FLAGS == 0, "Rope flags must be 0"); masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1); masm.store32(temp1, Address(output, JSString::offsetOfFlags())); masm.store32(temp2, Address(output, JSString::offsetOfLength())); // Store left and right nodes. masm.storePtr(lhs, Address(output, JSRope::offsetOfLeft())); masm.storePtr(rhs, Address(output, JSRope::offsetOfRight())); masm.ret(); masm.bind(&leftEmpty); masm.mov(rhs, output); masm.ret(); masm.bind(&rightEmpty); masm.mov(lhs, output); masm.ret(); masm.bind(&isFatInlineTwoByte); ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3, &failure, &failurePopTemps, true); masm.bind(&isFatInlineLatin1); ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3, &failure, &failurePopTemps, false); masm.bind(&failurePopTemps); masm.pop(temp2); masm.pop(temp1); masm.bind(&failure); masm.movePtr(ImmPtr(nullptr), output); masm.ret(); Linker linker(masm); AutoFlushICache afc("StringConcatStub"); JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "StringConcatStub"); #endif return code; } JitCode* JitRuntime::generateMallocStub(JSContext* cx) { const Register regReturn = CallTempReg0; const Register regNBytes = CallTempReg0; MacroAssembler masm(cx); AllocatableRegisterSet regs(RegisterSet::Volatile()); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif regs.takeUnchecked(regNBytes); LiveRegisterSet save(regs.asLiveSet()); masm.PushRegsInMask(save); const Register regTemp = regs.takeAnyGeneral(); const Register regRuntime = regTemp; MOZ_ASSERT(regTemp != regNBytes); masm.setupUnalignedABICall(regTemp); masm.movePtr(ImmPtr(cx->runtime()), regRuntime); masm.passABIArg(regRuntime); masm.passABIArg(regNBytes); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MallocWrapper)); masm.storeCallPointerResult(regReturn); masm.PopRegsInMask(save); masm.ret(); Linker linker(masm); AutoFlushICache afc("MallocStub"); JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "MallocStub"); #endif return code; } JitCode* JitRuntime::generateFreeStub(JSContext* cx) { const Register regSlots = CallTempReg0; MacroAssembler masm(cx); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif AllocatableRegisterSet regs(RegisterSet::Volatile()); regs.takeUnchecked(regSlots); LiveRegisterSet save(regs.asLiveSet()); masm.PushRegsInMask(save); const Register regTemp = regs.takeAnyGeneral(); MOZ_ASSERT(regTemp != regSlots); masm.setupUnalignedABICall(regTemp); masm.passABIArg(regSlots); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js_free)); masm.PopRegsInMask(save); masm.ret(); Linker linker(masm); AutoFlushICache afc("FreeStub"); JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "FreeStub"); #endif return code; } JitCode* JitRuntime::generateLazyLinkStub(JSContext* cx) { MacroAssembler masm(cx); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); Register temp0 = regs.takeAny(); masm.enterFakeExitFrame(LazyLinkExitFrameLayoutToken); masm.PushStubCode(); masm.setupUnalignedABICall(temp0); masm.loadJSContext(temp0); masm.passABIArg(temp0); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, LazyLinkTopActivation)); masm.leaveExitFrame(/* stub code */ sizeof(JitCode*)); #ifdef JS_USE_LINK_REGISTER // Restore the return address such that the emitPrologue function of the // CodeGenerator can push it back on the stack with pushReturnAddress. masm.popReturnAddress(); #endif masm.jump(ReturnReg); Linker linker(masm); AutoFlushICache afc("LazyLinkStub"); JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "LazyLinkStub"); #endif return code; } bool JitRuntime::generateTLEventVM(JSContext* cx, MacroAssembler& masm, const VMFunction& f, bool enter) { #ifdef JS_TRACE_LOGGING TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime()); bool vmEventEnabled = TraceLogTextIdEnabled(TraceLogger_VM); bool vmSpecificEventEnabled = TraceLogTextIdEnabled(TraceLogger_VMSpecific); if (vmEventEnabled || vmSpecificEventEnabled) { AllocatableRegisterSet regs(RegisterSet::Volatile()); Register loggerReg = regs.takeAnyGeneral(); masm.Push(loggerReg); masm.movePtr(ImmPtr(logger), loggerReg); if (vmEventEnabled) { if (enter) masm.tracelogStartId(loggerReg, TraceLogger_VM, /* force = */ true); else masm.tracelogStopId(loggerReg, TraceLogger_VM, /* force = */ true); } if (vmSpecificEventEnabled) { TraceLoggerEvent event(logger, f.name()); if (!event.hasPayload()) return false; if (enter) masm.tracelogStartId(loggerReg, event.payload()->textId(), /* force = */ true); else masm.tracelogStopId(loggerReg, event.payload()->textId(), /* force = */ true); } masm.Pop(loggerReg); } #endif return true; } typedef bool (*CharCodeAtFn)(JSContext*, HandleString, int32_t, uint32_t*); static const VMFunction CharCodeAtInfo = FunctionInfo<CharCodeAtFn>(jit::CharCodeAt, "CharCodeAt"); void CodeGenerator::visitCharCodeAt(LCharCodeAt* lir) { Register str = ToRegister(lir->str()); Register index = ToRegister(lir->index()); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(CharCodeAtInfo, lir, ArgList(str, index), StoreRegisterTo(output)); masm.branchIfRope(str, ool->entry()); masm.loadStringChar(str, index, output); masm.bind(ool->rejoin()); } typedef JSFlatString* (*StringFromCharCodeFn)(JSContext*, int32_t); static const VMFunction StringFromCharCodeInfo = FunctionInfo<StringFromCharCodeFn>(jit::StringFromCharCode, "StringFromCharCode"); void CodeGenerator::visitFromCharCode(LFromCharCode* lir) { Register code = ToRegister(lir->code()); Register output = ToRegister(lir->output()); OutOfLineCode* ool = oolCallVM(StringFromCharCodeInfo, lir, ArgList(code), StoreRegisterTo(output)); // OOL path if code >= UNIT_STATIC_LIMIT. masm.branch32(Assembler::AboveOrEqual, code, Imm32(StaticStrings::UNIT_STATIC_LIMIT), ool->entry()); masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().unitStaticTable), output); masm.loadPtr(BaseIndex(output, code, ScalePointer), output); masm.bind(ool->rejoin()); } typedef JSString* (*StringFromCodePointFn)(JSContext*, int32_t); static const VMFunction StringFromCodePointInfo = FunctionInfo<StringFromCodePointFn>(jit::StringFromCodePoint, "StringFromCodePoint"); void CodeGenerator::visitFromCodePoint(LFromCodePoint* lir) { Register codePoint = ToRegister(lir->codePoint()); Register output = ToRegister(lir->output()); LSnapshot* snapshot = lir->snapshot(); OutOfLineCode* ool = oolCallVM(StringFromCodePointInfo, lir, ArgList(codePoint), StoreRegisterTo(output)); // Use a bailout if the input is not a valid code point, because // MFromCodePoint is movable and it'd be observable when a moved // fromCodePoint throws an exception before its actual call site. bailoutCmp32(Assembler::Above, codePoint, Imm32(unicode::NonBMPMax), snapshot); // OOL path if code point >= UNIT_STATIC_LIMIT. masm.branch32(Assembler::AboveOrEqual, codePoint, Imm32(StaticStrings::UNIT_STATIC_LIMIT), ool->entry()); masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().unitStaticTable), output); masm.loadPtr(BaseIndex(output, codePoint, ScalePointer), output); masm.bind(ool->rejoin()); } void CodeGenerator::visitSinCos(LSinCos *lir) { Register temp = ToRegister(lir->temp()); Register params = ToRegister(lir->temp2()); FloatRegister input = ToFloatRegister(lir->input()); FloatRegister outputSin = ToFloatRegister(lir->outputSin()); FloatRegister outputCos = ToFloatRegister(lir->outputCos()); masm.reserveStack(sizeof(double) * 2); masm.movePtr(masm.getStackPointer(), params); const MathCache* mathCache = lir->mir()->cache(); masm.setupUnalignedABICall(temp); if (mathCache) { masm.movePtr(ImmPtr(mathCache), temp); masm.passABIArg(temp); } #define MAYBE_CACHED_(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached) masm.passABIArg(input, MoveOp::DOUBLE); masm.passABIArg(MoveOperand(params, sizeof(double), MoveOperand::EFFECTIVE_ADDRESS), MoveOp::GENERAL); masm.passABIArg(MoveOperand(params, 0, MoveOperand::EFFECTIVE_ADDRESS), MoveOp::GENERAL); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED_(js::math_sincos))); #undef MAYBE_CACHED_ masm.loadDouble(Address(masm.getStackPointer(), 0), outputCos); masm.loadDouble(Address(masm.getStackPointer(), sizeof(double)), outputSin); masm.freeStack(sizeof(double) * 2); } typedef JSObject* (*StringSplitFn)(JSContext*, HandleObjectGroup, HandleString, HandleString, uint32_t); static const VMFunction StringSplitInfo = FunctionInfo<StringSplitFn>(js::str_split_string, "str_split_string"); void CodeGenerator::visitStringSplit(LStringSplit* lir) { pushArg(Imm32(INT32_MAX)); pushArg(ToRegister(lir->separator())); pushArg(ToRegister(lir->string())); pushArg(ImmGCPtr(lir->mir()->group())); callVM(StringSplitInfo, lir); } void CodeGenerator::visitInitializedLength(LInitializedLength* lir) { Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength()); masm.load32(initLength, ToRegister(lir->output())); } void CodeGenerator::visitSetInitializedLength(LSetInitializedLength* lir) { Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength()); RegisterOrInt32Constant index = ToRegisterOrInt32Constant(lir->index()); masm.inc32(&index); masm.store32(index, initLength); // Restore register value if it is used/captured after. masm.dec32(&index); } void CodeGenerator::visitUnboxedArrayLength(LUnboxedArrayLength* lir) { Register obj = ToRegister(lir->object()); Register result = ToRegister(lir->output()); masm.load32(Address(obj, UnboxedArrayObject::offsetOfLength()), result); } void CodeGenerator::visitUnboxedArrayInitializedLength(LUnboxedArrayInitializedLength* lir) { Register obj = ToRegister(lir->object()); Register result = ToRegister(lir->output()); masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), result); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), result); } void CodeGenerator::visitIncrementUnboxedArrayInitializedLength(LIncrementUnboxedArrayInitializedLength* lir) { Register obj = ToRegister(lir->object()); masm.add32(Imm32(1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength())); } void CodeGenerator::visitSetUnboxedArrayInitializedLength(LSetUnboxedArrayInitializedLength* lir) { Register obj = ToRegister(lir->object()); RegisterOrInt32Constant key = ToRegisterOrInt32Constant(lir->length()); Register temp = ToRegister(lir->temp()); Address initLengthAddr(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()); masm.load32(initLengthAddr, temp); masm.and32(Imm32(UnboxedArrayObject::CapacityMask), temp); if (key.isRegister()) masm.or32(key.reg(), temp); else masm.or32(Imm32(key.constant()), temp); masm.store32(temp, initLengthAddr); } void CodeGenerator::visitNotO(LNotO* lir) { MOZ_ASSERT(lir->mir()->operandMightEmulateUndefined(), "This should be constant-folded if the object can't emulate undefined."); OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels(); addOutOfLineCode(ool, lir->mir()); Label* ifEmulatesUndefined = ool->label1(); Label* ifDoesntEmulateUndefined = ool->label2(); Register objreg = ToRegister(lir->input()); Register output = ToRegister(lir->output()); branchTestObjectEmulatesUndefined(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, output, ool); // fall through Label join; masm.move32(Imm32(0), output); masm.jump(&join); masm.bind(ifEmulatesUndefined); masm.move32(Imm32(1), output); masm.bind(&join); } void CodeGenerator::visitNotV(LNotV* lir) { Maybe<Label> ifTruthyLabel, ifFalsyLabel; Label* ifTruthy; Label* ifFalsy; OutOfLineTestObjectWithLabels* ool = nullptr; MDefinition* operand = lir->mir()->input(); // Unfortunately, it's possible that someone (e.g. phi elimination) switched // out our operand after we did cacheOperandMightEmulateUndefined. So we // might think it can emulate undefined _and_ know that it can't be an // object. if (lir->mir()->operandMightEmulateUndefined() && operand->mightBeType(MIRType::Object)) { ool = new(alloc()) OutOfLineTestObjectWithLabels(); addOutOfLineCode(ool, lir->mir()); ifTruthy = ool->label1(); ifFalsy = ool->label2(); } else { ifTruthyLabel.emplace(); ifFalsyLabel.emplace(); ifTruthy = ifTruthyLabel.ptr(); ifFalsy = ifFalsyLabel.ptr(); } testValueTruthyKernel(ToValue(lir, LNotV::Input), lir->temp1(), lir->temp2(), ToFloatRegister(lir->tempFloat()), ifTruthy, ifFalsy, ool, operand); Label join; Register output = ToRegister(lir->output()); // Note that the testValueTruthyKernel call above may choose to fall through // to ifTruthy instead of branching there. masm.bind(ifTruthy); masm.move32(Imm32(0), output); masm.jump(&join); masm.bind(ifFalsy); masm.move32(Imm32(1), output); // both branches meet here. masm.bind(&join); } void CodeGenerator::visitBoundsCheck(LBoundsCheck* lir) { const LAllocation* index = lir->index(); const LAllocation* length = lir->length(); LSnapshot* snapshot = lir->snapshot(); if (index->isConstant()) { // Use uint32 so that the comparison is unsigned. uint32_t idx = ToInt32(index); if (length->isConstant()) { uint32_t len = ToInt32(lir->length()); if (idx < len) return; bailout(snapshot); return; } if (length->isRegister()) bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(idx), snapshot); else bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(idx), snapshot); return; } Register indexReg = ToRegister(index); if (length->isConstant()) bailoutCmp32(Assembler::AboveOrEqual, indexReg, Imm32(ToInt32(length)), snapshot); else if (length->isRegister()) bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), indexReg, snapshot); else bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), indexReg, snapshot); } void CodeGenerator::visitBoundsCheckRange(LBoundsCheckRange* lir) { int32_t min = lir->mir()->minimum(); int32_t max = lir->mir()->maximum(); MOZ_ASSERT(max >= min); const LAllocation* length = lir->length(); LSnapshot* snapshot = lir->snapshot(); Register temp = ToRegister(lir->getTemp(0)); if (lir->index()->isConstant()) { int32_t nmin, nmax; int32_t index = ToInt32(lir->index()); if (SafeAdd(index, min, &nmin) && SafeAdd(index, max, &nmax) && nmin >= 0) { if (length->isRegister()) bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(nmax), snapshot); else bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(nmax), snapshot); return; } masm.mov(ImmWord(index), temp); } else { masm.mov(ToRegister(lir->index()), temp); } // If the minimum and maximum differ then do an underflow check first. // If the two are the same then doing an unsigned comparison on the // length will also catch a negative index. if (min != max) { if (min != 0) { Label bail; masm.branchAdd32(Assembler::Overflow, Imm32(min), temp, &bail); bailoutFrom(&bail, snapshot); } bailoutCmp32(Assembler::LessThan, temp, Imm32(0), snapshot); if (min != 0) { int32_t diff; if (SafeSub(max, min, &diff)) max = diff; else masm.sub32(Imm32(min), temp); } } // Compute the maximum possible index. No overflow check is needed when // max > 0. We can only wraparound to a negative number, which will test as // larger than all nonnegative numbers in the unsigned comparison, and the // length is required to be nonnegative (else testing a negative length // would succeed on any nonnegative index). if (max != 0) { if (max < 0) { Label bail; masm.branchAdd32(Assembler::Overflow, Imm32(max), temp, &bail); bailoutFrom(&bail, snapshot); } else { masm.add32(Imm32(max), temp); } } if (length->isRegister()) bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), temp, snapshot); else bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), temp, snapshot); } void CodeGenerator::visitBoundsCheckLower(LBoundsCheckLower* lir) { int32_t min = lir->mir()->minimum(); bailoutCmp32(Assembler::LessThan, ToRegister(lir->index()), Imm32(min), lir->snapshot()); } class OutOfLineStoreElementHole : public OutOfLineCodeBase<CodeGenerator> { LInstruction* ins_; Label rejoinStore_; public: explicit OutOfLineStoreElementHole(LInstruction* ins) : ins_(ins) { MOZ_ASSERT(ins->isStoreElementHoleV() || ins->isStoreElementHoleT() || ins->isFallibleStoreElementV() || ins->isFallibleStoreElementT()); } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineStoreElementHole(this); } LInstruction* ins() const { return ins_; } Label* rejoinStore() { return &rejoinStore_; } }; void CodeGenerator::emitStoreHoleCheck(Register elements, const LAllocation* index, int32_t offsetAdjustment, LSnapshot* snapshot) { Label bail; if (index->isConstant()) { Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment); masm.branchTestMagic(Assembler::Equal, dest, &bail); } else { BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment); masm.branchTestMagic(Assembler::Equal, dest, &bail); } bailoutFrom(&bail, snapshot); } static ConstantOrRegister ToConstantOrRegister(const LAllocation* value, MIRType valueType) { if (value->isConstant()) return ConstantOrRegister(value->toConstant()->toJSValue()); return TypedOrValueRegister(valueType, ToAnyRegister(value)); } void CodeGenerator::emitStoreElementTyped(const LAllocation* value, MIRType valueType, MIRType elementType, Register elements, const LAllocation* index, int32_t offsetAdjustment) { ConstantOrRegister v = ToConstantOrRegister(value, valueType); if (index->isConstant()) { Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment); masm.storeUnboxedValue(v, valueType, dest, elementType); } else { BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment); masm.storeUnboxedValue(v, valueType, dest, elementType); } } void CodeGenerator::visitStoreElementT(LStoreElementT* store) { Register elements = ToRegister(store->elements()); const LAllocation* index = store->index(); if (store->mir()->needsBarrier()) emitPreBarrier(elements, index, store->mir()->offsetAdjustment()); if (store->mir()->needsHoleCheck()) emitStoreHoleCheck(elements, index, store->mir()->offsetAdjustment(), store->snapshot()); emitStoreElementTyped(store->value(), store->mir()->value()->type(), store->mir()->elementType(), elements, index, store->mir()->offsetAdjustment()); } void CodeGenerator::visitStoreElementV(LStoreElementV* lir) { const ValueOperand value = ToValue(lir, LStoreElementV::Value); Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); if (lir->mir()->needsBarrier()) emitPreBarrier(elements, index, lir->mir()->offsetAdjustment()); if (lir->mir()->needsHoleCheck()) emitStoreHoleCheck(elements, index, lir->mir()->offsetAdjustment(), lir->snapshot()); if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * sizeof(js::Value) + lir->mir()->offsetAdjustment()); masm.storeValue(value, dest); } else { BaseIndex dest(elements, ToRegister(lir->index()), TimesEight, lir->mir()->offsetAdjustment()); masm.storeValue(value, dest); } } template <typename T> void CodeGenerator::emitStoreElementHoleT(T* lir) { static_assert(std::is_same<T, LStoreElementHoleT>::value || std::is_same<T, LFallibleStoreElementT>::value, "emitStoreElementHoleT called with unexpected argument type"); OutOfLineStoreElementHole* ool = new(alloc()) OutOfLineStoreElementHole(lir); addOutOfLineCode(ool, lir->mir()); Register obj = ToRegister(lir->object()); Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index); JSValueType unboxedType = lir->mir()->unboxedType(); if (unboxedType == JSVAL_TYPE_MAGIC) { Address initLength(elements, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::BelowOrEqual, initLength, key, ool->entry()); if (lir->mir()->needsBarrier()) emitPreBarrier(elements, index, 0); masm.bind(ool->rejoinStore()); emitStoreElementTyped(lir->value(), lir->mir()->value()->type(), lir->mir()->elementType(), elements, index, 0); } else { Register temp = ToRegister(lir->getTemp(0)); Address initLength(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()); masm.load32(initLength, temp); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), temp); masm.branch32(Assembler::BelowOrEqual, temp, key, ool->entry()); ConstantOrRegister v = ToConstantOrRegister(lir->value(), lir->mir()->value()->type()); if (index->isConstant()) { Address address(elements, ToInt32(index) * UnboxedTypeSize(unboxedType)); EmitUnboxedPreBarrier(masm, address, unboxedType); masm.bind(ool->rejoinStore()); masm.storeUnboxedProperty(address, unboxedType, v, nullptr); } else { BaseIndex address(elements, ToRegister(index), ScaleFromElemWidth(UnboxedTypeSize(unboxedType))); EmitUnboxedPreBarrier(masm, address, unboxedType); masm.bind(ool->rejoinStore()); masm.storeUnboxedProperty(address, unboxedType, v, nullptr); } } masm.bind(ool->rejoin()); } void CodeGenerator::visitStoreElementHoleT(LStoreElementHoleT* lir) { emitStoreElementHoleT(lir); } template <typename T> void CodeGenerator::emitStoreElementHoleV(T* lir) { static_assert(std::is_same<T, LStoreElementHoleV>::value || std::is_same<T, LFallibleStoreElementV>::value, "emitStoreElementHoleV called with unexpected parameter type"); OutOfLineStoreElementHole* ool = new(alloc()) OutOfLineStoreElementHole(lir); addOutOfLineCode(ool, lir->mir()); Register obj = ToRegister(lir->object()); Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); const ValueOperand value = ToValue(lir, T::Value); RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index); JSValueType unboxedType = lir->mir()->unboxedType(); if (unboxedType == JSVAL_TYPE_MAGIC) { Address initLength(elements, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::BelowOrEqual, initLength, key, ool->entry()); if (lir->mir()->needsBarrier()) emitPreBarrier(elements, index, 0); masm.bind(ool->rejoinStore()); if (index->isConstant()) masm.storeValue(value, Address(elements, ToInt32(index) * sizeof(js::Value))); else masm.storeValue(value, BaseIndex(elements, ToRegister(index), TimesEight)); } else { Register temp = ToRegister(lir->getTemp(0)); Address initLength(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()); masm.load32(initLength, temp); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), temp); masm.branch32(Assembler::BelowOrEqual, temp, key, ool->entry()); if (index->isConstant()) { Address address(elements, ToInt32(index) * UnboxedTypeSize(unboxedType)); EmitUnboxedPreBarrier(masm, address, unboxedType); masm.bind(ool->rejoinStore()); masm.storeUnboxedProperty(address, unboxedType, ConstantOrRegister(value), nullptr); } else { BaseIndex address(elements, ToRegister(index), ScaleFromElemWidth(UnboxedTypeSize(unboxedType))); EmitUnboxedPreBarrier(masm, address, unboxedType); masm.bind(ool->rejoinStore()); masm.storeUnboxedProperty(address, unboxedType, ConstantOrRegister(value), nullptr); } } masm.bind(ool->rejoin()); } void CodeGenerator::visitStoreElementHoleV(LStoreElementHoleV* lir) { emitStoreElementHoleV(lir); } typedef bool (*ThrowReadOnlyFn)(JSContext*, int32_t); static const VMFunction ThrowReadOnlyInfo = FunctionInfo<ThrowReadOnlyFn>(ThrowReadOnlyError, "ThrowReadOnlyError"); void CodeGenerator::visitFallibleStoreElementT(LFallibleStoreElementT* lir) { Register elements = ToRegister(lir->elements()); // Handle frozen objects Label isFrozen; Address flags(elements, ObjectElements::offsetOfFlags()); if (!lir->mir()->strict()) { masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), &isFrozen); } else { const LAllocation* index = lir->index(); OutOfLineCode* ool; if (index->isConstant()) ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(Imm32(ToInt32(index))), StoreNothing()); else ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(ToRegister(index)), StoreNothing()); masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), ool->entry()); // This OOL code should have thrown an exception, so will never return. // So, do not bind ool->rejoin() anywhere, so that it implicitly (and without the cost // of a jump) does a masm.assumeUnreachable(). } emitStoreElementHoleT(lir); masm.bind(&isFrozen); } void CodeGenerator::visitFallibleStoreElementV(LFallibleStoreElementV* lir) { Register elements = ToRegister(lir->elements()); // Handle frozen objects Label isFrozen; Address flags(elements, ObjectElements::offsetOfFlags()); if (!lir->mir()->strict()) { masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), &isFrozen); } else { const LAllocation* index = lir->index(); OutOfLineCode* ool; if (index->isConstant()) ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(Imm32(ToInt32(index))), StoreNothing()); else ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(ToRegister(index)), StoreNothing()); masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), ool->entry()); // This OOL code should have thrown an exception, so will never return. // So, do not bind ool->rejoin() anywhere, so that it implicitly (and without the cost // of a jump) does a masm.assumeUnreachable(). } emitStoreElementHoleV(lir); masm.bind(&isFrozen); } typedef bool (*SetDenseOrUnboxedArrayElementFn)(JSContext*, HandleObject, int32_t, HandleValue, bool strict); static const VMFunction SetDenseOrUnboxedArrayElementInfo = FunctionInfo<SetDenseOrUnboxedArrayElementFn>(SetDenseOrUnboxedArrayElement, "SetDenseOrUnboxedArrayElement"); void CodeGenerator::visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool) { Register object, elements; LInstruction* ins = ool->ins(); const LAllocation* index; MIRType valueType; ConstantOrRegister value; JSValueType unboxedType; LDefinition *temp = nullptr; if (ins->isStoreElementHoleV()) { LStoreElementHoleV* store = ins->toStoreElementHoleV(); object = ToRegister(store->object()); elements = ToRegister(store->elements()); index = store->index(); valueType = store->mir()->value()->type(); value = TypedOrValueRegister(ToValue(store, LStoreElementHoleV::Value)); unboxedType = store->mir()->unboxedType(); temp = store->getTemp(0); } else if (ins->isFallibleStoreElementV()) { LFallibleStoreElementV* store = ins->toFallibleStoreElementV(); object = ToRegister(store->object()); elements = ToRegister(store->elements()); index = store->index(); valueType = store->mir()->value()->type(); value = TypedOrValueRegister(ToValue(store, LFallibleStoreElementV::Value)); unboxedType = store->mir()->unboxedType(); temp = store->getTemp(0); } else if (ins->isStoreElementHoleT()) { LStoreElementHoleT* store = ins->toStoreElementHoleT(); object = ToRegister(store->object()); elements = ToRegister(store->elements()); index = store->index(); valueType = store->mir()->value()->type(); if (store->value()->isConstant()) value = ConstantOrRegister(store->value()->toConstant()->toJSValue()); else value = TypedOrValueRegister(valueType, ToAnyRegister(store->value())); unboxedType = store->mir()->unboxedType(); temp = store->getTemp(0); } else { // ins->isFallibleStoreElementT() LFallibleStoreElementT* store = ins->toFallibleStoreElementT(); object = ToRegister(store->object()); elements = ToRegister(store->elements()); index = store->index(); valueType = store->mir()->value()->type(); if (store->value()->isConstant()) value = ConstantOrRegister(store->value()->toConstant()->toJSValue()); else value = TypedOrValueRegister(valueType, ToAnyRegister(store->value())); unboxedType = store->mir()->unboxedType(); temp = store->getTemp(0); } RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index); // If index == initializedLength, try to bump the initialized length inline. // If index > initializedLength, call a stub. Note that this relies on the // condition flags sticking from the incoming branch. Label callStub; #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) // Had to reimplement for MIPS because there are no flags. if (unboxedType == JSVAL_TYPE_MAGIC) { Address initLength(elements, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::NotEqual, initLength, key, &callStub); } else { Address initLength(object, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()); masm.load32(initLength, ToRegister(temp)); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), ToRegister(temp)); masm.branch32(Assembler::NotEqual, ToRegister(temp), key, &callStub); } #else masm.j(Assembler::NotEqual, &callStub); #endif if (unboxedType == JSVAL_TYPE_MAGIC) { // Check array capacity. masm.branch32(Assembler::BelowOrEqual, Address(elements, ObjectElements::offsetOfCapacity()), key, &callStub); // Update initialized length. The capacity guard above ensures this won't overflow, // due to MAX_DENSE_ELEMENTS_COUNT. masm.inc32(&key); masm.store32(key, Address(elements, ObjectElements::offsetOfInitializedLength())); // Update length if length < initializedLength. Label dontUpdate; masm.branch32(Assembler::AboveOrEqual, Address(elements, ObjectElements::offsetOfLength()), key, &dontUpdate); masm.store32(key, Address(elements, ObjectElements::offsetOfLength())); masm.bind(&dontUpdate); masm.dec32(&key); } else { // Check array capacity. masm.checkUnboxedArrayCapacity(object, key, ToRegister(temp), &callStub); // Update initialized length. masm.add32(Imm32(1), Address(object, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength())); // Update length if length < initializedLength. Address lengthAddr(object, UnboxedArrayObject::offsetOfLength()); Label dontUpdate; masm.branch32(Assembler::Above, lengthAddr, key, &dontUpdate); masm.add32(Imm32(1), lengthAddr); masm.bind(&dontUpdate); } if ((ins->isStoreElementHoleT() || ins->isFallibleStoreElementT()) && unboxedType == JSVAL_TYPE_MAGIC && valueType != MIRType::Double) { // The inline path for StoreElementHoleT and FallibleStoreElementT does not always store // the type tag, so we do the store on the OOL path. We use MIRType::None for the element // type so that storeElementTyped will always store the type tag. if (ins->isStoreElementHoleT()) { emitStoreElementTyped(ins->toStoreElementHoleT()->value(), valueType, MIRType::None, elements, index, 0); masm.jump(ool->rejoin()); } else if (ins->isFallibleStoreElementT()) { emitStoreElementTyped(ins->toFallibleStoreElementT()->value(), valueType, MIRType::None, elements, index, 0); masm.jump(ool->rejoin()); } } else { // Jump to the inline path where we will store the value. masm.jump(ool->rejoinStore()); } masm.bind(&callStub); saveLive(ins); pushArg(Imm32(current->mir()->strict())); pushArg(value); if (index->isConstant()) pushArg(Imm32(ToInt32(index))); else pushArg(ToRegister(index)); pushArg(object); callVM(SetDenseOrUnboxedArrayElementInfo, ins); restoreLive(ins); masm.jump(ool->rejoin()); } template <typename T> static void StoreUnboxedPointer(MacroAssembler& masm, T address, MIRType type, const LAllocation* value, bool preBarrier) { if (preBarrier) masm.patchableCallPreBarrier(address, type); if (value->isConstant()) { Value v = value->toConstant()->toJSValue(); if (v.isMarkable()) { masm.storePtr(ImmGCPtr(v.toMarkablePointer()), address); } else { MOZ_ASSERT(v.isNull()); masm.storePtr(ImmWord(0), address); } } else { masm.storePtr(ToRegister(value), address); } } void CodeGenerator::visitStoreUnboxedPointer(LStoreUnboxedPointer* lir) { MIRType type; int32_t offsetAdjustment; bool preBarrier; if (lir->mir()->isStoreUnboxedObjectOrNull()) { type = MIRType::Object; offsetAdjustment = lir->mir()->toStoreUnboxedObjectOrNull()->offsetAdjustment(); preBarrier = lir->mir()->toStoreUnboxedObjectOrNull()->preBarrier(); } else if (lir->mir()->isStoreUnboxedString()) { type = MIRType::String; offsetAdjustment = lir->mir()->toStoreUnboxedString()->offsetAdjustment(); preBarrier = lir->mir()->toStoreUnboxedString()->preBarrier(); } else { MOZ_CRASH(); } Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); const LAllocation* value = lir->value(); if (index->isConstant()) { Address address(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment); StoreUnboxedPointer(masm, address, type, value, preBarrier); } else { BaseIndex address(elements, ToRegister(index), ScalePointer, offsetAdjustment); StoreUnboxedPointer(masm, address, type, value, preBarrier); } } typedef bool (*ConvertUnboxedObjectToNativeFn)(JSContext*, JSObject*); static const VMFunction ConvertUnboxedPlainObjectToNativeInfo = FunctionInfo<ConvertUnboxedObjectToNativeFn>(UnboxedPlainObject::convertToNative, "UnboxedPlainObject::convertToNative"); static const VMFunction ConvertUnboxedArrayObjectToNativeInfo = FunctionInfo<ConvertUnboxedObjectToNativeFn>(UnboxedArrayObject::convertToNative, "UnboxedArrayObject::convertToNative"); void CodeGenerator::visitConvertUnboxedObjectToNative(LConvertUnboxedObjectToNative* lir) { Register object = ToRegister(lir->getOperand(0)); OutOfLineCode* ool = oolCallVM(lir->mir()->group()->unboxedLayoutDontCheckGeneration().isArray() ? ConvertUnboxedArrayObjectToNativeInfo : ConvertUnboxedPlainObjectToNativeInfo, lir, ArgList(object), StoreNothing()); masm.branchPtr(Assembler::Equal, Address(object, JSObject::offsetOfGroup()), ImmGCPtr(lir->mir()->group()), ool->entry()); masm.bind(ool->rejoin()); } typedef bool (*ArrayPopShiftFn)(JSContext*, HandleObject, MutableHandleValue); static const VMFunction ArrayPopDenseInfo = FunctionInfo<ArrayPopShiftFn>(jit::ArrayPopDense, "ArrayPopDense"); static const VMFunction ArrayShiftDenseInfo = FunctionInfo<ArrayPopShiftFn>(jit::ArrayShiftDense, "ArrayShiftDense"); void CodeGenerator::emitArrayPopShift(LInstruction* lir, const MArrayPopShift* mir, Register obj, Register elementsTemp, Register lengthTemp, TypedOrValueRegister out) { OutOfLineCode* ool; if (mir->mode() == MArrayPopShift::Pop) { ool = oolCallVM(ArrayPopDenseInfo, lir, ArgList(obj), StoreValueTo(out)); } else { MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift); ool = oolCallVM(ArrayShiftDenseInfo, lir, ArgList(obj), StoreValueTo(out)); } // VM call if a write barrier is necessary. masm.branchTestNeedsIncrementalBarrier(Assembler::NonZero, ool->entry()); // Load elements and length, and VM call if length != initializedLength. RegisterOrInt32Constant key = RegisterOrInt32Constant(lengthTemp); if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp); masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), lengthTemp); Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::NotEqual, initLength, key, ool->entry()); } else { masm.loadPtr(Address(obj, UnboxedArrayObject::offsetOfElements()), elementsTemp); masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), lengthTemp); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), lengthTemp); Address lengthAddr(obj, UnboxedArrayObject::offsetOfLength()); masm.branch32(Assembler::NotEqual, lengthAddr, key, ool->entry()); } // Test for length != 0. On zero length either take a VM call or generate // an undefined value, depending on whether the call is known to produce // undefined. Label done; if (mir->maybeUndefined()) { Label notEmpty; masm.branchTest32(Assembler::NonZero, lengthTemp, lengthTemp, ¬Empty); // According to the spec we need to set the length 0 (which is already 0). // This is observable when the array length is made non-writable. // Handle this case in the OOL. When freezing an unboxed array it is converted // to an normal array. if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags()); Imm32 bit(ObjectElements::NONWRITABLE_ARRAY_LENGTH); masm.branchTest32(Assembler::NonZero, elementFlags, bit, ool->entry()); } masm.moveValue(UndefinedValue(), out.valueReg()); masm.jump(&done); masm.bind(¬Empty); } else { masm.branchTest32(Assembler::Zero, lengthTemp, lengthTemp, ool->entry()); } masm.dec32(&key); if (mir->mode() == MArrayPopShift::Pop) { if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { BaseIndex addr(elementsTemp, lengthTemp, TimesEight); masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry()); } else { size_t elemSize = UnboxedTypeSize(mir->unboxedType()); BaseIndex addr(elementsTemp, lengthTemp, ScaleFromElemWidth(elemSize)); masm.loadUnboxedProperty(addr, mir->unboxedType(), out); } } else { MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift); Address addr(elementsTemp, 0); if (mir->unboxedType() == JSVAL_TYPE_MAGIC) masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry()); else masm.loadUnboxedProperty(addr, mir->unboxedType(), out); } if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { // Handle the failure case when the array length is non-writable in the // OOL path. (Unlike in the adding-an-element cases, we can't rely on the // capacity <= length invariant for such arrays to avoid an explicit // check.) Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags()); Imm32 bit(ObjectElements::NONWRITABLE_ARRAY_LENGTH); masm.branchTest32(Assembler::NonZero, elementFlags, bit, ool->entry()); // Now adjust length and initializedLength. masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfLength())); masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfInitializedLength())); } else { // Unboxed arrays always have writable lengths. Adjust length and // initializedLength. masm.store32(lengthTemp, Address(obj, UnboxedArrayObject::offsetOfLength())); masm.add32(Imm32(-1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength())); } if (mir->mode() == MArrayPopShift::Shift) { // Don't save the temp registers. LiveRegisterSet temps; temps.add(elementsTemp); temps.add(lengthTemp); saveVolatile(temps); masm.setupUnalignedABICall(lengthTemp); masm.passABIArg(obj); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::ArrayShiftMoveElements)); restoreVolatile(temps); } masm.bind(&done); masm.bind(ool->rejoin()); } void CodeGenerator::visitArrayPopShiftV(LArrayPopShiftV* lir) { Register obj = ToRegister(lir->object()); Register elements = ToRegister(lir->temp0()); Register length = ToRegister(lir->temp1()); TypedOrValueRegister out(ToOutValue(lir)); emitArrayPopShift(lir, lir->mir(), obj, elements, length, out); } void CodeGenerator::visitArrayPopShiftT(LArrayPopShiftT* lir) { Register obj = ToRegister(lir->object()); Register elements = ToRegister(lir->temp0()); Register length = ToRegister(lir->temp1()); TypedOrValueRegister out(lir->mir()->type(), ToAnyRegister(lir->output())); emitArrayPopShift(lir, lir->mir(), obj, elements, length, out); } typedef bool (*ArrayPushDenseFn)(JSContext*, HandleObject, HandleValue, uint32_t*); static const VMFunction ArrayPushDenseInfo = FunctionInfo<ArrayPushDenseFn>(jit::ArrayPushDense, "ArrayPushDense"); void CodeGenerator::emitArrayPush(LInstruction* lir, const MArrayPush* mir, Register obj, const ConstantOrRegister& value, Register elementsTemp, Register length) { OutOfLineCode* ool = oolCallVM(ArrayPushDenseInfo, lir, ArgList(obj, value), StoreRegisterTo(length)); RegisterOrInt32Constant key = RegisterOrInt32Constant(length); if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { // Load elements and length. masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp); masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), length); // Guard length == initializedLength. Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength()); masm.branch32(Assembler::NotEqual, initLength, key, ool->entry()); // Guard length < capacity. Address capacity(elementsTemp, ObjectElements::offsetOfCapacity()); masm.branch32(Assembler::BelowOrEqual, capacity, key, ool->entry()); // Do the store. masm.storeConstantOrRegister(value, BaseIndex(elementsTemp, length, TimesEight)); } else { // Load initialized length. masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), length); masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), length); // Guard length == initializedLength. Address lengthAddr(obj, UnboxedArrayObject::offsetOfLength()); masm.branch32(Assembler::NotEqual, lengthAddr, key, ool->entry()); // Guard length < capacity. masm.checkUnboxedArrayCapacity(obj, key, elementsTemp, ool->entry()); // Load elements and do the store. masm.loadPtr(Address(obj, UnboxedArrayObject::offsetOfElements()), elementsTemp); size_t elemSize = UnboxedTypeSize(mir->unboxedType()); BaseIndex addr(elementsTemp, length, ScaleFromElemWidth(elemSize)); masm.storeUnboxedProperty(addr, mir->unboxedType(), value, nullptr); } masm.inc32(&key); // Update length and initialized length. if (mir->unboxedType() == JSVAL_TYPE_MAGIC) { masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfLength())); masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfInitializedLength())); } else { masm.store32(length, Address(obj, UnboxedArrayObject::offsetOfLength())); masm.add32(Imm32(1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength())); } masm.bind(ool->rejoin()); } void CodeGenerator::visitArrayPushV(LArrayPushV* lir) { Register obj = ToRegister(lir->object()); Register elementsTemp = ToRegister(lir->temp()); Register length = ToRegister(lir->output()); ConstantOrRegister value = TypedOrValueRegister(ToValue(lir, LArrayPushV::Value)); emitArrayPush(lir, lir->mir(), obj, value, elementsTemp, length); } void CodeGenerator::visitArrayPushT(LArrayPushT* lir) { Register obj = ToRegister(lir->object()); Register elementsTemp = ToRegister(lir->temp()); Register length = ToRegister(lir->output()); ConstantOrRegister value; if (lir->value()->isConstant()) value = ConstantOrRegister(lir->value()->toConstant()->toJSValue()); else value = TypedOrValueRegister(lir->mir()->value()->type(), ToAnyRegister(lir->value())); emitArrayPush(lir, lir->mir(), obj, value, elementsTemp, length); } typedef JSObject* (*ArraySliceDenseFn)(JSContext*, HandleObject, int32_t, int32_t, HandleObject); static const VMFunction ArraySliceDenseInfo = FunctionInfo<ArraySliceDenseFn>(array_slice_dense, "array_slice_dense"); void CodeGenerator::visitArraySlice(LArraySlice* lir) { Register object = ToRegister(lir->object()); Register begin = ToRegister(lir->begin()); Register end = ToRegister(lir->end()); Register temp1 = ToRegister(lir->temp1()); Register temp2 = ToRegister(lir->temp2()); Label call, fail; // Try to allocate an object. masm.createGCObject(temp1, temp2, lir->mir()->templateObj(), lir->mir()->initialHeap(), &fail); // Fixup the group of the result in case it doesn't match the template object. masm.loadPtr(Address(object, JSObject::offsetOfGroup()), temp2); masm.storePtr(temp2, Address(temp1, JSObject::offsetOfGroup())); masm.jump(&call); { masm.bind(&fail); masm.movePtr(ImmPtr(nullptr), temp1); } masm.bind(&call); pushArg(temp1); pushArg(end); pushArg(begin); pushArg(object); callVM(ArraySliceDenseInfo, lir); } typedef JSString* (*ArrayJoinFn)(JSContext*, HandleObject, HandleString); static const VMFunction ArrayJoinInfo = FunctionInfo<ArrayJoinFn>(jit::ArrayJoin, "ArrayJoin"); void CodeGenerator::visitArrayJoin(LArrayJoin* lir) { pushArg(ToRegister(lir->separator())); pushArg(ToRegister(lir->array())); callVM(ArrayJoinInfo, lir); } typedef JSObject* (*ValueToIteratorFn)(JSContext*, uint32_t, HandleValue); static const VMFunction ValueToIteratorInfo = FunctionInfo<ValueToIteratorFn>(ValueToIterator, "ValueToIterator"); void CodeGenerator::visitCallIteratorStartV(LCallIteratorStartV* lir) { pushArg(ToValue(lir, LCallIteratorStartV::Value)); pushArg(Imm32(lir->mir()->flags())); callVM(ValueToIteratorInfo, lir); } typedef JSObject* (*GetIteratorObjectFn)(JSContext*, HandleObject, uint32_t); static const VMFunction GetIteratorObjectInfo = FunctionInfo<GetIteratorObjectFn>(GetIteratorObject, "GetIteratorObject"); void CodeGenerator::visitCallIteratorStartO(LCallIteratorStartO* lir) { pushArg(Imm32(lir->mir()->flags())); pushArg(ToRegister(lir->object())); callVM(GetIteratorObjectInfo, lir); } void CodeGenerator::branchIfNotEmptyObjectElements(Register obj, Label* target) { Label emptyObj; masm.branchPtr(Assembler::Equal, Address(obj, NativeObject::offsetOfElements()), ImmPtr(js::emptyObjectElements), &emptyObj); masm.branchPtr(Assembler::NotEqual, Address(obj, NativeObject::offsetOfElements()), ImmPtr(js::emptyObjectElementsShared), target); masm.bind(&emptyObj); } void CodeGenerator::visitIteratorStartO(LIteratorStartO* lir) { const Register obj = ToRegister(lir->object()); const Register output = ToRegister(lir->output()); uint32_t flags = lir->mir()->flags(); OutOfLineCode* ool = oolCallVM(GetIteratorObjectInfo, lir, ArgList(obj, Imm32(flags)), StoreRegisterTo(output)); const Register temp1 = ToRegister(lir->temp1()); const Register temp2 = ToRegister(lir->temp2()); const Register niTemp = ToRegister(lir->temp3()); // Holds the NativeIterator object. // Iterators other than for-in should use LCallIteratorStart. MOZ_ASSERT(flags == JSITER_ENUMERATE); // Fetch the most recent iterator and ensure it's not nullptr. masm.loadPtr(AbsoluteAddress(gen->compartment->addressOfLastCachedNativeIterator()), output); masm.branchTestPtr(Assembler::Zero, output, output, ool->entry()); // Load NativeIterator. masm.loadObjPrivate(output, JSObject::ITER_CLASS_NFIXED_SLOTS, niTemp); // Ensure the |active| and |unreusable| bits are not set. masm.branchTest32(Assembler::NonZero, Address(niTemp, offsetof(NativeIterator, flags)), Imm32(JSITER_ACTIVE|JSITER_UNREUSABLE), ool->entry()); // Load the iterator's receiver guard array. masm.loadPtr(Address(niTemp, offsetof(NativeIterator, guard_array)), temp2); // Compare object with the first receiver guard. The last iterator can only // match for native objects and unboxed objects. { Address groupAddr(temp2, offsetof(ReceiverGuard, group)); Address shapeAddr(temp2, offsetof(ReceiverGuard, shape)); Label guardDone, shapeMismatch, noExpando; masm.loadObjShape(obj, temp1); masm.branchPtr(Assembler::NotEqual, shapeAddr, temp1, &shapeMismatch); // Ensure the object does not have any elements. The presence of dense // elements is not captured by the shape tests above. branchIfNotEmptyObjectElements(obj, ool->entry()); masm.jump(&guardDone); masm.bind(&shapeMismatch); masm.loadObjGroup(obj, temp1); masm.branchPtr(Assembler::NotEqual, groupAddr, temp1, ool->entry()); masm.loadPtr(Address(obj, UnboxedPlainObject::offsetOfExpando()), temp1); masm.branchTestPtr(Assembler::Zero, temp1, temp1, &noExpando); branchIfNotEmptyObjectElements(temp1, ool->entry()); masm.loadObjShape(temp1, temp1); masm.bind(&noExpando); masm.branchPtr(Assembler::NotEqual, shapeAddr, temp1, ool->entry()); masm.bind(&guardDone); } // Compare shape of object's prototype with the second shape. The prototype // must be native, as unboxed objects cannot be prototypes (they cannot // have the delegate flag set). Also check for the absence of dense elements. Address prototypeShapeAddr(temp2, sizeof(ReceiverGuard) + offsetof(ReceiverGuard, shape)); masm.loadObjProto(obj, temp1); branchIfNotEmptyObjectElements(temp1, ool->entry()); masm.loadObjShape(temp1, temp1); masm.branchPtr(Assembler::NotEqual, prototypeShapeAddr, temp1, ool->entry()); // Ensure the object's prototype's prototype is nullptr. The last native // iterator will always have a prototype chain length of one (i.e. it must // be a plain object), so we do not need to generate a loop here. masm.loadObjProto(obj, temp1); masm.loadObjProto(temp1, temp1); masm.branchTestPtr(Assembler::NonZero, temp1, temp1, ool->entry()); // Write barrier for stores to the iterator. We only need to take a write // barrier if NativeIterator::obj is actually going to change. { // Bug 867815: Unconditionally take this out- of-line so that we do not // have to post-barrier the store to NativeIter::obj. This just needs // JIT support for the Cell* buffer. Address objAddr(niTemp, offsetof(NativeIterator, obj)); masm.branchPtr(Assembler::NotEqual, objAddr, obj, ool->entry()); } // Mark iterator as active. masm.storePtr(obj, Address(niTemp, offsetof(NativeIterator, obj))); masm.or32(Imm32(JSITER_ACTIVE), Address(niTemp, offsetof(NativeIterator, flags))); // Chain onto the active iterator stack. masm.loadPtr(AbsoluteAddress(gen->compartment->addressOfEnumerators()), temp1); // ni->next = list masm.storePtr(temp1, Address(niTemp, NativeIterator::offsetOfNext())); // ni->prev = list->prev masm.loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), temp2); masm.storePtr(temp2, Address(niTemp, NativeIterator::offsetOfPrev())); // list->prev->next = ni masm.storePtr(niTemp, Address(temp2, NativeIterator::offsetOfNext())); // list->prev = ni masm.storePtr(niTemp, Address(temp1, NativeIterator::offsetOfPrev())); masm.bind(ool->rejoin()); } static void LoadNativeIterator(MacroAssembler& masm, Register obj, Register dest, Label* failures) { MOZ_ASSERT(obj != dest); // Test class. masm.branchTestObjClass(Assembler::NotEqual, obj, dest, &PropertyIteratorObject::class_, failures); // Load NativeIterator object. masm.loadObjPrivate(obj, JSObject::ITER_CLASS_NFIXED_SLOTS, dest); } typedef bool (*IteratorMoreFn)(JSContext*, HandleObject, MutableHandleValue); static const VMFunction IteratorMoreInfo = FunctionInfo<IteratorMoreFn>(IteratorMore, "IteratorMore"); void CodeGenerator::visitIteratorMore(LIteratorMore* lir) { const Register obj = ToRegister(lir->object()); const ValueOperand output = ToOutValue(lir); const Register temp = ToRegister(lir->temp()); OutOfLineCode* ool = oolCallVM(IteratorMoreInfo, lir, ArgList(obj), StoreValueTo(output)); Register outputScratch = output.scratchReg(); LoadNativeIterator(masm, obj, outputScratch, ool->entry()); masm.branchTest32(Assembler::NonZero, Address(outputScratch, offsetof(NativeIterator, flags)), Imm32(JSITER_FOREACH), ool->entry()); // If props_cursor < props_end, load the next string and advance the cursor. // Else, return MagicValue(JS_NO_ITER_VALUE). Label iterDone; Address cursorAddr(outputScratch, offsetof(NativeIterator, props_cursor)); Address cursorEndAddr(outputScratch, offsetof(NativeIterator, props_end)); masm.loadPtr(cursorAddr, temp); masm.branchPtr(Assembler::BelowOrEqual, cursorEndAddr, temp, &iterDone); // Get next string. masm.loadPtr(Address(temp, 0), temp); // Increase the cursor. masm.addPtr(Imm32(sizeof(JSString*)), cursorAddr); masm.tagValue(JSVAL_TYPE_STRING, temp, output); masm.jump(ool->rejoin()); masm.bind(&iterDone); masm.moveValue(MagicValue(JS_NO_ITER_VALUE), output); masm.bind(ool->rejoin()); } void CodeGenerator::visitIsNoIterAndBranch(LIsNoIterAndBranch* lir) { ValueOperand input = ToValue(lir, LIsNoIterAndBranch::Input); Label* ifTrue = getJumpLabelForBranch(lir->ifTrue()); Label* ifFalse = getJumpLabelForBranch(lir->ifFalse()); masm.branchTestMagic(Assembler::Equal, input, ifTrue); if (!isNextBlock(lir->ifFalse()->lir())) masm.jump(ifFalse); } typedef bool (*CloseIteratorFn)(JSContext*, HandleObject); static const VMFunction CloseIteratorInfo = FunctionInfo<CloseIteratorFn>(CloseIterator, "CloseIterator"); void CodeGenerator::visitIteratorEnd(LIteratorEnd* lir) { const Register obj = ToRegister(lir->object()); const Register temp1 = ToRegister(lir->temp1()); const Register temp2 = ToRegister(lir->temp2()); const Register temp3 = ToRegister(lir->temp3()); OutOfLineCode* ool = oolCallVM(CloseIteratorInfo, lir, ArgList(obj), StoreNothing()); LoadNativeIterator(masm, obj, temp1, ool->entry()); masm.branchTest32(Assembler::Zero, Address(temp1, offsetof(NativeIterator, flags)), Imm32(JSITER_ENUMERATE), ool->entry()); // Clear active bit. masm.and32(Imm32(~JSITER_ACTIVE), Address(temp1, offsetof(NativeIterator, flags))); // Reset property cursor. masm.loadPtr(Address(temp1, offsetof(NativeIterator, props_array)), temp2); masm.storePtr(temp2, Address(temp1, offsetof(NativeIterator, props_cursor))); // Unlink from the iterator list. const Register next = temp2; const Register prev = temp3; masm.loadPtr(Address(temp1, NativeIterator::offsetOfNext()), next); masm.loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), prev); masm.storePtr(prev, Address(next, NativeIterator::offsetOfPrev())); masm.storePtr(next, Address(prev, NativeIterator::offsetOfNext())); #ifdef DEBUG masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfNext())); masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfPrev())); #endif masm.bind(ool->rejoin()); } void CodeGenerator::visitArgumentsLength(LArgumentsLength* lir) { // read number of actual arguments from the JS frame. Register argc = ToRegister(lir->output()); Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs()); masm.loadPtr(ptr, argc); } void CodeGenerator::visitGetFrameArgument(LGetFrameArgument* lir) { ValueOperand result = GetValueOutput(lir); const LAllocation* index = lir->index(); size_t argvOffset = frameSize() + JitFrameLayout::offsetOfActualArgs(); if (index->isConstant()) { int32_t i = index->toConstant()->toInt32(); Address argPtr(masm.getStackPointer(), sizeof(Value) * i + argvOffset); masm.loadValue(argPtr, result); } else { Register i = ToRegister(index); BaseValueIndex argPtr(masm.getStackPointer(), i, argvOffset); masm.loadValue(argPtr, result); } } void CodeGenerator::visitSetFrameArgumentT(LSetFrameArgumentT* lir) { size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() + (sizeof(Value) * lir->mir()->argno()); MIRType type = lir->mir()->value()->type(); if (type == MIRType::Double) { // Store doubles directly. FloatRegister input = ToFloatRegister(lir->input()); masm.storeDouble(input, Address(masm.getStackPointer(), argOffset)); } else { Register input = ToRegister(lir->input()); masm.storeValue(ValueTypeFromMIRType(type), input, Address(masm.getStackPointer(), argOffset)); } } void CodeGenerator:: visitSetFrameArgumentC(LSetFrameArgumentC* lir) { size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() + (sizeof(Value) * lir->mir()->argno()); masm.storeValue(lir->val(), Address(masm.getStackPointer(), argOffset)); } void CodeGenerator:: visitSetFrameArgumentV(LSetFrameArgumentV* lir) { const ValueOperand val = ToValue(lir, LSetFrameArgumentV::Input); size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() + (sizeof(Value) * lir->mir()->argno()); masm.storeValue(val, Address(masm.getStackPointer(), argOffset)); } typedef bool (*RunOnceScriptPrologueFn)(JSContext*, HandleScript); static const VMFunction RunOnceScriptPrologueInfo = FunctionInfo<RunOnceScriptPrologueFn>(js::RunOnceScriptPrologue, "RunOnceScriptPrologue"); void CodeGenerator::visitRunOncePrologue(LRunOncePrologue* lir) { pushArg(ImmGCPtr(lir->mir()->block()->info().script())); callVM(RunOnceScriptPrologueInfo, lir); } typedef JSObject* (*InitRestParameterFn)(JSContext*, uint32_t, Value*, HandleObject, HandleObject); static const VMFunction InitRestParameterInfo = FunctionInfo<InitRestParameterFn>(InitRestParameter, "InitRestParameter"); void CodeGenerator::emitRest(LInstruction* lir, Register array, Register numActuals, Register temp0, Register temp1, unsigned numFormals, JSObject* templateObject, bool saveAndRestore, Register resultreg) { // Compute actuals() + numFormals. size_t actualsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs(); masm.moveStackPtrTo(temp1); masm.addPtr(Imm32(sizeof(Value) * numFormals + actualsOffset), temp1); // Compute numActuals - numFormals. Label emptyLength, joinLength; masm.movePtr(numActuals, temp0); masm.branch32(Assembler::LessThanOrEqual, temp0, Imm32(numFormals), &emptyLength); masm.sub32(Imm32(numFormals), temp0); masm.jump(&joinLength); { masm.bind(&emptyLength); masm.move32(Imm32(0), temp0); } masm.bind(&joinLength); if (saveAndRestore) saveLive(lir); pushArg(array); pushArg(ImmGCPtr(templateObject)); pushArg(temp1); pushArg(temp0); callVM(InitRestParameterInfo, lir); if (saveAndRestore) { storePointerResultTo(resultreg); restoreLive(lir); } } void CodeGenerator::visitRest(LRest* lir) { Register numActuals = ToRegister(lir->numActuals()); Register temp0 = ToRegister(lir->getTemp(0)); Register temp1 = ToRegister(lir->getTemp(1)); Register temp2 = ToRegister(lir->getTemp(2)); unsigned numFormals = lir->mir()->numFormals(); ArrayObject* templateObject = lir->mir()->templateObject(); Label joinAlloc, failAlloc; masm.createGCObject(temp2, temp0, templateObject, gc::DefaultHeap, &failAlloc); masm.jump(&joinAlloc); { masm.bind(&failAlloc); masm.movePtr(ImmPtr(nullptr), temp2); } masm.bind(&joinAlloc); emitRest(lir, temp2, numActuals, temp0, temp1, numFormals, templateObject, false, ToRegister(lir->output())); } bool CodeGenerator::generateWasm(wasm::SigIdDesc sigId, wasm::TrapOffset trapOffset, wasm::FuncOffsets* offsets) { JitSpew(JitSpew_Codegen, "# Emitting wasm code"); wasm::GenerateFunctionPrologue(masm, frameSize(), sigId, offsets); // Overflow checks are omitted by CodeGenerator in some cases (leaf // functions with small framePushed). Perform overflow-checking after // pushing framePushed to catch cases with really large frames. Label onOverflow; if (!omitOverRecursedCheck()) { masm.branchPtr(Assembler::AboveOrEqual, Address(WasmTlsReg, offsetof(wasm::TlsData, stackLimit)), masm.getStackPointer(), &onOverflow); } if (!generateBody()) return false; masm.bind(&returnLabel_); wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets); if (!omitOverRecursedCheck()) { // Since we just overflowed the stack, to be on the safe side, pop the // stack so that, when the trap exit stub executes, it is a safe // distance away from the end of the native stack. wasm::TrapDesc trap(trapOffset, wasm::Trap::StackOverflow, /* framePushed = */ 0); if (frameSize() > 0) { masm.bind(&onOverflow); masm.addToStackPtr(Imm32(frameSize())); masm.jump(trap); } else { masm.bindLater(&onOverflow, trap); } } #if defined(JS_ION_PERF) // Note the end of the inline code and start of the OOL code. gen->perfSpewer().noteEndInlineCode(masm); #endif if (!generateOutOfLineCode()) return false; masm.wasmEmitTrapOutOfLineCode(); masm.flush(); if (masm.oom()) return false; offsets->end = masm.currentOffset(); MOZ_ASSERT(!masm.failureLabel()->used()); MOZ_ASSERT(snapshots_.listSize() == 0); MOZ_ASSERT(snapshots_.RVATableSize() == 0); MOZ_ASSERT(recovers_.size() == 0); MOZ_ASSERT(bailouts_.empty()); MOZ_ASSERT(graph.numConstants() == 0); MOZ_ASSERT(safepointIndices_.empty()); MOZ_ASSERT(osiIndices_.empty()); MOZ_ASSERT(cacheList_.empty()); MOZ_ASSERT(safepoints_.size() == 0); MOZ_ASSERT(!scriptCounts_); return true; } bool CodeGenerator::generate() { JitSpew(JitSpew_Codegen, "# Emitting code for script %s:%" PRIuSIZE, gen->info().script()->filename(), gen->info().script()->lineno()); // Initialize native code table with an entry to the start of // top-level script. InlineScriptTree* tree = gen->info().inlineScriptTree(); jsbytecode* startPC = tree->script()->code(); BytecodeSite* startSite = new(gen->alloc()) BytecodeSite(tree, startPC); if (!addNativeToBytecodeEntry(startSite)) return false; if (!snapshots_.init()) return false; if (!safepoints_.init(gen->alloc())) return false; if (!generatePrologue()) return false; // Before generating any code, we generate type checks for all parameters. // This comes before deoptTable_, because we can't use deopt tables without // creating the actual frame. generateArgumentsChecks(); if (frameClass_ != FrameSizeClass::None()) { deoptTable_ = gen->jitRuntime()->getBailoutTable(frameClass_); if (!deoptTable_) return false; } // Skip over the alternative entry to IonScript code. Label skipPrologue; masm.jump(&skipPrologue); // An alternative entry to the IonScript code, which doesn't test the // arguments. masm.flushBuffer(); setSkipArgCheckEntryOffset(masm.size()); masm.setFramePushed(0); if (!generatePrologue()) return false; masm.bind(&skipPrologue); #ifdef DEBUG // Assert that the argument types are correct. generateArgumentsChecks(/* bailout = */ false); #endif // Reset native => bytecode map table with top-level script and startPc. if (!addNativeToBytecodeEntry(startSite)) return false; if (!generateBody()) return false; // Reset native => bytecode map table with top-level script and startPc. if (!addNativeToBytecodeEntry(startSite)) return false; if (!generateEpilogue()) return false; // Reset native => bytecode map table with top-level script and startPc. if (!addNativeToBytecodeEntry(startSite)) return false; generateInvalidateEpilogue(); #if defined(JS_ION_PERF) // Note the end of the inline code and start of the OOL code. perfSpewer_.noteEndInlineCode(masm); #endif // native => bytecode entries for OOL code will be added // by CodeGeneratorShared::generateOutOfLineCode if (!generateOutOfLineCode()) return false; // Add terminal entry. if (!addNativeToBytecodeEntry(startSite)) return false; // Dump Native to bytecode entries to spew. dumpNativeToBytecodeEntries(); return !masm.oom(); } bool CodeGenerator::linkSharedStubs(JSContext* cx) { for (uint32_t i = 0; i < sharedStubs_.length(); i++) { ICStub *stub = nullptr; switch (sharedStubs_[i].kind) { case ICStub::Kind::BinaryArith_Fallback: { ICBinaryArith_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } case ICStub::Kind::UnaryArith_Fallback: { ICUnaryArith_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } case ICStub::Kind::Compare_Fallback: { ICCompare_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } case ICStub::Kind::GetProp_Fallback: { ICGetProp_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } case ICStub::Kind::NewArray_Fallback: { JSScript* script = sharedStubs_[i].entry.script(); jsbytecode* pc = sharedStubs_[i].entry.pc(script); ObjectGroup* group = ObjectGroup::allocationSiteGroup(cx, script, pc, JSProto_Array); if (!group) return false; ICNewArray_Fallback::Compiler stubCompiler(cx, group, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } case ICStub::Kind::NewObject_Fallback: { ICNewObject_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey); stub = stubCompiler.getStub(&stubSpace_); break; } default: MOZ_CRASH("Unsupported shared stub."); } if (!stub) return false; sharedStubs_[i].entry.setFirstStub(stub); } return true; } bool CodeGenerator::link(JSContext* cx, CompilerConstraintList* constraints) { RootedScript script(cx, gen->info().script()); OptimizationLevel optimizationLevel = gen->optimizationInfo().level(); // Capture the SIMD template objects which are used during the // compilation. This iterates over the template objects, using read-barriers // to let the GC know that the generated code relies on these template // objects. captureSimdTemplate(cx); // We finished the new IonScript. Invalidate the current active IonScript, // so we can replace it with this new (probably higher optimized) version. if (script->hasIonScript()) { MOZ_ASSERT(script->ionScript()->isRecompiling()); // Do a normal invalidate, except don't cancel offThread compilations, // since that will cancel this compilation too. Invalidate(cx, script, /* resetUses */ false, /* cancelOffThread*/ false); } if (scriptCounts_ && !script->hasScriptCounts() && !script->initScriptCounts(cx)) return false; if (!linkSharedStubs(cx)) return false; // Check to make sure we didn't have a mid-build invalidation. If so, we // will trickle to jit::Compile() and return Method_Skipped. uint32_t warmUpCount = script->getWarmUpCount(); // Record constraints. If an error occured, returns false and potentially // prevent future compilations. Otherwise, if an invalidation occured, then // skip the current compilation. RecompileInfo recompileInfo; bool validRecompiledInfo = false; if (!FinishCompilation(cx, script, constraints, &recompileInfo, &validRecompiledInfo)) return false; if (!validRecompiledInfo) return true; auto guardRecordedConstraints = mozilla::MakeScopeExit([&] { // In case of error, invalidate the current recompileInfo. recompileInfo.compilerOutput(cx->zone()->types)->invalidate(); }); // IonMonkey could have inferred better type information during // compilation. Since adding the new information to the actual type // information can reset the usecount, increase it back to what it was // before. if (warmUpCount > script->getWarmUpCount()) script->incWarmUpCounter(warmUpCount - script->getWarmUpCount()); uint32_t argumentSlots = (gen->info().nargs() + 1) * sizeof(Value); uint32_t scriptFrameSize = frameClass_ == FrameSizeClass::None() ? frameDepth_ : FrameSizeClass::FromDepth(frameDepth_).frameSize(); // We encode safepoints after the OSI-point offsets have been determined. if (!encodeSafepoints()) return false; IonScript* ionScript = IonScript::New(cx, recompileInfo, graph.totalSlotCount(), argumentSlots, scriptFrameSize, snapshots_.listSize(), snapshots_.RVATableSize(), recovers_.size(), bailouts_.length(), graph.numConstants(), safepointIndices_.length(), osiIndices_.length(), cacheList_.length(), runtimeData_.length(), safepoints_.size(), patchableBackedges_.length(), sharedStubs_.length(), optimizationLevel); if (!ionScript) return false; auto guardIonScript = mozilla::MakeScopeExit([&ionScript] { // Use js_free instead of IonScript::Destroy: the cache list and // backedge list are still uninitialized. js_free(ionScript); }); // Also, note that creating the code here during an incremental GC will // trace the code and mark all GC things it refers to. This captures any // read barriers which were skipped while compiling the script off thread. Linker linker(masm); AutoFlushICache afc("IonLink"); JitCode* code = linker.newCode<CanGC>(cx, ION_CODE, !patchableBackedges_.empty()); if (!code) return false; // Encode native to bytecode map if profiling is enabled. if (isProfilerInstrumentationEnabled()) { // Generate native-to-bytecode main table. if (!generateCompactNativeToBytecodeMap(cx, code)) return false; uint8_t* ionTableAddr = ((uint8_t*) nativeToBytecodeMap_) + nativeToBytecodeTableOffset_; JitcodeIonTable* ionTable = (JitcodeIonTable*) ionTableAddr; // Construct the IonEntry that will go into the global table. JitcodeGlobalEntry::IonEntry entry; if (!ionTable->makeIonEntry(cx, code, nativeToBytecodeScriptListLength_, nativeToBytecodeScriptList_, entry)) { js_free(nativeToBytecodeScriptList_); js_free(nativeToBytecodeMap_); return false; } // nativeToBytecodeScriptList_ is no longer needed. js_free(nativeToBytecodeScriptList_); // Generate the tracked optimizations map. if (isOptimizationTrackingEnabled()) { // Treat OOMs and failures as if optimization tracking were turned off. IonTrackedTypeVector* allTypes = cx->new_<IonTrackedTypeVector>(); if (allTypes && generateCompactTrackedOptimizationsMap(cx, code, allTypes)) { const uint8_t* optsRegionTableAddr = trackedOptimizationsMap_ + trackedOptimizationsRegionTableOffset_; const IonTrackedOptimizationsRegionTable* optsRegionTable = (const IonTrackedOptimizationsRegionTable*) optsRegionTableAddr; const uint8_t* optsTypesTableAddr = trackedOptimizationsMap_ + trackedOptimizationsTypesTableOffset_; const IonTrackedOptimizationsTypesTable* optsTypesTable = (const IonTrackedOptimizationsTypesTable*) optsTypesTableAddr; const uint8_t* optsAttemptsTableAddr = trackedOptimizationsMap_ + trackedOptimizationsAttemptsTableOffset_; const IonTrackedOptimizationsAttemptsTable* optsAttemptsTable = (const IonTrackedOptimizationsAttemptsTable*) optsAttemptsTableAddr; entry.initTrackedOptimizations(optsRegionTable, optsTypesTable, optsAttemptsTable, allTypes); } else { cx->recoverFromOutOfMemory(); } } // Add entry to the global table. JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); if (!globalTable->addEntry(entry, cx->runtime())) { // Memory may have been allocated for the entry. entry.destroy(); return false; } // Mark the jitcode as having a bytecode map. code->setHasBytecodeMap(); } else { // Add a dumy jitcodeGlobalTable entry. JitcodeGlobalEntry::DummyEntry entry; entry.init(code, code->raw(), code->rawEnd()); // Add entry to the global table. JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); if (!globalTable->addEntry(entry, cx->runtime())) { // Memory may have been allocated for the entry. entry.destroy(); return false; } // Mark the jitcode as having a bytecode map. code->setHasBytecodeMap(); } ionScript->setMethod(code); ionScript->setSkipArgCheckEntryOffset(getSkipArgCheckEntryOffset()); // If SPS is enabled, mark IonScript as having been instrumented with SPS if (isProfilerInstrumentationEnabled()) ionScript->setHasProfilingInstrumentation(); script->setIonScript(cx->runtime(), ionScript); // Adopt fallback shared stubs from the compiler into the ion script. ionScript->adoptFallbackStubs(&stubSpace_); Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, invalidateEpilogueData_), ImmPtr(ionScript), ImmPtr((void*)-1)); for (size_t i = 0; i < ionScriptLabels_.length(); i++) { Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, ionScriptLabels_[i]), ImmPtr(ionScript), ImmPtr((void*)-1)); } #ifdef JS_TRACE_LOGGING bool TLFailed = false; TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime()); for (uint32_t i = 0; i < patchableTLEvents_.length(); i++) { // Create an event on the mainthread. TraceLoggerEvent event(logger, patchableTLEvents_[i].event); if (!event.hasPayload() || !ionScript->addTraceLoggerEvent(event)) { TLFailed = true; break; } Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLEvents_[i].offset), ImmPtr((void*) uintptr_t(event.payload()->textId())), ImmPtr((void*)0)); } if (!TLFailed && patchableTLScripts_.length() > 0) { MOZ_ASSERT(TraceLogTextIdEnabled(TraceLogger_Scripts)); TraceLoggerEvent event(logger, TraceLogger_Scripts, script); if (!event.hasPayload() || !ionScript->addTraceLoggerEvent(event)) TLFailed = true; if (!TLFailed) { uint32_t textId = event.payload()->textId(); for (uint32_t i = 0; i < patchableTLScripts_.length(); i++) { Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLScripts_[i]), ImmPtr((void*) uintptr_t(textId)), ImmPtr((void*)0)); } } } if (!TLFailed) { for (uint32_t i = 0; i < patchableTraceLoggers_.length(); i++) { Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTraceLoggers_[i]), ImmPtr(logger), ImmPtr(nullptr)); } } #endif // Patch shared stub IC loads using IC entries for (size_t i = 0; i < sharedStubs_.length(); i++) { CodeOffset label = sharedStubs_[i].label; IonICEntry& entry = ionScript->sharedStubList()[i]; entry = sharedStubs_[i].entry; Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, label), ImmPtr(&entry), ImmPtr((void*)-1)); MOZ_ASSERT(entry.hasStub()); MOZ_ASSERT(entry.firstStub()->isFallback()); entry.firstStub()->toFallbackStub()->fixupICEntry(&entry); } // for generating inline caches during the execution. if (runtimeData_.length()) ionScript->copyRuntimeData(&runtimeData_[0]); if (cacheList_.length()) ionScript->copyCacheEntries(&cacheList_[0], masm); JitSpew(JitSpew_Codegen, "Created IonScript %p (raw %p)", (void*) ionScript, (void*) code->raw()); ionScript->setInvalidationEpilogueDataOffset(invalidateEpilogueData_.offset()); ionScript->setOsrPc(gen->info().osrPc()); ionScript->setOsrEntryOffset(getOsrEntryOffset()); ionScript->setInvalidationEpilogueOffset(invalidate_.offset()); ionScript->setDeoptTable(deoptTable_); #if defined(JS_ION_PERF) if (PerfEnabled()) perfSpewer_.writeProfile(script, code, masm); #endif // for marking during GC. if (safepointIndices_.length()) ionScript->copySafepointIndices(&safepointIndices_[0], masm); if (safepoints_.size()) ionScript->copySafepoints(&safepoints_); // for reconvering from an Ion Frame. if (bailouts_.length()) ionScript->copyBailoutTable(&bailouts_[0]); if (osiIndices_.length()) ionScript->copyOsiIndices(&osiIndices_[0], masm); if (snapshots_.listSize()) ionScript->copySnapshots(&snapshots_); MOZ_ASSERT_IF(snapshots_.listSize(), recovers_.size()); if (recovers_.size()) ionScript->copyRecovers(&recovers_); if (graph.numConstants()) { const Value* vp = graph.constantPool(); ionScript->copyConstants(vp); for (size_t i = 0; i < graph.numConstants(); i++) { const Value& v = vp[i]; if (v.isObject() && IsInsideNursery(&v.toObject())) { cx->runtime()->gc.storeBuffer.putWholeCell(script); break; } } } if (patchableBackedges_.length() > 0) ionScript->copyPatchableBackedges(cx, code, patchableBackedges_.begin(), masm); // The correct state for prebarriers is unknown until the end of compilation, // since a GC can occur during code generation. All barriers are emitted // off-by-default, and are toggled on here if necessary. if (cx->zone()->needsIncrementalBarrier()) ionScript->toggleBarriers(true, DontReprotect); // Attach any generated script counts to the script. if (IonScriptCounts* counts = extractScriptCounts()) script->addIonCounts(counts); guardIonScript.release(); guardRecordedConstraints.release(); return true; } // An out-of-line path to convert a boxed int32 to either a float or double. class OutOfLineUnboxFloatingPoint : public OutOfLineCodeBase<CodeGenerator> { LUnboxFloatingPoint* unboxFloatingPoint_; public: explicit OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint* unboxFloatingPoint) : unboxFloatingPoint_(unboxFloatingPoint) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineUnboxFloatingPoint(this); } LUnboxFloatingPoint* unboxFloatingPoint() const { return unboxFloatingPoint_; } }; void CodeGenerator::visitUnboxFloatingPoint(LUnboxFloatingPoint* lir) { const ValueOperand box = ToValue(lir, LUnboxFloatingPoint::Input); const LDefinition* result = lir->output(); // Out-of-line path to convert int32 to double or bailout // if this instruction is fallible. OutOfLineUnboxFloatingPoint* ool = new(alloc()) OutOfLineUnboxFloatingPoint(lir); addOutOfLineCode(ool, lir->mir()); FloatRegister resultReg = ToFloatRegister(result); masm.branchTestDouble(Assembler::NotEqual, box, ool->entry()); masm.unboxDouble(box, resultReg); if (lir->type() == MIRType::Float32) masm.convertDoubleToFloat32(resultReg, resultReg); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool) { LUnboxFloatingPoint* ins = ool->unboxFloatingPoint(); const ValueOperand value = ToValue(ins, LUnboxFloatingPoint::Input); if (ins->mir()->fallible()) { Label bail; masm.branchTestInt32(Assembler::NotEqual, value, &bail); bailoutFrom(&bail, ins->snapshot()); } masm.int32ValueToFloatingPoint(value, ToFloatRegister(ins->output()), ins->type()); masm.jump(ool->rejoin()); } typedef JSObject* (*BindVarFn)(JSContext*, HandleObject); static const VMFunction BindVarInfo = FunctionInfo<BindVarFn>(jit::BindVar, "BindVar"); void CodeGenerator::visitCallBindVar(LCallBindVar* lir) { pushArg(ToRegister(lir->environmentChain())); callVM(BindVarInfo, lir); } typedef bool (*GetPropertyFn)(JSContext*, HandleValue, HandlePropertyName, MutableHandleValue); static const VMFunction GetPropertyInfo = FunctionInfo<GetPropertyFn>(GetProperty, "GetProperty"); void CodeGenerator::visitCallGetProperty(LCallGetProperty* lir) { pushArg(ImmGCPtr(lir->mir()->name())); pushArg(ToValue(lir, LCallGetProperty::Value)); callVM(GetPropertyInfo, lir); } typedef bool (*GetOrCallElementFn)(JSContext*, MutableHandleValue, HandleValue, MutableHandleValue); static const VMFunction GetElementInfo = FunctionInfo<GetOrCallElementFn>(js::GetElement, "GetElement"); static const VMFunction CallElementInfo = FunctionInfo<GetOrCallElementFn>(js::CallElement, "CallElement"); void CodeGenerator::visitCallGetElement(LCallGetElement* lir) { pushArg(ToValue(lir, LCallGetElement::RhsInput)); pushArg(ToValue(lir, LCallGetElement::LhsInput)); JSOp op = JSOp(*lir->mir()->resumePoint()->pc()); if (op == JSOP_GETELEM) { callVM(GetElementInfo, lir); } else { MOZ_ASSERT(op == JSOP_CALLELEM); callVM(CallElementInfo, lir); } } typedef bool (*SetObjectElementFn)(JSContext*, HandleObject, HandleValue, HandleValue, bool strict); static const VMFunction SetObjectElementInfo = FunctionInfo<SetObjectElementFn>(SetObjectElement, "SetObjectElement"); void CodeGenerator::visitCallSetElement(LCallSetElement* lir) { pushArg(Imm32(lir->mir()->strict())); pushArg(ToValue(lir, LCallSetElement::Value)); pushArg(ToValue(lir, LCallSetElement::Index)); pushArg(ToRegister(lir->getOperand(0))); callVM(SetObjectElementInfo, lir); } typedef bool (*InitElementArrayFn)(JSContext*, jsbytecode*, HandleObject, uint32_t, HandleValue); static const VMFunction InitElementArrayInfo = FunctionInfo<InitElementArrayFn>(js::InitElementArray, "InitElementArray"); void CodeGenerator::visitCallInitElementArray(LCallInitElementArray* lir) { pushArg(ToValue(lir, LCallInitElementArray::Value)); pushArg(Imm32(lir->mir()->index())); pushArg(ToRegister(lir->getOperand(0))); pushArg(ImmPtr(lir->mir()->resumePoint()->pc())); callVM(InitElementArrayInfo, lir); } void CodeGenerator::visitLoadFixedSlotV(LLoadFixedSlotV* ins) { const Register obj = ToRegister(ins->getOperand(0)); size_t slot = ins->mir()->slot(); ValueOperand result = GetValueOutput(ins); masm.loadValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), result); } void CodeGenerator::visitLoadFixedSlotT(LLoadFixedSlotT* ins) { const Register obj = ToRegister(ins->getOperand(0)); size_t slot = ins->mir()->slot(); AnyRegister result = ToAnyRegister(ins->getDef(0)); MIRType type = ins->mir()->type(); masm.loadUnboxedValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), type, result); } void CodeGenerator::visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox* ins) { const MLoadFixedSlotAndUnbox* mir = ins->mir(); MIRType type = mir->type(); const Register input = ToRegister(ins->getOperand(0)); AnyRegister result = ToAnyRegister(ins->output()); size_t slot = mir->slot(); Address address(input, NativeObject::getFixedSlotOffset(slot)); Label bail; if (type == MIRType::Double) { MOZ_ASSERT(result.isFloat()); masm.ensureDouble(address, result.fpu(), &bail); if (mir->fallible()) bailoutFrom(&bail, ins->snapshot()); return; } if (mir->fallible()) { switch (type) { case MIRType::Int32: masm.branchTestInt32(Assembler::NotEqual, address, &bail); break; case MIRType::Boolean: masm.branchTestBoolean(Assembler::NotEqual, address, &bail); break; default: MOZ_CRASH("Given MIRType cannot be unboxed."); } bailoutFrom(&bail, ins->snapshot()); } masm.loadUnboxedValue(address, type, result); } void CodeGenerator::visitStoreFixedSlotV(LStoreFixedSlotV* ins) { const Register obj = ToRegister(ins->getOperand(0)); size_t slot = ins->mir()->slot(); const ValueOperand value = ToValue(ins, LStoreFixedSlotV::Value); Address address(obj, NativeObject::getFixedSlotOffset(slot)); if (ins->mir()->needsBarrier()) emitPreBarrier(address); masm.storeValue(value, address); } void CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT* ins) { const Register obj = ToRegister(ins->getOperand(0)); size_t slot = ins->mir()->slot(); const LAllocation* value = ins->value(); MIRType valueType = ins->mir()->value()->type(); Address address(obj, NativeObject::getFixedSlotOffset(slot)); if (ins->mir()->needsBarrier()) emitPreBarrier(address); if (valueType == MIRType::ObjectOrNull) { Register nvalue = ToRegister(value); masm.storeObjectOrNull(nvalue, address); } else { ConstantOrRegister nvalue = value->isConstant() ? ConstantOrRegister(value->toConstant()->toJSValue()) : TypedOrValueRegister(valueType, ToAnyRegister(value)); masm.storeConstantOrRegister(nvalue, address); } } void CodeGenerator::visitGetNameCache(LGetNameCache* ins) { LiveRegisterSet liveRegs = ins->safepoint()->liveRegs(); Register envChain = ToRegister(ins->envObj()); TypedOrValueRegister output(GetValueOutput(ins)); bool isTypeOf = ins->mir()->accessKind() != MGetNameCache::NAME; NameIC cache(liveRegs, isTypeOf, envChain, ins->mir()->name(), output); cache.setProfilerLeavePC(ins->mir()->profilerLeavePc()); addCache(ins, allocateCache(cache)); } typedef bool (*NameICFn)(JSContext*, HandleScript, size_t, HandleObject, MutableHandleValue); const VMFunction NameIC::UpdateInfo = FunctionInfo<NameICFn>(NameIC::update, "NameIC::update"); void CodeGenerator::visitNameIC(OutOfLineUpdateCache* ool, DataPtr<NameIC>& ic) { LInstruction* lir = ool->lir(); saveLive(lir); pushArg(ic->environmentChainReg()); pushArg(Imm32(ool->getCacheIndex())); pushArg(ImmGCPtr(gen->info().script())); callVM(NameIC::UpdateInfo, lir); StoreValueTo(ic->outputReg()).generate(this); restoreLiveIgnore(lir, StoreValueTo(ic->outputReg()).clobbered()); masm.jump(ool->rejoin()); } void CodeGenerator::addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, Register objReg, const ConstantOrRegister& id, TypedOrValueRegister output, bool monitoredResult, bool allowDoubleResult, jsbytecode* profilerLeavePc) { GetPropertyIC cache(liveRegs, objReg, id, output, monitoredResult, allowDoubleResult); cache.setProfilerLeavePC(profilerLeavePc); addCache(ins, allocateCache(cache)); } void CodeGenerator::addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, Register objReg, Register temp, Register tempUnbox, FloatRegister tempDouble, FloatRegister tempF32, const ConstantOrRegister& id, const ConstantOrRegister& value, bool strict, bool needsTypeBarrier, bool guardHoles, jsbytecode* profilerLeavePc) { SetPropertyIC cache(liveRegs, objReg, temp, tempUnbox, tempDouble, tempF32, id, value, strict, needsTypeBarrier, guardHoles); cache.setProfilerLeavePC(profilerLeavePc); addCache(ins, allocateCache(cache)); } ConstantOrRegister CodeGenerator::toConstantOrRegister(LInstruction* lir, size_t n, MIRType type) { if (type == MIRType::Value) return TypedOrValueRegister(ToValue(lir, n)); const LAllocation* value = lir->getOperand(n); if (value->isConstant()) return ConstantOrRegister(value->toConstant()->toJSValue()); return TypedOrValueRegister(type, ToAnyRegister(value)); } void CodeGenerator::visitGetPropertyCacheV(LGetPropertyCacheV* ins) { LiveRegisterSet liveRegs = ins->safepoint()->liveRegs(); Register objReg = ToRegister(ins->getOperand(0)); ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheV::Id, ins->mir()->idval()->type()); bool monitoredResult = ins->mir()->monitoredResult(); TypedOrValueRegister output = TypedOrValueRegister(GetValueOutput(ins)); addGetPropertyCache(ins, liveRegs, objReg, id, output, monitoredResult, ins->mir()->allowDoubleResult(), ins->mir()->profilerLeavePc()); } void CodeGenerator::visitGetPropertyCacheT(LGetPropertyCacheT* ins) { LiveRegisterSet liveRegs = ins->safepoint()->liveRegs(); Register objReg = ToRegister(ins->getOperand(0)); ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheT::Id, ins->mir()->idval()->type()); bool monitoredResult = ins->mir()->monitoredResult(); TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->getDef(0))); addGetPropertyCache(ins, liveRegs, objReg, id, output, monitoredResult, ins->mir()->allowDoubleResult(), ins->mir()->profilerLeavePc()); } typedef bool (*GetPropertyICFn)(JSContext*, HandleScript, size_t, HandleObject, HandleValue, MutableHandleValue); const VMFunction GetPropertyIC::UpdateInfo = FunctionInfo<GetPropertyICFn>(GetPropertyIC::update, "GetPropertyIC::update"); void CodeGenerator::visitGetPropertyIC(OutOfLineUpdateCache* ool, DataPtr<GetPropertyIC>& ic) { LInstruction* lir = ool->lir(); if (ic->idempotent()) { size_t numLocs; CacheLocationList& cacheLocs = lir->mirRaw()->toGetPropertyCache()->location(); size_t locationBase; if (!addCacheLocations(cacheLocs, &numLocs, &locationBase)) return; ic->setLocationInfo(locationBase, numLocs); } saveLive(lir); pushArg(ic->id()); pushArg(ic->object()); pushArg(Imm32(ool->getCacheIndex())); pushArg(ImmGCPtr(gen->info().script())); callVM(GetPropertyIC::UpdateInfo, lir); StoreValueTo(ic->output()).generate(this); restoreLiveIgnore(lir, StoreValueTo(ic->output()).clobbered()); masm.jump(ool->rejoin()); } void CodeGenerator::visitBindNameCache(LBindNameCache* ins) { Register envChain = ToRegister(ins->environmentChain()); Register output = ToRegister(ins->output()); BindNameIC cache(envChain, ins->mir()->name(), output); cache.setProfilerLeavePC(ins->mir()->profilerLeavePc()); addCache(ins, allocateCache(cache)); } typedef JSObject* (*BindNameICFn)(JSContext*, HandleScript, size_t, HandleObject); const VMFunction BindNameIC::UpdateInfo = FunctionInfo<BindNameICFn>(BindNameIC::update, "BindNameIC::update"); void CodeGenerator::visitBindNameIC(OutOfLineUpdateCache* ool, DataPtr<BindNameIC>& ic) { LInstruction* lir = ool->lir(); saveLive(lir); pushArg(ic->environmentChainReg()); pushArg(Imm32(ool->getCacheIndex())); pushArg(ImmGCPtr(gen->info().script())); callVM(BindNameIC::UpdateInfo, lir); StoreRegisterTo(ic->outputReg()).generate(this); restoreLiveIgnore(lir, StoreRegisterTo(ic->outputReg()).clobbered()); masm.jump(ool->rejoin()); } typedef bool (*SetPropertyFn)(JSContext*, HandleObject, HandlePropertyName, const HandleValue, bool, jsbytecode*); static const VMFunction SetPropertyInfo = FunctionInfo<SetPropertyFn>(SetProperty, "SetProperty"); void CodeGenerator::visitCallSetProperty(LCallSetProperty* ins) { ConstantOrRegister value = TypedOrValueRegister(ToValue(ins, LCallSetProperty::Value)); const Register objReg = ToRegister(ins->getOperand(0)); pushArg(ImmPtr(ins->mir()->resumePoint()->pc())); pushArg(Imm32(ins->mir()->strict())); pushArg(value); pushArg(ImmGCPtr(ins->mir()->name())); pushArg(objReg); callVM(SetPropertyInfo, ins); } typedef bool (*DeletePropertyFn)(JSContext*, HandleValue, HandlePropertyName, bool*); static const VMFunction DeletePropertyStrictInfo = FunctionInfo<DeletePropertyFn>(DeletePropertyJit<true>, "DeletePropertyStrictJit"); static const VMFunction DeletePropertyNonStrictInfo = FunctionInfo<DeletePropertyFn>(DeletePropertyJit<false>, "DeletePropertyNonStrictJit"); void CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty* lir) { pushArg(ImmGCPtr(lir->mir()->name())); pushArg(ToValue(lir, LCallDeleteProperty::Value)); if (lir->mir()->strict()) callVM(DeletePropertyStrictInfo, lir); else callVM(DeletePropertyNonStrictInfo, lir); } typedef bool (*DeleteElementFn)(JSContext*, HandleValue, HandleValue, bool*); static const VMFunction DeleteElementStrictInfo = FunctionInfo<DeleteElementFn>(DeleteElementJit<true>, "DeleteElementStrictJit"); static const VMFunction DeleteElementNonStrictInfo = FunctionInfo<DeleteElementFn>(DeleteElementJit<false>, "DeleteElementNonStrictJit"); void CodeGenerator::visitCallDeleteElement(LCallDeleteElement* lir) { pushArg(ToValue(lir, LCallDeleteElement::Index)); pushArg(ToValue(lir, LCallDeleteElement::Value)); if (lir->mir()->strict()) callVM(DeleteElementStrictInfo, lir); else callVM(DeleteElementNonStrictInfo, lir); } void CodeGenerator::visitSetPropertyCache(LSetPropertyCache* ins) { LiveRegisterSet liveRegs = ins->safepoint()->liveRegs(); Register objReg = ToRegister(ins->getOperand(0)); Register temp = ToRegister(ins->temp()); Register tempUnbox = ToTempUnboxRegister(ins->tempToUnboxIndex()); FloatRegister tempDouble = ToTempFloatRegisterOrInvalid(ins->tempDouble()); FloatRegister tempF32 = ToTempFloatRegisterOrInvalid(ins->tempFloat32()); ConstantOrRegister id = toConstantOrRegister(ins, LSetPropertyCache::Id, ins->mir()->idval()->type()); ConstantOrRegister value = toConstantOrRegister(ins, LSetPropertyCache::Value, ins->mir()->value()->type()); addSetPropertyCache(ins, liveRegs, objReg, temp, tempUnbox, tempDouble, tempF32, id, value, ins->mir()->strict(), ins->mir()->needsTypeBarrier(), ins->mir()->guardHoles(), ins->mir()->profilerLeavePc()); } typedef bool (*SetPropertyICFn)(JSContext*, HandleScript, size_t, HandleObject, HandleValue, HandleValue); const VMFunction SetPropertyIC::UpdateInfo = FunctionInfo<SetPropertyICFn>(SetPropertyIC::update, "SetPropertyIC::update"); void CodeGenerator::visitSetPropertyIC(OutOfLineUpdateCache* ool, DataPtr<SetPropertyIC>& ic) { LInstruction* lir = ool->lir(); saveLive(lir); pushArg(ic->value()); pushArg(ic->id()); pushArg(ic->object()); pushArg(Imm32(ool->getCacheIndex())); pushArg(ImmGCPtr(gen->info().script())); callVM(SetPropertyIC::UpdateInfo, lir); restoreLive(lir); masm.jump(ool->rejoin()); } typedef bool (*ThrowFn)(JSContext*, HandleValue); static const VMFunction ThrowInfoCodeGen = FunctionInfo<ThrowFn>(js::Throw, "Throw"); void CodeGenerator::visitThrow(LThrow* lir) { pushArg(ToValue(lir, LThrow::Value)); callVM(ThrowInfoCodeGen, lir); } typedef bool (*BitNotFn)(JSContext*, HandleValue, int* p); static const VMFunction BitNotInfo = FunctionInfo<BitNotFn>(BitNot, "BitNot"); void CodeGenerator::visitBitNotV(LBitNotV* lir) { pushArg(ToValue(lir, LBitNotV::Input)); callVM(BitNotInfo, lir); } typedef bool (*BitopFn)(JSContext*, HandleValue, HandleValue, int* p); static const VMFunction BitAndInfo = FunctionInfo<BitopFn>(BitAnd, "BitAnd"); static const VMFunction BitOrInfo = FunctionInfo<BitopFn>(BitOr, "BitOr"); static const VMFunction BitXorInfo = FunctionInfo<BitopFn>(BitXor, "BitXor"); static const VMFunction BitLhsInfo = FunctionInfo<BitopFn>(BitLsh, "BitLsh"); static const VMFunction BitRhsInfo = FunctionInfo<BitopFn>(BitRsh, "BitRsh"); void CodeGenerator::visitBitOpV(LBitOpV* lir) { pushArg(ToValue(lir, LBitOpV::RhsInput)); pushArg(ToValue(lir, LBitOpV::LhsInput)); switch (lir->jsop()) { case JSOP_BITAND: callVM(BitAndInfo, lir); break; case JSOP_BITOR: callVM(BitOrInfo, lir); break; case JSOP_BITXOR: callVM(BitXorInfo, lir); break; case JSOP_LSH: callVM(BitLhsInfo, lir); break; case JSOP_RSH: callVM(BitRhsInfo, lir); break; default: MOZ_CRASH("unexpected bitop"); } } class OutOfLineTypeOfV : public OutOfLineCodeBase<CodeGenerator> { LTypeOfV* ins_; public: explicit OutOfLineTypeOfV(LTypeOfV* ins) : ins_(ins) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineTypeOfV(this); } LTypeOfV* ins() const { return ins_; } }; void CodeGenerator::visitTypeOfV(LTypeOfV* lir) { const ValueOperand value = ToValue(lir, LTypeOfV::Input); Register output = ToRegister(lir->output()); Register tag = masm.splitTagForTest(value); const JSAtomState& names = GetJitContext()->runtime->names(); Label done; MDefinition* input = lir->mir()->input(); bool testObject = input->mightBeType(MIRType::Object); bool testNumber = input->mightBeType(MIRType::Int32) || input->mightBeType(MIRType::Double); bool testBoolean = input->mightBeType(MIRType::Boolean); bool testUndefined = input->mightBeType(MIRType::Undefined); bool testNull = input->mightBeType(MIRType::Null); bool testString = input->mightBeType(MIRType::String); bool testSymbol = input->mightBeType(MIRType::Symbol); unsigned numTests = unsigned(testObject) + unsigned(testNumber) + unsigned(testBoolean) + unsigned(testUndefined) + unsigned(testNull) + unsigned(testString) + unsigned(testSymbol); MOZ_ASSERT_IF(!input->emptyResultTypeSet(), numTests > 0); OutOfLineTypeOfV* ool = nullptr; if (testObject) { if (lir->mir()->inputMaybeCallableOrEmulatesUndefined()) { // The input may be a callable object (result is "function") or may // emulate undefined (result is "undefined"). Use an OOL path. ool = new(alloc()) OutOfLineTypeOfV(lir); addOutOfLineCode(ool, lir->mir()); if (numTests > 1) masm.branchTestObject(Assembler::Equal, tag, ool->entry()); else masm.jump(ool->entry()); } else { // Input is not callable and does not emulate undefined, so if // it's an object the result is always "object". Label notObject; if (numTests > 1) masm.branchTestObject(Assembler::NotEqual, tag, ¬Object); masm.movePtr(ImmGCPtr(names.object), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Object); } numTests--; } if (testNumber) { Label notNumber; if (numTests > 1) masm.branchTestNumber(Assembler::NotEqual, tag, ¬Number); masm.movePtr(ImmGCPtr(names.number), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Number); numTests--; } if (testUndefined) { Label notUndefined; if (numTests > 1) masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined); masm.movePtr(ImmGCPtr(names.undefined), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Undefined); numTests--; } if (testNull) { Label notNull; if (numTests > 1) masm.branchTestNull(Assembler::NotEqual, tag, ¬Null); masm.movePtr(ImmGCPtr(names.object), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Null); numTests--; } if (testBoolean) { Label notBoolean; if (numTests > 1) masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean); masm.movePtr(ImmGCPtr(names.boolean), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Boolean); numTests--; } if (testString) { Label notString; if (numTests > 1) masm.branchTestString(Assembler::NotEqual, tag, ¬String); masm.movePtr(ImmGCPtr(names.string), output); if (numTests > 1) masm.jump(&done); masm.bind(¬String); numTests--; } if (testSymbol) { Label notSymbol; if (numTests > 1) masm.branchTestSymbol(Assembler::NotEqual, tag, ¬Symbol); masm.movePtr(ImmGCPtr(names.symbol), output); if (numTests > 1) masm.jump(&done); masm.bind(¬Symbol); numTests--; } MOZ_ASSERT(numTests == 0); masm.bind(&done); if (ool) masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool) { LTypeOfV* ins = ool->ins(); ValueOperand input = ToValue(ins, LTypeOfV::Input); Register temp = ToTempUnboxRegister(ins->tempToUnbox()); Register output = ToRegister(ins->output()); Register obj = masm.extractObject(input, temp); saveVolatile(output); masm.setupUnalignedABICall(output); masm.passABIArg(obj); masm.movePtr(ImmPtr(GetJitContext()->runtime), output); masm.passABIArg(output); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::TypeOfObjectOperation)); masm.storeCallPointerResult(output); restoreVolatile(output); masm.jump(ool->rejoin()); } typedef JSObject* (*ToAsyncFn)(JSContext*, HandleFunction); static const VMFunction ToAsyncInfo = FunctionInfo<ToAsyncFn>(js::WrapAsyncFunction, "ToAsync"); void CodeGenerator::visitToAsync(LToAsync* lir) { pushArg(ToRegister(lir->unwrapped())); callVM(ToAsyncInfo, lir); } typedef bool (*ToIdFn)(JSContext*, HandleScript, jsbytecode*, HandleValue, MutableHandleValue); static const VMFunction ToIdInfo = FunctionInfo<ToIdFn>(ToIdOperation, "ToIdOperation"); void CodeGenerator::visitToIdV(LToIdV* lir) { Label notInt32; FloatRegister temp = ToFloatRegister(lir->tempFloat()); const ValueOperand out = ToOutValue(lir); ValueOperand input = ToValue(lir, LToIdV::Input); OutOfLineCode* ool = oolCallVM(ToIdInfo, lir, ArgList(ImmGCPtr(current->mir()->info().script()), ImmPtr(lir->mir()->resumePoint()->pc()), ToValue(lir, LToIdV::Input)), StoreValueTo(out)); Register tag = masm.splitTagForTest(input); masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32); masm.moveValue(input, out); masm.jump(ool->rejoin()); masm.bind(¬Int32); masm.branchTestDouble(Assembler::NotEqual, tag, ool->entry()); masm.unboxDouble(input, temp); masm.convertDoubleToInt32(temp, out.scratchReg(), ool->entry(), true); masm.tagValue(JSVAL_TYPE_INT32, out.scratchReg(), out); masm.bind(ool->rejoin()); } template<typename T> void CodeGenerator::emitLoadElementT(LLoadElementT* lir, const T& source) { if (LIRGenerator::allowTypedElementHoleCheck()) { if (lir->mir()->needsHoleCheck()) { Label bail; masm.branchTestMagic(Assembler::Equal, source, &bail); bailoutFrom(&bail, lir->snapshot()); } } else { MOZ_ASSERT(!lir->mir()->needsHoleCheck()); } AnyRegister output = ToAnyRegister(lir->output()); if (lir->mir()->loadDoubles()) masm.loadDouble(source, output.fpu()); else masm.loadUnboxedValue(source, lir->mir()->type(), output); } void CodeGenerator::visitLoadElementT(LLoadElementT* lir) { Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); if (index->isConstant()) { int32_t offset = ToInt32(index) * sizeof(js::Value) + lir->mir()->offsetAdjustment(); emitLoadElementT(lir, Address(elements, offset)); } else { emitLoadElementT(lir, BaseIndex(elements, ToRegister(index), TimesEight, lir->mir()->offsetAdjustment())); } } void CodeGenerator::visitLoadElementV(LLoadElementV* load) { Register elements = ToRegister(load->elements()); const ValueOperand out = ToOutValue(load); if (load->index()->isConstant()) { NativeObject::elementsSizeMustNotOverflow(); int32_t offset = ToInt32(load->index()) * sizeof(Value) + load->mir()->offsetAdjustment(); masm.loadValue(Address(elements, offset), out); } else { masm.loadValue(BaseObjectElementIndex(elements, ToRegister(load->index()), load->mir()->offsetAdjustment()), out); } if (load->mir()->needsHoleCheck()) { Label testMagic; masm.branchTestMagic(Assembler::Equal, out, &testMagic); bailoutFrom(&testMagic, load->snapshot()); } } void CodeGenerator::visitLoadElementHole(LLoadElementHole* lir) { Register elements = ToRegister(lir->elements()); Register initLength = ToRegister(lir->initLength()); const ValueOperand out = ToOutValue(lir); const MLoadElementHole* mir = lir->mir(); // If the index is out of bounds, load |undefined|. Otherwise, load the // value. Label undefined, done; if (lir->index()->isConstant()) masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(ToInt32(lir->index())), &undefined); else masm.branch32(Assembler::BelowOrEqual, initLength, ToRegister(lir->index()), &undefined); if (mir->unboxedType() != JSVAL_TYPE_MAGIC) { size_t width = UnboxedTypeSize(mir->unboxedType()); if (lir->index()->isConstant()) { Address addr(elements, ToInt32(lir->index()) * width); masm.loadUnboxedProperty(addr, mir->unboxedType(), out); } else { BaseIndex addr(elements, ToRegister(lir->index()), ScaleFromElemWidth(width)); masm.loadUnboxedProperty(addr, mir->unboxedType(), out); } } else { if (lir->index()->isConstant()) { NativeObject::elementsSizeMustNotOverflow(); masm.loadValue(Address(elements, ToInt32(lir->index()) * sizeof(Value)), out); } else { masm.loadValue(BaseObjectElementIndex(elements, ToRegister(lir->index())), out); } } // If a hole check is needed, and the value wasn't a hole, we're done. // Otherwise, we'll load undefined. if (lir->mir()->needsHoleCheck()) masm.branchTestMagic(Assembler::NotEqual, out, &done); else masm.jump(&done); masm.bind(&undefined); if (mir->needsNegativeIntCheck()) { if (lir->index()->isConstant()) { if (ToInt32(lir->index()) < 0) bailout(lir->snapshot()); } else { Label negative; masm.branch32(Assembler::LessThan, ToRegister(lir->index()), Imm32(0), &negative); bailoutFrom(&negative, lir->snapshot()); } } masm.moveValue(UndefinedValue(), out); masm.bind(&done); } void CodeGenerator::visitLoadUnboxedPointerV(LLoadUnboxedPointerV* lir) { Register elements = ToRegister(lir->elements()); const ValueOperand out = ToOutValue(lir); if (lir->index()->isConstant()) { int32_t offset = ToInt32(lir->index()) * sizeof(uintptr_t) + lir->mir()->offsetAdjustment(); masm.loadPtr(Address(elements, offset), out.scratchReg()); } else { masm.loadPtr(BaseIndex(elements, ToRegister(lir->index()), ScalePointer, lir->mir()->offsetAdjustment()), out.scratchReg()); } Label notNull, done; masm.branchPtr(Assembler::NotEqual, out.scratchReg(), ImmWord(0), ¬Null); masm.moveValue(NullValue(), out); masm.jump(&done); masm.bind(¬Null); masm.tagValue(JSVAL_TYPE_OBJECT, out.scratchReg(), out); masm.bind(&done); } void CodeGenerator::visitLoadUnboxedPointerT(LLoadUnboxedPointerT* lir) { Register elements = ToRegister(lir->elements()); const LAllocation* index = lir->index(); Register out = ToRegister(lir->output()); bool bailOnNull; int32_t offsetAdjustment; if (lir->mir()->isLoadUnboxedObjectOrNull()) { bailOnNull = lir->mir()->toLoadUnboxedObjectOrNull()->nullBehavior() == MLoadUnboxedObjectOrNull::BailOnNull; offsetAdjustment = lir->mir()->toLoadUnboxedObjectOrNull()->offsetAdjustment(); } else if (lir->mir()->isLoadUnboxedString()) { bailOnNull = false; offsetAdjustment = lir->mir()->toLoadUnboxedString()->offsetAdjustment(); } else { MOZ_CRASH(); } if (index->isConstant()) { Address source(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment); masm.loadPtr(source, out); } else { BaseIndex source(elements, ToRegister(index), ScalePointer, offsetAdjustment); masm.loadPtr(source, out); } if (bailOnNull) { Label bail; masm.branchTestPtr(Assembler::Zero, out, out, &bail); bailoutFrom(&bail, lir->snapshot()); } } void CodeGenerator::visitUnboxObjectOrNull(LUnboxObjectOrNull* lir) { Register obj = ToRegister(lir->input()); if (lir->mir()->fallible()) { Label bail; masm.branchTestPtr(Assembler::Zero, obj, obj, &bail); bailoutFrom(&bail, lir->snapshot()); } } void CodeGenerator::visitLoadUnboxedScalar(LLoadUnboxedScalar* lir) { Register elements = ToRegister(lir->elements()); Register temp = lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp()); AnyRegister out = ToAnyRegister(lir->output()); const MLoadUnboxedScalar* mir = lir->mir(); Scalar::Type readType = mir->readType(); unsigned numElems = mir->numElems(); int width = Scalar::byteSize(mir->storageType()); bool canonicalizeDouble = mir->canonicalizeDoubles(); Label fail; if (lir->index()->isConstant()) { Address source(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment()); masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems); } else { BaseIndex source(elements, ToRegister(lir->index()), ScaleFromElemWidth(width), mir->offsetAdjustment()); masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems); } if (fail.used()) bailoutFrom(&fail, lir->snapshot()); } void CodeGenerator::visitLoadTypedArrayElementHole(LLoadTypedArrayElementHole* lir) { Register object = ToRegister(lir->object()); const ValueOperand out = ToOutValue(lir); // Load the length. Register scratch = out.scratchReg(); RegisterOrInt32Constant key = ToRegisterOrInt32Constant(lir->index()); masm.unboxInt32(Address(object, TypedArrayObject::lengthOffset()), scratch); // Load undefined unless length > key. Label inbounds, done; masm.branch32(Assembler::Above, scratch, key, &inbounds); masm.moveValue(UndefinedValue(), out); masm.jump(&done); // Load the elements vector. masm.bind(&inbounds); masm.loadPtr(Address(object, TypedArrayObject::dataOffset()), scratch); Scalar::Type arrayType = lir->mir()->arrayType(); int width = Scalar::byteSize(arrayType); Label fail; if (key.isConstant()) { Address source(scratch, key.constant() * width); masm.loadFromTypedArray(arrayType, source, out, lir->mir()->allowDouble(), out.scratchReg(), &fail); } else { BaseIndex source(scratch, key.reg(), ScaleFromElemWidth(width)); masm.loadFromTypedArray(arrayType, source, out, lir->mir()->allowDouble(), out.scratchReg(), &fail); } if (fail.used()) bailoutFrom(&fail, lir->snapshot()); masm.bind(&done); } template <typename T> static inline void StoreToTypedArray(MacroAssembler& masm, Scalar::Type writeType, const LAllocation* value, const T& dest, unsigned numElems = 0) { if (Scalar::isSimdType(writeType) || writeType == Scalar::Float32 || writeType == Scalar::Float64) { masm.storeToTypedFloatArray(writeType, ToFloatRegister(value), dest, numElems); } else { if (value->isConstant()) masm.storeToTypedIntArray(writeType, Imm32(ToInt32(value)), dest); else masm.storeToTypedIntArray(writeType, ToRegister(value), dest); } } void CodeGenerator::visitStoreUnboxedScalar(LStoreUnboxedScalar* lir) { Register elements = ToRegister(lir->elements()); const LAllocation* value = lir->value(); const MStoreUnboxedScalar* mir = lir->mir(); Scalar::Type writeType = mir->writeType(); unsigned numElems = mir->numElems(); int width = Scalar::byteSize(mir->storageType()); if (lir->index()->isConstant()) { Address dest(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment()); StoreToTypedArray(masm, writeType, value, dest, numElems); } else { BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width), mir->offsetAdjustment()); StoreToTypedArray(masm, writeType, value, dest, numElems); } } void CodeGenerator::visitStoreTypedArrayElementHole(LStoreTypedArrayElementHole* lir) { Register elements = ToRegister(lir->elements()); const LAllocation* value = lir->value(); Scalar::Type arrayType = lir->mir()->arrayType(); int width = Scalar::byteSize(arrayType); const LAllocation* index = lir->index(); const LAllocation* length = lir->length(); bool guardLength = true; if (index->isConstant() && length->isConstant()) { uint32_t idx = ToInt32(index); uint32_t len = ToInt32(length); if (idx >= len) return; guardLength = false; } Label skip; if (index->isConstant()) { uint32_t idx = ToInt32(index); if (guardLength) { if (length->isRegister()) masm.branch32(Assembler::BelowOrEqual, ToRegister(length), Imm32(idx), &skip); else masm.branch32(Assembler::BelowOrEqual, ToAddress(length), Imm32(idx), &skip); } Address dest(elements, idx * width); StoreToTypedArray(masm, arrayType, value, dest); } else { Register idxReg = ToRegister(index); MOZ_ASSERT(guardLength); if (length->isConstant()) masm.branch32(Assembler::AboveOrEqual, idxReg, Imm32(ToInt32(length)), &skip); else if (length->isRegister()) masm.branch32(Assembler::BelowOrEqual, ToRegister(length), idxReg, &skip); else masm.branch32(Assembler::BelowOrEqual, ToAddress(length), idxReg, &skip); BaseIndex dest(elements, ToRegister(index), ScaleFromElemWidth(width)); StoreToTypedArray(masm, arrayType, value, dest); } if (guardLength) masm.bind(&skip); } void CodeGenerator::visitAtomicIsLockFree(LAtomicIsLockFree* lir) { Register value = ToRegister(lir->value()); Register output = ToRegister(lir->output()); // Keep this in sync with isLockfree() in jit/AtomicOperations.h. MOZ_ASSERT(AtomicOperations::isLockfree(1)); // Implementation artifact MOZ_ASSERT(AtomicOperations::isLockfree(2)); // Implementation artifact MOZ_ASSERT(AtomicOperations::isLockfree(4)); // Spec requirement MOZ_ASSERT(!AtomicOperations::isLockfree(8)); // Implementation invariant, for now Label Ldone, Lfailed; masm.move32(Imm32(1), output); masm.branch32(Assembler::Equal, value, Imm32(4), &Ldone); masm.branch32(Assembler::Equal, value, Imm32(2), &Ldone); masm.branch32(Assembler::Equal, value, Imm32(1), &Ldone); masm.move32(Imm32(0), output); masm.bind(&Ldone); } void CodeGenerator::visitGuardSharedTypedArray(LGuardSharedTypedArray* guard) { Register obj = ToRegister(guard->input()); Register tmp = ToRegister(guard->tempInt()); // The shared-memory flag is a bit in the ObjectElements header // that is set if the TypedArray is mapping a SharedArrayBuffer. // The flag is set at construction and does not change subsequently. masm.loadPtr(Address(obj, TypedArrayObject::offsetOfElements()), tmp); masm.load32(Address(tmp, ObjectElements::offsetOfFlags()), tmp); bailoutTest32(Assembler::Zero, tmp, Imm32(ObjectElements::SHARED_MEMORY), guard->snapshot()); } void CodeGenerator::visitClampIToUint8(LClampIToUint8* lir) { Register output = ToRegister(lir->output()); MOZ_ASSERT(output == ToRegister(lir->input())); masm.clampIntToUint8(output); } void CodeGenerator::visitClampDToUint8(LClampDToUint8* lir) { FloatRegister input = ToFloatRegister(lir->input()); Register output = ToRegister(lir->output()); masm.clampDoubleToUint8(input, output); } void CodeGenerator::visitClampVToUint8(LClampVToUint8* lir) { ValueOperand operand = ToValue(lir, LClampVToUint8::Input); FloatRegister tempFloat = ToFloatRegister(lir->tempFloat()); Register output = ToRegister(lir->output()); MDefinition* input = lir->mir()->input(); Label* stringEntry; Label* stringRejoin; if (input->mightBeType(MIRType::String)) { OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(output), StoreFloatRegisterTo(tempFloat)); stringEntry = oolString->entry(); stringRejoin = oolString->rejoin(); } else { stringEntry = nullptr; stringRejoin = nullptr; } Label fails; masm.clampValueToUint8(operand, input, stringEntry, stringRejoin, output, tempFloat, output, &fails); bailoutFrom(&fails, lir->snapshot()); } typedef bool (*OperatorInFn)(JSContext*, HandleValue, HandleObject, bool*); static const VMFunction OperatorInInfo = FunctionInfo<OperatorInFn>(OperatorIn, "OperatorIn"); void CodeGenerator::visitIn(LIn* ins) { pushArg(ToRegister(ins->rhs())); pushArg(ToValue(ins, LIn::LHS)); callVM(OperatorInInfo, ins); } typedef bool (*OperatorInIFn)(JSContext*, uint32_t, HandleObject, bool*); static const VMFunction OperatorInIInfo = FunctionInfo<OperatorInIFn>(OperatorInI, "OperatorInI"); void CodeGenerator::visitInArray(LInArray* lir) { const MInArray* mir = lir->mir(); Register elements = ToRegister(lir->elements()); Register initLength = ToRegister(lir->initLength()); Register output = ToRegister(lir->output()); // When the array is not packed we need to do a hole check in addition to the bounds check. Label falseBranch, done, trueBranch; OutOfLineCode* ool = nullptr; Label* failedInitLength = &falseBranch; if (lir->index()->isConstant()) { int32_t index = ToInt32(lir->index()); MOZ_ASSERT_IF(index < 0, mir->needsNegativeIntCheck()); if (mir->needsNegativeIntCheck()) { ool = oolCallVM(OperatorInIInfo, lir, ArgList(Imm32(index), ToRegister(lir->object())), StoreRegisterTo(output)); failedInitLength = ool->entry(); } masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(index), failedInitLength); if (mir->needsHoleCheck() && mir->unboxedType() == JSVAL_TYPE_MAGIC) { NativeObject::elementsSizeMustNotOverflow(); Address address = Address(elements, index * sizeof(Value)); masm.branchTestMagic(Assembler::Equal, address, &falseBranch); } } else { Label negativeIntCheck; Register index = ToRegister(lir->index()); if (mir->needsNegativeIntCheck()) failedInitLength = &negativeIntCheck; masm.branch32(Assembler::BelowOrEqual, initLength, index, failedInitLength); if (mir->needsHoleCheck() && mir->unboxedType() == JSVAL_TYPE_MAGIC) { BaseIndex address = BaseIndex(elements, ToRegister(lir->index()), TimesEight); masm.branchTestMagic(Assembler::Equal, address, &falseBranch); } masm.jump(&trueBranch); if (mir->needsNegativeIntCheck()) { masm.bind(&negativeIntCheck); ool = oolCallVM(OperatorInIInfo, lir, ArgList(index, ToRegister(lir->object())), StoreRegisterTo(output)); masm.branch32(Assembler::LessThan, index, Imm32(0), ool->entry()); masm.jump(&falseBranch); } } masm.bind(&trueBranch); masm.move32(Imm32(1), output); masm.jump(&done); masm.bind(&falseBranch); masm.move32(Imm32(0), output); masm.bind(&done); if (ool) masm.bind(ool->rejoin()); } void CodeGenerator::visitInstanceOfO(LInstanceOfO* ins) { emitInstanceOf(ins, ins->mir()->prototypeObject()); } void CodeGenerator::visitInstanceOfV(LInstanceOfV* ins) { emitInstanceOf(ins, ins->mir()->prototypeObject()); } // Wrap IsDelegateOfObject, which takes a JSObject*, not a HandleObject static bool IsDelegateObject(JSContext* cx, HandleObject protoObj, HandleObject obj, bool* res) { return IsDelegateOfObject(cx, protoObj, obj, res); } typedef bool (*IsDelegateObjectFn)(JSContext*, HandleObject, HandleObject, bool*); static const VMFunction IsDelegateObjectInfo = FunctionInfo<IsDelegateObjectFn>(IsDelegateObject, "IsDelegateObject"); void CodeGenerator::emitInstanceOf(LInstruction* ins, JSObject* prototypeObject) { // This path implements fun_hasInstance when the function's prototype is // known to be prototypeObject. Label done; Register output = ToRegister(ins->getDef(0)); // If the lhs is a primitive, the result is false. Register objReg; if (ins->isInstanceOfV()) { Label isObject; ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS); masm.branchTestObject(Assembler::Equal, lhsValue, &isObject); masm.mov(ImmWord(0), output); masm.jump(&done); masm.bind(&isObject); objReg = masm.extractObject(lhsValue, output); } else { objReg = ToRegister(ins->toInstanceOfO()->lhs()); } // Crawl the lhs's prototype chain in a loop to search for prototypeObject. // This follows the main loop of js::IsDelegate, though additionally breaks // out of the loop on Proxy::LazyProto. // Load the lhs's prototype. masm.loadObjProto(objReg, output); Label testLazy; { Label loopPrototypeChain; masm.bind(&loopPrototypeChain); // Test for the target prototype object. Label notPrototypeObject; masm.branchPtr(Assembler::NotEqual, output, ImmGCPtr(prototypeObject), ¬PrototypeObject); masm.mov(ImmWord(1), output); masm.jump(&done); masm.bind(¬PrototypeObject); MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1); // Test for nullptr or Proxy::LazyProto masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), &testLazy); // Load the current object's prototype. masm.loadObjProto(output, output); masm.jump(&loopPrototypeChain); } // Make a VM call if an object with a lazy proto was found on the prototype // chain. This currently occurs only for cross compartment wrappers, which // we do not expect to be compared with non-wrapper functions from this // compartment. Otherwise, we stopped on a nullptr prototype and the output // register is already correct. OutOfLineCode* ool = oolCallVM(IsDelegateObjectInfo, ins, ArgList(ImmGCPtr(prototypeObject), objReg), StoreRegisterTo(output)); // Regenerate the original lhs object for the VM call. Label regenerate, *lazyEntry; if (objReg != output) { lazyEntry = ool->entry(); } else { masm.bind(®enerate); lazyEntry = ®enerate; if (ins->isInstanceOfV()) { ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS); objReg = masm.extractObject(lhsValue, output); } else { objReg = ToRegister(ins->toInstanceOfO()->lhs()); } MOZ_ASSERT(objReg == output); masm.jump(ool->entry()); } masm.bind(&testLazy); masm.branchPtr(Assembler::Equal, output, ImmWord(1), lazyEntry); masm.bind(&done); masm.bind(ool->rejoin()); } typedef bool (*HasInstanceFn)(JSContext*, HandleObject, HandleValue, bool*); static const VMFunction HasInstanceInfo = FunctionInfo<HasInstanceFn>(js::HasInstance, "HasInstance"); void CodeGenerator::visitCallInstanceOf(LCallInstanceOf* ins) { ValueOperand lhs = ToValue(ins, LCallInstanceOf::LHS); Register rhs = ToRegister(ins->rhs()); MOZ_ASSERT(ToRegister(ins->output()) == ReturnReg); pushArg(lhs); pushArg(rhs); callVM(HasInstanceInfo, ins); } void CodeGenerator::visitGetDOMProperty(LGetDOMProperty* ins) { const Register JSContextReg = ToRegister(ins->getJSContextReg()); const Register ObjectReg = ToRegister(ins->getObjectReg()); const Register PrivateReg = ToRegister(ins->getPrivReg()); const Register ValueReg = ToRegister(ins->getValueReg()); Label haveValue; if (ins->mir()->valueMayBeInSlot()) { size_t slot = ins->mir()->domMemberSlotIndex(); // It's a bit annoying to redo these slot calculations, which duplcate // LSlots and a few other things like that, but I'm not sure there's a // way to reuse those here. if (slot < NativeObject::MAX_FIXED_SLOTS) { masm.loadValue(Address(ObjectReg, NativeObject::getFixedSlotOffset(slot)), JSReturnOperand); } else { // It's a dynamic slot. slot -= NativeObject::MAX_FIXED_SLOTS; // Use PrivateReg as a scratch register for the slots pointer. masm.loadPtr(Address(ObjectReg, NativeObject::offsetOfSlots()), PrivateReg); masm.loadValue(Address(PrivateReg, slot*sizeof(js::Value)), JSReturnOperand); } masm.branchTestUndefined(Assembler::NotEqual, JSReturnOperand, &haveValue); } DebugOnly<uint32_t> initialStack = masm.framePushed(); masm.checkStackAlignment(); // Make space for the outparam. Pre-initialize it to UndefinedValue so we // can trace it at GC time. masm.Push(UndefinedValue()); // We pass the pointer to our out param as an instance of // JSJitGetterCallArgs, since on the binary level it's the same thing. JS_STATIC_ASSERT(sizeof(JSJitGetterCallArgs) == sizeof(Value*)); masm.moveStackPtrTo(ValueReg); masm.Push(ObjectReg); LoadDOMPrivate(masm, ObjectReg, PrivateReg); // Rooting will happen at GC time. masm.moveStackPtrTo(ObjectReg); uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg); masm.enterFakeExitFrame(IonDOMExitFrameLayoutGetterToken); markSafepointAt(safepointOffset, ins); masm.setupUnalignedABICall(JSContextReg); masm.loadJSContext(JSContextReg); masm.passABIArg(JSContextReg); masm.passABIArg(ObjectReg); masm.passABIArg(PrivateReg); masm.passABIArg(ValueReg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun())); if (ins->mir()->isInfallible()) { masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()), JSReturnOperand); } else { masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()), JSReturnOperand); } masm.adjustStack(IonDOMExitFrameLayout::Size()); masm.bind(&haveValue); MOZ_ASSERT(masm.framePushed() == initialStack); } void CodeGenerator::visitGetDOMMemberV(LGetDOMMemberV* ins) { // It's simpler to duplicate visitLoadFixedSlotV here than it is to try to // use an LLoadFixedSlotV or some subclass of it for this case: that would // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then // we'd have to duplicate a bunch of stuff we now get for free from // MGetDOMProperty. Register object = ToRegister(ins->object()); size_t slot = ins->mir()->domMemberSlotIndex(); ValueOperand result = GetValueOutput(ins); masm.loadValue(Address(object, NativeObject::getFixedSlotOffset(slot)), result); } void CodeGenerator::visitGetDOMMemberT(LGetDOMMemberT* ins) { // It's simpler to duplicate visitLoadFixedSlotT here than it is to try to // use an LLoadFixedSlotT or some subclass of it for this case: that would // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then // we'd have to duplicate a bunch of stuff we now get for free from // MGetDOMProperty. Register object = ToRegister(ins->object()); size_t slot = ins->mir()->domMemberSlotIndex(); AnyRegister result = ToAnyRegister(ins->getDef(0)); MIRType type = ins->mir()->type(); masm.loadUnboxedValue(Address(object, NativeObject::getFixedSlotOffset(slot)), type, result); } void CodeGenerator::visitSetDOMProperty(LSetDOMProperty* ins) { const Register JSContextReg = ToRegister(ins->getJSContextReg()); const Register ObjectReg = ToRegister(ins->getObjectReg()); const Register PrivateReg = ToRegister(ins->getPrivReg()); const Register ValueReg = ToRegister(ins->getValueReg()); DebugOnly<uint32_t> initialStack = masm.framePushed(); masm.checkStackAlignment(); // Push the argument. Rooting will happen at GC time. ValueOperand argVal = ToValue(ins, LSetDOMProperty::Value); masm.Push(argVal); // We pass the pointer to our out param as an instance of // JSJitGetterCallArgs, since on the binary level it's the same thing. JS_STATIC_ASSERT(sizeof(JSJitSetterCallArgs) == sizeof(Value*)); masm.moveStackPtrTo(ValueReg); masm.Push(ObjectReg); LoadDOMPrivate(masm, ObjectReg, PrivateReg); // Rooting will happen at GC time. masm.moveStackPtrTo(ObjectReg); uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg); masm.enterFakeExitFrame(IonDOMExitFrameLayoutSetterToken); markSafepointAt(safepointOffset, ins); masm.setupUnalignedABICall(JSContextReg); masm.loadJSContext(JSContextReg); masm.passABIArg(JSContextReg); masm.passABIArg(ObjectReg); masm.passABIArg(PrivateReg); masm.passABIArg(ValueReg); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun())); masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); masm.adjustStack(IonDOMExitFrameLayout::Size()); MOZ_ASSERT(masm.framePushed() == initialStack); } class OutOfLineIsCallable : public OutOfLineCodeBase<CodeGenerator> { LIsCallable* ins_; public: explicit OutOfLineIsCallable(LIsCallable* ins) : ins_(ins) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineIsCallable(this); } LIsCallable* ins() const { return ins_; } }; void CodeGenerator::visitIsCallable(LIsCallable* ins) { Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); OutOfLineIsCallable* ool = new(alloc()) OutOfLineIsCallable(ins); addOutOfLineCode(ool, ins->mir()); Label notFunction, hasCOps, done; masm.loadObjClass(object, output); // Just skim proxies off. Their notion of isCallable() is more complicated. masm.branchTestClassIsProxy(true, output, ool->entry()); // An object is callable iff: // is<JSFunction>() || (getClass()->cOps && getClass()->cOps->call). masm.branchPtr(Assembler::NotEqual, output, ImmPtr(&JSFunction::class_), ¬Function); masm.move32(Imm32(1), output); masm.jump(&done); masm.bind(¬Function); masm.branchPtr(Assembler::NonZero, Address(output, offsetof(js::Class, cOps)), ImmPtr(nullptr), &hasCOps); masm.move32(Imm32(0), output); masm.jump(&done); masm.bind(&hasCOps); masm.loadPtr(Address(output, offsetof(js::Class, cOps)), output); masm.cmpPtrSet(Assembler::NonZero, Address(output, offsetof(js::ClassOps, call)), ImmPtr(nullptr), output); masm.bind(&done); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineIsCallable(OutOfLineIsCallable* ool) { LIsCallable* ins = ool->ins(); Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); saveVolatile(output); masm.setupUnalignedABICall(output); masm.passABIArg(object); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsCallable)); masm.storeCallBoolResult(output); restoreVolatile(output); masm.jump(ool->rejoin()); } class OutOfLineIsConstructor : public OutOfLineCodeBase<CodeGenerator> { LIsConstructor* ins_; public: explicit OutOfLineIsConstructor(LIsConstructor* ins) : ins_(ins) { } void accept(CodeGenerator* codegen) { codegen->visitOutOfLineIsConstructor(this); } LIsConstructor* ins() const { return ins_; } }; void CodeGenerator::visitIsConstructor(LIsConstructor* ins) { Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); OutOfLineIsConstructor* ool = new(alloc()) OutOfLineIsConstructor(ins); addOutOfLineCode(ool, ins->mir()); Label notFunction, notConstructor, hasCOps, done; masm.loadObjClass(object, output); // Just skim proxies off. Their notion of isConstructor() is more complicated. masm.branchTestClassIsProxy(true, output, ool->entry()); // An object is constructor iff // ((is<JSFunction>() && as<JSFunction>().isConstructor) || // (getClass()->cOps && getClass()->cOps->construct)). masm.branchPtr(Assembler::NotEqual, output, ImmPtr(&JSFunction::class_), ¬Function); masm.load16ZeroExtend(Address(object, JSFunction::offsetOfFlags()), output); masm.and32(Imm32(JSFunction::CONSTRUCTOR), output); masm.branchTest32(Assembler::Zero, output, output, ¬Constructor); masm.move32(Imm32(1), output); masm.jump(&done); masm.bind(¬Constructor); masm.move32(Imm32(0), output); masm.jump(&done); masm.bind(¬Function); masm.branchPtr(Assembler::NonZero, Address(output, offsetof(js::Class, cOps)), ImmPtr(nullptr), &hasCOps); masm.move32(Imm32(0), output); masm.jump(&done); masm.bind(&hasCOps); masm.loadPtr(Address(output, offsetof(js::Class, cOps)), output); masm.cmpPtrSet(Assembler::NonZero, Address(output, offsetof(js::ClassOps, construct)), ImmPtr(nullptr), output); masm.bind(&done); masm.bind(ool->rejoin()); } void CodeGenerator::visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool) { LIsConstructor* ins = ool->ins(); Register object = ToRegister(ins->object()); Register output = ToRegister(ins->output()); saveVolatile(output); masm.setupUnalignedABICall(output); masm.passABIArg(object); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsConstructor)); masm.storeCallBoolResult(output); restoreVolatile(output); masm.jump(ool->rejoin()); } void CodeGenerator::visitIsObject(LIsObject* ins) { Register output = ToRegister(ins->output()); ValueOperand value = ToValue(ins, LIsObject::Input); masm.testObjectSet(Assembler::Equal, value, output); } void CodeGenerator::visitIsObjectAndBranch(LIsObjectAndBranch* ins) { ValueOperand value = ToValue(ins, LIsObjectAndBranch::Input); testObjectEmitBranch(Assembler::Equal, value, ins->ifTrue(), ins->ifFalse()); } void CodeGenerator::loadOutermostJSScript(Register reg) { // The "outermost" JSScript means the script that we are compiling // basically; this is not always the script associated with the // current basic block, which might be an inlined script. MIRGraph& graph = current->mir()->graph(); MBasicBlock* entryBlock = graph.entryBlock(); masm.movePtr(ImmGCPtr(entryBlock->info().script()), reg); } void CodeGenerator::loadJSScriptForBlock(MBasicBlock* block, Register reg) { // The current JSScript means the script for the current // basic block. This may be an inlined script. JSScript* script = block->info().script(); masm.movePtr(ImmGCPtr(script), reg); } void CodeGenerator::visitHasClass(LHasClass* ins) { Register lhs = ToRegister(ins->lhs()); Register output = ToRegister(ins->output()); masm.loadObjClass(lhs, output); masm.cmpPtrSet(Assembler::Equal, output, ImmPtr(ins->mir()->getClass()), output); } void CodeGenerator::visitWasmParameter(LWasmParameter* lir) { } void CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir) { } void CodeGenerator::visitWasmReturn(LWasmReturn* lir) { // Don't emit a jump to the return label if this is the last block. if (current->mir() != *gen->graph().poBegin()) masm.jump(&returnLabel_); } void CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir) { // Don't emit a jump to the return label if this is the last block. if (current->mir() != *gen->graph().poBegin()) masm.jump(&returnLabel_); } void CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir) { // Don't emit a jump to the return label if this is the last block. if (current->mir() != *gen->graph().poBegin()) masm.jump(&returnLabel_); } void CodeGenerator::emitAssertRangeI(const Range* r, Register input) { // Check the lower bound. if (r->hasInt32LowerBound() && r->lower() > INT32_MIN) { Label success; masm.branch32(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()), &success); masm.assumeUnreachable("Integer input should be equal or higher than Lowerbound."); masm.bind(&success); } // Check the upper bound. if (r->hasInt32UpperBound() && r->upper() < INT32_MAX) { Label success; masm.branch32(Assembler::LessThanOrEqual, input, Imm32(r->upper()), &success); masm.assumeUnreachable("Integer input should be lower or equal than Upperbound."); masm.bind(&success); } // For r->canHaveFractionalPart(), r->canBeNegativeZero(), and // r->exponent(), there's nothing to check, because if we ended up in the // integer range checking code, the value is already in an integer register // in the integer range. } void CodeGenerator::emitAssertRangeD(const Range* r, FloatRegister input, FloatRegister temp) { // Check the lower bound. if (r->hasInt32LowerBound()) { Label success; masm.loadConstantDouble(r->lower(), temp); if (r->canBeNaN()) masm.branchDouble(Assembler::DoubleUnordered, input, input, &success); masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &success); masm.assumeUnreachable("Double input should be equal or higher than Lowerbound."); masm.bind(&success); } // Check the upper bound. if (r->hasInt32UpperBound()) { Label success; masm.loadConstantDouble(r->upper(), temp); if (r->canBeNaN()) masm.branchDouble(Assembler::DoubleUnordered, input, input, &success); masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &success); masm.assumeUnreachable("Double input should be lower or equal than Upperbound."); masm.bind(&success); } // This code does not yet check r->canHaveFractionalPart(). This would require new // assembler interfaces to make rounding instructions available. if (!r->canBeNegativeZero()) { Label success; // First, test for being equal to 0.0, which also includes -0.0. masm.loadConstantDouble(0.0, temp); masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, temp, &success); // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0 is // -Infinity instead of Infinity. masm.loadConstantDouble(1.0, temp); masm.divDouble(input, temp); masm.branchDouble(Assembler::DoubleGreaterThan, temp, input, &success); masm.assumeUnreachable("Input shouldn't be negative zero."); masm.bind(&success); } if (!r->hasInt32Bounds() && !r->canBeInfiniteOrNaN() && r->exponent() < FloatingPoint<double>::kExponentBias) { // Check the bounds implied by the maximum exponent. Label exponentLoOk; masm.loadConstantDouble(pow(2.0, r->exponent() + 1), temp); masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentLoOk); masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &exponentLoOk); masm.assumeUnreachable("Check for exponent failed."); masm.bind(&exponentLoOk); Label exponentHiOk; masm.loadConstantDouble(-pow(2.0, r->exponent() + 1), temp); masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentHiOk); masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &exponentHiOk); masm.assumeUnreachable("Check for exponent failed."); masm.bind(&exponentHiOk); } else if (!r->hasInt32Bounds() && !r->canBeNaN()) { // If we think the value can't be NaN, check that it isn't. Label notnan; masm.branchDouble(Assembler::DoubleOrdered, input, input, ¬nan); masm.assumeUnreachable("Input shouldn't be NaN."); masm.bind(¬nan); // If we think the value also can't be an infinity, check that it isn't. if (!r->canBeInfiniteOrNaN()) { Label notposinf; masm.loadConstantDouble(PositiveInfinity<double>(), temp); masm.branchDouble(Assembler::DoubleLessThan, input, temp, ¬posinf); masm.assumeUnreachable("Input shouldn't be +Inf."); masm.bind(¬posinf); Label notneginf; masm.loadConstantDouble(NegativeInfinity<double>(), temp); masm.branchDouble(Assembler::DoubleGreaterThan, input, temp, ¬neginf); masm.assumeUnreachable("Input shouldn't be -Inf."); masm.bind(¬neginf); } } } void CodeGenerator::visitAssertResultV(LAssertResultV* ins) { const ValueOperand value = ToValue(ins, LAssertResultV::Input); emitAssertResultV(value, ins->mirRaw()->resultTypeSet()); } void CodeGenerator::visitAssertResultT(LAssertResultT* ins) { Register input = ToRegister(ins->input()); MDefinition* mir = ins->mirRaw(); emitAssertObjectOrStringResult(input, mir->type(), mir->resultTypeSet()); } void CodeGenerator::visitAssertRangeI(LAssertRangeI* ins) { Register input = ToRegister(ins->input()); const Range* r = ins->range(); emitAssertRangeI(r, input); } void CodeGenerator::visitAssertRangeD(LAssertRangeD* ins) { FloatRegister input = ToFloatRegister(ins->input()); FloatRegister temp = ToFloatRegister(ins->temp()); const Range* r = ins->range(); emitAssertRangeD(r, input, temp); } void CodeGenerator::visitAssertRangeF(LAssertRangeF* ins) { FloatRegister input = ToFloatRegister(ins->input()); FloatRegister temp = ToFloatRegister(ins->temp()); FloatRegister temp2 = ToFloatRegister(ins->temp2()); const Range* r = ins->range(); masm.convertFloat32ToDouble(input, temp); emitAssertRangeD(r, temp, temp2); } void CodeGenerator::visitAssertRangeV(LAssertRangeV* ins) { const Range* r = ins->range(); const ValueOperand value = ToValue(ins, LAssertRangeV::Input); Register tag = masm.splitTagForTest(value); Label done; { Label isNotInt32; masm.branchTestInt32(Assembler::NotEqual, tag, &isNotInt32); Register unboxInt32 = ToTempUnboxRegister(ins->temp()); Register input = masm.extractInt32(value, unboxInt32); emitAssertRangeI(r, input); masm.jump(&done); masm.bind(&isNotInt32); } { Label isNotDouble; masm.branchTestDouble(Assembler::NotEqual, tag, &isNotDouble); FloatRegister input = ToFloatRegister(ins->floatTemp1()); FloatRegister temp = ToFloatRegister(ins->floatTemp2()); masm.unboxDouble(value, input); emitAssertRangeD(r, input, temp); masm.jump(&done); masm.bind(&isNotDouble); } masm.assumeUnreachable("Incorrect range for Value."); masm.bind(&done); } void CodeGenerator::visitInterruptCheck(LInterruptCheck* lir) { if (lir->implicit()) { OutOfLineInterruptCheckImplicit* ool = new(alloc()) OutOfLineInterruptCheckImplicit(current, lir); addOutOfLineCode(ool, lir->mir()); lir->setOolEntry(ool->entry()); masm.bind(ool->rejoin()); return; } OutOfLineCode* ool = oolCallVM(InterruptCheckInfo, lir, ArgList(), StoreNothing()); AbsoluteAddress interruptAddr(GetJitContext()->runtime->addressOfInterruptUint32()); masm.branch32(Assembler::NotEqual, interruptAddr, Imm32(0), ool->entry()); masm.bind(ool->rejoin()); } void CodeGenerator::visitWasmTrap(LWasmTrap* lir) { MOZ_ASSERT(gen->compilingWasm()); const MWasmTrap* mir = lir->mir(); masm.jump(trap(mir, mir->trap())); } void CodeGenerator::visitWasmBoundsCheck(LWasmBoundsCheck* ins) { const MWasmBoundsCheck* mir = ins->mir(); Register ptr = ToRegister(ins->ptr()); masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr, trap(mir, wasm::Trap::OutOfBounds)); } typedef bool (*RecompileFn)(JSContext*); static const VMFunction RecompileFnInfo = FunctionInfo<RecompileFn>(Recompile, "Recompile"); typedef bool (*ForcedRecompileFn)(JSContext*); static const VMFunction ForcedRecompileFnInfo = FunctionInfo<ForcedRecompileFn>(ForcedRecompile, "ForcedRecompile"); void CodeGenerator::visitRecompileCheck(LRecompileCheck* ins) { Label done; Register tmp = ToRegister(ins->scratch()); OutOfLineCode* ool; if (ins->mir()->forceRecompilation()) ool = oolCallVM(ForcedRecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp)); else ool = oolCallVM(RecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp)); // Check if warm-up counter is high enough. AbsoluteAddress warmUpCount = AbsoluteAddress(ins->mir()->script()->addressOfWarmUpCounter()); if (ins->mir()->increaseWarmUpCounter()) { masm.load32(warmUpCount, tmp); masm.add32(Imm32(1), tmp); masm.store32(tmp, warmUpCount); masm.branch32(Assembler::BelowOrEqual, tmp, Imm32(ins->mir()->recompileThreshold()), &done); } else { masm.branch32(Assembler::BelowOrEqual, warmUpCount, Imm32(ins->mir()->recompileThreshold()), &done); } // Check if not yet recompiling. CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp); masm.propagateOOM(ionScriptLabels_.append(label)); masm.branch32(Assembler::Equal, Address(tmp, IonScript::offsetOfRecompiling()), Imm32(0), ool->entry()); masm.bind(ool->rejoin()); masm.bind(&done); } void CodeGenerator::visitLexicalCheck(LLexicalCheck* ins) { ValueOperand inputValue = ToValue(ins, LLexicalCheck::Input); Label bail; masm.branchTestMagicValue(Assembler::Equal, inputValue, JS_UNINITIALIZED_LEXICAL, &bail); bailoutFrom(&bail, ins->snapshot()); } typedef bool (*ThrowRuntimeLexicalErrorFn)(JSContext*, unsigned); static const VMFunction ThrowRuntimeLexicalErrorInfo = FunctionInfo<ThrowRuntimeLexicalErrorFn>(ThrowRuntimeLexicalError, "ThrowRuntimeLexicalError"); void CodeGenerator::visitThrowRuntimeLexicalError(LThrowRuntimeLexicalError* ins) { pushArg(Imm32(ins->mir()->errorNumber())); callVM(ThrowRuntimeLexicalErrorInfo, ins); } typedef bool (*GlobalNameConflictsCheckFromIonFn)(JSContext*, HandleScript); static const VMFunction GlobalNameConflictsCheckFromIonInfo = FunctionInfo<GlobalNameConflictsCheckFromIonFn>(GlobalNameConflictsCheckFromIon, "GlobalNameConflictsCheckFromIon"); void CodeGenerator::visitGlobalNameConflictsCheck(LGlobalNameConflictsCheck* ins) { pushArg(ImmGCPtr(ins->mirRaw()->block()->info().script())); callVM(GlobalNameConflictsCheckFromIonInfo, ins); } void CodeGenerator::visitDebugger(LDebugger* ins) { Register cx = ToRegister(ins->getTemp(0)); Register temp = ToRegister(ins->getTemp(1)); masm.loadJSContext(cx); masm.setupUnalignedABICall(temp); masm.passABIArg(cx); masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GlobalHasLiveOnDebuggerStatement)); Label bail; masm.branchIfTrueBool(ReturnReg, &bail); bailoutFrom(&bail, ins->snapshot()); } void CodeGenerator::visitNewTarget(LNewTarget *ins) { ValueOperand output = GetValueOutput(ins); // if (isConstructing) output = argv[Max(numActualArgs, numFormalArgs)] Label notConstructing, done; Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken()); masm.branchTestPtr(Assembler::Zero, calleeToken, Imm32(CalleeToken_FunctionConstructing), ¬Constructing); Register argvLen = output.scratchReg(); Address actualArgsPtr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs()); masm.loadPtr(actualArgsPtr, argvLen); Label useNFormals; size_t numFormalArgs = ins->mirRaw()->block()->info().funMaybeLazy()->nargs(); masm.branchPtr(Assembler::Below, argvLen, Imm32(numFormalArgs), &useNFormals); size_t argsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs(); { BaseValueIndex newTarget(masm.getStackPointer(), argvLen, argsOffset); masm.loadValue(newTarget, output); masm.jump(&done); } masm.bind(&useNFormals); { Address newTarget(masm.getStackPointer(), argsOffset + (numFormalArgs * sizeof(Value))); masm.loadValue(newTarget, output); masm.jump(&done); } // else output = undefined masm.bind(¬Constructing); masm.moveValue(UndefinedValue(), output); masm.bind(&done); } void CodeGenerator::visitCheckReturn(LCheckReturn* ins) { ValueOperand returnValue = ToValue(ins, LCheckReturn::ReturnValue); ValueOperand thisValue = ToValue(ins, LCheckReturn::ThisValue); Label bail, noChecks; masm.branchTestObject(Assembler::Equal, returnValue, &noChecks); masm.branchTestUndefined(Assembler::NotEqual, returnValue, &bail); masm.branchTestMagicValue(Assembler::Equal, thisValue, JS_UNINITIALIZED_LEXICAL, &bail); bailoutFrom(&bail, ins->snapshot()); masm.bind(&noChecks); } typedef bool (*ThrowCheckIsObjectFn)(JSContext*, CheckIsObjectKind); static const VMFunction ThrowCheckIsObjectInfo = FunctionInfo<ThrowCheckIsObjectFn>(ThrowCheckIsObject, "ThrowCheckIsObject"); void CodeGenerator::visitCheckIsObj(LCheckIsObj* ins) { ValueOperand checkValue = ToValue(ins, LCheckIsObj::CheckValue); OutOfLineCode* ool = oolCallVM(ThrowCheckIsObjectInfo, ins, ArgList(Imm32(ins->mir()->checkKind())), StoreNothing()); masm.branchTestObject(Assembler::NotEqual, checkValue, ool->entry()); masm.bind(ool->rejoin()); } typedef bool (*ThrowObjCoercibleFn)(JSContext*, HandleValue); static const VMFunction ThrowObjectCoercibleInfo = FunctionInfo<ThrowObjCoercibleFn>(ThrowObjectCoercible, "ThrowObjectCoercible"); void CodeGenerator::visitCheckObjCoercible(LCheckObjCoercible* ins) { ValueOperand checkValue = ToValue(ins, LCheckObjCoercible::CheckValue); Label fail, done; masm.branchTestNull(Assembler::Equal, checkValue, &fail); masm.branchTestUndefined(Assembler::NotEqual, checkValue, &done); masm.bind(&fail); pushArg(checkValue); callVM(ThrowObjectCoercibleInfo, ins); masm.bind(&done); } typedef bool (*CheckSelfHostedFn)(JSContext*, HandleValue); static const VMFunction CheckSelfHostedInfo = FunctionInfo<CheckSelfHostedFn>(js::Debug_CheckSelfHosted, "Debug_CheckSelfHosted"); void CodeGenerator::visitDebugCheckSelfHosted(LDebugCheckSelfHosted* ins) { ValueOperand checkValue = ToValue(ins, LDebugCheckSelfHosted::CheckValue); pushArg(checkValue); callVM(CheckSelfHostedInfo, ins); } void CodeGenerator::visitRandom(LRandom* ins) { using mozilla::non_crypto::XorShift128PlusRNG; FloatRegister output = ToFloatRegister(ins->output()); Register tempReg = ToRegister(ins->temp0()); #ifdef JS_PUNBOX64 Register64 s0Reg(ToRegister(ins->temp1())); Register64 s1Reg(ToRegister(ins->temp2())); #else Register64 s0Reg(ToRegister(ins->temp1()), ToRegister(ins->temp2())); Register64 s1Reg(ToRegister(ins->temp3()), ToRegister(ins->temp4())); #endif const void* rng = gen->compartment->addressOfRandomNumberGenerator(); masm.movePtr(ImmPtr(rng), tempReg); static_assert(sizeof(XorShift128PlusRNG) == 2 * sizeof(uint64_t), "Code below assumes XorShift128PlusRNG contains two uint64_t values"); Address state0Addr(tempReg, XorShift128PlusRNG::offsetOfState0()); Address state1Addr(tempReg, XorShift128PlusRNG::offsetOfState1()); // uint64_t s1 = mState[0]; masm.load64(state0Addr, s1Reg); // s1 ^= s1 << 23; masm.move64(s1Reg, s0Reg); masm.lshift64(Imm32(23), s1Reg); masm.xor64(s0Reg, s1Reg); // s1 ^= s1 >> 17 masm.move64(s1Reg, s0Reg); masm.rshift64(Imm32(17), s1Reg); masm.xor64(s0Reg, s1Reg); // const uint64_t s0 = mState[1]; masm.load64(state1Addr, s0Reg); // mState[0] = s0; masm.store64(s0Reg, state0Addr); // s1 ^= s0 masm.xor64(s0Reg, s1Reg); // s1 ^= s0 >> 26 masm.rshift64(Imm32(26), s0Reg); masm.xor64(s0Reg, s1Reg); // mState[1] = s1 masm.store64(s1Reg, state1Addr); // s1 += mState[0] masm.load64(state0Addr, s0Reg); masm.add64(s0Reg, s1Reg); // See comment in XorShift128PlusRNG::nextDouble(). static const int MantissaBits = FloatingPoint<double>::kExponentShift + 1; static const double ScaleInv = double(1) / (1ULL << MantissaBits); masm.and64(Imm64((1ULL << MantissaBits) - 1), s1Reg); if (masm.convertUInt64ToDoubleNeedsTemp()) masm.convertUInt64ToDouble(s1Reg, output, tempReg); else masm.convertUInt64ToDouble(s1Reg, output, Register::Invalid()); // output *= ScaleInv masm.mulDoublePtr(ImmPtr(&ScaleInv), tempReg, output); } void CodeGenerator::visitSignExtend(LSignExtend* ins) { Register input = ToRegister(ins->input()); Register output = ToRegister(ins->output()); switch (ins->mode()) { case MSignExtend::Byte: masm.move8SignExtend(input, output); break; case MSignExtend::Half: masm.move16SignExtend(input, output); break; } } void CodeGenerator::visitRotate(LRotate* ins) { MRotate* mir = ins->mir(); Register input = ToRegister(ins->input()); Register dest = ToRegister(ins->output()); const LAllocation* count = ins->count(); if (count->isConstant()) { int32_t c = ToInt32(count) & 0x1F; if (mir->isLeftRotate()) masm.rotateLeft(Imm32(c), input, dest); else masm.rotateRight(Imm32(c), input, dest); } else { Register creg = ToRegister(count); if (mir->isLeftRotate()) masm.rotateLeft(creg, input, dest); else masm.rotateRight(creg, input, dest); } } class OutOfLineNaNToZero : public OutOfLineCodeBase<CodeGenerator> { LNaNToZero* lir_; public: explicit OutOfLineNaNToZero(LNaNToZero* lir) : lir_(lir) {} void accept(CodeGenerator* codegen) { codegen->visitOutOfLineNaNToZero(this); } LNaNToZero* lir() const { return lir_; } }; void CodeGenerator::visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool) { FloatRegister output = ToFloatRegister(ool->lir()->output()); masm.loadConstantDouble(0.0, output); masm.jump(ool->rejoin()); } void CodeGenerator::visitNaNToZero(LNaNToZero* lir) { FloatRegister input = ToFloatRegister(lir->input()); OutOfLineNaNToZero* ool = new(alloc()) OutOfLineNaNToZero(lir); addOutOfLineCode(ool, lir->mir()); if (lir->mir()->operandIsNeverNegativeZero()){ masm.branchDouble(Assembler::DoubleUnordered, input, input, ool->entry()); } else { FloatRegister scratch = ToFloatRegister(lir->tempDouble()); masm.loadConstantDouble(0.0, scratch); masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch, ool->entry()); } masm.bind(ool->rejoin()); } } // namespace jit } // namespace js