/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * * Copyright 2015 Mozilla Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "wasm/WasmGenerator.h" #include "mozilla/CheckedInt.h" #include "mozilla/EnumeratedRange.h" #include <algorithm> #include "wasm/WasmBaselineCompile.h" #include "wasm/WasmIonCompile.h" #include "wasm/WasmStubs.h" #include "jit/MacroAssembler-inl.h" using namespace js; using namespace js::jit; using namespace js::wasm; using mozilla::CheckedInt; using mozilla::MakeEnumeratedRange; // **************************************************************************** // ModuleGenerator static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024; static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024; static const uint32_t BAD_CODE_RANGE = UINT32_MAX; ModuleGenerator::ModuleGenerator(ImportVector&& imports) : alwaysBaseline_(false), imports_(Move(imports)), numSigs_(0), numTables_(0), lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE), masmAlloc_(&lifo_), masm_(MacroAssembler::WasmToken(), masmAlloc_), lastPatchedCallsite_(0), startOfUnpatchedCallsites_(0), parallel_(false), outstanding_(0), activeFuncDef_(nullptr), startedFuncDefs_(false), finishedFuncDefs_(false), numFinishedFuncDefs_(0) { MOZ_ASSERT(IsCompilingWasm()); } ModuleGenerator::~ModuleGenerator() { if (parallel_) { // Wait for any outstanding jobs to fail or complete. if (outstanding_) { AutoLockHelperThreadState lock; while (true) { IonCompileTaskPtrVector& worklist = HelperThreadState().wasmWorklist(lock); MOZ_ASSERT(outstanding_ >= worklist.length()); outstanding_ -= worklist.length(); worklist.clear(); IonCompileTaskPtrVector& finished = HelperThreadState().wasmFinishedList(lock); MOZ_ASSERT(outstanding_ >= finished.length()); outstanding_ -= finished.length(); finished.clear(); uint32_t numFailed = HelperThreadState().harvestFailedWasmJobs(lock); MOZ_ASSERT(outstanding_ >= numFailed); outstanding_ -= numFailed; if (!outstanding_) break; HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER); } } MOZ_ASSERT(HelperThreadState().wasmCompilationInProgress); HelperThreadState().wasmCompilationInProgress = false; } else { MOZ_ASSERT(!outstanding_); } } bool ModuleGenerator::init(UniqueModuleGeneratorData shared, const CompileArgs& args, Metadata* maybeAsmJSMetadata) { shared_ = Move(shared); alwaysBaseline_ = args.alwaysBaseline; if (!exportedFuncs_.init()) return false; if (!funcToCodeRange_.appendN(BAD_CODE_RANGE, shared_->funcSigs.length())) return false; linkData_.globalDataLength = AlignBytes(InitialGlobalDataBytes, sizeof(void*));; // asm.js passes in an AsmJSMetadata subclass to use instead. if (maybeAsmJSMetadata) { metadata_ = maybeAsmJSMetadata; MOZ_ASSERT(isAsmJS()); } else { metadata_ = js_new<Metadata>(); if (!metadata_) return false; MOZ_ASSERT(!isAsmJS()); } if (args.scriptedCaller.filename) { metadata_->filename = DuplicateString(args.scriptedCaller.filename.get()); if (!metadata_->filename) return false; } if (!assumptions_.clone(args.assumptions)) return false; // For asm.js, the Vectors in ModuleGeneratorData are max-sized reservations // and will be initialized in a linear order via init* functions as the // module is generated. For wasm, the Vectors are correctly-sized and // already initialized. if (!isAsmJS()) { numSigs_ = shared_->sigs.length(); numTables_ = shared_->tables.length(); for (size_t i = 0; i < shared_->funcImportGlobalDataOffsets.length(); i++) { shared_->funcImportGlobalDataOffsets[i] = linkData_.globalDataLength; linkData_.globalDataLength += sizeof(FuncImportTls); if (!addFuncImport(*shared_->funcSigs[i], shared_->funcImportGlobalDataOffsets[i])) return false; } for (const Import& import : imports_) { if (import.kind == DefinitionKind::Table) { MOZ_ASSERT(shared_->tables.length() == 1); shared_->tables[0].external = true; break; } } for (TableDesc& table : shared_->tables) { if (!allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset)) return false; } for (uint32_t i = 0; i < numSigs_; i++) { SigWithId& sig = shared_->sigs[i]; if (SigIdDesc::isGlobal(sig)) { uint32_t globalDataOffset; if (!allocateGlobalBytes(sizeof(void*), sizeof(void*), &globalDataOffset)) return false; sig.id = SigIdDesc::global(sig, globalDataOffset); Sig copy; if (!copy.clone(sig)) return false; if (!metadata_->sigIds.emplaceBack(Move(copy), sig.id)) return false; } else { sig.id = SigIdDesc::immediate(sig); } } for (GlobalDesc& global : shared_->globals) { if (global.isConstant()) continue; if (!allocateGlobal(&global)) return false; } } else { MOZ_ASSERT(shared_->sigs.length() == MaxSigs); MOZ_ASSERT(shared_->tables.length() == MaxTables); MOZ_ASSERT(shared_->asmJSSigToTableIndex.length() == MaxSigs); } return true; } bool ModuleGenerator::finishOutstandingTask() { MOZ_ASSERT(parallel_); IonCompileTask* task = nullptr; { AutoLockHelperThreadState lock; while (true) { MOZ_ASSERT(outstanding_ > 0); if (HelperThreadState().wasmFailed(lock)) return false; if (!HelperThreadState().wasmFinishedList(lock).empty()) { outstanding_--; task = HelperThreadState().wasmFinishedList(lock).popCopy(); break; } HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER); } } return finishTask(task); } bool ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const { return funcToCodeRange_[funcIndex] != BAD_CODE_RANGE; } const CodeRange& ModuleGenerator::funcCodeRange(uint32_t funcIndex) const { MOZ_ASSERT(funcIsCompiled(funcIndex)); const CodeRange& cr = metadata_->codeRanges[funcToCodeRange_[funcIndex]]; MOZ_ASSERT(cr.isFunction()); return cr; } static uint32_t JumpRange() { return Min(JitOptions.jumpThreshold, JumpImmediateRange); } typedef HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy> OffsetMap; bool ModuleGenerator::patchCallSites(TrapExitOffsetArray* maybeTrapExits) { MacroAssembler::AutoPrepareForPatching patching(masm_); masm_.haltingAlign(CodeAlignment); // Create far jumps for calls that have relative offsets that may otherwise // go out of range. Far jumps are created for two cases: direct calls // between function definitions and calls to trap exits by trap out-of-line // paths. Far jump code is shared when possible to reduce bloat. This method // is called both between function bodies (at a frequency determined by the // ISA's jump range) and once at the very end of a module's codegen after // all possible calls/traps have been emitted. OffsetMap existingCallFarJumps; if (!existingCallFarJumps.init()) return false; EnumeratedArray<Trap, Trap::Limit, Maybe<uint32_t>> existingTrapFarJumps; for (; lastPatchedCallsite_ < masm_.callSites().length(); lastPatchedCallsite_++) { const CallSiteAndTarget& cs = masm_.callSites()[lastPatchedCallsite_]; uint32_t callerOffset = cs.returnAddressOffset(); MOZ_RELEASE_ASSERT(callerOffset < INT32_MAX); switch (cs.kind()) { case CallSiteDesc::Dynamic: case CallSiteDesc::Symbolic: break; case CallSiteDesc::Func: { if (funcIsCompiled(cs.funcIndex())) { uint32_t calleeOffset = funcCodeRange(cs.funcIndex()).funcNonProfilingEntry(); MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX); if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) { masm_.patchCall(callerOffset, calleeOffset); break; } } OffsetMap::AddPtr p = existingCallFarJumps.lookupForAdd(cs.funcIndex()); if (!p) { Offsets offsets; offsets.begin = masm_.currentOffset(); uint32_t jumpOffset = masm_.farJumpWithPatch().offset(); offsets.end = masm_.currentOffset(); if (masm_.oom()) return false; if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets)) return false; if (!existingCallFarJumps.add(p, cs.funcIndex(), offsets.begin)) return false; // Record calls' far jumps in metadata since they must be // repatched at runtime when profiling mode is toggled. if (!metadata_->callThunks.emplaceBack(jumpOffset, cs.funcIndex())) return false; } masm_.patchCall(callerOffset, p->value()); break; } case CallSiteDesc::TrapExit: { if (maybeTrapExits) { uint32_t calleeOffset = (*maybeTrapExits)[cs.trap()].begin; MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX); if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) { masm_.patchCall(callerOffset, calleeOffset); break; } } if (!existingTrapFarJumps[cs.trap()]) { Offsets offsets; offsets.begin = masm_.currentOffset(); masm_.append(TrapFarJump(cs.trap(), masm_.farJumpWithPatch())); offsets.end = masm_.currentOffset(); if (masm_.oom()) return false; if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets)) return false; existingTrapFarJumps[cs.trap()] = Some(offsets.begin); } masm_.patchCall(callerOffset, *existingTrapFarJumps[cs.trap()]); break; } } } return true; } bool ModuleGenerator::patchFarJumps(const TrapExitOffsetArray& trapExits) { MacroAssembler::AutoPrepareForPatching patching(masm_); for (CallThunk& callThunk : metadata_->callThunks) { uint32_t funcIndex = callThunk.u.funcIndex; callThunk.u.codeRangeIndex = funcToCodeRange_[funcIndex]; CodeOffset farJump(callThunk.offset); masm_.patchFarJump(farJump, funcCodeRange(funcIndex).funcNonProfilingEntry()); } for (const TrapFarJump& farJump : masm_.trapFarJumps()) masm_.patchFarJump(farJump.jump, trapExits[farJump.trap].begin); return true; } bool ModuleGenerator::finishTask(IonCompileTask* task) { const FuncBytes& func = task->func(); FuncCompileResults& results = task->results(); masm_.haltingAlign(CodeAlignment); // Before merging in the new function's code, if calls in a prior function // body might go out of range, insert far jumps to extend the range. if ((masm_.size() - startOfUnpatchedCallsites_) + results.masm().size() > JumpRange()) { startOfUnpatchedCallsites_ = masm_.size(); if (!patchCallSites()) return false; } // Offset the recorded FuncOffsets by the offset of the function in the // whole module's code segment. uint32_t offsetInWhole = masm_.size(); results.offsets().offsetBy(offsetInWhole); // Add the CodeRange for this function. uint32_t funcCodeRangeIndex = metadata_->codeRanges.length(); if (!metadata_->codeRanges.emplaceBack(func.index(), func.lineOrBytecode(), results.offsets())) return false; MOZ_ASSERT(!funcIsCompiled(func.index())); funcToCodeRange_[func.index()] = funcCodeRangeIndex; // Merge the compiled results into the whole-module masm. mozilla::DebugOnly<size_t> sizeBefore = masm_.size(); if (!masm_.asmMergeWith(results.masm())) return false; MOZ_ASSERT(masm_.size() == offsetInWhole + results.masm().size()); freeTasks_.infallibleAppend(task); return true; } bool ModuleGenerator::finishFuncExports() { // In addition to all the functions that were explicitly exported, any // element of an exported table is also exported. for (ElemSegment& elems : elemSegments_) { if (shared_->tables[elems.tableIndex].external) { for (uint32_t funcIndex : elems.elemFuncIndices) { if (!exportedFuncs_.put(funcIndex)) return false; } } } // ModuleGenerator::exportedFuncs_ is an unordered HashSet. The // FuncExportVector stored in Metadata needs to be stored sorted by // function index to allow O(log(n)) lookup at runtime. Uint32Vector sorted; if (!sorted.reserve(exportedFuncs_.count())) return false; for (Uint32Set::Range r = exportedFuncs_.all(); !r.empty(); r.popFront()) sorted.infallibleAppend(r.front()); std::sort(sorted.begin(), sorted.end()); MOZ_ASSERT(metadata_->funcExports.empty()); if (!metadata_->funcExports.reserve(sorted.length())) return false; for (uint32_t funcIndex : sorted) { Sig sig; if (!sig.clone(funcSig(funcIndex))) return false; uint32_t codeRangeIndex = funcToCodeRange_[funcIndex]; metadata_->funcExports.infallibleEmplaceBack(Move(sig), funcIndex, codeRangeIndex); } return true; } typedef Vector<Offsets, 0, SystemAllocPolicy> OffsetVector; typedef Vector<ProfilingOffsets, 0, SystemAllocPolicy> ProfilingOffsetVector; bool ModuleGenerator::finishCodegen() { masm_.haltingAlign(CodeAlignment); uint32_t offsetInWhole = masm_.size(); uint32_t numFuncExports = metadata_->funcExports.length(); MOZ_ASSERT(numFuncExports == exportedFuncs_.count()); // Generate stubs in a separate MacroAssembler since, otherwise, for modules // larger than the JumpImmediateRange, even local uses of Label will fail // due to the large absolute offsets temporarily stored by Label::bind(). OffsetVector entries; ProfilingOffsetVector interpExits; ProfilingOffsetVector jitExits; TrapExitOffsetArray trapExits; Offsets outOfBoundsExit; Offsets unalignedAccessExit; Offsets interruptExit; Offsets throwStub; { TempAllocator alloc(&lifo_); MacroAssembler masm(MacroAssembler::WasmToken(), alloc); Label throwLabel; if (!entries.resize(numFuncExports)) return false; for (uint32_t i = 0; i < numFuncExports; i++) entries[i] = GenerateEntry(masm, metadata_->funcExports[i]); if (!interpExits.resize(numFuncImports())) return false; if (!jitExits.resize(numFuncImports())) return false; for (uint32_t i = 0; i < numFuncImports(); i++) { interpExits[i] = GenerateImportInterpExit(masm, metadata_->funcImports[i], i, &throwLabel); jitExits[i] = GenerateImportJitExit(masm, metadata_->funcImports[i], &throwLabel); } for (Trap trap : MakeEnumeratedRange(Trap::Limit)) trapExits[trap] = GenerateTrapExit(masm, trap, &throwLabel); outOfBoundsExit = GenerateOutOfBoundsExit(masm, &throwLabel); unalignedAccessExit = GenerateUnalignedExit(masm, &throwLabel); interruptExit = GenerateInterruptExit(masm, &throwLabel); throwStub = GenerateThrowStub(masm, &throwLabel); if (masm.oom() || !masm_.asmMergeWith(masm)) return false; } // Adjust each of the resulting Offsets (to account for being merged into // masm_) and then create code ranges for all the stubs. for (uint32_t i = 0; i < numFuncExports; i++) { entries[i].offsetBy(offsetInWhole); metadata_->funcExports[i].initEntryOffset(entries[i].begin); if (!metadata_->codeRanges.emplaceBack(CodeRange::Entry, entries[i])) return false; } for (uint32_t i = 0; i < numFuncImports(); i++) { interpExits[i].offsetBy(offsetInWhole); metadata_->funcImports[i].initInterpExitOffset(interpExits[i].begin); if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportInterpExit, interpExits[i])) return false; jitExits[i].offsetBy(offsetInWhole); metadata_->funcImports[i].initJitExitOffset(jitExits[i].begin); if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportJitExit, jitExits[i])) return false; } for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { trapExits[trap].offsetBy(offsetInWhole); if (!metadata_->codeRanges.emplaceBack(CodeRange::TrapExit, trapExits[trap])) return false; } outOfBoundsExit.offsetBy(offsetInWhole); if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, outOfBoundsExit)) return false; unalignedAccessExit.offsetBy(offsetInWhole); if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, unalignedAccessExit)) return false; interruptExit.offsetBy(offsetInWhole); if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, interruptExit)) return false; throwStub.offsetBy(offsetInWhole); if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, throwStub)) return false; // Fill in LinkData with the offsets of these stubs. linkData_.outOfBoundsOffset = outOfBoundsExit.begin; linkData_.interruptOffset = interruptExit.begin; // Now that all other code has been emitted, patch all remaining callsites // then far jumps. Patching callsites can generate far jumps so there is an // ordering dependency. if (!patchCallSites(&trapExits)) return false; if (!patchFarJumps(trapExits)) return false; // Code-generation is complete! masm_.finish(); return !masm_.oom(); } bool ModuleGenerator::finishLinkData(Bytes& code) { // Inflate the global bytes up to page size so that the total bytes are a // page size (as required by the allocator functions). linkData_.globalDataLength = AlignBytes(linkData_.globalDataLength, gc::SystemPageSize()); // Add links to absolute addresses identified symbolically. for (size_t i = 0; i < masm_.numSymbolicAccesses(); i++) { SymbolicAccess src = masm_.symbolicAccess(i); if (!linkData_.symbolicLinks[src.target].append(src.patchAt.offset())) return false; } // Relative link metadata: absolute addresses that refer to another point within // the asm.js module. // CodeLabels are used for switch cases and loads from floating-point / // SIMD values in the constant pool. for (size_t i = 0; i < masm_.numCodeLabels(); i++) { CodeLabel cl = masm_.codeLabel(i); LinkData::InternalLink inLink(LinkData::InternalLink::CodeLabel); inLink.patchAtOffset = masm_.labelToPatchOffset(*cl.patchAt()); inLink.targetOffset = cl.target()->offset(); if (!linkData_.internalLinks.append(inLink)) return false; } #if defined(JS_CODEGEN_X86) // Global data accesses in x86 need to be patched with the absolute // address of the global. Globals are allocated sequentially after the // code section so we can just use an InternalLink. for (GlobalAccess a : masm_.globalAccesses()) { LinkData::InternalLink inLink(LinkData::InternalLink::RawPointer); inLink.patchAtOffset = masm_.labelToPatchOffset(a.patchAt); inLink.targetOffset = code.length() + a.globalDataOffset; if (!linkData_.internalLinks.append(inLink)) return false; } #elif defined(JS_CODEGEN_X64) // Global data accesses on x64 use rip-relative addressing and thus we can // patch here, now that we know the final codeLength. for (GlobalAccess a : masm_.globalAccesses()) { void* from = code.begin() + a.patchAt.offset(); void* to = code.end() + a.globalDataOffset; X86Encoding::SetRel32(from, to); } #else // Global access is performed using the GlobalReg and requires no patching. MOZ_ASSERT(masm_.globalAccesses().length() == 0); #endif return true; } bool ModuleGenerator::addFuncImport(const Sig& sig, uint32_t globalDataOffset) { MOZ_ASSERT(!finishedFuncDefs_); Sig copy; if (!copy.clone(sig)) return false; return metadata_->funcImports.emplaceBack(Move(copy), globalDataOffset); } bool ModuleGenerator::allocateGlobalBytes(uint32_t bytes, uint32_t align, uint32_t* globalDataOffset) { CheckedInt<uint32_t> newGlobalDataLength(linkData_.globalDataLength); newGlobalDataLength += ComputeByteAlignment(newGlobalDataLength.value(), align); if (!newGlobalDataLength.isValid()) return false; *globalDataOffset = newGlobalDataLength.value(); newGlobalDataLength += bytes; if (!newGlobalDataLength.isValid()) return false; linkData_.globalDataLength = newGlobalDataLength.value(); return true; } bool ModuleGenerator::allocateGlobal(GlobalDesc* global) { MOZ_ASSERT(!startedFuncDefs_); unsigned width = 0; switch (global->type()) { case ValType::I32: case ValType::F32: width = 4; break; case ValType::I64: case ValType::F64: width = 8; break; case ValType::I8x16: case ValType::I16x8: case ValType::I32x4: case ValType::F32x4: case ValType::B8x16: case ValType::B16x8: case ValType::B32x4: width = 16; break; } uint32_t offset; if (!allocateGlobalBytes(width, width, &offset)) return false; global->setOffset(offset); return true; } bool ModuleGenerator::addGlobal(ValType type, bool isConst, uint32_t* index) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(!startedFuncDefs_); *index = shared_->globals.length(); GlobalDesc global(type, !isConst, *index); if (!allocateGlobal(&global)) return false; return shared_->globals.append(global); } void ModuleGenerator::initSig(uint32_t sigIndex, Sig&& sig) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(sigIndex == numSigs_); numSigs_++; MOZ_ASSERT(shared_->sigs[sigIndex] == Sig()); shared_->sigs[sigIndex] = Move(sig); } const SigWithId& ModuleGenerator::sig(uint32_t index) const { MOZ_ASSERT(index < numSigs_); return shared_->sigs[index]; } void ModuleGenerator::initFuncSig(uint32_t funcIndex, uint32_t sigIndex) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(!shared_->funcSigs[funcIndex]); shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex]; } void ModuleGenerator::initMemoryUsage(MemoryUsage memoryUsage) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(shared_->memoryUsage == MemoryUsage::None); shared_->memoryUsage = memoryUsage; } void ModuleGenerator::bumpMinMemoryLength(uint32_t newMinMemoryLength) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(newMinMemoryLength >= shared_->minMemoryLength); shared_->minMemoryLength = newMinMemoryLength; } bool ModuleGenerator::initImport(uint32_t funcIndex, uint32_t sigIndex) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(!shared_->funcSigs[funcIndex]); shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex]; uint32_t globalDataOffset; if (!allocateGlobalBytes(sizeof(FuncImportTls), sizeof(void*), &globalDataOffset)) return false; MOZ_ASSERT(!shared_->funcImportGlobalDataOffsets[funcIndex]); shared_->funcImportGlobalDataOffsets[funcIndex] = globalDataOffset; MOZ_ASSERT(funcIndex == metadata_->funcImports.length()); return addFuncImport(sig(sigIndex), globalDataOffset); } uint32_t ModuleGenerator::numFuncImports() const { // Until all functions have been validated, asm.js doesn't know the total // number of imports. MOZ_ASSERT_IF(isAsmJS(), finishedFuncDefs_); return metadata_->funcImports.length(); } uint32_t ModuleGenerator::numFuncDefs() const { // asm.js overallocates the length of funcSigs and in general does not know // the number of function definitions until it's done compiling. MOZ_ASSERT(!isAsmJS()); return shared_->funcSigs.length() - numFuncImports(); } uint32_t ModuleGenerator::numFuncs() const { // asm.js pre-reserves a bunch of function index space which is // incrementally filled in during function-body validation. Thus, there are // a few possible interpretations of numFuncs() (total index space size vs. // exact number of imports/definitions encountered so far) and to simplify // things we simply only define this quantity for wasm. MOZ_ASSERT(!isAsmJS()); return shared_->funcSigs.length(); } const SigWithId& ModuleGenerator::funcSig(uint32_t funcIndex) const { MOZ_ASSERT(shared_->funcSigs[funcIndex]); return *shared_->funcSigs[funcIndex]; } bool ModuleGenerator::addFuncExport(UniqueChars fieldName, uint32_t funcIndex) { return exportedFuncs_.put(funcIndex) && exports_.emplaceBack(Move(fieldName), funcIndex, DefinitionKind::Function); } bool ModuleGenerator::addTableExport(UniqueChars fieldName) { MOZ_ASSERT(!startedFuncDefs_); MOZ_ASSERT(shared_->tables.length() == 1); shared_->tables[0].external = true; return exports_.emplaceBack(Move(fieldName), DefinitionKind::Table); } bool ModuleGenerator::addMemoryExport(UniqueChars fieldName) { return exports_.emplaceBack(Move(fieldName), DefinitionKind::Memory); } bool ModuleGenerator::addGlobalExport(UniqueChars fieldName, uint32_t globalIndex) { return exports_.emplaceBack(Move(fieldName), globalIndex, DefinitionKind::Global); } bool ModuleGenerator::setStartFunction(uint32_t funcIndex) { metadata_->startFuncIndex.emplace(funcIndex); return exportedFuncs_.put(funcIndex); } bool ModuleGenerator::addElemSegment(InitExpr offset, Uint32Vector&& elemFuncIndices) { MOZ_ASSERT(!isAsmJS()); MOZ_ASSERT(!startedFuncDefs_); MOZ_ASSERT(shared_->tables.length() == 1); for (uint32_t funcIndex : elemFuncIndices) { if (funcIndex < numFuncImports()) { shared_->tables[0].external = true; break; } } return elemSegments_.emplaceBack(0, offset, Move(elemFuncIndices)); } void ModuleGenerator::setDataSegments(DataSegmentVector&& segments) { MOZ_ASSERT(dataSegments_.empty()); dataSegments_ = Move(segments); } bool ModuleGenerator::startFuncDefs() { MOZ_ASSERT(!startedFuncDefs_); MOZ_ASSERT(!finishedFuncDefs_); // The wasmCompilationInProgress atomic ensures that there is only one // parallel compilation in progress at a time. In the special case of // asm.js, where the ModuleGenerator itself can be on a helper thread, this // avoids the possibility of deadlock since at most 1 helper thread will be // blocking on other helper threads and there are always >1 helper threads. // With wasm, this restriction could be relaxed by moving the worklist state // out of HelperThreadState since each independent compilation needs its own // worklist pair. Alternatively, the deadlock could be avoided by having the // ModuleGenerator thread make progress (on compile tasks) instead of // blocking. GlobalHelperThreadState& threads = HelperThreadState(); MOZ_ASSERT(threads.threadCount > 1); uint32_t numTasks; if (CanUseExtraThreads() && threads.wasmCompilationInProgress.compareExchange(false, true)) { #ifdef DEBUG { AutoLockHelperThreadState lock; MOZ_ASSERT(!HelperThreadState().wasmFailed(lock)); MOZ_ASSERT(HelperThreadState().wasmWorklist(lock).empty()); MOZ_ASSERT(HelperThreadState().wasmFinishedList(lock).empty()); } #endif parallel_ = true; numTasks = 2 * threads.maxWasmCompilationThreads(); } else { numTasks = 1; } if (!tasks_.initCapacity(numTasks)) return false; for (size_t i = 0; i < numTasks; i++) tasks_.infallibleEmplaceBack(*shared_, COMPILATION_LIFO_DEFAULT_CHUNK_SIZE); if (!freeTasks_.reserve(numTasks)) return false; for (size_t i = 0; i < numTasks; i++) freeTasks_.infallibleAppend(&tasks_[i]); startedFuncDefs_ = true; MOZ_ASSERT(!finishedFuncDefs_); return true; } bool ModuleGenerator::startFuncDef(uint32_t lineOrBytecode, FunctionGenerator* fg) { MOZ_ASSERT(startedFuncDefs_); MOZ_ASSERT(!activeFuncDef_); MOZ_ASSERT(!finishedFuncDefs_); if (freeTasks_.empty() && !finishOutstandingTask()) return false; IonCompileTask* task = freeTasks_.popCopy(); task->reset(&fg->bytes_); fg->bytes_.clear(); fg->lineOrBytecode_ = lineOrBytecode; fg->m_ = this; fg->task_ = task; activeFuncDef_ = fg; return true; } bool ModuleGenerator::finishFuncDef(uint32_t funcIndex, FunctionGenerator* fg) { MOZ_ASSERT(activeFuncDef_ == fg); auto func = js::MakeUnique<FuncBytes>(Move(fg->bytes_), funcIndex, funcSig(funcIndex), fg->lineOrBytecode_, Move(fg->callSiteLineNums_)); if (!func) return false; auto mode = alwaysBaseline_ && BaselineCanCompile(fg) ? IonCompileTask::CompileMode::Baseline : IonCompileTask::CompileMode::Ion; fg->task_->init(Move(func), mode); if (parallel_) { if (!StartOffThreadWasmCompile(fg->task_)) return false; outstanding_++; } else { if (!CompileFunction(fg->task_)) return false; if (!finishTask(fg->task_)) return false; } fg->m_ = nullptr; fg->task_ = nullptr; activeFuncDef_ = nullptr; numFinishedFuncDefs_++; return true; } bool ModuleGenerator::finishFuncDefs() { MOZ_ASSERT(startedFuncDefs_); MOZ_ASSERT(!activeFuncDef_); MOZ_ASSERT(!finishedFuncDefs_); while (outstanding_ > 0) { if (!finishOutstandingTask()) return false; } linkData_.functionCodeLength = masm_.size(); finishedFuncDefs_ = true; // Generate wrapper functions for every import. These wrappers turn imports // into plain functions so they can be put into tables and re-exported. // asm.js cannot do either and so no wrappers are generated. if (!isAsmJS()) { for (size_t funcIndex = 0; funcIndex < numFuncImports(); funcIndex++) { const FuncImport& funcImport = metadata_->funcImports[funcIndex]; const SigWithId& sig = funcSig(funcIndex); FuncOffsets offsets = GenerateImportFunction(masm_, funcImport, sig.id); if (masm_.oom()) return false; uint32_t codeRangeIndex = metadata_->codeRanges.length(); if (!metadata_->codeRanges.emplaceBack(funcIndex, /* bytecodeOffset = */ 0, offsets)) return false; MOZ_ASSERT(!funcIsCompiled(funcIndex)); funcToCodeRange_[funcIndex] = codeRangeIndex; } } // All function indices should have an associated code range at this point // (except in asm.js, which doesn't have import wrapper functions). #ifdef DEBUG if (isAsmJS()) { MOZ_ASSERT(numFuncImports() < AsmJSFirstDefFuncIndex); for (uint32_t i = 0; i < AsmJSFirstDefFuncIndex; i++) MOZ_ASSERT(funcToCodeRange_[i] == BAD_CODE_RANGE); for (uint32_t i = AsmJSFirstDefFuncIndex; i < numFinishedFuncDefs_; i++) MOZ_ASSERT(funcCodeRange(i).funcIndex() == i); } else { MOZ_ASSERT(numFinishedFuncDefs_ == numFuncDefs()); for (uint32_t i = 0; i < numFuncs(); i++) MOZ_ASSERT(funcCodeRange(i).funcIndex() == i); } #endif // Complete element segments with the code range index of every element, now // that all functions have been compiled. for (ElemSegment& elems : elemSegments_) { Uint32Vector& codeRangeIndices = elems.elemCodeRangeIndices; MOZ_ASSERT(codeRangeIndices.empty()); if (!codeRangeIndices.reserve(elems.elemFuncIndices.length())) return false; for (uint32_t funcIndex : elems.elemFuncIndices) codeRangeIndices.infallibleAppend(funcToCodeRange_[funcIndex]); } return true; } void ModuleGenerator::setFuncNames(NameInBytecodeVector&& funcNames) { MOZ_ASSERT(metadata_->funcNames.empty()); metadata_->funcNames = Move(funcNames); } bool ModuleGenerator::initSigTableLength(uint32_t sigIndex, uint32_t length) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(length != 0); MOZ_ASSERT(length <= MaxTableElems); MOZ_ASSERT(shared_->asmJSSigToTableIndex[sigIndex] == 0); shared_->asmJSSigToTableIndex[sigIndex] = numTables_; TableDesc& table = shared_->tables[numTables_++]; table.kind = TableKind::TypedFunction; table.limits.initial = length; table.limits.maximum = Some(length); return allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset); } bool ModuleGenerator::initSigTableElems(uint32_t sigIndex, Uint32Vector&& elemFuncIndices) { MOZ_ASSERT(isAsmJS()); MOZ_ASSERT(finishedFuncDefs_); uint32_t tableIndex = shared_->asmJSSigToTableIndex[sigIndex]; MOZ_ASSERT(shared_->tables[tableIndex].limits.initial == elemFuncIndices.length()); Uint32Vector codeRangeIndices; if (!codeRangeIndices.resize(elemFuncIndices.length())) return false; for (size_t i = 0; i < elemFuncIndices.length(); i++) codeRangeIndices[i] = funcToCodeRange_[elemFuncIndices[i]]; InitExpr offset(Val(uint32_t(0))); if (!elemSegments_.emplaceBack(tableIndex, offset, Move(elemFuncIndices))) return false; elemSegments_.back().elemCodeRangeIndices = Move(codeRangeIndices); return true; } SharedModule ModuleGenerator::finish(const ShareableBytes& bytecode) { MOZ_ASSERT(!activeFuncDef_); MOZ_ASSERT(finishedFuncDefs_); if (!finishFuncExports()) return nullptr; if (!finishCodegen()) return nullptr; // Round up the code size to page size since this is eventually required by // the executable-code allocator and for setting memory protection. uint32_t bytesNeeded = masm_.bytesNeeded(); uint32_t padding = ComputeByteAlignment(bytesNeeded, gc::SystemPageSize()); // Use initLengthUninitialized so there is no round-up allocation nor time // wasted zeroing memory. Bytes code; if (!code.initLengthUninitialized(bytesNeeded + padding)) return nullptr; // Delay flushing of the icache until CodeSegment::create since there is // more patching to do before this code becomes executable. { AutoFlushICache afc("ModuleGenerator::finish", /* inhibit = */ true); masm_.executableCopy(code.begin()); } // Zero the padding, since we used resizeUninitialized above. memset(code.begin() + bytesNeeded, 0, padding); // Convert the CallSiteAndTargetVector (needed during generation) to a // CallSiteVector (what is stored in the Module). if (!metadata_->callSites.appendAll(masm_.callSites())) return nullptr; // The MacroAssembler has accumulated all the memory accesses during codegen. metadata_->memoryAccesses = masm_.extractMemoryAccesses(); metadata_->memoryPatches = masm_.extractMemoryPatches(); metadata_->boundsChecks = masm_.extractBoundsChecks(); // Copy over data from the ModuleGeneratorData. metadata_->memoryUsage = shared_->memoryUsage; metadata_->minMemoryLength = shared_->minMemoryLength; metadata_->maxMemoryLength = shared_->maxMemoryLength; metadata_->tables = Move(shared_->tables); metadata_->globals = Move(shared_->globals); // These Vectors can get large and the excess capacity can be significant, // so realloc them down to size. metadata_->memoryAccesses.podResizeToFit(); metadata_->memoryPatches.podResizeToFit(); metadata_->boundsChecks.podResizeToFit(); metadata_->codeRanges.podResizeToFit(); metadata_->callSites.podResizeToFit(); metadata_->callThunks.podResizeToFit(); // For asm.js, the tables vector is over-allocated (to avoid resize during // parallel copilation). Shrink it back down to fit. if (isAsmJS() && !metadata_->tables.resize(numTables_)) return nullptr; // Assert CodeRanges are sorted. #ifdef DEBUG uint32_t lastEnd = 0; for (const CodeRange& codeRange : metadata_->codeRanges) { MOZ_ASSERT(codeRange.begin() >= lastEnd); lastEnd = codeRange.end(); } #endif if (!finishLinkData(code)) return nullptr; return SharedModule(js_new<Module>(Move(assumptions_), Move(code), Move(linkData_), Move(imports_), Move(exports_), Move(dataSegments_), Move(elemSegments_), *metadata_, bytecode)); }