diff options
Diffstat (limited to 'js/src/wasm/WasmGenerator.cpp')
| -rw-r--r-- | js/src/wasm/WasmGenerator.cpp | 1174 |
1 files changed, 1174 insertions, 0 deletions
diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp new file mode 100644 index 000000000..e6f1edd99 --- /dev/null +++ b/js/src/wasm/WasmGenerator.cpp @@ -0,0 +1,1174 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmGenerator.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/EnumeratedRange.h" + +#include <algorithm> + +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmStubs.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::CheckedInt; +using mozilla::MakeEnumeratedRange; + +// **************************************************************************** +// ModuleGenerator + +static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024; +static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024; +static const uint32_t BAD_CODE_RANGE = UINT32_MAX; + +ModuleGenerator::ModuleGenerator(ImportVector&& imports) + : alwaysBaseline_(false), + imports_(Move(imports)), + numSigs_(0), + numTables_(0), + lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE), + masmAlloc_(&lifo_), + masm_(MacroAssembler::WasmToken(), masmAlloc_), + lastPatchedCallsite_(0), + startOfUnpatchedCallsites_(0), + parallel_(false), + outstanding_(0), + activeFuncDef_(nullptr), + startedFuncDefs_(false), + finishedFuncDefs_(false), + numFinishedFuncDefs_(0) +{ + MOZ_ASSERT(IsCompilingWasm()); +} + +ModuleGenerator::~ModuleGenerator() +{ + if (parallel_) { + // Wait for any outstanding jobs to fail or complete. + if (outstanding_) { + AutoLockHelperThreadState lock; + while (true) { + IonCompileTaskPtrVector& worklist = HelperThreadState().wasmWorklist(lock); + MOZ_ASSERT(outstanding_ >= worklist.length()); + outstanding_ -= worklist.length(); + worklist.clear(); + + IonCompileTaskPtrVector& finished = HelperThreadState().wasmFinishedList(lock); + MOZ_ASSERT(outstanding_ >= finished.length()); + outstanding_ -= finished.length(); + finished.clear(); + + uint32_t numFailed = HelperThreadState().harvestFailedWasmJobs(lock); + MOZ_ASSERT(outstanding_ >= numFailed); + outstanding_ -= numFailed; + + if (!outstanding_) + break; + + HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER); + } + } + + MOZ_ASSERT(HelperThreadState().wasmCompilationInProgress); + HelperThreadState().wasmCompilationInProgress = false; + } else { + MOZ_ASSERT(!outstanding_); + } +} + +bool +ModuleGenerator::init(UniqueModuleGeneratorData shared, const CompileArgs& args, + Metadata* maybeAsmJSMetadata) +{ + shared_ = Move(shared); + alwaysBaseline_ = args.alwaysBaseline; + + if (!exportedFuncs_.init()) + return false; + + if (!funcToCodeRange_.appendN(BAD_CODE_RANGE, shared_->funcSigs.length())) + return false; + + linkData_.globalDataLength = AlignBytes(InitialGlobalDataBytes, sizeof(void*));; + + // asm.js passes in an AsmJSMetadata subclass to use instead. + if (maybeAsmJSMetadata) { + metadata_ = maybeAsmJSMetadata; + MOZ_ASSERT(isAsmJS()); + } else { + metadata_ = js_new<Metadata>(); + if (!metadata_) + return false; + MOZ_ASSERT(!isAsmJS()); + } + + if (args.scriptedCaller.filename) { + metadata_->filename = DuplicateString(args.scriptedCaller.filename.get()); + if (!metadata_->filename) + return false; + } + + if (!assumptions_.clone(args.assumptions)) + return false; + + // For asm.js, the Vectors in ModuleGeneratorData are max-sized reservations + // and will be initialized in a linear order via init* functions as the + // module is generated. For wasm, the Vectors are correctly-sized and + // already initialized. + + if (!isAsmJS()) { + numSigs_ = shared_->sigs.length(); + numTables_ = shared_->tables.length(); + + for (size_t i = 0; i < shared_->funcImportGlobalDataOffsets.length(); i++) { + shared_->funcImportGlobalDataOffsets[i] = linkData_.globalDataLength; + linkData_.globalDataLength += sizeof(FuncImportTls); + if (!addFuncImport(*shared_->funcSigs[i], shared_->funcImportGlobalDataOffsets[i])) + return false; + } + + for (const Import& import : imports_) { + if (import.kind == DefinitionKind::Table) { + MOZ_ASSERT(shared_->tables.length() == 1); + shared_->tables[0].external = true; + break; + } + } + + for (TableDesc& table : shared_->tables) { + if (!allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset)) + return false; + } + + for (uint32_t i = 0; i < numSigs_; i++) { + SigWithId& sig = shared_->sigs[i]; + if (SigIdDesc::isGlobal(sig)) { + uint32_t globalDataOffset; + if (!allocateGlobalBytes(sizeof(void*), sizeof(void*), &globalDataOffset)) + return false; + + sig.id = SigIdDesc::global(sig, globalDataOffset); + + Sig copy; + if (!copy.clone(sig)) + return false; + + if (!metadata_->sigIds.emplaceBack(Move(copy), sig.id)) + return false; + } else { + sig.id = SigIdDesc::immediate(sig); + } + } + + for (GlobalDesc& global : shared_->globals) { + if (global.isConstant()) + continue; + if (!allocateGlobal(&global)) + return false; + } + } else { + MOZ_ASSERT(shared_->sigs.length() == MaxSigs); + MOZ_ASSERT(shared_->tables.length() == MaxTables); + MOZ_ASSERT(shared_->asmJSSigToTableIndex.length() == MaxSigs); + } + + return true; +} + +bool +ModuleGenerator::finishOutstandingTask() +{ + MOZ_ASSERT(parallel_); + + IonCompileTask* task = nullptr; + { + AutoLockHelperThreadState lock; + while (true) { + MOZ_ASSERT(outstanding_ > 0); + + if (HelperThreadState().wasmFailed(lock)) + return false; + + if (!HelperThreadState().wasmFinishedList(lock).empty()) { + outstanding_--; + task = HelperThreadState().wasmFinishedList(lock).popCopy(); + break; + } + + HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER); + } + } + + return finishTask(task); +} + +bool +ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const +{ + return funcToCodeRange_[funcIndex] != BAD_CODE_RANGE; +} + +const CodeRange& +ModuleGenerator::funcCodeRange(uint32_t funcIndex) const +{ + MOZ_ASSERT(funcIsCompiled(funcIndex)); + const CodeRange& cr = metadata_->codeRanges[funcToCodeRange_[funcIndex]]; + MOZ_ASSERT(cr.isFunction()); + return cr; +} + +static uint32_t +JumpRange() +{ + return Min(JitOptions.jumpThreshold, JumpImmediateRange); +} + +typedef HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy> OffsetMap; + +bool +ModuleGenerator::patchCallSites(TrapExitOffsetArray* maybeTrapExits) +{ + MacroAssembler::AutoPrepareForPatching patching(masm_); + + masm_.haltingAlign(CodeAlignment); + + // Create far jumps for calls that have relative offsets that may otherwise + // go out of range. Far jumps are created for two cases: direct calls + // between function definitions and calls to trap exits by trap out-of-line + // paths. Far jump code is shared when possible to reduce bloat. This method + // is called both between function bodies (at a frequency determined by the + // ISA's jump range) and once at the very end of a module's codegen after + // all possible calls/traps have been emitted. + + OffsetMap existingCallFarJumps; + if (!existingCallFarJumps.init()) + return false; + + EnumeratedArray<Trap, Trap::Limit, Maybe<uint32_t>> existingTrapFarJumps; + + for (; lastPatchedCallsite_ < masm_.callSites().length(); lastPatchedCallsite_++) { + const CallSiteAndTarget& cs = masm_.callSites()[lastPatchedCallsite_]; + uint32_t callerOffset = cs.returnAddressOffset(); + MOZ_RELEASE_ASSERT(callerOffset < INT32_MAX); + + switch (cs.kind()) { + case CallSiteDesc::Dynamic: + case CallSiteDesc::Symbolic: + break; + case CallSiteDesc::Func: { + if (funcIsCompiled(cs.funcIndex())) { + uint32_t calleeOffset = funcCodeRange(cs.funcIndex()).funcNonProfilingEntry(); + MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX); + + if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) { + masm_.patchCall(callerOffset, calleeOffset); + break; + } + } + + OffsetMap::AddPtr p = existingCallFarJumps.lookupForAdd(cs.funcIndex()); + if (!p) { + Offsets offsets; + offsets.begin = masm_.currentOffset(); + uint32_t jumpOffset = masm_.farJumpWithPatch().offset(); + offsets.end = masm_.currentOffset(); + if (masm_.oom()) + return false; + + if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets)) + return false; + if (!existingCallFarJumps.add(p, cs.funcIndex(), offsets.begin)) + return false; + + // Record calls' far jumps in metadata since they must be + // repatched at runtime when profiling mode is toggled. + if (!metadata_->callThunks.emplaceBack(jumpOffset, cs.funcIndex())) + return false; + } + + masm_.patchCall(callerOffset, p->value()); + break; + } + case CallSiteDesc::TrapExit: { + if (maybeTrapExits) { + uint32_t calleeOffset = (*maybeTrapExits)[cs.trap()].begin; + MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX); + + if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) { + masm_.patchCall(callerOffset, calleeOffset); + break; + } + } + + if (!existingTrapFarJumps[cs.trap()]) { + Offsets offsets; + offsets.begin = masm_.currentOffset(); + masm_.append(TrapFarJump(cs.trap(), masm_.farJumpWithPatch())); + offsets.end = masm_.currentOffset(); + if (masm_.oom()) + return false; + + if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets)) + return false; + existingTrapFarJumps[cs.trap()] = Some(offsets.begin); + } + + masm_.patchCall(callerOffset, *existingTrapFarJumps[cs.trap()]); + break; + } + } + } + + return true; +} + +bool +ModuleGenerator::patchFarJumps(const TrapExitOffsetArray& trapExits) +{ + MacroAssembler::AutoPrepareForPatching patching(masm_); + + for (CallThunk& callThunk : metadata_->callThunks) { + uint32_t funcIndex = callThunk.u.funcIndex; + callThunk.u.codeRangeIndex = funcToCodeRange_[funcIndex]; + CodeOffset farJump(callThunk.offset); + masm_.patchFarJump(farJump, funcCodeRange(funcIndex).funcNonProfilingEntry()); + } + + for (const TrapFarJump& farJump : masm_.trapFarJumps()) + masm_.patchFarJump(farJump.jump, trapExits[farJump.trap].begin); + + return true; +} + +bool +ModuleGenerator::finishTask(IonCompileTask* task) +{ + const FuncBytes& func = task->func(); + FuncCompileResults& results = task->results(); + + masm_.haltingAlign(CodeAlignment); + + // Before merging in the new function's code, if calls in a prior function + // body might go out of range, insert far jumps to extend the range. + if ((masm_.size() - startOfUnpatchedCallsites_) + results.masm().size() > JumpRange()) { + startOfUnpatchedCallsites_ = masm_.size(); + if (!patchCallSites()) + return false; + } + + // Offset the recorded FuncOffsets by the offset of the function in the + // whole module's code segment. + uint32_t offsetInWhole = masm_.size(); + results.offsets().offsetBy(offsetInWhole); + + // Add the CodeRange for this function. + uint32_t funcCodeRangeIndex = metadata_->codeRanges.length(); + if (!metadata_->codeRanges.emplaceBack(func.index(), func.lineOrBytecode(), results.offsets())) + return false; + + MOZ_ASSERT(!funcIsCompiled(func.index())); + funcToCodeRange_[func.index()] = funcCodeRangeIndex; + + // Merge the compiled results into the whole-module masm. + mozilla::DebugOnly<size_t> sizeBefore = masm_.size(); + if (!masm_.asmMergeWith(results.masm())) + return false; + MOZ_ASSERT(masm_.size() == offsetInWhole + results.masm().size()); + + freeTasks_.infallibleAppend(task); + return true; +} + +bool +ModuleGenerator::finishFuncExports() +{ + // In addition to all the functions that were explicitly exported, any + // element of an exported table is also exported. + + for (ElemSegment& elems : elemSegments_) { + if (shared_->tables[elems.tableIndex].external) { + for (uint32_t funcIndex : elems.elemFuncIndices) { + if (!exportedFuncs_.put(funcIndex)) + return false; + } + } + } + + // ModuleGenerator::exportedFuncs_ is an unordered HashSet. The + // FuncExportVector stored in Metadata needs to be stored sorted by + // function index to allow O(log(n)) lookup at runtime. + + Uint32Vector sorted; + if (!sorted.reserve(exportedFuncs_.count())) + return false; + + for (Uint32Set::Range r = exportedFuncs_.all(); !r.empty(); r.popFront()) + sorted.infallibleAppend(r.front()); + + std::sort(sorted.begin(), sorted.end()); + + MOZ_ASSERT(metadata_->funcExports.empty()); + if (!metadata_->funcExports.reserve(sorted.length())) + return false; + + for (uint32_t funcIndex : sorted) { + Sig sig; + if (!sig.clone(funcSig(funcIndex))) + return false; + + uint32_t codeRangeIndex = funcToCodeRange_[funcIndex]; + metadata_->funcExports.infallibleEmplaceBack(Move(sig), funcIndex, codeRangeIndex); + } + + return true; +} + +typedef Vector<Offsets, 0, SystemAllocPolicy> OffsetVector; +typedef Vector<ProfilingOffsets, 0, SystemAllocPolicy> ProfilingOffsetVector; + +bool +ModuleGenerator::finishCodegen() +{ + masm_.haltingAlign(CodeAlignment); + uint32_t offsetInWhole = masm_.size(); + + uint32_t numFuncExports = metadata_->funcExports.length(); + MOZ_ASSERT(numFuncExports == exportedFuncs_.count()); + + // Generate stubs in a separate MacroAssembler since, otherwise, for modules + // larger than the JumpImmediateRange, even local uses of Label will fail + // due to the large absolute offsets temporarily stored by Label::bind(). + + OffsetVector entries; + ProfilingOffsetVector interpExits; + ProfilingOffsetVector jitExits; + TrapExitOffsetArray trapExits; + Offsets outOfBoundsExit; + Offsets unalignedAccessExit; + Offsets interruptExit; + Offsets throwStub; + + { + TempAllocator alloc(&lifo_); + MacroAssembler masm(MacroAssembler::WasmToken(), alloc); + Label throwLabel; + + if (!entries.resize(numFuncExports)) + return false; + for (uint32_t i = 0; i < numFuncExports; i++) + entries[i] = GenerateEntry(masm, metadata_->funcExports[i]); + + if (!interpExits.resize(numFuncImports())) + return false; + if (!jitExits.resize(numFuncImports())) + return false; + for (uint32_t i = 0; i < numFuncImports(); i++) { + interpExits[i] = GenerateImportInterpExit(masm, metadata_->funcImports[i], i, &throwLabel); + jitExits[i] = GenerateImportJitExit(masm, metadata_->funcImports[i], &throwLabel); + } + + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) + trapExits[trap] = GenerateTrapExit(masm, trap, &throwLabel); + + outOfBoundsExit = GenerateOutOfBoundsExit(masm, &throwLabel); + unalignedAccessExit = GenerateUnalignedExit(masm, &throwLabel); + interruptExit = GenerateInterruptExit(masm, &throwLabel); + throwStub = GenerateThrowStub(masm, &throwLabel); + + if (masm.oom() || !masm_.asmMergeWith(masm)) + return false; + } + + // Adjust each of the resulting Offsets (to account for being merged into + // masm_) and then create code ranges for all the stubs. + + for (uint32_t i = 0; i < numFuncExports; i++) { + entries[i].offsetBy(offsetInWhole); + metadata_->funcExports[i].initEntryOffset(entries[i].begin); + if (!metadata_->codeRanges.emplaceBack(CodeRange::Entry, entries[i])) + return false; + } + + for (uint32_t i = 0; i < numFuncImports(); i++) { + interpExits[i].offsetBy(offsetInWhole); + metadata_->funcImports[i].initInterpExitOffset(interpExits[i].begin); + if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportInterpExit, interpExits[i])) + return false; + + jitExits[i].offsetBy(offsetInWhole); + metadata_->funcImports[i].initJitExitOffset(jitExits[i].begin); + if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportJitExit, jitExits[i])) + return false; + } + + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + trapExits[trap].offsetBy(offsetInWhole); + if (!metadata_->codeRanges.emplaceBack(CodeRange::TrapExit, trapExits[trap])) + return false; + } + + outOfBoundsExit.offsetBy(offsetInWhole); + if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, outOfBoundsExit)) + return false; + + unalignedAccessExit.offsetBy(offsetInWhole); + if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, unalignedAccessExit)) + return false; + + interruptExit.offsetBy(offsetInWhole); + if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, interruptExit)) + return false; + + throwStub.offsetBy(offsetInWhole); + if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, throwStub)) + return false; + + // Fill in LinkData with the offsets of these stubs. + + linkData_.outOfBoundsOffset = outOfBoundsExit.begin; + linkData_.interruptOffset = interruptExit.begin; + + // Now that all other code has been emitted, patch all remaining callsites + // then far jumps. Patching callsites can generate far jumps so there is an + // ordering dependency. + + if (!patchCallSites(&trapExits)) + return false; + + if (!patchFarJumps(trapExits)) + return false; + + // Code-generation is complete! + + masm_.finish(); + return !masm_.oom(); +} + +bool +ModuleGenerator::finishLinkData(Bytes& code) +{ + // Inflate the global bytes up to page size so that the total bytes are a + // page size (as required by the allocator functions). + linkData_.globalDataLength = AlignBytes(linkData_.globalDataLength, gc::SystemPageSize()); + + // Add links to absolute addresses identified symbolically. + for (size_t i = 0; i < masm_.numSymbolicAccesses(); i++) { + SymbolicAccess src = masm_.symbolicAccess(i); + if (!linkData_.symbolicLinks[src.target].append(src.patchAt.offset())) + return false; + } + + // Relative link metadata: absolute addresses that refer to another point within + // the asm.js module. + + // CodeLabels are used for switch cases and loads from floating-point / + // SIMD values in the constant pool. + for (size_t i = 0; i < masm_.numCodeLabels(); i++) { + CodeLabel cl = masm_.codeLabel(i); + LinkData::InternalLink inLink(LinkData::InternalLink::CodeLabel); + inLink.patchAtOffset = masm_.labelToPatchOffset(*cl.patchAt()); + inLink.targetOffset = cl.target()->offset(); + if (!linkData_.internalLinks.append(inLink)) + return false; + } + +#if defined(JS_CODEGEN_X86) + // Global data accesses in x86 need to be patched with the absolute + // address of the global. Globals are allocated sequentially after the + // code section so we can just use an InternalLink. + for (GlobalAccess a : masm_.globalAccesses()) { + LinkData::InternalLink inLink(LinkData::InternalLink::RawPointer); + inLink.patchAtOffset = masm_.labelToPatchOffset(a.patchAt); + inLink.targetOffset = code.length() + a.globalDataOffset; + if (!linkData_.internalLinks.append(inLink)) + return false; + } +#elif defined(JS_CODEGEN_X64) + // Global data accesses on x64 use rip-relative addressing and thus we can + // patch here, now that we know the final codeLength. + for (GlobalAccess a : masm_.globalAccesses()) { + void* from = code.begin() + a.patchAt.offset(); + void* to = code.end() + a.globalDataOffset; + X86Encoding::SetRel32(from, to); + } +#else + // Global access is performed using the GlobalReg and requires no patching. + MOZ_ASSERT(masm_.globalAccesses().length() == 0); +#endif + + return true; +} + +bool +ModuleGenerator::addFuncImport(const Sig& sig, uint32_t globalDataOffset) +{ + MOZ_ASSERT(!finishedFuncDefs_); + + Sig copy; + if (!copy.clone(sig)) + return false; + + return metadata_->funcImports.emplaceBack(Move(copy), globalDataOffset); +} + +bool +ModuleGenerator::allocateGlobalBytes(uint32_t bytes, uint32_t align, uint32_t* globalDataOffset) +{ + CheckedInt<uint32_t> newGlobalDataLength(linkData_.globalDataLength); + + newGlobalDataLength += ComputeByteAlignment(newGlobalDataLength.value(), align); + if (!newGlobalDataLength.isValid()) + return false; + + *globalDataOffset = newGlobalDataLength.value(); + newGlobalDataLength += bytes; + + if (!newGlobalDataLength.isValid()) + return false; + + linkData_.globalDataLength = newGlobalDataLength.value(); + return true; +} + +bool +ModuleGenerator::allocateGlobal(GlobalDesc* global) +{ + MOZ_ASSERT(!startedFuncDefs_); + unsigned width = 0; + switch (global->type()) { + case ValType::I32: + case ValType::F32: + width = 4; + break; + case ValType::I64: + case ValType::F64: + width = 8; + break; + case ValType::I8x16: + case ValType::I16x8: + case ValType::I32x4: + case ValType::F32x4: + case ValType::B8x16: + case ValType::B16x8: + case ValType::B32x4: + width = 16; + break; + } + + uint32_t offset; + if (!allocateGlobalBytes(width, width, &offset)) + return false; + + global->setOffset(offset); + return true; +} + +bool +ModuleGenerator::addGlobal(ValType type, bool isConst, uint32_t* index) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(!startedFuncDefs_); + + *index = shared_->globals.length(); + GlobalDesc global(type, !isConst, *index); + if (!allocateGlobal(&global)) + return false; + + return shared_->globals.append(global); +} + +void +ModuleGenerator::initSig(uint32_t sigIndex, Sig&& sig) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(sigIndex == numSigs_); + numSigs_++; + + MOZ_ASSERT(shared_->sigs[sigIndex] == Sig()); + shared_->sigs[sigIndex] = Move(sig); +} + +const SigWithId& +ModuleGenerator::sig(uint32_t index) const +{ + MOZ_ASSERT(index < numSigs_); + return shared_->sigs[index]; +} + +void +ModuleGenerator::initFuncSig(uint32_t funcIndex, uint32_t sigIndex) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(!shared_->funcSigs[funcIndex]); + + shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex]; +} + +void +ModuleGenerator::initMemoryUsage(MemoryUsage memoryUsage) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(shared_->memoryUsage == MemoryUsage::None); + + shared_->memoryUsage = memoryUsage; +} + +void +ModuleGenerator::bumpMinMemoryLength(uint32_t newMinMemoryLength) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(newMinMemoryLength >= shared_->minMemoryLength); + + shared_->minMemoryLength = newMinMemoryLength; +} + +bool +ModuleGenerator::initImport(uint32_t funcIndex, uint32_t sigIndex) +{ + MOZ_ASSERT(isAsmJS()); + + MOZ_ASSERT(!shared_->funcSigs[funcIndex]); + shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex]; + + uint32_t globalDataOffset; + if (!allocateGlobalBytes(sizeof(FuncImportTls), sizeof(void*), &globalDataOffset)) + return false; + + MOZ_ASSERT(!shared_->funcImportGlobalDataOffsets[funcIndex]); + shared_->funcImportGlobalDataOffsets[funcIndex] = globalDataOffset; + + MOZ_ASSERT(funcIndex == metadata_->funcImports.length()); + return addFuncImport(sig(sigIndex), globalDataOffset); +} + +uint32_t +ModuleGenerator::numFuncImports() const +{ + // Until all functions have been validated, asm.js doesn't know the total + // number of imports. + MOZ_ASSERT_IF(isAsmJS(), finishedFuncDefs_); + return metadata_->funcImports.length(); +} + +uint32_t +ModuleGenerator::numFuncDefs() const +{ + // asm.js overallocates the length of funcSigs and in general does not know + // the number of function definitions until it's done compiling. + MOZ_ASSERT(!isAsmJS()); + return shared_->funcSigs.length() - numFuncImports(); +} + +uint32_t +ModuleGenerator::numFuncs() const +{ + // asm.js pre-reserves a bunch of function index space which is + // incrementally filled in during function-body validation. Thus, there are + // a few possible interpretations of numFuncs() (total index space size vs. + // exact number of imports/definitions encountered so far) and to simplify + // things we simply only define this quantity for wasm. + MOZ_ASSERT(!isAsmJS()); + return shared_->funcSigs.length(); +} + +const SigWithId& +ModuleGenerator::funcSig(uint32_t funcIndex) const +{ + MOZ_ASSERT(shared_->funcSigs[funcIndex]); + return *shared_->funcSigs[funcIndex]; +} + +bool +ModuleGenerator::addFuncExport(UniqueChars fieldName, uint32_t funcIndex) +{ + return exportedFuncs_.put(funcIndex) && + exports_.emplaceBack(Move(fieldName), funcIndex, DefinitionKind::Function); +} + +bool +ModuleGenerator::addTableExport(UniqueChars fieldName) +{ + MOZ_ASSERT(!startedFuncDefs_); + MOZ_ASSERT(shared_->tables.length() == 1); + shared_->tables[0].external = true; + return exports_.emplaceBack(Move(fieldName), DefinitionKind::Table); +} + +bool +ModuleGenerator::addMemoryExport(UniqueChars fieldName) +{ + return exports_.emplaceBack(Move(fieldName), DefinitionKind::Memory); +} + +bool +ModuleGenerator::addGlobalExport(UniqueChars fieldName, uint32_t globalIndex) +{ + return exports_.emplaceBack(Move(fieldName), globalIndex, DefinitionKind::Global); +} + +bool +ModuleGenerator::setStartFunction(uint32_t funcIndex) +{ + metadata_->startFuncIndex.emplace(funcIndex); + return exportedFuncs_.put(funcIndex); +} + +bool +ModuleGenerator::addElemSegment(InitExpr offset, Uint32Vector&& elemFuncIndices) +{ + MOZ_ASSERT(!isAsmJS()); + MOZ_ASSERT(!startedFuncDefs_); + MOZ_ASSERT(shared_->tables.length() == 1); + + for (uint32_t funcIndex : elemFuncIndices) { + if (funcIndex < numFuncImports()) { + shared_->tables[0].external = true; + break; + } + } + + return elemSegments_.emplaceBack(0, offset, Move(elemFuncIndices)); +} + +void +ModuleGenerator::setDataSegments(DataSegmentVector&& segments) +{ + MOZ_ASSERT(dataSegments_.empty()); + dataSegments_ = Move(segments); +} + +bool +ModuleGenerator::startFuncDefs() +{ + MOZ_ASSERT(!startedFuncDefs_); + MOZ_ASSERT(!finishedFuncDefs_); + + // The wasmCompilationInProgress atomic ensures that there is only one + // parallel compilation in progress at a time. In the special case of + // asm.js, where the ModuleGenerator itself can be on a helper thread, this + // avoids the possibility of deadlock since at most 1 helper thread will be + // blocking on other helper threads and there are always >1 helper threads. + // With wasm, this restriction could be relaxed by moving the worklist state + // out of HelperThreadState since each independent compilation needs its own + // worklist pair. Alternatively, the deadlock could be avoided by having the + // ModuleGenerator thread make progress (on compile tasks) instead of + // blocking. + + GlobalHelperThreadState& threads = HelperThreadState(); + MOZ_ASSERT(threads.threadCount > 1); + + uint32_t numTasks; + if (CanUseExtraThreads() && threads.wasmCompilationInProgress.compareExchange(false, true)) { +#ifdef DEBUG + { + AutoLockHelperThreadState lock; + MOZ_ASSERT(!HelperThreadState().wasmFailed(lock)); + MOZ_ASSERT(HelperThreadState().wasmWorklist(lock).empty()); + MOZ_ASSERT(HelperThreadState().wasmFinishedList(lock).empty()); + } +#endif + parallel_ = true; + numTasks = 2 * threads.maxWasmCompilationThreads(); + } else { + numTasks = 1; + } + + if (!tasks_.initCapacity(numTasks)) + return false; + for (size_t i = 0; i < numTasks; i++) + tasks_.infallibleEmplaceBack(*shared_, COMPILATION_LIFO_DEFAULT_CHUNK_SIZE); + + if (!freeTasks_.reserve(numTasks)) + return false; + for (size_t i = 0; i < numTasks; i++) + freeTasks_.infallibleAppend(&tasks_[i]); + + startedFuncDefs_ = true; + MOZ_ASSERT(!finishedFuncDefs_); + return true; +} + +bool +ModuleGenerator::startFuncDef(uint32_t lineOrBytecode, FunctionGenerator* fg) +{ + MOZ_ASSERT(startedFuncDefs_); + MOZ_ASSERT(!activeFuncDef_); + MOZ_ASSERT(!finishedFuncDefs_); + + if (freeTasks_.empty() && !finishOutstandingTask()) + return false; + + IonCompileTask* task = freeTasks_.popCopy(); + + task->reset(&fg->bytes_); + fg->bytes_.clear(); + fg->lineOrBytecode_ = lineOrBytecode; + fg->m_ = this; + fg->task_ = task; + activeFuncDef_ = fg; + return true; +} + +bool +ModuleGenerator::finishFuncDef(uint32_t funcIndex, FunctionGenerator* fg) +{ + MOZ_ASSERT(activeFuncDef_ == fg); + + auto func = js::MakeUnique<FuncBytes>(Move(fg->bytes_), + funcIndex, + funcSig(funcIndex), + fg->lineOrBytecode_, + Move(fg->callSiteLineNums_)); + if (!func) + return false; + + auto mode = alwaysBaseline_ && BaselineCanCompile(fg) + ? IonCompileTask::CompileMode::Baseline + : IonCompileTask::CompileMode::Ion; + + fg->task_->init(Move(func), mode); + + if (parallel_) { + if (!StartOffThreadWasmCompile(fg->task_)) + return false; + outstanding_++; + } else { + if (!CompileFunction(fg->task_)) + return false; + if (!finishTask(fg->task_)) + return false; + } + + fg->m_ = nullptr; + fg->task_ = nullptr; + activeFuncDef_ = nullptr; + numFinishedFuncDefs_++; + return true; +} + +bool +ModuleGenerator::finishFuncDefs() +{ + MOZ_ASSERT(startedFuncDefs_); + MOZ_ASSERT(!activeFuncDef_); + MOZ_ASSERT(!finishedFuncDefs_); + + while (outstanding_ > 0) { + if (!finishOutstandingTask()) + return false; + } + + linkData_.functionCodeLength = masm_.size(); + finishedFuncDefs_ = true; + + // Generate wrapper functions for every import. These wrappers turn imports + // into plain functions so they can be put into tables and re-exported. + // asm.js cannot do either and so no wrappers are generated. + + if (!isAsmJS()) { + for (size_t funcIndex = 0; funcIndex < numFuncImports(); funcIndex++) { + const FuncImport& funcImport = metadata_->funcImports[funcIndex]; + const SigWithId& sig = funcSig(funcIndex); + + FuncOffsets offsets = GenerateImportFunction(masm_, funcImport, sig.id); + if (masm_.oom()) + return false; + + uint32_t codeRangeIndex = metadata_->codeRanges.length(); + if (!metadata_->codeRanges.emplaceBack(funcIndex, /* bytecodeOffset = */ 0, offsets)) + return false; + + MOZ_ASSERT(!funcIsCompiled(funcIndex)); + funcToCodeRange_[funcIndex] = codeRangeIndex; + } + } + + // All function indices should have an associated code range at this point + // (except in asm.js, which doesn't have import wrapper functions). + +#ifdef DEBUG + if (isAsmJS()) { + MOZ_ASSERT(numFuncImports() < AsmJSFirstDefFuncIndex); + for (uint32_t i = 0; i < AsmJSFirstDefFuncIndex; i++) + MOZ_ASSERT(funcToCodeRange_[i] == BAD_CODE_RANGE); + for (uint32_t i = AsmJSFirstDefFuncIndex; i < numFinishedFuncDefs_; i++) + MOZ_ASSERT(funcCodeRange(i).funcIndex() == i); + } else { + MOZ_ASSERT(numFinishedFuncDefs_ == numFuncDefs()); + for (uint32_t i = 0; i < numFuncs(); i++) + MOZ_ASSERT(funcCodeRange(i).funcIndex() == i); + } +#endif + + // Complete element segments with the code range index of every element, now + // that all functions have been compiled. + + for (ElemSegment& elems : elemSegments_) { + Uint32Vector& codeRangeIndices = elems.elemCodeRangeIndices; + + MOZ_ASSERT(codeRangeIndices.empty()); + if (!codeRangeIndices.reserve(elems.elemFuncIndices.length())) + return false; + + for (uint32_t funcIndex : elems.elemFuncIndices) + codeRangeIndices.infallibleAppend(funcToCodeRange_[funcIndex]); + } + + return true; +} + +void +ModuleGenerator::setFuncNames(NameInBytecodeVector&& funcNames) +{ + MOZ_ASSERT(metadata_->funcNames.empty()); + metadata_->funcNames = Move(funcNames); +} + +bool +ModuleGenerator::initSigTableLength(uint32_t sigIndex, uint32_t length) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(length != 0); + MOZ_ASSERT(length <= MaxTableElems); + + MOZ_ASSERT(shared_->asmJSSigToTableIndex[sigIndex] == 0); + shared_->asmJSSigToTableIndex[sigIndex] = numTables_; + + TableDesc& table = shared_->tables[numTables_++]; + table.kind = TableKind::TypedFunction; + table.limits.initial = length; + table.limits.maximum = Some(length); + return allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset); +} + +bool +ModuleGenerator::initSigTableElems(uint32_t sigIndex, Uint32Vector&& elemFuncIndices) +{ + MOZ_ASSERT(isAsmJS()); + MOZ_ASSERT(finishedFuncDefs_); + + uint32_t tableIndex = shared_->asmJSSigToTableIndex[sigIndex]; + MOZ_ASSERT(shared_->tables[tableIndex].limits.initial == elemFuncIndices.length()); + + Uint32Vector codeRangeIndices; + if (!codeRangeIndices.resize(elemFuncIndices.length())) + return false; + for (size_t i = 0; i < elemFuncIndices.length(); i++) + codeRangeIndices[i] = funcToCodeRange_[elemFuncIndices[i]]; + + InitExpr offset(Val(uint32_t(0))); + if (!elemSegments_.emplaceBack(tableIndex, offset, Move(elemFuncIndices))) + return false; + + elemSegments_.back().elemCodeRangeIndices = Move(codeRangeIndices); + return true; +} + +SharedModule +ModuleGenerator::finish(const ShareableBytes& bytecode) +{ + MOZ_ASSERT(!activeFuncDef_); + MOZ_ASSERT(finishedFuncDefs_); + + if (!finishFuncExports()) + return nullptr; + + if (!finishCodegen()) + return nullptr; + + // Round up the code size to page size since this is eventually required by + // the executable-code allocator and for setting memory protection. + uint32_t bytesNeeded = masm_.bytesNeeded(); + uint32_t padding = ComputeByteAlignment(bytesNeeded, gc::SystemPageSize()); + + // Use initLengthUninitialized so there is no round-up allocation nor time + // wasted zeroing memory. + Bytes code; + if (!code.initLengthUninitialized(bytesNeeded + padding)) + return nullptr; + + // Delay flushing of the icache until CodeSegment::create since there is + // more patching to do before this code becomes executable. + { + AutoFlushICache afc("ModuleGenerator::finish", /* inhibit = */ true); + masm_.executableCopy(code.begin()); + } + + // Zero the padding, since we used resizeUninitialized above. + memset(code.begin() + bytesNeeded, 0, padding); + + // Convert the CallSiteAndTargetVector (needed during generation) to a + // CallSiteVector (what is stored in the Module). + if (!metadata_->callSites.appendAll(masm_.callSites())) + return nullptr; + + // The MacroAssembler has accumulated all the memory accesses during codegen. + metadata_->memoryAccesses = masm_.extractMemoryAccesses(); + metadata_->memoryPatches = masm_.extractMemoryPatches(); + metadata_->boundsChecks = masm_.extractBoundsChecks(); + + // Copy over data from the ModuleGeneratorData. + metadata_->memoryUsage = shared_->memoryUsage; + metadata_->minMemoryLength = shared_->minMemoryLength; + metadata_->maxMemoryLength = shared_->maxMemoryLength; + metadata_->tables = Move(shared_->tables); + metadata_->globals = Move(shared_->globals); + + // These Vectors can get large and the excess capacity can be significant, + // so realloc them down to size. + metadata_->memoryAccesses.podResizeToFit(); + metadata_->memoryPatches.podResizeToFit(); + metadata_->boundsChecks.podResizeToFit(); + metadata_->codeRanges.podResizeToFit(); + metadata_->callSites.podResizeToFit(); + metadata_->callThunks.podResizeToFit(); + + // For asm.js, the tables vector is over-allocated (to avoid resize during + // parallel copilation). Shrink it back down to fit. + if (isAsmJS() && !metadata_->tables.resize(numTables_)) + return nullptr; + + // Assert CodeRanges are sorted. +#ifdef DEBUG + uint32_t lastEnd = 0; + for (const CodeRange& codeRange : metadata_->codeRanges) { + MOZ_ASSERT(codeRange.begin() >= lastEnd); + lastEnd = codeRange.end(); + } +#endif + + if (!finishLinkData(code)) + return nullptr; + + return SharedModule(js_new<Module>(Move(assumptions_), + Move(code), + Move(linkData_), + Move(imports_), + Move(exports_), + Move(dataSegments_), + Move(elemSegments_), + *metadata_, + bytecode)); +} |