summaryrefslogtreecommitdiffstats
path: root/js/src/wasm/WasmGenerator.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/wasm/WasmGenerator.cpp')
-rw-r--r--js/src/wasm/WasmGenerator.cpp1174
1 files changed, 1174 insertions, 0 deletions
diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp
new file mode 100644
index 000000000..e6f1edd99
--- /dev/null
+++ b/js/src/wasm/WasmGenerator.cpp
@@ -0,0 +1,1174 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ *
+ * Copyright 2015 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "wasm/WasmGenerator.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/EnumeratedRange.h"
+
+#include <algorithm>
+
+#include "wasm/WasmBaselineCompile.h"
+#include "wasm/WasmIonCompile.h"
+#include "wasm/WasmStubs.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+using namespace js::wasm;
+
+using mozilla::CheckedInt;
+using mozilla::MakeEnumeratedRange;
+
+// ****************************************************************************
+// ModuleGenerator
+
+static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024;
+static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024;
+static const uint32_t BAD_CODE_RANGE = UINT32_MAX;
+
+ModuleGenerator::ModuleGenerator(ImportVector&& imports)
+ : alwaysBaseline_(false),
+ imports_(Move(imports)),
+ numSigs_(0),
+ numTables_(0),
+ lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE),
+ masmAlloc_(&lifo_),
+ masm_(MacroAssembler::WasmToken(), masmAlloc_),
+ lastPatchedCallsite_(0),
+ startOfUnpatchedCallsites_(0),
+ parallel_(false),
+ outstanding_(0),
+ activeFuncDef_(nullptr),
+ startedFuncDefs_(false),
+ finishedFuncDefs_(false),
+ numFinishedFuncDefs_(0)
+{
+ MOZ_ASSERT(IsCompilingWasm());
+}
+
+ModuleGenerator::~ModuleGenerator()
+{
+ if (parallel_) {
+ // Wait for any outstanding jobs to fail or complete.
+ if (outstanding_) {
+ AutoLockHelperThreadState lock;
+ while (true) {
+ IonCompileTaskPtrVector& worklist = HelperThreadState().wasmWorklist(lock);
+ MOZ_ASSERT(outstanding_ >= worklist.length());
+ outstanding_ -= worklist.length();
+ worklist.clear();
+
+ IonCompileTaskPtrVector& finished = HelperThreadState().wasmFinishedList(lock);
+ MOZ_ASSERT(outstanding_ >= finished.length());
+ outstanding_ -= finished.length();
+ finished.clear();
+
+ uint32_t numFailed = HelperThreadState().harvestFailedWasmJobs(lock);
+ MOZ_ASSERT(outstanding_ >= numFailed);
+ outstanding_ -= numFailed;
+
+ if (!outstanding_)
+ break;
+
+ HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER);
+ }
+ }
+
+ MOZ_ASSERT(HelperThreadState().wasmCompilationInProgress);
+ HelperThreadState().wasmCompilationInProgress = false;
+ } else {
+ MOZ_ASSERT(!outstanding_);
+ }
+}
+
+bool
+ModuleGenerator::init(UniqueModuleGeneratorData shared, const CompileArgs& args,
+ Metadata* maybeAsmJSMetadata)
+{
+ shared_ = Move(shared);
+ alwaysBaseline_ = args.alwaysBaseline;
+
+ if (!exportedFuncs_.init())
+ return false;
+
+ if (!funcToCodeRange_.appendN(BAD_CODE_RANGE, shared_->funcSigs.length()))
+ return false;
+
+ linkData_.globalDataLength = AlignBytes(InitialGlobalDataBytes, sizeof(void*));;
+
+ // asm.js passes in an AsmJSMetadata subclass to use instead.
+ if (maybeAsmJSMetadata) {
+ metadata_ = maybeAsmJSMetadata;
+ MOZ_ASSERT(isAsmJS());
+ } else {
+ metadata_ = js_new<Metadata>();
+ if (!metadata_)
+ return false;
+ MOZ_ASSERT(!isAsmJS());
+ }
+
+ if (args.scriptedCaller.filename) {
+ metadata_->filename = DuplicateString(args.scriptedCaller.filename.get());
+ if (!metadata_->filename)
+ return false;
+ }
+
+ if (!assumptions_.clone(args.assumptions))
+ return false;
+
+ // For asm.js, the Vectors in ModuleGeneratorData are max-sized reservations
+ // and will be initialized in a linear order via init* functions as the
+ // module is generated. For wasm, the Vectors are correctly-sized and
+ // already initialized.
+
+ if (!isAsmJS()) {
+ numSigs_ = shared_->sigs.length();
+ numTables_ = shared_->tables.length();
+
+ for (size_t i = 0; i < shared_->funcImportGlobalDataOffsets.length(); i++) {
+ shared_->funcImportGlobalDataOffsets[i] = linkData_.globalDataLength;
+ linkData_.globalDataLength += sizeof(FuncImportTls);
+ if (!addFuncImport(*shared_->funcSigs[i], shared_->funcImportGlobalDataOffsets[i]))
+ return false;
+ }
+
+ for (const Import& import : imports_) {
+ if (import.kind == DefinitionKind::Table) {
+ MOZ_ASSERT(shared_->tables.length() == 1);
+ shared_->tables[0].external = true;
+ break;
+ }
+ }
+
+ for (TableDesc& table : shared_->tables) {
+ if (!allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset))
+ return false;
+ }
+
+ for (uint32_t i = 0; i < numSigs_; i++) {
+ SigWithId& sig = shared_->sigs[i];
+ if (SigIdDesc::isGlobal(sig)) {
+ uint32_t globalDataOffset;
+ if (!allocateGlobalBytes(sizeof(void*), sizeof(void*), &globalDataOffset))
+ return false;
+
+ sig.id = SigIdDesc::global(sig, globalDataOffset);
+
+ Sig copy;
+ if (!copy.clone(sig))
+ return false;
+
+ if (!metadata_->sigIds.emplaceBack(Move(copy), sig.id))
+ return false;
+ } else {
+ sig.id = SigIdDesc::immediate(sig);
+ }
+ }
+
+ for (GlobalDesc& global : shared_->globals) {
+ if (global.isConstant())
+ continue;
+ if (!allocateGlobal(&global))
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(shared_->sigs.length() == MaxSigs);
+ MOZ_ASSERT(shared_->tables.length() == MaxTables);
+ MOZ_ASSERT(shared_->asmJSSigToTableIndex.length() == MaxSigs);
+ }
+
+ return true;
+}
+
+bool
+ModuleGenerator::finishOutstandingTask()
+{
+ MOZ_ASSERT(parallel_);
+
+ IonCompileTask* task = nullptr;
+ {
+ AutoLockHelperThreadState lock;
+ while (true) {
+ MOZ_ASSERT(outstanding_ > 0);
+
+ if (HelperThreadState().wasmFailed(lock))
+ return false;
+
+ if (!HelperThreadState().wasmFinishedList(lock).empty()) {
+ outstanding_--;
+ task = HelperThreadState().wasmFinishedList(lock).popCopy();
+ break;
+ }
+
+ HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER);
+ }
+ }
+
+ return finishTask(task);
+}
+
+bool
+ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const
+{
+ return funcToCodeRange_[funcIndex] != BAD_CODE_RANGE;
+}
+
+const CodeRange&
+ModuleGenerator::funcCodeRange(uint32_t funcIndex) const
+{
+ MOZ_ASSERT(funcIsCompiled(funcIndex));
+ const CodeRange& cr = metadata_->codeRanges[funcToCodeRange_[funcIndex]];
+ MOZ_ASSERT(cr.isFunction());
+ return cr;
+}
+
+static uint32_t
+JumpRange()
+{
+ return Min(JitOptions.jumpThreshold, JumpImmediateRange);
+}
+
+typedef HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy> OffsetMap;
+
+bool
+ModuleGenerator::patchCallSites(TrapExitOffsetArray* maybeTrapExits)
+{
+ MacroAssembler::AutoPrepareForPatching patching(masm_);
+
+ masm_.haltingAlign(CodeAlignment);
+
+ // Create far jumps for calls that have relative offsets that may otherwise
+ // go out of range. Far jumps are created for two cases: direct calls
+ // between function definitions and calls to trap exits by trap out-of-line
+ // paths. Far jump code is shared when possible to reduce bloat. This method
+ // is called both between function bodies (at a frequency determined by the
+ // ISA's jump range) and once at the very end of a module's codegen after
+ // all possible calls/traps have been emitted.
+
+ OffsetMap existingCallFarJumps;
+ if (!existingCallFarJumps.init())
+ return false;
+
+ EnumeratedArray<Trap, Trap::Limit, Maybe<uint32_t>> existingTrapFarJumps;
+
+ for (; lastPatchedCallsite_ < masm_.callSites().length(); lastPatchedCallsite_++) {
+ const CallSiteAndTarget& cs = masm_.callSites()[lastPatchedCallsite_];
+ uint32_t callerOffset = cs.returnAddressOffset();
+ MOZ_RELEASE_ASSERT(callerOffset < INT32_MAX);
+
+ switch (cs.kind()) {
+ case CallSiteDesc::Dynamic:
+ case CallSiteDesc::Symbolic:
+ break;
+ case CallSiteDesc::Func: {
+ if (funcIsCompiled(cs.funcIndex())) {
+ uint32_t calleeOffset = funcCodeRange(cs.funcIndex()).funcNonProfilingEntry();
+ MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX);
+
+ if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) {
+ masm_.patchCall(callerOffset, calleeOffset);
+ break;
+ }
+ }
+
+ OffsetMap::AddPtr p = existingCallFarJumps.lookupForAdd(cs.funcIndex());
+ if (!p) {
+ Offsets offsets;
+ offsets.begin = masm_.currentOffset();
+ uint32_t jumpOffset = masm_.farJumpWithPatch().offset();
+ offsets.end = masm_.currentOffset();
+ if (masm_.oom())
+ return false;
+
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets))
+ return false;
+ if (!existingCallFarJumps.add(p, cs.funcIndex(), offsets.begin))
+ return false;
+
+ // Record calls' far jumps in metadata since they must be
+ // repatched at runtime when profiling mode is toggled.
+ if (!metadata_->callThunks.emplaceBack(jumpOffset, cs.funcIndex()))
+ return false;
+ }
+
+ masm_.patchCall(callerOffset, p->value());
+ break;
+ }
+ case CallSiteDesc::TrapExit: {
+ if (maybeTrapExits) {
+ uint32_t calleeOffset = (*maybeTrapExits)[cs.trap()].begin;
+ MOZ_RELEASE_ASSERT(calleeOffset < INT32_MAX);
+
+ if (uint32_t(abs(int32_t(calleeOffset) - int32_t(callerOffset))) < JumpRange()) {
+ masm_.patchCall(callerOffset, calleeOffset);
+ break;
+ }
+ }
+
+ if (!existingTrapFarJumps[cs.trap()]) {
+ Offsets offsets;
+ offsets.begin = masm_.currentOffset();
+ masm_.append(TrapFarJump(cs.trap(), masm_.farJumpWithPatch()));
+ offsets.end = masm_.currentOffset();
+ if (masm_.oom())
+ return false;
+
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, offsets))
+ return false;
+ existingTrapFarJumps[cs.trap()] = Some(offsets.begin);
+ }
+
+ masm_.patchCall(callerOffset, *existingTrapFarJumps[cs.trap()]);
+ break;
+ }
+ }
+ }
+
+ return true;
+}
+
+bool
+ModuleGenerator::patchFarJumps(const TrapExitOffsetArray& trapExits)
+{
+ MacroAssembler::AutoPrepareForPatching patching(masm_);
+
+ for (CallThunk& callThunk : metadata_->callThunks) {
+ uint32_t funcIndex = callThunk.u.funcIndex;
+ callThunk.u.codeRangeIndex = funcToCodeRange_[funcIndex];
+ CodeOffset farJump(callThunk.offset);
+ masm_.patchFarJump(farJump, funcCodeRange(funcIndex).funcNonProfilingEntry());
+ }
+
+ for (const TrapFarJump& farJump : masm_.trapFarJumps())
+ masm_.patchFarJump(farJump.jump, trapExits[farJump.trap].begin);
+
+ return true;
+}
+
+bool
+ModuleGenerator::finishTask(IonCompileTask* task)
+{
+ const FuncBytes& func = task->func();
+ FuncCompileResults& results = task->results();
+
+ masm_.haltingAlign(CodeAlignment);
+
+ // Before merging in the new function's code, if calls in a prior function
+ // body might go out of range, insert far jumps to extend the range.
+ if ((masm_.size() - startOfUnpatchedCallsites_) + results.masm().size() > JumpRange()) {
+ startOfUnpatchedCallsites_ = masm_.size();
+ if (!patchCallSites())
+ return false;
+ }
+
+ // Offset the recorded FuncOffsets by the offset of the function in the
+ // whole module's code segment.
+ uint32_t offsetInWhole = masm_.size();
+ results.offsets().offsetBy(offsetInWhole);
+
+ // Add the CodeRange for this function.
+ uint32_t funcCodeRangeIndex = metadata_->codeRanges.length();
+ if (!metadata_->codeRanges.emplaceBack(func.index(), func.lineOrBytecode(), results.offsets()))
+ return false;
+
+ MOZ_ASSERT(!funcIsCompiled(func.index()));
+ funcToCodeRange_[func.index()] = funcCodeRangeIndex;
+
+ // Merge the compiled results into the whole-module masm.
+ mozilla::DebugOnly<size_t> sizeBefore = masm_.size();
+ if (!masm_.asmMergeWith(results.masm()))
+ return false;
+ MOZ_ASSERT(masm_.size() == offsetInWhole + results.masm().size());
+
+ freeTasks_.infallibleAppend(task);
+ return true;
+}
+
+bool
+ModuleGenerator::finishFuncExports()
+{
+ // In addition to all the functions that were explicitly exported, any
+ // element of an exported table is also exported.
+
+ for (ElemSegment& elems : elemSegments_) {
+ if (shared_->tables[elems.tableIndex].external) {
+ for (uint32_t funcIndex : elems.elemFuncIndices) {
+ if (!exportedFuncs_.put(funcIndex))
+ return false;
+ }
+ }
+ }
+
+ // ModuleGenerator::exportedFuncs_ is an unordered HashSet. The
+ // FuncExportVector stored in Metadata needs to be stored sorted by
+ // function index to allow O(log(n)) lookup at runtime.
+
+ Uint32Vector sorted;
+ if (!sorted.reserve(exportedFuncs_.count()))
+ return false;
+
+ for (Uint32Set::Range r = exportedFuncs_.all(); !r.empty(); r.popFront())
+ sorted.infallibleAppend(r.front());
+
+ std::sort(sorted.begin(), sorted.end());
+
+ MOZ_ASSERT(metadata_->funcExports.empty());
+ if (!metadata_->funcExports.reserve(sorted.length()))
+ return false;
+
+ for (uint32_t funcIndex : sorted) {
+ Sig sig;
+ if (!sig.clone(funcSig(funcIndex)))
+ return false;
+
+ uint32_t codeRangeIndex = funcToCodeRange_[funcIndex];
+ metadata_->funcExports.infallibleEmplaceBack(Move(sig), funcIndex, codeRangeIndex);
+ }
+
+ return true;
+}
+
+typedef Vector<Offsets, 0, SystemAllocPolicy> OffsetVector;
+typedef Vector<ProfilingOffsets, 0, SystemAllocPolicy> ProfilingOffsetVector;
+
+bool
+ModuleGenerator::finishCodegen()
+{
+ masm_.haltingAlign(CodeAlignment);
+ uint32_t offsetInWhole = masm_.size();
+
+ uint32_t numFuncExports = metadata_->funcExports.length();
+ MOZ_ASSERT(numFuncExports == exportedFuncs_.count());
+
+ // Generate stubs in a separate MacroAssembler since, otherwise, for modules
+ // larger than the JumpImmediateRange, even local uses of Label will fail
+ // due to the large absolute offsets temporarily stored by Label::bind().
+
+ OffsetVector entries;
+ ProfilingOffsetVector interpExits;
+ ProfilingOffsetVector jitExits;
+ TrapExitOffsetArray trapExits;
+ Offsets outOfBoundsExit;
+ Offsets unalignedAccessExit;
+ Offsets interruptExit;
+ Offsets throwStub;
+
+ {
+ TempAllocator alloc(&lifo_);
+ MacroAssembler masm(MacroAssembler::WasmToken(), alloc);
+ Label throwLabel;
+
+ if (!entries.resize(numFuncExports))
+ return false;
+ for (uint32_t i = 0; i < numFuncExports; i++)
+ entries[i] = GenerateEntry(masm, metadata_->funcExports[i]);
+
+ if (!interpExits.resize(numFuncImports()))
+ return false;
+ if (!jitExits.resize(numFuncImports()))
+ return false;
+ for (uint32_t i = 0; i < numFuncImports(); i++) {
+ interpExits[i] = GenerateImportInterpExit(masm, metadata_->funcImports[i], i, &throwLabel);
+ jitExits[i] = GenerateImportJitExit(masm, metadata_->funcImports[i], &throwLabel);
+ }
+
+ for (Trap trap : MakeEnumeratedRange(Trap::Limit))
+ trapExits[trap] = GenerateTrapExit(masm, trap, &throwLabel);
+
+ outOfBoundsExit = GenerateOutOfBoundsExit(masm, &throwLabel);
+ unalignedAccessExit = GenerateUnalignedExit(masm, &throwLabel);
+ interruptExit = GenerateInterruptExit(masm, &throwLabel);
+ throwStub = GenerateThrowStub(masm, &throwLabel);
+
+ if (masm.oom() || !masm_.asmMergeWith(masm))
+ return false;
+ }
+
+ // Adjust each of the resulting Offsets (to account for being merged into
+ // masm_) and then create code ranges for all the stubs.
+
+ for (uint32_t i = 0; i < numFuncExports; i++) {
+ entries[i].offsetBy(offsetInWhole);
+ metadata_->funcExports[i].initEntryOffset(entries[i].begin);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::Entry, entries[i]))
+ return false;
+ }
+
+ for (uint32_t i = 0; i < numFuncImports(); i++) {
+ interpExits[i].offsetBy(offsetInWhole);
+ metadata_->funcImports[i].initInterpExitOffset(interpExits[i].begin);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportInterpExit, interpExits[i]))
+ return false;
+
+ jitExits[i].offsetBy(offsetInWhole);
+ metadata_->funcImports[i].initJitExitOffset(jitExits[i].begin);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::ImportJitExit, jitExits[i]))
+ return false;
+ }
+
+ for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
+ trapExits[trap].offsetBy(offsetInWhole);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::TrapExit, trapExits[trap]))
+ return false;
+ }
+
+ outOfBoundsExit.offsetBy(offsetInWhole);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, outOfBoundsExit))
+ return false;
+
+ unalignedAccessExit.offsetBy(offsetInWhole);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, unalignedAccessExit))
+ return false;
+
+ interruptExit.offsetBy(offsetInWhole);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, interruptExit))
+ return false;
+
+ throwStub.offsetBy(offsetInWhole);
+ if (!metadata_->codeRanges.emplaceBack(CodeRange::Inline, throwStub))
+ return false;
+
+ // Fill in LinkData with the offsets of these stubs.
+
+ linkData_.outOfBoundsOffset = outOfBoundsExit.begin;
+ linkData_.interruptOffset = interruptExit.begin;
+
+ // Now that all other code has been emitted, patch all remaining callsites
+ // then far jumps. Patching callsites can generate far jumps so there is an
+ // ordering dependency.
+
+ if (!patchCallSites(&trapExits))
+ return false;
+
+ if (!patchFarJumps(trapExits))
+ return false;
+
+ // Code-generation is complete!
+
+ masm_.finish();
+ return !masm_.oom();
+}
+
+bool
+ModuleGenerator::finishLinkData(Bytes& code)
+{
+ // Inflate the global bytes up to page size so that the total bytes are a
+ // page size (as required by the allocator functions).
+ linkData_.globalDataLength = AlignBytes(linkData_.globalDataLength, gc::SystemPageSize());
+
+ // Add links to absolute addresses identified symbolically.
+ for (size_t i = 0; i < masm_.numSymbolicAccesses(); i++) {
+ SymbolicAccess src = masm_.symbolicAccess(i);
+ if (!linkData_.symbolicLinks[src.target].append(src.patchAt.offset()))
+ return false;
+ }
+
+ // Relative link metadata: absolute addresses that refer to another point within
+ // the asm.js module.
+
+ // CodeLabels are used for switch cases and loads from floating-point /
+ // SIMD values in the constant pool.
+ for (size_t i = 0; i < masm_.numCodeLabels(); i++) {
+ CodeLabel cl = masm_.codeLabel(i);
+ LinkData::InternalLink inLink(LinkData::InternalLink::CodeLabel);
+ inLink.patchAtOffset = masm_.labelToPatchOffset(*cl.patchAt());
+ inLink.targetOffset = cl.target()->offset();
+ if (!linkData_.internalLinks.append(inLink))
+ return false;
+ }
+
+#if defined(JS_CODEGEN_X86)
+ // Global data accesses in x86 need to be patched with the absolute
+ // address of the global. Globals are allocated sequentially after the
+ // code section so we can just use an InternalLink.
+ for (GlobalAccess a : masm_.globalAccesses()) {
+ LinkData::InternalLink inLink(LinkData::InternalLink::RawPointer);
+ inLink.patchAtOffset = masm_.labelToPatchOffset(a.patchAt);
+ inLink.targetOffset = code.length() + a.globalDataOffset;
+ if (!linkData_.internalLinks.append(inLink))
+ return false;
+ }
+#elif defined(JS_CODEGEN_X64)
+ // Global data accesses on x64 use rip-relative addressing and thus we can
+ // patch here, now that we know the final codeLength.
+ for (GlobalAccess a : masm_.globalAccesses()) {
+ void* from = code.begin() + a.patchAt.offset();
+ void* to = code.end() + a.globalDataOffset;
+ X86Encoding::SetRel32(from, to);
+ }
+#else
+ // Global access is performed using the GlobalReg and requires no patching.
+ MOZ_ASSERT(masm_.globalAccesses().length() == 0);
+#endif
+
+ return true;
+}
+
+bool
+ModuleGenerator::addFuncImport(const Sig& sig, uint32_t globalDataOffset)
+{
+ MOZ_ASSERT(!finishedFuncDefs_);
+
+ Sig copy;
+ if (!copy.clone(sig))
+ return false;
+
+ return metadata_->funcImports.emplaceBack(Move(copy), globalDataOffset);
+}
+
+bool
+ModuleGenerator::allocateGlobalBytes(uint32_t bytes, uint32_t align, uint32_t* globalDataOffset)
+{
+ CheckedInt<uint32_t> newGlobalDataLength(linkData_.globalDataLength);
+
+ newGlobalDataLength += ComputeByteAlignment(newGlobalDataLength.value(), align);
+ if (!newGlobalDataLength.isValid())
+ return false;
+
+ *globalDataOffset = newGlobalDataLength.value();
+ newGlobalDataLength += bytes;
+
+ if (!newGlobalDataLength.isValid())
+ return false;
+
+ linkData_.globalDataLength = newGlobalDataLength.value();
+ return true;
+}
+
+bool
+ModuleGenerator::allocateGlobal(GlobalDesc* global)
+{
+ MOZ_ASSERT(!startedFuncDefs_);
+ unsigned width = 0;
+ switch (global->type()) {
+ case ValType::I32:
+ case ValType::F32:
+ width = 4;
+ break;
+ case ValType::I64:
+ case ValType::F64:
+ width = 8;
+ break;
+ case ValType::I8x16:
+ case ValType::I16x8:
+ case ValType::I32x4:
+ case ValType::F32x4:
+ case ValType::B8x16:
+ case ValType::B16x8:
+ case ValType::B32x4:
+ width = 16;
+ break;
+ }
+
+ uint32_t offset;
+ if (!allocateGlobalBytes(width, width, &offset))
+ return false;
+
+ global->setOffset(offset);
+ return true;
+}
+
+bool
+ModuleGenerator::addGlobal(ValType type, bool isConst, uint32_t* index)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(!startedFuncDefs_);
+
+ *index = shared_->globals.length();
+ GlobalDesc global(type, !isConst, *index);
+ if (!allocateGlobal(&global))
+ return false;
+
+ return shared_->globals.append(global);
+}
+
+void
+ModuleGenerator::initSig(uint32_t sigIndex, Sig&& sig)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(sigIndex == numSigs_);
+ numSigs_++;
+
+ MOZ_ASSERT(shared_->sigs[sigIndex] == Sig());
+ shared_->sigs[sigIndex] = Move(sig);
+}
+
+const SigWithId&
+ModuleGenerator::sig(uint32_t index) const
+{
+ MOZ_ASSERT(index < numSigs_);
+ return shared_->sigs[index];
+}
+
+void
+ModuleGenerator::initFuncSig(uint32_t funcIndex, uint32_t sigIndex)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(!shared_->funcSigs[funcIndex]);
+
+ shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex];
+}
+
+void
+ModuleGenerator::initMemoryUsage(MemoryUsage memoryUsage)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(shared_->memoryUsage == MemoryUsage::None);
+
+ shared_->memoryUsage = memoryUsage;
+}
+
+void
+ModuleGenerator::bumpMinMemoryLength(uint32_t newMinMemoryLength)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(newMinMemoryLength >= shared_->minMemoryLength);
+
+ shared_->minMemoryLength = newMinMemoryLength;
+}
+
+bool
+ModuleGenerator::initImport(uint32_t funcIndex, uint32_t sigIndex)
+{
+ MOZ_ASSERT(isAsmJS());
+
+ MOZ_ASSERT(!shared_->funcSigs[funcIndex]);
+ shared_->funcSigs[funcIndex] = &shared_->sigs[sigIndex];
+
+ uint32_t globalDataOffset;
+ if (!allocateGlobalBytes(sizeof(FuncImportTls), sizeof(void*), &globalDataOffset))
+ return false;
+
+ MOZ_ASSERT(!shared_->funcImportGlobalDataOffsets[funcIndex]);
+ shared_->funcImportGlobalDataOffsets[funcIndex] = globalDataOffset;
+
+ MOZ_ASSERT(funcIndex == metadata_->funcImports.length());
+ return addFuncImport(sig(sigIndex), globalDataOffset);
+}
+
+uint32_t
+ModuleGenerator::numFuncImports() const
+{
+ // Until all functions have been validated, asm.js doesn't know the total
+ // number of imports.
+ MOZ_ASSERT_IF(isAsmJS(), finishedFuncDefs_);
+ return metadata_->funcImports.length();
+}
+
+uint32_t
+ModuleGenerator::numFuncDefs() const
+{
+ // asm.js overallocates the length of funcSigs and in general does not know
+ // the number of function definitions until it's done compiling.
+ MOZ_ASSERT(!isAsmJS());
+ return shared_->funcSigs.length() - numFuncImports();
+}
+
+uint32_t
+ModuleGenerator::numFuncs() const
+{
+ // asm.js pre-reserves a bunch of function index space which is
+ // incrementally filled in during function-body validation. Thus, there are
+ // a few possible interpretations of numFuncs() (total index space size vs.
+ // exact number of imports/definitions encountered so far) and to simplify
+ // things we simply only define this quantity for wasm.
+ MOZ_ASSERT(!isAsmJS());
+ return shared_->funcSigs.length();
+}
+
+const SigWithId&
+ModuleGenerator::funcSig(uint32_t funcIndex) const
+{
+ MOZ_ASSERT(shared_->funcSigs[funcIndex]);
+ return *shared_->funcSigs[funcIndex];
+}
+
+bool
+ModuleGenerator::addFuncExport(UniqueChars fieldName, uint32_t funcIndex)
+{
+ return exportedFuncs_.put(funcIndex) &&
+ exports_.emplaceBack(Move(fieldName), funcIndex, DefinitionKind::Function);
+}
+
+bool
+ModuleGenerator::addTableExport(UniqueChars fieldName)
+{
+ MOZ_ASSERT(!startedFuncDefs_);
+ MOZ_ASSERT(shared_->tables.length() == 1);
+ shared_->tables[0].external = true;
+ return exports_.emplaceBack(Move(fieldName), DefinitionKind::Table);
+}
+
+bool
+ModuleGenerator::addMemoryExport(UniqueChars fieldName)
+{
+ return exports_.emplaceBack(Move(fieldName), DefinitionKind::Memory);
+}
+
+bool
+ModuleGenerator::addGlobalExport(UniqueChars fieldName, uint32_t globalIndex)
+{
+ return exports_.emplaceBack(Move(fieldName), globalIndex, DefinitionKind::Global);
+}
+
+bool
+ModuleGenerator::setStartFunction(uint32_t funcIndex)
+{
+ metadata_->startFuncIndex.emplace(funcIndex);
+ return exportedFuncs_.put(funcIndex);
+}
+
+bool
+ModuleGenerator::addElemSegment(InitExpr offset, Uint32Vector&& elemFuncIndices)
+{
+ MOZ_ASSERT(!isAsmJS());
+ MOZ_ASSERT(!startedFuncDefs_);
+ MOZ_ASSERT(shared_->tables.length() == 1);
+
+ for (uint32_t funcIndex : elemFuncIndices) {
+ if (funcIndex < numFuncImports()) {
+ shared_->tables[0].external = true;
+ break;
+ }
+ }
+
+ return elemSegments_.emplaceBack(0, offset, Move(elemFuncIndices));
+}
+
+void
+ModuleGenerator::setDataSegments(DataSegmentVector&& segments)
+{
+ MOZ_ASSERT(dataSegments_.empty());
+ dataSegments_ = Move(segments);
+}
+
+bool
+ModuleGenerator::startFuncDefs()
+{
+ MOZ_ASSERT(!startedFuncDefs_);
+ MOZ_ASSERT(!finishedFuncDefs_);
+
+ // The wasmCompilationInProgress atomic ensures that there is only one
+ // parallel compilation in progress at a time. In the special case of
+ // asm.js, where the ModuleGenerator itself can be on a helper thread, this
+ // avoids the possibility of deadlock since at most 1 helper thread will be
+ // blocking on other helper threads and there are always >1 helper threads.
+ // With wasm, this restriction could be relaxed by moving the worklist state
+ // out of HelperThreadState since each independent compilation needs its own
+ // worklist pair. Alternatively, the deadlock could be avoided by having the
+ // ModuleGenerator thread make progress (on compile tasks) instead of
+ // blocking.
+
+ GlobalHelperThreadState& threads = HelperThreadState();
+ MOZ_ASSERT(threads.threadCount > 1);
+
+ uint32_t numTasks;
+ if (CanUseExtraThreads() && threads.wasmCompilationInProgress.compareExchange(false, true)) {
+#ifdef DEBUG
+ {
+ AutoLockHelperThreadState lock;
+ MOZ_ASSERT(!HelperThreadState().wasmFailed(lock));
+ MOZ_ASSERT(HelperThreadState().wasmWorklist(lock).empty());
+ MOZ_ASSERT(HelperThreadState().wasmFinishedList(lock).empty());
+ }
+#endif
+ parallel_ = true;
+ numTasks = 2 * threads.maxWasmCompilationThreads();
+ } else {
+ numTasks = 1;
+ }
+
+ if (!tasks_.initCapacity(numTasks))
+ return false;
+ for (size_t i = 0; i < numTasks; i++)
+ tasks_.infallibleEmplaceBack(*shared_, COMPILATION_LIFO_DEFAULT_CHUNK_SIZE);
+
+ if (!freeTasks_.reserve(numTasks))
+ return false;
+ for (size_t i = 0; i < numTasks; i++)
+ freeTasks_.infallibleAppend(&tasks_[i]);
+
+ startedFuncDefs_ = true;
+ MOZ_ASSERT(!finishedFuncDefs_);
+ return true;
+}
+
+bool
+ModuleGenerator::startFuncDef(uint32_t lineOrBytecode, FunctionGenerator* fg)
+{
+ MOZ_ASSERT(startedFuncDefs_);
+ MOZ_ASSERT(!activeFuncDef_);
+ MOZ_ASSERT(!finishedFuncDefs_);
+
+ if (freeTasks_.empty() && !finishOutstandingTask())
+ return false;
+
+ IonCompileTask* task = freeTasks_.popCopy();
+
+ task->reset(&fg->bytes_);
+ fg->bytes_.clear();
+ fg->lineOrBytecode_ = lineOrBytecode;
+ fg->m_ = this;
+ fg->task_ = task;
+ activeFuncDef_ = fg;
+ return true;
+}
+
+bool
+ModuleGenerator::finishFuncDef(uint32_t funcIndex, FunctionGenerator* fg)
+{
+ MOZ_ASSERT(activeFuncDef_ == fg);
+
+ auto func = js::MakeUnique<FuncBytes>(Move(fg->bytes_),
+ funcIndex,
+ funcSig(funcIndex),
+ fg->lineOrBytecode_,
+ Move(fg->callSiteLineNums_));
+ if (!func)
+ return false;
+
+ auto mode = alwaysBaseline_ && BaselineCanCompile(fg)
+ ? IonCompileTask::CompileMode::Baseline
+ : IonCompileTask::CompileMode::Ion;
+
+ fg->task_->init(Move(func), mode);
+
+ if (parallel_) {
+ if (!StartOffThreadWasmCompile(fg->task_))
+ return false;
+ outstanding_++;
+ } else {
+ if (!CompileFunction(fg->task_))
+ return false;
+ if (!finishTask(fg->task_))
+ return false;
+ }
+
+ fg->m_ = nullptr;
+ fg->task_ = nullptr;
+ activeFuncDef_ = nullptr;
+ numFinishedFuncDefs_++;
+ return true;
+}
+
+bool
+ModuleGenerator::finishFuncDefs()
+{
+ MOZ_ASSERT(startedFuncDefs_);
+ MOZ_ASSERT(!activeFuncDef_);
+ MOZ_ASSERT(!finishedFuncDefs_);
+
+ while (outstanding_ > 0) {
+ if (!finishOutstandingTask())
+ return false;
+ }
+
+ linkData_.functionCodeLength = masm_.size();
+ finishedFuncDefs_ = true;
+
+ // Generate wrapper functions for every import. These wrappers turn imports
+ // into plain functions so they can be put into tables and re-exported.
+ // asm.js cannot do either and so no wrappers are generated.
+
+ if (!isAsmJS()) {
+ for (size_t funcIndex = 0; funcIndex < numFuncImports(); funcIndex++) {
+ const FuncImport& funcImport = metadata_->funcImports[funcIndex];
+ const SigWithId& sig = funcSig(funcIndex);
+
+ FuncOffsets offsets = GenerateImportFunction(masm_, funcImport, sig.id);
+ if (masm_.oom())
+ return false;
+
+ uint32_t codeRangeIndex = metadata_->codeRanges.length();
+ if (!metadata_->codeRanges.emplaceBack(funcIndex, /* bytecodeOffset = */ 0, offsets))
+ return false;
+
+ MOZ_ASSERT(!funcIsCompiled(funcIndex));
+ funcToCodeRange_[funcIndex] = codeRangeIndex;
+ }
+ }
+
+ // All function indices should have an associated code range at this point
+ // (except in asm.js, which doesn't have import wrapper functions).
+
+#ifdef DEBUG
+ if (isAsmJS()) {
+ MOZ_ASSERT(numFuncImports() < AsmJSFirstDefFuncIndex);
+ for (uint32_t i = 0; i < AsmJSFirstDefFuncIndex; i++)
+ MOZ_ASSERT(funcToCodeRange_[i] == BAD_CODE_RANGE);
+ for (uint32_t i = AsmJSFirstDefFuncIndex; i < numFinishedFuncDefs_; i++)
+ MOZ_ASSERT(funcCodeRange(i).funcIndex() == i);
+ } else {
+ MOZ_ASSERT(numFinishedFuncDefs_ == numFuncDefs());
+ for (uint32_t i = 0; i < numFuncs(); i++)
+ MOZ_ASSERT(funcCodeRange(i).funcIndex() == i);
+ }
+#endif
+
+ // Complete element segments with the code range index of every element, now
+ // that all functions have been compiled.
+
+ for (ElemSegment& elems : elemSegments_) {
+ Uint32Vector& codeRangeIndices = elems.elemCodeRangeIndices;
+
+ MOZ_ASSERT(codeRangeIndices.empty());
+ if (!codeRangeIndices.reserve(elems.elemFuncIndices.length()))
+ return false;
+
+ for (uint32_t funcIndex : elems.elemFuncIndices)
+ codeRangeIndices.infallibleAppend(funcToCodeRange_[funcIndex]);
+ }
+
+ return true;
+}
+
+void
+ModuleGenerator::setFuncNames(NameInBytecodeVector&& funcNames)
+{
+ MOZ_ASSERT(metadata_->funcNames.empty());
+ metadata_->funcNames = Move(funcNames);
+}
+
+bool
+ModuleGenerator::initSigTableLength(uint32_t sigIndex, uint32_t length)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(length != 0);
+ MOZ_ASSERT(length <= MaxTableElems);
+
+ MOZ_ASSERT(shared_->asmJSSigToTableIndex[sigIndex] == 0);
+ shared_->asmJSSigToTableIndex[sigIndex] = numTables_;
+
+ TableDesc& table = shared_->tables[numTables_++];
+ table.kind = TableKind::TypedFunction;
+ table.limits.initial = length;
+ table.limits.maximum = Some(length);
+ return allocateGlobalBytes(sizeof(TableTls), sizeof(void*), &table.globalDataOffset);
+}
+
+bool
+ModuleGenerator::initSigTableElems(uint32_t sigIndex, Uint32Vector&& elemFuncIndices)
+{
+ MOZ_ASSERT(isAsmJS());
+ MOZ_ASSERT(finishedFuncDefs_);
+
+ uint32_t tableIndex = shared_->asmJSSigToTableIndex[sigIndex];
+ MOZ_ASSERT(shared_->tables[tableIndex].limits.initial == elemFuncIndices.length());
+
+ Uint32Vector codeRangeIndices;
+ if (!codeRangeIndices.resize(elemFuncIndices.length()))
+ return false;
+ for (size_t i = 0; i < elemFuncIndices.length(); i++)
+ codeRangeIndices[i] = funcToCodeRange_[elemFuncIndices[i]];
+
+ InitExpr offset(Val(uint32_t(0)));
+ if (!elemSegments_.emplaceBack(tableIndex, offset, Move(elemFuncIndices)))
+ return false;
+
+ elemSegments_.back().elemCodeRangeIndices = Move(codeRangeIndices);
+ return true;
+}
+
+SharedModule
+ModuleGenerator::finish(const ShareableBytes& bytecode)
+{
+ MOZ_ASSERT(!activeFuncDef_);
+ MOZ_ASSERT(finishedFuncDefs_);
+
+ if (!finishFuncExports())
+ return nullptr;
+
+ if (!finishCodegen())
+ return nullptr;
+
+ // Round up the code size to page size since this is eventually required by
+ // the executable-code allocator and for setting memory protection.
+ uint32_t bytesNeeded = masm_.bytesNeeded();
+ uint32_t padding = ComputeByteAlignment(bytesNeeded, gc::SystemPageSize());
+
+ // Use initLengthUninitialized so there is no round-up allocation nor time
+ // wasted zeroing memory.
+ Bytes code;
+ if (!code.initLengthUninitialized(bytesNeeded + padding))
+ return nullptr;
+
+ // Delay flushing of the icache until CodeSegment::create since there is
+ // more patching to do before this code becomes executable.
+ {
+ AutoFlushICache afc("ModuleGenerator::finish", /* inhibit = */ true);
+ masm_.executableCopy(code.begin());
+ }
+
+ // Zero the padding, since we used resizeUninitialized above.
+ memset(code.begin() + bytesNeeded, 0, padding);
+
+ // Convert the CallSiteAndTargetVector (needed during generation) to a
+ // CallSiteVector (what is stored in the Module).
+ if (!metadata_->callSites.appendAll(masm_.callSites()))
+ return nullptr;
+
+ // The MacroAssembler has accumulated all the memory accesses during codegen.
+ metadata_->memoryAccesses = masm_.extractMemoryAccesses();
+ metadata_->memoryPatches = masm_.extractMemoryPatches();
+ metadata_->boundsChecks = masm_.extractBoundsChecks();
+
+ // Copy over data from the ModuleGeneratorData.
+ metadata_->memoryUsage = shared_->memoryUsage;
+ metadata_->minMemoryLength = shared_->minMemoryLength;
+ metadata_->maxMemoryLength = shared_->maxMemoryLength;
+ metadata_->tables = Move(shared_->tables);
+ metadata_->globals = Move(shared_->globals);
+
+ // These Vectors can get large and the excess capacity can be significant,
+ // so realloc them down to size.
+ metadata_->memoryAccesses.podResizeToFit();
+ metadata_->memoryPatches.podResizeToFit();
+ metadata_->boundsChecks.podResizeToFit();
+ metadata_->codeRanges.podResizeToFit();
+ metadata_->callSites.podResizeToFit();
+ metadata_->callThunks.podResizeToFit();
+
+ // For asm.js, the tables vector is over-allocated (to avoid resize during
+ // parallel copilation). Shrink it back down to fit.
+ if (isAsmJS() && !metadata_->tables.resize(numTables_))
+ return nullptr;
+
+ // Assert CodeRanges are sorted.
+#ifdef DEBUG
+ uint32_t lastEnd = 0;
+ for (const CodeRange& codeRange : metadata_->codeRanges) {
+ MOZ_ASSERT(codeRange.begin() >= lastEnd);
+ lastEnd = codeRange.end();
+ }
+#endif
+
+ if (!finishLinkData(code))
+ return nullptr;
+
+ return SharedModule(js_new<Module>(Move(assumptions_),
+ Move(code),
+ Move(linkData_),
+ Move(imports_),
+ Move(exports_),
+ Move(dataSegments_),
+ Move(elemSegments_),
+ *metadata_,
+ bytecode));
+}