summaryrefslogtreecommitdiffstats
path: root/js/src/vm/HelperThreads.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/vm/HelperThreads.cpp')
-rw-r--r--js/src/vm/HelperThreads.cpp1896
1 files changed, 1896 insertions, 0 deletions
diff --git a/js/src/vm/HelperThreads.cpp b/js/src/vm/HelperThreads.cpp
new file mode 100644
index 000000000..7381a97b5
--- /dev/null
+++ b/js/src/vm/HelperThreads.cpp
@@ -0,0 +1,1896 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "vm/HelperThreads.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Unused.h"
+
+#include "jsnativestack.h"
+#include "jsnum.h" // For FIX_FPU()
+
+#include "builtin/Promise.h"
+#include "frontend/BytecodeCompiler.h"
+#include "gc/GCInternals.h"
+#include "jit/IonBuilder.h"
+#include "vm/Debugger.h"
+#include "vm/SharedImmutableStringsCache.h"
+#include "vm/Time.h"
+#include "vm/TraceLogging.h"
+#include "wasm/WasmIonCompile.h"
+
+#include "jscntxtinlines.h"
+#include "jscompartmentinlines.h"
+#include "jsobjinlines.h"
+#include "jsscriptinlines.h"
+
+using namespace js;
+
+using mozilla::ArrayLength;
+using mozilla::DebugOnly;
+using mozilla::Unused;
+using mozilla::TimeDuration;
+
+namespace js {
+
+GlobalHelperThreadState* gHelperThreadState = nullptr;
+
+} // namespace js
+
+bool
+js::CreateHelperThreadsState()
+{
+ MOZ_ASSERT(!gHelperThreadState);
+ gHelperThreadState = js_new<GlobalHelperThreadState>();
+ return gHelperThreadState != nullptr;
+}
+
+void
+js::DestroyHelperThreadsState()
+{
+ MOZ_ASSERT(gHelperThreadState);
+ gHelperThreadState->finish();
+ js_delete(gHelperThreadState);
+ gHelperThreadState = nullptr;
+}
+
+bool
+js::EnsureHelperThreadsInitialized()
+{
+ MOZ_ASSERT(gHelperThreadState);
+ return gHelperThreadState->ensureInitialized();
+}
+
+static size_t
+ThreadCountForCPUCount(size_t cpuCount)
+{
+ // Create additional threads on top of the number of cores available, to
+ // provide some excess capacity in case threads pause each other.
+ static const uint32_t EXCESS_THREADS = 4;
+ return cpuCount + EXCESS_THREADS;
+}
+
+void
+js::SetFakeCPUCount(size_t count)
+{
+ // This must be called before the threads have been initialized.
+ MOZ_ASSERT(!HelperThreadState().threads);
+
+ HelperThreadState().cpuCount = count;
+ HelperThreadState().threadCount = ThreadCountForCPUCount(count);
+}
+
+bool
+js::StartOffThreadWasmCompile(wasm::IonCompileTask* task)
+{
+ AutoLockHelperThreadState lock;
+
+ // Don't append this task if another failed.
+ if (HelperThreadState().wasmFailed(lock))
+ return false;
+
+ if (!HelperThreadState().wasmWorklist(lock).append(task))
+ return false;
+
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+ return true;
+}
+
+bool
+js::StartOffThreadIonCompile(JSContext* cx, jit::IonBuilder* builder)
+{
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().ionWorklist(lock).append(builder))
+ return false;
+
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+ return true;
+}
+
+/*
+ * Move an IonBuilder for which compilation has either finished, failed, or
+ * been cancelled into the global finished compilation list. All off thread
+ * compilations which are started must eventually be finished.
+ */
+static void
+FinishOffThreadIonCompile(jit::IonBuilder* builder, const AutoLockHelperThreadState& lock)
+{
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!HelperThreadState().ionFinishedList(lock).append(builder))
+ oomUnsafe.crash("FinishOffThreadIonCompile");
+}
+
+static JSRuntime*
+GetSelectorRuntime(CompilationSelector selector)
+{
+ struct Matcher
+ {
+ JSRuntime* match(JSScript* script) { return script->runtimeFromMainThread(); }
+ JSRuntime* match(JSCompartment* comp) { return comp->runtimeFromMainThread(); }
+ JSRuntime* match(ZonesInState zbs) { return zbs.runtime; }
+ JSRuntime* match(JSRuntime* runtime) { return runtime; }
+ JSRuntime* match(AllCompilations all) { return nullptr; }
+ };
+
+ return selector.match(Matcher());
+}
+
+static bool
+JitDataStructuresExist(CompilationSelector selector)
+{
+ struct Matcher
+ {
+ bool match(JSScript* script) { return !!script->compartment()->jitCompartment(); }
+ bool match(JSCompartment* comp) { return !!comp->jitCompartment(); }
+ bool match(ZonesInState zbs) { return !!zbs.runtime->jitRuntime(); }
+ bool match(JSRuntime* runtime) { return !!runtime->jitRuntime(); }
+ bool match(AllCompilations all) { return true; }
+ };
+
+ return selector.match(Matcher());
+}
+
+static bool
+CompiledScriptMatches(CompilationSelector selector, JSScript* target)
+{
+ struct ScriptMatches
+ {
+ JSScript* target_;
+
+ bool match(JSScript* script) { return script == target_; }
+ bool match(JSCompartment* comp) { return comp == target_->compartment(); }
+ bool match(JSRuntime* runtime) { return runtime == target_->runtimeFromAnyThread(); }
+ bool match(AllCompilations all) { return true; }
+ bool match(ZonesInState zbs) {
+ return zbs.runtime == target_->runtimeFromAnyThread() &&
+ zbs.state == target_->zoneFromAnyThread()->gcState();
+ }
+ };
+
+ return selector.match(ScriptMatches{target});
+}
+
+void
+js::CancelOffThreadIonCompile(CompilationSelector selector, bool discardLazyLinkList)
+{
+ if (!JitDataStructuresExist(selector))
+ return;
+
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().threads)
+ return;
+
+ /* Cancel any pending entries for which processing hasn't started. */
+ GlobalHelperThreadState::IonBuilderVector& worklist = HelperThreadState().ionWorklist(lock);
+ for (size_t i = 0; i < worklist.length(); i++) {
+ jit::IonBuilder* builder = worklist[i];
+ if (CompiledScriptMatches(selector, builder->script())) {
+ FinishOffThreadIonCompile(builder, lock);
+ HelperThreadState().remove(worklist, &i);
+ }
+ }
+
+ /* Wait for in progress entries to finish up. */
+ bool cancelled;
+ do {
+ cancelled = false;
+ bool unpaused = false;
+ for (auto& helper : *HelperThreadState().threads) {
+ if (helper.ionBuilder() &&
+ CompiledScriptMatches(selector, helper.ionBuilder()->script()))
+ {
+ helper.ionBuilder()->cancel();
+ if (helper.pause) {
+ helper.pause = false;
+ unpaused = true;
+ }
+ cancelled = true;
+ }
+ }
+ if (unpaused)
+ HelperThreadState().notifyAll(GlobalHelperThreadState::PAUSE, lock);
+ if (cancelled)
+ HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER);
+ } while (cancelled);
+
+ /* Cancel code generation for any completed entries. */
+ GlobalHelperThreadState::IonBuilderVector& finished = HelperThreadState().ionFinishedList(lock);
+ for (size_t i = 0; i < finished.length(); i++) {
+ jit::IonBuilder* builder = finished[i];
+ if (CompiledScriptMatches(selector, builder->script())) {
+ jit::FinishOffThreadBuilder(nullptr, builder, lock);
+ HelperThreadState().remove(finished, &i);
+ }
+ }
+
+ /* Cancel lazy linking for pending builders (attached to the ionScript). */
+ if (discardLazyLinkList) {
+ MOZ_ASSERT(!selector.is<AllCompilations>());
+ JSRuntime* runtime = GetSelectorRuntime(selector);
+ jit::IonBuilder* builder = runtime->ionLazyLinkList().getFirst();
+ while (builder) {
+ jit::IonBuilder* next = builder->getNext();
+ if (CompiledScriptMatches(selector, builder->script()))
+ jit::FinishOffThreadBuilder(runtime, builder, lock);
+ builder = next;
+ }
+ }
+}
+
+#ifdef DEBUG
+bool
+js::HasOffThreadIonCompile(JSCompartment* comp)
+{
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().threads)
+ return false;
+
+ GlobalHelperThreadState::IonBuilderVector& worklist = HelperThreadState().ionWorklist(lock);
+ for (size_t i = 0; i < worklist.length(); i++) {
+ jit::IonBuilder* builder = worklist[i];
+ if (builder->script()->compartment() == comp)
+ return true;
+ }
+
+ for (auto& helper : *HelperThreadState().threads) {
+ if (helper.ionBuilder() && helper.ionBuilder()->script()->compartment() == comp)
+ return true;
+ }
+
+ GlobalHelperThreadState::IonBuilderVector& finished = HelperThreadState().ionFinishedList(lock);
+ for (size_t i = 0; i < finished.length(); i++) {
+ jit::IonBuilder* builder = finished[i];
+ if (builder->script()->compartment() == comp)
+ return true;
+ }
+
+ jit::IonBuilder* builder = comp->runtimeFromMainThread()->ionLazyLinkList().getFirst();
+ while (builder) {
+ if (builder->script()->compartment() == comp)
+ return true;
+ builder = builder->getNext();
+ }
+
+ return false;
+}
+#endif
+
+static const JSClassOps parseTaskGlobalClassOps = {
+ nullptr, nullptr, nullptr, nullptr,
+ nullptr, nullptr, nullptr, nullptr,
+ nullptr, nullptr, nullptr,
+ JS_GlobalObjectTraceHook
+};
+
+static const JSClass parseTaskGlobalClass = {
+ "internal-parse-task-global", JSCLASS_GLOBAL_FLAGS,
+ &parseTaskGlobalClassOps
+};
+
+ParseTask::ParseTask(ParseTaskKind kind, ExclusiveContext* cx, JSObject* exclusiveContextGlobal,
+ JSContext* initCx, const char16_t* chars, size_t length,
+ JS::OffThreadCompileCallback callback, void* callbackData)
+ : kind(kind), cx(cx), options(initCx), chars(chars), length(length),
+ alloc(JSRuntime::TEMP_LIFO_ALLOC_PRIMARY_CHUNK_SIZE),
+ exclusiveContextGlobal(exclusiveContextGlobal),
+ callback(callback), callbackData(callbackData),
+ script(nullptr), sourceObject(nullptr),
+ errors(cx), overRecursed(false), outOfMemory(false)
+{
+}
+
+bool
+ParseTask::init(JSContext* cx, const ReadOnlyCompileOptions& options)
+{
+ if (!this->options.copy(cx, options))
+ return false;
+
+ return true;
+}
+
+void
+ParseTask::activate(JSRuntime* rt)
+{
+ rt->setUsedByExclusiveThread(exclusiveContextGlobal->zone());
+ cx->enterCompartment(exclusiveContextGlobal->compartment());
+}
+
+bool
+ParseTask::finish(JSContext* cx)
+{
+ if (sourceObject) {
+ RootedScriptSource sso(cx, sourceObject);
+ if (!ScriptSourceObject::initFromOptions(cx, sso, options))
+ return false;
+ }
+
+ return true;
+}
+
+ParseTask::~ParseTask()
+{
+ // ParseTask takes over ownership of its input exclusive context.
+ js_delete(cx);
+
+ for (size_t i = 0; i < errors.length(); i++)
+ js_delete(errors[i]);
+}
+
+void
+ParseTask::trace(JSTracer* trc)
+{
+ if (!cx->runtimeMatches(trc->runtime()))
+ return;
+
+ TraceManuallyBarrieredEdge(trc, &exclusiveContextGlobal, "ParseTask::exclusiveContextGlobal");
+ if (script)
+ TraceManuallyBarrieredEdge(trc, &script, "ParseTask::script");
+ if (sourceObject)
+ TraceManuallyBarrieredEdge(trc, &sourceObject, "ParseTask::sourceObject");
+}
+
+ScriptParseTask::ScriptParseTask(ExclusiveContext* cx, JSObject* exclusiveContextGlobal,
+ JSContext* initCx, const char16_t* chars, size_t length,
+ JS::OffThreadCompileCallback callback, void* callbackData)
+ : ParseTask(ParseTaskKind::Script, cx, exclusiveContextGlobal, initCx, chars, length, callback,
+ callbackData)
+{
+}
+
+void
+ScriptParseTask::parse()
+{
+ SourceBufferHolder srcBuf(chars, length, SourceBufferHolder::NoOwnership);
+ script = frontend::CompileGlobalScript(cx, alloc, ScopeKind::Global,
+ options, srcBuf,
+ /* extraSct = */ nullptr,
+ /* sourceObjectOut = */ &sourceObject);
+}
+
+ModuleParseTask::ModuleParseTask(ExclusiveContext* cx, JSObject* exclusiveContextGlobal,
+ JSContext* initCx, const char16_t* chars, size_t length,
+ JS::OffThreadCompileCallback callback, void* callbackData)
+ : ParseTask(ParseTaskKind::Module, cx, exclusiveContextGlobal, initCx, chars, length, callback,
+ callbackData)
+{
+}
+
+void
+ModuleParseTask::parse()
+{
+ SourceBufferHolder srcBuf(chars, length, SourceBufferHolder::NoOwnership);
+ ModuleObject* module = frontend::CompileModule(cx, options, srcBuf, alloc, &sourceObject);
+ if (module)
+ script = module->script();
+}
+
+void
+js::CancelOffThreadParses(JSRuntime* rt)
+{
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().threads)
+ return;
+
+#ifdef DEBUG
+ GlobalHelperThreadState::ParseTaskVector& waitingOnGC =
+ HelperThreadState().parseWaitingOnGC(lock);
+ for (size_t i = 0; i < waitingOnGC.length(); i++)
+ MOZ_ASSERT(!waitingOnGC[i]->runtimeMatches(rt));
+#endif
+
+ // Instead of forcibly canceling pending parse tasks, just wait for all scheduled
+ // and in progress ones to complete. Otherwise the final GC may not collect
+ // everything due to zones being used off thread.
+ while (true) {
+ bool pending = false;
+ GlobalHelperThreadState::ParseTaskVector& worklist = HelperThreadState().parseWorklist(lock);
+ for (size_t i = 0; i < worklist.length(); i++) {
+ ParseTask* task = worklist[i];
+ if (task->runtimeMatches(rt))
+ pending = true;
+ }
+ if (!pending) {
+ bool inProgress = false;
+ for (auto& thread : *HelperThreadState().threads) {
+ ParseTask* task = thread.parseTask();
+ if (task && task->runtimeMatches(rt))
+ inProgress = true;
+ }
+ if (!inProgress)
+ break;
+ }
+ HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER);
+ }
+
+ // Clean up any parse tasks which haven't been finished by the main thread.
+ GlobalHelperThreadState::ParseTaskVector& finished = HelperThreadState().parseFinishedList(lock);
+ while (true) {
+ bool found = false;
+ for (size_t i = 0; i < finished.length(); i++) {
+ ParseTask* task = finished[i];
+ if (task->runtimeMatches(rt)) {
+ found = true;
+ AutoUnlockHelperThreadState unlock(lock);
+ HelperThreadState().cancelParseTask(rt->contextFromMainThread(), task->kind, task);
+ }
+ }
+ if (!found)
+ break;
+ }
+}
+
+bool
+js::OffThreadParsingMustWaitForGC(JSRuntime* rt)
+{
+ // Off thread parsing can't occur during incremental collections on the
+ // atoms compartment, to avoid triggering barriers. (Outside the atoms
+ // compartment, the compilation will use a new zone that is never
+ // collected.) If an atoms-zone GC is in progress, hold off on executing the
+ // parse task until the atoms-zone GC completes (see
+ // EnqueuePendingParseTasksAfterGC).
+ return rt->activeGCInAtomsZone();
+}
+
+static bool
+EnsureConstructor(JSContext* cx, Handle<GlobalObject*> global, JSProtoKey key)
+{
+ if (!GlobalObject::ensureConstructor(cx, global, key))
+ return false;
+
+ MOZ_ASSERT(global->getPrototype(key).toObject().isDelegate(),
+ "standard class prototype wasn't a delegate from birth");
+ return true;
+}
+
+// Initialize all classes potentially created during parsing for use in parser
+// data structures, template objects, &c.
+static bool
+EnsureParserCreatedClasses(JSContext* cx, ParseTaskKind kind)
+{
+ Handle<GlobalObject*> global = cx->global();
+
+ if (!EnsureConstructor(cx, global, JSProto_Function))
+ return false; // needed by functions, also adds object literals' proto
+
+ if (!EnsureConstructor(cx, global, JSProto_Array))
+ return false; // needed by array literals
+
+ if (!EnsureConstructor(cx, global, JSProto_RegExp))
+ return false; // needed by regular expression literals
+
+ if (!EnsureConstructor(cx, global, JSProto_Iterator))
+ return false; // needed by ???
+
+ if (!GlobalObject::initStarGenerators(cx, global))
+ return false; // needed by function*() {} and generator comprehensions
+
+ if (kind == ParseTaskKind::Module && !GlobalObject::ensureModulePrototypesCreated(cx, global))
+ return false;
+
+ return true;
+}
+
+static JSObject*
+CreateGlobalForOffThreadParse(JSContext* cx, ParseTaskKind kind, const gc::AutoSuppressGC& nogc)
+{
+ JSCompartment* currentCompartment = cx->compartment();
+
+ JS::CompartmentOptions compartmentOptions(currentCompartment->creationOptions(),
+ currentCompartment->behaviors());
+
+ auto& creationOptions = compartmentOptions.creationOptions();
+
+ creationOptions.setInvisibleToDebugger(true)
+ .setMergeable(true)
+ .setZone(JS::FreshZone);
+
+ // Don't falsely inherit the host's global trace hook.
+ creationOptions.setTrace(nullptr);
+
+ JSObject* global = JS_NewGlobalObject(cx, &parseTaskGlobalClass, nullptr,
+ JS::FireOnNewGlobalHook, compartmentOptions);
+ if (!global)
+ return nullptr;
+
+ JS_SetCompartmentPrincipals(global->compartment(), currentCompartment->principals());
+
+ // Initialize all classes required for parsing while still on the main
+ // thread, for both the target and the new global so that prototype
+ // pointers can be changed infallibly after parsing finishes.
+ if (!EnsureParserCreatedClasses(cx, kind))
+ return nullptr;
+ {
+ AutoCompartment ac(cx, global);
+ if (!EnsureParserCreatedClasses(cx, kind))
+ return nullptr;
+ }
+
+ return global;
+}
+
+static bool
+QueueOffThreadParseTask(JSContext* cx, ParseTask* task)
+{
+ if (OffThreadParsingMustWaitForGC(cx->runtime())) {
+ AutoLockHelperThreadState lock;
+ if (!HelperThreadState().parseWaitingOnGC(lock).append(task)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ } else {
+ AutoLockHelperThreadState lock;
+ if (!HelperThreadState().parseWorklist(lock).append(task)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ task->activate(cx->runtime());
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+ }
+
+ return true;
+}
+
+bool
+js::StartOffThreadParseScript(JSContext* cx, const ReadOnlyCompileOptions& options,
+ const char16_t* chars, size_t length,
+ JS::OffThreadCompileCallback callback, void* callbackData)
+{
+ // Suppress GC so that calls below do not trigger a new incremental GC
+ // which could require barriers on the atoms compartment.
+ gc::AutoSuppressGC nogc(cx);
+ gc::AutoAssertNoNurseryAlloc noNurseryAlloc(cx->runtime());
+ AutoSuppressAllocationMetadataBuilder suppressMetadata(cx);
+
+ JSObject* global = CreateGlobalForOffThreadParse(cx, ParseTaskKind::Script, nogc);
+ if (!global)
+ return false;
+
+ ScopedJSDeletePtr<ExclusiveContext> helpercx(
+ cx->new_<ExclusiveContext>(cx->runtime(), (PerThreadData*) nullptr,
+ ExclusiveContext::Context_Exclusive, cx->options()));
+ if (!helpercx)
+ return false;
+
+ ScopedJSDeletePtr<ParseTask> task(
+ cx->new_<ScriptParseTask>(helpercx.get(), global, cx, chars, length,
+ callback, callbackData));
+ if (!task)
+ return false;
+
+ helpercx.forget();
+
+ if (!task->init(cx, options) || !QueueOffThreadParseTask(cx, task))
+ return false;
+
+ task.forget();
+
+ return true;
+}
+
+bool
+js::StartOffThreadParseModule(JSContext* cx, const ReadOnlyCompileOptions& options,
+ const char16_t* chars, size_t length,
+ JS::OffThreadCompileCallback callback, void* callbackData)
+{
+ // Suppress GC so that calls below do not trigger a new incremental GC
+ // which could require barriers on the atoms compartment.
+ gc::AutoSuppressGC nogc(cx);
+ gc::AutoAssertNoNurseryAlloc noNurseryAlloc(cx->runtime());
+ AutoSuppressAllocationMetadataBuilder suppressMetadata(cx);
+
+ JSObject* global = CreateGlobalForOffThreadParse(cx, ParseTaskKind::Module, nogc);
+ if (!global)
+ return false;
+
+ ScopedJSDeletePtr<ExclusiveContext> helpercx(
+ cx->new_<ExclusiveContext>(cx->runtime(), (PerThreadData*) nullptr,
+ ExclusiveContext::Context_Exclusive, cx->options()));
+ if (!helpercx)
+ return false;
+
+ ScopedJSDeletePtr<ParseTask> task(
+ cx->new_<ModuleParseTask>(helpercx.get(), global, cx, chars, length,
+ callback, callbackData));
+ if (!task)
+ return false;
+
+ helpercx.forget();
+
+ if (!task->init(cx, options) || !QueueOffThreadParseTask(cx, task))
+ return false;
+
+ task.forget();
+
+ return true;
+}
+
+void
+js::EnqueuePendingParseTasksAfterGC(JSRuntime* rt)
+{
+ MOZ_ASSERT(!OffThreadParsingMustWaitForGC(rt));
+
+ GlobalHelperThreadState::ParseTaskVector newTasks;
+ {
+ AutoLockHelperThreadState lock;
+ GlobalHelperThreadState::ParseTaskVector& waiting =
+ HelperThreadState().parseWaitingOnGC(lock);
+
+ for (size_t i = 0; i < waiting.length(); i++) {
+ ParseTask* task = waiting[i];
+ if (task->runtimeMatches(rt)) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!newTasks.append(task))
+ oomUnsafe.crash("EnqueuePendingParseTasksAfterGC");
+ HelperThreadState().remove(waiting, &i);
+ }
+ }
+ }
+
+ if (newTasks.empty())
+ return;
+
+ // This logic should mirror the contents of the !activeGCInAtomsZone()
+ // branch in StartOffThreadParseScript:
+
+ for (size_t i = 0; i < newTasks.length(); i++)
+ newTasks[i]->activate(rt);
+
+ AutoLockHelperThreadState lock;
+
+ {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!HelperThreadState().parseWorklist(lock).appendAll(newTasks))
+ oomUnsafe.crash("EnqueuePendingParseTasksAfterGC");
+ }
+
+ HelperThreadState().notifyAll(GlobalHelperThreadState::PRODUCER, lock);
+}
+
+static const uint32_t kDefaultHelperStackSize = 2048 * 1024;
+static const uint32_t kDefaultHelperStackQuota = 1800 * 1024;
+
+// TSan enforces a minimum stack size that's just slightly larger than our
+// default helper stack size. It does this to store blobs of TSan-specific
+// data on each thread's stack. Unfortunately, that means that even though
+// we'll actually receive a larger stack than we requested, the effective
+// usable space of that stack is significantly less than what we expect.
+// To offset TSan stealing our stack space from underneath us, double the
+// default.
+//
+// Note that we don't need this for ASan/MOZ_ASAN because ASan doesn't
+// require all the thread-specific state that TSan does.
+#if defined(MOZ_TSAN)
+static const uint32_t HELPER_STACK_SIZE = 2 * kDefaultHelperStackSize;
+static const uint32_t HELPER_STACK_QUOTA = 2 * kDefaultHelperStackQuota;
+#else
+static const uint32_t HELPER_STACK_SIZE = kDefaultHelperStackSize;
+static const uint32_t HELPER_STACK_QUOTA = kDefaultHelperStackQuota;
+#endif
+
+bool
+GlobalHelperThreadState::ensureInitialized()
+{
+ MOZ_ASSERT(CanUseExtraThreads());
+
+ MOZ_ASSERT(this == &HelperThreadState());
+ AutoLockHelperThreadState lock;
+
+ if (threads)
+ return true;
+
+ threads = js::UniquePtr<HelperThreadVector>(js_new<HelperThreadVector>());
+ if (!threads || !threads->initCapacity(threadCount))
+ return false;
+
+ for (size_t i = 0; i < threadCount; i++) {
+ threads->infallibleEmplaceBack();
+ HelperThread& helper = (*threads)[i];
+
+ helper.threadData.emplace(static_cast<JSRuntime*>(nullptr));
+ if (!helper.threadData->init())
+ goto error;
+
+ helper.thread = mozilla::Some(Thread(Thread::Options().setStackSize(HELPER_STACK_SIZE)));
+ if (!helper.thread->init(HelperThread::ThreadMain, &helper))
+ goto error;
+
+ continue;
+
+ error:
+ // Ensure that we do not leave uninitialized threads in the `threads`
+ // vector.
+ threads->popBack();
+ finishThreads();
+ return false;
+ }
+
+ return true;
+}
+
+GlobalHelperThreadState::GlobalHelperThreadState()
+ : cpuCount(0),
+ threadCount(0),
+ threads(nullptr),
+ wasmCompilationInProgress(false),
+ numWasmFailedJobs(0),
+ helperLock(mutexid::GlobalHelperThreadState)
+{
+ cpuCount = GetCPUCount();
+ threadCount = ThreadCountForCPUCount(cpuCount);
+
+ MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
+}
+
+void
+GlobalHelperThreadState::finish()
+{
+ finishThreads();
+}
+
+void
+GlobalHelperThreadState::finishThreads()
+{
+ if (!threads)
+ return;
+
+ MOZ_ASSERT(CanUseExtraThreads());
+ for (auto& thread : *threads)
+ thread.destroy();
+ threads.reset(nullptr);
+}
+
+void
+GlobalHelperThreadState::lock()
+{
+ helperLock.lock();
+}
+
+void
+GlobalHelperThreadState::unlock()
+{
+ helperLock.unlock();
+}
+
+void
+GlobalHelperThreadState::wait(AutoLockHelperThreadState& locked, CondVar which,
+ TimeDuration timeout /* = TimeDuration::Forever() */)
+{
+ whichWakeup(which).wait_for(locked, timeout);
+}
+
+void
+GlobalHelperThreadState::notifyAll(CondVar which, const AutoLockHelperThreadState&)
+{
+ whichWakeup(which).notify_all();
+}
+
+void
+GlobalHelperThreadState::notifyOne(CondVar which, const AutoLockHelperThreadState&)
+{
+ whichWakeup(which).notify_one();
+}
+
+bool
+GlobalHelperThreadState::hasActiveThreads(const AutoLockHelperThreadState&)
+{
+ if (!threads)
+ return false;
+
+ for (auto& thread : *threads) {
+ if (!thread.idle())
+ return true;
+ }
+
+ return false;
+}
+
+void
+GlobalHelperThreadState::waitForAllThreads()
+{
+ CancelOffThreadIonCompile();
+
+ AutoLockHelperThreadState lock;
+ while (hasActiveThreads(lock))
+ wait(lock, CONSUMER);
+}
+
+template <typename T>
+bool
+GlobalHelperThreadState::checkTaskThreadLimit(size_t maxThreads) const
+{
+ if (maxThreads >= threadCount)
+ return true;
+
+ size_t count = 0;
+ for (auto& thread : *threads) {
+ if (thread.currentTask.isSome() && thread.currentTask->is<T>())
+ count++;
+ if (count >= maxThreads)
+ return false;
+ }
+
+ return true;
+}
+
+static inline bool
+IsHelperThreadSimulatingOOM(js::oom::ThreadType threadType)
+{
+#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
+ return js::oom::targetThread == threadType;
+#else
+ return false;
+#endif
+}
+
+size_t
+GlobalHelperThreadState::maxIonCompilationThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_ION))
+ return 1;
+ return threadCount;
+}
+
+size_t
+GlobalHelperThreadState::maxUnpausedIonCompilationThreads() const
+{
+ return 1;
+}
+
+size_t
+GlobalHelperThreadState::maxWasmCompilationThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_ASMJS))
+ return 1;
+ if (cpuCount < 2)
+ return 2;
+ return cpuCount;
+}
+
+size_t
+GlobalHelperThreadState::maxParseThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_PARSE))
+ return 1;
+
+ // Don't allow simultaneous off thread parses, to reduce contention on the
+ // atoms table. Note that wasm compilation depends on this to avoid
+ // stalling the helper thread, as off thread parse tasks can trigger and
+ // block on other off thread wasm compilation tasks.
+ return 1;
+}
+
+size_t
+GlobalHelperThreadState::maxCompressionThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_COMPRESS))
+ return 1;
+ return threadCount;
+}
+
+size_t
+GlobalHelperThreadState::maxGCHelperThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_GCHELPER))
+ return 1;
+ return threadCount;
+}
+
+size_t
+GlobalHelperThreadState::maxGCParallelThreads() const
+{
+ if (IsHelperThreadSimulatingOOM(js::oom::THREAD_TYPE_GCPARALLEL))
+ return 1;
+ return threadCount;
+}
+
+bool
+GlobalHelperThreadState::canStartWasmCompile(const AutoLockHelperThreadState& lock)
+{
+ // Don't execute an wasm job if an earlier one failed.
+ if (wasmWorklist(lock).empty() || numWasmFailedJobs)
+ return false;
+
+ // Honor the maximum allowed threads to compile wasm jobs at once,
+ // to avoid oversaturating the machine.
+ if (!checkTaskThreadLimit<wasm::IonCompileTask*>(maxWasmCompilationThreads()))
+ return false;
+
+ return true;
+}
+
+bool
+GlobalHelperThreadState::canStartPromiseTask(const AutoLockHelperThreadState& lock)
+{
+ return !promiseTasks(lock).empty();
+}
+
+static bool
+IonBuilderHasHigherPriority(jit::IonBuilder* first, jit::IonBuilder* second)
+{
+ // This method can return whatever it wants, though it really ought to be a
+ // total order. The ordering is allowed to race (change on the fly), however.
+
+ // A lower optimization level indicates a higher priority.
+ if (first->optimizationInfo().level() != second->optimizationInfo().level())
+ return first->optimizationInfo().level() < second->optimizationInfo().level();
+
+ // A script without an IonScript has precedence on one with.
+ if (first->scriptHasIonScript() != second->scriptHasIonScript())
+ return !first->scriptHasIonScript();
+
+ // A higher warm-up counter indicates a higher priority.
+ return first->script()->getWarmUpCount() / first->script()->length() >
+ second->script()->getWarmUpCount() / second->script()->length();
+}
+
+bool
+GlobalHelperThreadState::canStartIonCompile(const AutoLockHelperThreadState& lock)
+{
+ return !ionWorklist(lock).empty() &&
+ checkTaskThreadLimit<jit::IonBuilder*>(maxIonCompilationThreads());
+}
+
+jit::IonBuilder*
+GlobalHelperThreadState::highestPriorityPendingIonCompile(const AutoLockHelperThreadState& lock,
+ bool remove /* = false */)
+{
+ auto& worklist = ionWorklist(lock);
+ if (worklist.empty()) {
+ MOZ_ASSERT(!remove);
+ return nullptr;
+ }
+
+ // Get the highest priority IonBuilder which has not started compilation yet.
+ size_t index = 0;
+ for (size_t i = 1; i < worklist.length(); i++) {
+ if (IonBuilderHasHigherPriority(worklist[i], worklist[index]))
+ index = i;
+ }
+ jit::IonBuilder* builder = worklist[index];
+ if (remove)
+ worklist.erase(&worklist[index]);
+ return builder;
+}
+
+HelperThread*
+GlobalHelperThreadState::lowestPriorityUnpausedIonCompileAtThreshold(
+ const AutoLockHelperThreadState& lock)
+{
+ // Get the lowest priority IonBuilder which has started compilation and
+ // isn't paused, unless there are still fewer than the maximum number of
+ // such builders permitted.
+ size_t numBuilderThreads = 0;
+ HelperThread* thread = nullptr;
+ for (auto& thisThread : *threads) {
+ if (thisThread.ionBuilder() && !thisThread.pause) {
+ numBuilderThreads++;
+ if (!thread ||
+ IonBuilderHasHigherPriority(thread->ionBuilder(), thisThread.ionBuilder()))
+ {
+ thread = &thisThread;
+ }
+ }
+ }
+ if (numBuilderThreads < maxUnpausedIonCompilationThreads())
+ return nullptr;
+ return thread;
+}
+
+HelperThread*
+GlobalHelperThreadState::highestPriorityPausedIonCompile(const AutoLockHelperThreadState& lock)
+{
+ // Get the highest priority IonBuilder which has started compilation but
+ // which was subsequently paused.
+ HelperThread* thread = nullptr;
+ for (auto& thisThread : *threads) {
+ if (thisThread.pause) {
+ // Currently, only threads with IonBuilders can be paused.
+ MOZ_ASSERT(thisThread.ionBuilder());
+ if (!thread ||
+ IonBuilderHasHigherPriority(thisThread.ionBuilder(), thread->ionBuilder()))
+ {
+ thread = &thisThread;
+ }
+ }
+ }
+ return thread;
+}
+
+bool
+GlobalHelperThreadState::pendingIonCompileHasSufficientPriority(
+ const AutoLockHelperThreadState& lock)
+{
+ // Can't compile anything if there are no scripts to compile.
+ if (!canStartIonCompile(lock))
+ return false;
+
+ // Count the number of threads currently compiling scripts, and look for
+ // the thread with the lowest priority.
+ HelperThread* lowestPriorityThread = lowestPriorityUnpausedIonCompileAtThreshold(lock);
+
+ // If the number of threads building scripts is less than the maximum, the
+ // compilation can start immediately.
+ if (!lowestPriorityThread)
+ return true;
+
+ // If there is a builder in the worklist with higher priority than some
+ // builder currently being compiled, then that current compilation can be
+ // paused, so allow the compilation.
+ if (IonBuilderHasHigherPriority(highestPriorityPendingIonCompile(lock),
+ lowestPriorityThread->ionBuilder()))
+ return true;
+
+ // Compilation will have to wait until one of the active compilations finishes.
+ return false;
+}
+
+bool
+GlobalHelperThreadState::canStartParseTask(const AutoLockHelperThreadState& lock)
+{
+ return !parseWorklist(lock).empty() && checkTaskThreadLimit<ParseTask*>(maxParseThreads());
+}
+
+bool
+GlobalHelperThreadState::canStartCompressionTask(const AutoLockHelperThreadState& lock)
+{
+ return !compressionWorklist(lock).empty() &&
+ checkTaskThreadLimit<SourceCompressionTask*>(maxCompressionThreads());
+}
+
+bool
+GlobalHelperThreadState::canStartGCHelperTask(const AutoLockHelperThreadState& lock)
+{
+ return !gcHelperWorklist(lock).empty() &&
+ checkTaskThreadLimit<GCHelperState*>(maxGCHelperThreads());
+}
+
+bool
+GlobalHelperThreadState::canStartGCParallelTask(const AutoLockHelperThreadState& lock)
+{
+ return !gcParallelWorklist(lock).empty() &&
+ checkTaskThreadLimit<GCParallelTask*>(maxGCParallelThreads());
+}
+
+js::GCParallelTask::~GCParallelTask()
+{
+ // Only most-derived classes' destructors may do the join: base class
+ // destructors run after those for derived classes' members, so a join in a
+ // base class can't ensure that the task is done using the members. All we
+ // can do now is check that someone has previously stopped the task.
+#ifdef DEBUG
+ AutoLockHelperThreadState helperLock;
+ MOZ_ASSERT(state == NotStarted);
+#endif
+}
+
+bool
+js::GCParallelTask::startWithLockHeld(AutoLockHelperThreadState& lock)
+{
+ // Tasks cannot be started twice.
+ MOZ_ASSERT(state == NotStarted);
+
+ // If we do the shutdown GC before running anything, we may never
+ // have initialized the helper threads. Just use the serial path
+ // since we cannot safely intialize them at this point.
+ if (!HelperThreadState().threads)
+ return false;
+
+ if (!HelperThreadState().gcParallelWorklist(lock).append(this))
+ return false;
+ state = Dispatched;
+
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+
+ return true;
+}
+
+bool
+js::GCParallelTask::start()
+{
+ AutoLockHelperThreadState helperLock;
+ return startWithLockHeld(helperLock);
+}
+
+void
+js::GCParallelTask::joinWithLockHeld(AutoLockHelperThreadState& locked)
+{
+ if (state == NotStarted)
+ return;
+
+ while (state != Finished)
+ HelperThreadState().wait(locked, GlobalHelperThreadState::CONSUMER);
+ state = NotStarted;
+ cancel_ = false;
+}
+
+void
+js::GCParallelTask::join()
+{
+ AutoLockHelperThreadState helperLock;
+ joinWithLockHeld(helperLock);
+}
+
+void
+js::GCParallelTask::runFromMainThread(JSRuntime* rt)
+{
+ MOZ_ASSERT(state == NotStarted);
+ MOZ_ASSERT(js::CurrentThreadCanAccessRuntime(rt));
+ uint64_t timeStart = PRMJ_Now();
+ run();
+ duration_ = PRMJ_Now() - timeStart;
+}
+
+void
+js::GCParallelTask::runFromHelperThread(AutoLockHelperThreadState& locked)
+{
+ {
+ AutoUnlockHelperThreadState parallelSection(locked);
+ gc::AutoSetThreadIsPerformingGC performingGC;
+ uint64_t timeStart = PRMJ_Now();
+ run();
+ duration_ = PRMJ_Now() - timeStart;
+ }
+
+ state = Finished;
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+}
+
+bool
+js::GCParallelTask::isRunningWithLockHeld(const AutoLockHelperThreadState& locked) const
+{
+ return state == Dispatched;
+}
+
+bool
+js::GCParallelTask::isRunning() const
+{
+ AutoLockHelperThreadState helperLock;
+ return isRunningWithLockHeld(helperLock);
+}
+
+void
+HelperThread::handleGCParallelWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartGCParallelTask(locked));
+ MOZ_ASSERT(idle());
+
+ TraceLoggerThread* logger = TraceLoggerForCurrentThread();
+ AutoTraceLog logCompile(logger, TraceLogger_GC);
+
+ currentTask.emplace(HelperThreadState().gcParallelWorklist(locked).popCopy());
+ gcParallelTask()->runFromHelperThread(locked);
+ currentTask.reset();
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+}
+
+static void
+LeaveParseTaskZone(JSRuntime* rt, ParseTask* task)
+{
+ // Mark the zone as no longer in use by an ExclusiveContext, and available
+ // to be collected by the GC.
+ task->cx->leaveCompartment(task->cx->compartment());
+ rt->clearUsedByExclusiveThread(task->cx->zone());
+}
+
+ParseTask*
+GlobalHelperThreadState::removeFinishedParseTask(ParseTaskKind kind, void* token)
+{
+ // The token is a ParseTask* which should be in the finished list.
+ // Find and remove its entry.
+
+ AutoLockHelperThreadState lock;
+ ParseTaskVector& finished = parseFinishedList(lock);
+
+ for (size_t i = 0; i < finished.length(); i++) {
+ if (finished[i] == token) {
+ ParseTask* parseTask = finished[i];
+ remove(finished, &i);
+ MOZ_ASSERT(parseTask);
+ MOZ_ASSERT(parseTask->kind == kind);
+ return parseTask;
+ }
+ }
+
+ MOZ_CRASH("Invalid ParseTask token");
+}
+
+JSScript*
+GlobalHelperThreadState::finishParseTask(JSContext* cx, ParseTaskKind kind, void* token)
+{
+ MOZ_ASSERT(cx->compartment());
+
+ ScopedJSDeletePtr<ParseTask> parseTask(removeFinishedParseTask(kind, token));
+
+ // Make sure we have all the constructors we need for the prototype
+ // remapping below, since we can't GC while that's happening.
+ Rooted<GlobalObject*> global(cx, &cx->global()->as<GlobalObject>());
+ if (!EnsureParserCreatedClasses(cx, kind)) {
+ LeaveParseTaskZone(cx, parseTask);
+ return nullptr;
+ }
+
+ mergeParseTaskCompartment(cx, parseTask, global, cx->compartment());
+
+ RootedScript script(cx, parseTask->script);
+ releaseAssertSameCompartment(cx, script);
+
+ if (!parseTask->finish(cx))
+ return nullptr;
+
+ // Report out of memory errors eagerly, or errors could be malformed.
+ if (parseTask->outOfMemory) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ // Report any error or warnings generated during the parse, and inform the
+ // debugger about the compiled scripts.
+ for (size_t i = 0; i < parseTask->errors.length(); i++)
+ parseTask->errors[i]->throwError(cx);
+ if (parseTask->overRecursed)
+ ReportOverRecursed(cx);
+ if (cx->isExceptionPending())
+ return nullptr;
+
+ if (!script) {
+ // No error was reported, but no script produced. Assume we hit out of
+ // memory.
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ // The Debugger only needs to be told about the topmost script that was compiled.
+ Debugger::onNewScript(cx, script);
+
+ return script;
+}
+
+JSScript*
+GlobalHelperThreadState::finishScriptParseTask(JSContext* cx, void* token)
+{
+ JSScript* script = finishParseTask(cx, ParseTaskKind::Script, token);
+ MOZ_ASSERT_IF(script, script->isGlobalCode());
+ return script;
+}
+
+JSObject*
+GlobalHelperThreadState::finishModuleParseTask(JSContext* cx, void* token)
+{
+ JSScript* script = finishParseTask(cx, ParseTaskKind::Module, token);
+ if (!script)
+ return nullptr;
+
+ MOZ_ASSERT(script->module());
+
+ RootedModuleObject module(cx, script->module());
+ module->fixEnvironmentsAfterCompartmentMerge(cx);
+ if (!ModuleObject::Freeze(cx, module))
+ return nullptr;
+
+ return module;
+}
+
+void
+GlobalHelperThreadState::cancelParseTask(JSContext* cx, ParseTaskKind kind, void* token)
+{
+ ScopedJSDeletePtr<ParseTask> parseTask(removeFinishedParseTask(kind, token));
+ LeaveParseTaskZone(cx, parseTask);
+}
+
+JSObject*
+GlobalObject::getStarGeneratorFunctionPrototype()
+{
+ const Value& v = getReservedSlot(STAR_GENERATOR_FUNCTION_PROTO);
+ return v.isObject() ? &v.toObject() : nullptr;
+}
+
+void
+GlobalHelperThreadState::mergeParseTaskCompartment(JSContext* cx, ParseTask* parseTask,
+ Handle<GlobalObject*> global,
+ JSCompartment* dest)
+{
+ // After we call LeaveParseTaskZone() it's not safe to GC until we have
+ // finished merging the contents of the parse task's compartment into the
+ // destination compartment. Finish any ongoing incremental GC first and
+ // assert that no allocation can occur.
+ gc::FinishGC(cx);
+ JS::AutoAssertNoGC nogc(cx);
+
+ LeaveParseTaskZone(cx, parseTask);
+
+ {
+ // Generator functions don't have Function.prototype as prototype but a
+ // different function object, so the IdentifyStandardPrototype trick
+ // below won't work. Just special-case it.
+ GlobalObject* parseGlobal = &parseTask->exclusiveContextGlobal->as<GlobalObject>();
+ JSObject* parseTaskStarGenFunctionProto = parseGlobal->getStarGeneratorFunctionPrototype();
+
+ // Module objects don't have standard prototypes either.
+ JSObject* moduleProto = parseGlobal->maybeGetModulePrototype();
+ JSObject* importEntryProto = parseGlobal->maybeGetImportEntryPrototype();
+ JSObject* exportEntryProto = parseGlobal->maybeGetExportEntryPrototype();
+
+ // Point the prototypes of any objects in the script's compartment to refer
+ // to the corresponding prototype in the new compartment. This will briefly
+ // create cross compartment pointers, which will be fixed by the
+ // MergeCompartments call below.
+ for (auto group = parseTask->cx->zone()->cellIter<ObjectGroup>(); !group.done(); group.next()) {
+ TaggedProto proto(group->proto());
+ if (!proto.isObject())
+ continue;
+
+ JSObject* protoObj = proto.toObject();
+
+ JSObject* newProto;
+ JSProtoKey key = JS::IdentifyStandardPrototype(protoObj);
+ if (key != JSProto_Null) {
+ MOZ_ASSERT(key == JSProto_Object || key == JSProto_Array ||
+ key == JSProto_Function || key == JSProto_RegExp ||
+ key == JSProto_Iterator);
+ newProto = GetBuiltinPrototypePure(global, key);
+ } else if (protoObj == parseTaskStarGenFunctionProto) {
+ newProto = global->getStarGeneratorFunctionPrototype();
+ } else if (protoObj == moduleProto) {
+ newProto = global->getModulePrototype();
+ } else if (protoObj == importEntryProto) {
+ newProto = global->getImportEntryPrototype();
+ } else if (protoObj == exportEntryProto) {
+ newProto = global->getExportEntryPrototype();
+ } else {
+ continue;
+ }
+
+ group->setProtoUnchecked(TaggedProto(newProto));
+ }
+ }
+
+ // Move the parsed script and all its contents into the desired compartment.
+ gc::MergeCompartments(parseTask->cx->compartment(), dest);
+}
+
+void
+HelperThread::destroy()
+{
+ if (thread.isSome()) {
+ {
+ AutoLockHelperThreadState lock;
+ terminate = true;
+
+ /* Notify all helpers, to ensure that this thread wakes up. */
+ HelperThreadState().notifyAll(GlobalHelperThreadState::PRODUCER, lock);
+ }
+
+ thread->join();
+ thread.reset();
+ }
+
+ threadData.reset();
+}
+
+/* static */
+void
+HelperThread::ThreadMain(void* arg)
+{
+ ThisThread::SetName("JS Helper");
+
+ //See bug 1104658.
+ //Set the FPU control word to be the same as the main thread's, or math
+ //computations on this thread may use incorrect precision rules during
+ //Ion compilation.
+ FIX_FPU();
+
+ static_cast<HelperThread*>(arg)->threadLoop();
+}
+
+void
+HelperThread::handleWasmWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartWasmCompile(locked));
+ MOZ_ASSERT(idle());
+
+ currentTask.emplace(HelperThreadState().wasmWorklist(locked).popCopy());
+ bool success = false;
+
+ wasm::IonCompileTask* task = wasmTask();
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+ success = wasm::CompileFunction(task);
+ }
+
+ // On success, try to move work to the finished list.
+ if (success)
+ success = HelperThreadState().wasmFinishedList(locked).append(task);
+
+ // On failure, note the failure for harvesting by the parent.
+ if (!success)
+ HelperThreadState().noteWasmFailure(locked);
+
+ // Notify the main thread in case it's waiting.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+ currentTask.reset();
+}
+
+void
+HelperThread::handlePromiseTaskWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartPromiseTask(locked));
+ MOZ_ASSERT(idle());
+
+ PromiseTask* task = HelperThreadState().promiseTasks(locked).popCopy();
+ currentTask.emplace(task);
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+
+ task->execute();
+
+ if (!task->runtime()->finishAsyncTaskCallback(task)) {
+ // We cannot simply delete the task now because the PromiseTask must
+ // be destroyed on its runtime's thread. Add it to a list of tasks
+ // to delete before the next GC.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!task->runtime()->promiseTasksToDestroy.lock()->append(task))
+ oomUnsafe.crash("handlePromiseTaskWorkload");
+ }
+ }
+
+ // Notify the main thread in case it's waiting.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+ currentTask.reset();
+}
+
+void
+HelperThread::handleIonWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartIonCompile(locked));
+ MOZ_ASSERT(idle());
+
+ // Find the IonBuilder in the worklist with the highest priority, and
+ // remove it from the worklist.
+ jit::IonBuilder* builder =
+ HelperThreadState().highestPriorityPendingIonCompile(locked, /* remove = */ true);
+
+ // If there are now too many threads with active IonBuilders, indicate to
+ // the one with the lowest priority that it should pause. Note that due to
+ // builder priorities changing since pendingIonCompileHasSufficientPriority
+ // was called, the builder we are pausing may actually be higher priority
+ // than the one we are about to start. Oh well.
+ HelperThread* other = HelperThreadState().lowestPriorityUnpausedIonCompileAtThreshold(locked);
+ if (other) {
+ MOZ_ASSERT(other->ionBuilder() && !other->pause);
+ other->pause = true;
+ }
+
+ currentTask.emplace(builder);
+ builder->setPauseFlag(&pause);
+
+ JSRuntime* rt = builder->script()->compartment()->runtimeFromAnyThread();
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+
+ TraceLoggerThread* logger = TraceLoggerForCurrentThread();
+ TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, builder->script());
+ AutoTraceLog logScript(logger, event);
+ AutoTraceLog logCompile(logger, TraceLogger_IonCompilation);
+
+ PerThreadData::AutoEnterRuntime enter(threadData.ptr(),
+ builder->script()->runtimeFromAnyThread());
+ jit::JitContext jctx(jit::CompileRuntime::get(rt),
+ jit::CompileCompartment::get(builder->script()->compartment()),
+ &builder->alloc());
+ builder->setBackgroundCodegen(jit::CompileBackEnd(builder));
+ }
+
+ FinishOffThreadIonCompile(builder, locked);
+ currentTask.reset();
+ pause = false;
+
+ // Ping the main thread so that the compiled code can be incorporated
+ // at the next interrupt callback. Don't interrupt Ion code for this, as
+ // this incorporation can be delayed indefinitely without affecting
+ // performance as long as the main thread is actually executing Ion code.
+ rt->requestInterrupt(JSRuntime::RequestInterruptCanWait);
+
+ // Notify the main thread in case it is waiting for the compilation to finish.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+
+ // When finishing Ion compilation jobs, we can start unpausing compilation
+ // threads that were paused to restrict the number of active compilations.
+ // Only unpause one at a time, to make sure we don't exceed the restriction.
+ // Since threads are currently only paused for Ion compilations, this
+ // strategy will eventually unpause all paused threads, regardless of how
+ // many there are, since each thread we unpause will eventually finish and
+ // end up back here.
+ if (HelperThread* other = HelperThreadState().highestPriorityPausedIonCompile(locked)) {
+ MOZ_ASSERT(other->ionBuilder() && other->pause);
+
+ // Only unpause the other thread if there isn't a higher priority
+ // builder which this thread or another can start on.
+ jit::IonBuilder* builder = HelperThreadState().highestPriorityPendingIonCompile(locked);
+ if (!builder || IonBuilderHasHigherPriority(other->ionBuilder(), builder)) {
+ other->pause = false;
+
+ // Notify all paused threads, to make sure the one we just
+ // unpaused wakes up.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::PAUSE, locked);
+ }
+ }
+}
+
+static HelperThread*
+CurrentHelperThread()
+{
+ auto threadId = ThisThread::GetId();
+ HelperThread* thread = nullptr;
+ for (auto& thisThread : *HelperThreadState().threads) {
+ if (thisThread.thread.isSome() && threadId == thisThread.thread->get_id()) {
+ thread = &thisThread;
+ break;
+ }
+ }
+ MOZ_ASSERT(thread);
+ return thread;
+}
+
+void
+js::PauseCurrentHelperThread()
+{
+ TraceLoggerThread* logger = TraceLoggerForCurrentThread();
+ AutoTraceLog logPaused(logger, TraceLogger_IonCompilationPaused);
+
+ HelperThread* thread = CurrentHelperThread();
+
+ AutoLockHelperThreadState lock;
+ while (thread->pause)
+ HelperThreadState().wait(lock, GlobalHelperThreadState::PAUSE);
+}
+
+void
+ExclusiveContext::setHelperThread(HelperThread* thread)
+{
+ helperThread_ = thread;
+ perThreadData = thread->threadData.ptr();
+}
+
+bool
+ExclusiveContext::addPendingCompileError(frontend::CompileError** error)
+{
+ UniquePtr<frontend::CompileError> errorPtr(new_<frontend::CompileError>());
+ if (!errorPtr)
+ return false;
+ if (!helperThread()->parseTask()->errors.append(errorPtr.get()))
+ return false;
+ *error = errorPtr.release();
+ return true;
+}
+
+void
+ExclusiveContext::addPendingOverRecursed()
+{
+ if (helperThread()->parseTask())
+ helperThread()->parseTask()->overRecursed = true;
+}
+
+void
+ExclusiveContext::addPendingOutOfMemory()
+{
+ // Keep in sync with recoverFromOutOfMemory.
+ if (helperThread()->parseTask())
+ helperThread()->parseTask()->outOfMemory = true;
+}
+
+void
+HelperThread::handleParseWorkload(AutoLockHelperThreadState& locked, uintptr_t stackLimit)
+{
+ MOZ_ASSERT(HelperThreadState().canStartParseTask(locked));
+ MOZ_ASSERT(idle());
+
+ currentTask.emplace(HelperThreadState().parseWorklist(locked).popCopy());
+ ParseTask* task = parseTask();
+ task->cx->setHelperThread(this);
+
+ for (size_t i = 0; i < ArrayLength(task->cx->nativeStackLimit); i++)
+ task->cx->nativeStackLimit[i] = stackLimit;
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+ PerThreadData::AutoEnterRuntime enter(threadData.ptr(),
+ task->exclusiveContextGlobal->runtimeFromAnyThread());
+ task->parse();
+ }
+
+ // The callback is invoked while we are still off the main thread.
+ task->callback(task, task->callbackData);
+
+ // FinishOffThreadScript will need to be called on the script to
+ // migrate it into the correct compartment.
+ {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!HelperThreadState().parseFinishedList(locked).append(task))
+ oomUnsafe.crash("handleParseWorkload");
+ }
+
+ currentTask.reset();
+
+ // Notify the main thread in case it is waiting for the parse/emit to finish.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+}
+
+void
+HelperThread::handleCompressionWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartCompressionTask(locked));
+ MOZ_ASSERT(idle());
+
+ currentTask.emplace(HelperThreadState().compressionWorklist(locked).popCopy());
+ SourceCompressionTask* task = compressionTask();
+ task->helperThread = this;
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+
+ TraceLoggerThread* logger = TraceLoggerForCurrentThread();
+ AutoTraceLog logCompile(logger, TraceLogger_CompressSource);
+
+ task->result = task->work();
+ }
+
+ task->helperThread = nullptr;
+ currentTask.reset();
+
+ // Notify the main thread in case it is waiting for the compression to finish.
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+}
+
+bool
+js::StartOffThreadCompression(ExclusiveContext* cx, SourceCompressionTask* task)
+{
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().compressionWorklist(lock).append(task)) {
+ if (JSContext* maybecx = cx->maybeJSContext())
+ ReportOutOfMemory(maybecx);
+ return false;
+ }
+
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+ return true;
+}
+
+bool
+js::StartPromiseTask(JSContext* cx, UniquePtr<PromiseTask> task)
+{
+ // Execute synchronously if there are no helper threads.
+ if (!CanUseExtraThreads())
+ return task->executeAndFinish(cx);
+
+ // If we fail to start, by interface contract, it is because the JSContext
+ // is in the process of shutting down. Since promise handlers are not
+ // necessarily run while shutting down *anyway*, we simply ignore the error.
+ // This is symmetric with the handling of errors in finishAsyncTaskCallback
+ // which, since it is off the JSContext's owner thread, cannot report an
+ // error anyway.
+ if (!cx->startAsyncTaskCallback(cx, task.get())) {
+ MOZ_ASSERT(!cx->isExceptionPending());
+ return true;
+ }
+
+ // Per interface contract, after startAsyncTaskCallback succeeds,
+ // finishAsyncTaskCallback *must* be called on all paths.
+
+ AutoLockHelperThreadState lock;
+
+ if (!HelperThreadState().promiseTasks(lock).append(task.get())) {
+ Unused << cx->finishAsyncTaskCallback(task.get());
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ Unused << task.release();
+
+ HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER, lock);
+ return true;
+}
+
+bool
+GlobalHelperThreadState::compressionInProgress(SourceCompressionTask* task,
+ const AutoLockHelperThreadState& lock)
+{
+ for (size_t i = 0; i < compressionWorklist(lock).length(); i++) {
+ if (compressionWorklist(lock)[i] == task)
+ return true;
+ }
+ for (auto& thread : *threads) {
+ if (thread.compressionTask() == task)
+ return true;
+ }
+ return false;
+}
+
+bool
+SourceCompressionTask::complete()
+{
+ if (!active())
+ return true;
+
+ {
+ AutoLockHelperThreadState lock;
+ while (HelperThreadState().compressionInProgress(this, lock))
+ HelperThreadState().wait(lock, GlobalHelperThreadState::CONSUMER);
+ }
+
+ if (result == Success) {
+ MOZ_ASSERT(resultString);
+ ss->setCompressedSource(mozilla::Move(*resultString), ss->length());
+ } else {
+ if (result == OOM)
+ ReportOutOfMemory(cx);
+ }
+
+ ss = nullptr;
+ MOZ_ASSERT(!active());
+
+ return result != OOM;
+}
+
+SourceCompressionTask*
+GlobalHelperThreadState::compressionTaskForSource(ScriptSource* ss,
+ const AutoLockHelperThreadState& lock)
+{
+ for (size_t i = 0; i < compressionWorklist(lock).length(); i++) {
+ SourceCompressionTask* task = compressionWorklist(lock)[i];
+ if (task->source() == ss)
+ return task;
+ }
+ for (auto& thread : *threads) {
+ SourceCompressionTask* task = thread.compressionTask();
+ if (task && task->source() == ss)
+ return task;
+ }
+ return nullptr;
+}
+
+void
+GlobalHelperThreadState::trace(JSTracer* trc)
+{
+ AutoLockHelperThreadState lock;
+ for (auto builder : ionWorklist(lock))
+ builder->trace(trc);
+ for (auto builder : ionFinishedList(lock))
+ builder->trace(trc);
+
+ if (HelperThreadState().threads) {
+ for (auto& helper : *HelperThreadState().threads) {
+ if (auto builder = helper.ionBuilder())
+ builder->trace(trc);
+ }
+ }
+
+ jit::IonBuilder* builder = trc->runtime()->ionLazyLinkList().getFirst();
+ while (builder) {
+ builder->trace(trc);
+ builder = builder->getNext();
+ }
+
+ for (auto parseTask : parseWorklist_)
+ parseTask->trace(trc);
+ for (auto parseTask : parseFinishedList_)
+ parseTask->trace(trc);
+ for (auto parseTask : parseWaitingOnGC_)
+ parseTask->trace(trc);
+}
+
+void
+HelperThread::handleGCHelperWorkload(AutoLockHelperThreadState& locked)
+{
+ MOZ_ASSERT(HelperThreadState().canStartGCHelperTask(locked));
+ MOZ_ASSERT(idle());
+
+ currentTask.emplace(HelperThreadState().gcHelperWorklist(locked).popCopy());
+ GCHelperState* task = gcHelperTask();
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+ task->work();
+ }
+
+ currentTask.reset();
+ HelperThreadState().notifyAll(GlobalHelperThreadState::CONSUMER, locked);
+}
+
+void
+HelperThread::threadLoop()
+{
+ MOZ_ASSERT(CanUseExtraThreads());
+
+ JS::AutoSuppressGCAnalysis nogc;
+ AutoLockHelperThreadState lock;
+
+ js::TlsPerThreadData.set(threadData.ptr());
+
+ // Compute the thread's stack limit, for over-recursed checks.
+ uintptr_t stackLimit = GetNativeStackBase();
+#if JS_STACK_GROWTH_DIRECTION > 0
+ stackLimit += HELPER_STACK_QUOTA;
+#else
+ stackLimit -= HELPER_STACK_QUOTA;
+#endif
+
+ while (true) {
+ MOZ_ASSERT(idle());
+
+ // Block until a task is available. Save the value of whether we are
+ // going to do an Ion compile, in case the value returned by the method
+ // changes.
+ bool ionCompile = false;
+ while (true) {
+ if (terminate)
+ return;
+ if ((ionCompile = HelperThreadState().pendingIonCompileHasSufficientPriority(lock)) ||
+ HelperThreadState().canStartWasmCompile(lock) ||
+ HelperThreadState().canStartPromiseTask(lock) ||
+ HelperThreadState().canStartParseTask(lock) ||
+ HelperThreadState().canStartCompressionTask(lock) ||
+ HelperThreadState().canStartGCHelperTask(lock) ||
+ HelperThreadState().canStartGCParallelTask(lock))
+ {
+ break;
+ }
+ HelperThreadState().wait(lock, GlobalHelperThreadState::PRODUCER);
+ }
+
+ if (ionCompile) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_ION);
+ handleIonWorkload(lock);
+ } else if (HelperThreadState().canStartWasmCompile(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_ASMJS);
+ handleWasmWorkload(lock);
+ } else if (HelperThreadState().canStartPromiseTask(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_PROMISE_TASK);
+ handlePromiseTaskWorkload(lock);
+ } else if (HelperThreadState().canStartParseTask(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_PARSE);
+ handleParseWorkload(lock, stackLimit);
+ } else if (HelperThreadState().canStartCompressionTask(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_COMPRESS);
+ handleCompressionWorkload(lock);
+ } else if (HelperThreadState().canStartGCHelperTask(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_GCHELPER);
+ handleGCHelperWorkload(lock);
+ } else if (HelperThreadState().canStartGCParallelTask(lock)) {
+ js::oom::SetThreadType(js::oom::THREAD_TYPE_GCPARALLEL);
+ handleGCParallelWorkload(lock);
+ } else {
+ MOZ_CRASH("No task to perform");
+ }
+ }
+}