diff options
Diffstat (limited to 'js/src/builtin/AtomicsObject.cpp')
| -rw-r--r-- | js/src/builtin/AtomicsObject.cpp | 1152 |
1 files changed, 1152 insertions, 0 deletions
diff --git a/js/src/builtin/AtomicsObject.cpp b/js/src/builtin/AtomicsObject.cpp new file mode 100644 index 000000000..08777fd51 --- /dev/null +++ b/js/src/builtin/AtomicsObject.cpp @@ -0,0 +1,1152 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * JS Atomics pseudo-module. + * + * See "Spec: JavaScript Shared Memory, Atomics, and Locks" for the + * full specification. + * + * In addition to what is specified there, we throw an Error object if + * the futex API hooks have not been installed on the runtime. + * Essentially that is an implementation error at a higher level. + * + * + * Note on the current implementation of atomic operations. + * + * The Mozilla atomics are not sufficient to implement these APIs + * because we need to support 8-bit, 16-bit, and 32-bit data: the + * Mozilla atomics only support 32-bit data. + * + * At the moment we include mozilla/Atomics.h, which will define + * MOZ_HAVE_CXX11_ATOMICS and include <atomic> if we have C++11 + * atomics. + * + * If MOZ_HAVE_CXX11_ATOMICS is set we'll use C++11 atomics. + * + * Otherwise, if the compiler has them we'll fall back on gcc/Clang + * intrinsics. + * + * Otherwise, if we're on VC++2012, we'll use C++11 atomics even if + * MOZ_HAVE_CXX11_ATOMICS is not defined. The compiler has the + * atomics but they are disabled in Mozilla due to a performance bug. + * That performance bug does not affect the Atomics code. See + * mozilla/Atomics.h for further comments on that bug. + * + * Otherwise, if we're on VC++2010 or VC++2008, we'll emulate the + * gcc/Clang intrinsics with simple code below using the VC++ + * intrinsics, like the VC++2012 solution this is a stopgap since + * we're about to start using VC++2013 anyway. + * + * If none of those options are available then the build must disable + * shared memory, or compilation will fail with a predictable error. + */ + +#include "builtin/AtomicsObject.h" + +#include "mozilla/Atomics.h" +#include "mozilla/FloatingPoint.h" +#include "mozilla/Maybe.h" +#include "mozilla/Unused.h" + +#include "jsapi.h" +#include "jsfriendapi.h" +#include "jsnum.h" + +#include "jit/AtomicOperations.h" +#include "jit/InlinableNatives.h" +#include "js/Class.h" +#include "vm/GlobalObject.h" +#include "vm/Time.h" +#include "vm/TypedArrayObject.h" +#include "wasm/WasmInstance.h" + +#include "jsobjinlines.h" + +using namespace js; + +const Class AtomicsObject::class_ = { + "Atomics", + JSCLASS_HAS_CACHED_PROTO(JSProto_Atomics) +}; + +static bool +ReportBadArrayType(JSContext* cx) +{ + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_ATOMICS_BAD_ARRAY); + return false; +} + +static bool +ReportOutOfRange(JSContext* cx) +{ + // Use JSMSG_BAD_INDEX here even if it is generic, since that is + // the message used by ToIntegerIndex for its initial range + // checking. + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_BAD_INDEX); + return false; +} + +static bool +ReportCannotWait(JSContext* cx) +{ + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_ATOMICS_WAIT_NOT_ALLOWED); + return false; +} + +static bool +GetSharedTypedArray(JSContext* cx, HandleValue v, + MutableHandle<TypedArrayObject*> viewp) +{ + if (!v.isObject()) + return ReportBadArrayType(cx); + if (!v.toObject().is<TypedArrayObject>()) + return ReportBadArrayType(cx); + viewp.set(&v.toObject().as<TypedArrayObject>()); + if (!viewp->isSharedMemory()) + return ReportBadArrayType(cx); + return true; +} + +static bool +GetTypedArrayIndex(JSContext* cx, HandleValue v, Handle<TypedArrayObject*> view, uint32_t* offset) +{ + uint64_t index; + if (!js::ToIntegerIndex(cx, v, &index)) + return false; + if (index >= view->length()) + return ReportOutOfRange(cx); + *offset = uint32_t(index); + return true; +} + +static int32_t +CompareExchange(Scalar::Type viewType, int32_t oldCandidate, int32_t newCandidate, + SharedMem<void*> viewData, uint32_t offset, bool* badArrayType = nullptr) +{ + switch (viewType) { + case Scalar::Int8: { + int8_t oldval = (int8_t)oldCandidate; + int8_t newval = (int8_t)newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<int8_t*>() + offset, + oldval, newval); + return oldval; + } + case Scalar::Uint8: { + uint8_t oldval = (uint8_t)oldCandidate; + uint8_t newval = (uint8_t)newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<uint8_t*>() + offset, + oldval, newval); + return oldval; + } + case Scalar::Int16: { + int16_t oldval = (int16_t)oldCandidate; + int16_t newval = (int16_t)newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<int16_t*>() + offset, + oldval, newval); + return oldval; + } + case Scalar::Uint16: { + uint16_t oldval = (uint16_t)oldCandidate; + uint16_t newval = (uint16_t)newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<uint16_t*>() + offset, + oldval, newval); + return oldval; + } + case Scalar::Int32: { + int32_t oldval = oldCandidate; + int32_t newval = newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<int32_t*>() + offset, + oldval, newval); + return oldval; + } + case Scalar::Uint32: { + uint32_t oldval = (uint32_t)oldCandidate; + uint32_t newval = (uint32_t)newCandidate; + oldval = jit::AtomicOperations::compareExchangeSeqCst(viewData.cast<uint32_t*>() + offset, + oldval, newval); + return (int32_t)oldval; + } + default: + if (badArrayType) + *badArrayType = true; + return 0; + } +} + +bool +js::atomics_compareExchange(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue objv = args.get(0); + HandleValue idxv = args.get(1); + HandleValue oldv = args.get(2); + HandleValue newv = args.get(3); + MutableHandleValue r = args.rval(); + + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + int32_t oldCandidate; + if (!ToInt32(cx, oldv, &oldCandidate)) + return false; + int32_t newCandidate; + if (!ToInt32(cx, newv, &newCandidate)) + return false; + + bool badType = false; + int32_t result = CompareExchange(view->type(), oldCandidate, newCandidate, + view->viewDataShared(), offset, &badType); + + if (badType) + return ReportBadArrayType(cx); + + if (view->type() == Scalar::Uint32) + r.setNumber((double)(uint32_t)result); + else + r.setInt32(result); + return true; +} + +bool +js::atomics_load(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue objv = args.get(0); + HandleValue idxv = args.get(1); + MutableHandleValue r = args.rval(); + + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + + SharedMem<void*> viewData = view->viewDataShared(); + switch (view->type()) { + case Scalar::Uint8: { + uint8_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<uint8_t*>() + offset); + r.setInt32(v); + return true; + } + case Scalar::Int8: { + int8_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<uint8_t*>() + offset); + r.setInt32(v); + return true; + } + case Scalar::Int16: { + int16_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<int16_t*>() + offset); + r.setInt32(v); + return true; + } + case Scalar::Uint16: { + uint16_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<uint16_t*>() + offset); + r.setInt32(v); + return true; + } + case Scalar::Int32: { + int32_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<int32_t*>() + offset); + r.setInt32(v); + return true; + } + case Scalar::Uint32: { + uint32_t v = jit::AtomicOperations::loadSeqCst(viewData.cast<uint32_t*>() + offset); + r.setNumber(v); + return true; + } + default: + return ReportBadArrayType(cx); + } +} + +enum XchgStoreOp { + DoExchange, + DoStore +}; + +template<XchgStoreOp op> +static int32_t +ExchangeOrStore(Scalar::Type viewType, int32_t numberValue, SharedMem<void*> viewData, + uint32_t offset, bool* badArrayType = nullptr) +{ +#define INT_OP(ptr, value) \ + JS_BEGIN_MACRO \ + if (op == DoStore) \ + jit::AtomicOperations::storeSeqCst(ptr, value); \ + else \ + value = jit::AtomicOperations::exchangeSeqCst(ptr, value); \ + JS_END_MACRO + + switch (viewType) { + case Scalar::Int8: { + int8_t value = (int8_t)numberValue; + INT_OP(viewData.cast<int8_t*>() + offset, value); + return value; + } + case Scalar::Uint8: { + uint8_t value = (uint8_t)numberValue; + INT_OP(viewData.cast<uint8_t*>() + offset, value); + return value; + } + case Scalar::Int16: { + int16_t value = (int16_t)numberValue; + INT_OP(viewData.cast<int16_t*>() + offset, value); + return value; + } + case Scalar::Uint16: { + uint16_t value = (uint16_t)numberValue; + INT_OP(viewData.cast<uint16_t*>() + offset, value); + return value; + } + case Scalar::Int32: { + int32_t value = numberValue; + INT_OP(viewData.cast<int32_t*>() + offset, value); + return value; + } + case Scalar::Uint32: { + uint32_t value = (uint32_t)numberValue; + INT_OP(viewData.cast<uint32_t*>() + offset, value); + return (int32_t)value; + } + default: + if (badArrayType) + *badArrayType = true; + return 0; + } +#undef INT_OP +} + +template<XchgStoreOp op> +static bool +ExchangeOrStore(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue objv = args.get(0); + HandleValue idxv = args.get(1); + HandleValue valv = args.get(2); + MutableHandleValue r = args.rval(); + + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + double integerValue; + if (!ToInteger(cx, valv, &integerValue)) + return false; + + bool badType = false; + int32_t result = ExchangeOrStore<op>(view->type(), JS::ToInt32(integerValue), + view->viewDataShared(), offset, &badType); + + if (badType) + return ReportBadArrayType(cx); + + if (op == DoStore) + r.setNumber(integerValue); + else if (view->type() == Scalar::Uint32) + r.setNumber((double)(uint32_t)result); + else + r.setInt32(result); + return true; +} + +bool +js::atomics_store(JSContext* cx, unsigned argc, Value* vp) +{ + return ExchangeOrStore<DoStore>(cx, argc, vp); +} + +bool +js::atomics_exchange(JSContext* cx, unsigned argc, Value* vp) +{ + return ExchangeOrStore<DoExchange>(cx, argc, vp); +} + +template<typename T> +static bool +AtomicsBinop(JSContext* cx, HandleValue objv, HandleValue idxv, HandleValue valv, + MutableHandleValue r) +{ + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + int32_t numberValue; + if (!ToInt32(cx, valv, &numberValue)) + return false; + + SharedMem<void*> viewData = view->viewDataShared(); + switch (view->type()) { + case Scalar::Int8: { + int8_t v = (int8_t)numberValue; + r.setInt32(T::operate(viewData.cast<int8_t*>() + offset, v)); + return true; + } + case Scalar::Uint8: { + uint8_t v = (uint8_t)numberValue; + r.setInt32(T::operate(viewData.cast<uint8_t*>() + offset, v)); + return true; + } + case Scalar::Int16: { + int16_t v = (int16_t)numberValue; + r.setInt32(T::operate(viewData.cast<int16_t*>() + offset, v)); + return true; + } + case Scalar::Uint16: { + uint16_t v = (uint16_t)numberValue; + r.setInt32(T::operate(viewData.cast<uint16_t*>() + offset, v)); + return true; + } + case Scalar::Int32: { + int32_t v = numberValue; + r.setInt32(T::operate(viewData.cast<int32_t*>() + offset, v)); + return true; + } + case Scalar::Uint32: { + uint32_t v = (uint32_t)numberValue; + r.setNumber((double)T::operate(viewData.cast<uint32_t*>() + offset, v)); + return true; + } + default: + return ReportBadArrayType(cx); + } +} + +#define INTEGRAL_TYPES_FOR_EACH(NAME) \ + static int8_t operate(SharedMem<int8_t*> addr, int8_t v) { return NAME(addr, v); } \ + static uint8_t operate(SharedMem<uint8_t*> addr, uint8_t v) { return NAME(addr, v); } \ + static int16_t operate(SharedMem<int16_t*> addr, int16_t v) { return NAME(addr, v); } \ + static uint16_t operate(SharedMem<uint16_t*> addr, uint16_t v) { return NAME(addr, v); } \ + static int32_t operate(SharedMem<int32_t*> addr, int32_t v) { return NAME(addr, v); } \ + static uint32_t operate(SharedMem<uint32_t*> addr, uint32_t v) { return NAME(addr, v); } + +class PerformAdd +{ +public: + INTEGRAL_TYPES_FOR_EACH(jit::AtomicOperations::fetchAddSeqCst) + static int32_t perform(int32_t x, int32_t y) { return x + y; } +}; + +bool +js::atomics_add(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + return AtomicsBinop<PerformAdd>(cx, args.get(0), args.get(1), args.get(2), args.rval()); +} + +class PerformSub +{ +public: + INTEGRAL_TYPES_FOR_EACH(jit::AtomicOperations::fetchSubSeqCst) + static int32_t perform(int32_t x, int32_t y) { return x - y; } +}; + +bool +js::atomics_sub(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + return AtomicsBinop<PerformSub>(cx, args.get(0), args.get(1), args.get(2), args.rval()); +} + +class PerformAnd +{ +public: + INTEGRAL_TYPES_FOR_EACH(jit::AtomicOperations::fetchAndSeqCst) + static int32_t perform(int32_t x, int32_t y) { return x & y; } +}; + +bool +js::atomics_and(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + return AtomicsBinop<PerformAnd>(cx, args.get(0), args.get(1), args.get(2), args.rval()); +} + +class PerformOr +{ +public: + INTEGRAL_TYPES_FOR_EACH(jit::AtomicOperations::fetchOrSeqCst) + static int32_t perform(int32_t x, int32_t y) { return x | y; } +}; + +bool +js::atomics_or(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + return AtomicsBinop<PerformOr>(cx, args.get(0), args.get(1), args.get(2), args.rval()); +} + +class PerformXor +{ +public: + INTEGRAL_TYPES_FOR_EACH(jit::AtomicOperations::fetchXorSeqCst) + static int32_t perform(int32_t x, int32_t y) { return x ^ y; } +}; + +bool +js::atomics_xor(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + return AtomicsBinop<PerformXor>(cx, args.get(0), args.get(1), args.get(2), args.rval()); +} + +bool +js::atomics_isLockFree(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue v = args.get(0); + int32_t size; + if (v.isInt32()) { + size = v.toInt32(); + } else { + double dsize; + if (!ToInteger(cx, v, &dsize)) + return false; + if (!mozilla::NumberIsInt32(dsize, &size)) { + args.rval().setBoolean(false); + return true; + } + } + args.rval().setBoolean(jit::AtomicOperations::isLockfree(size)); + return true; +} + +// asm.js callouts for platforms that do not have non-word-sized +// atomics where we don't want to inline the logic for the atomics. +// +// Memory will always be shared since the callouts are only called from +// code that checks that the memory is shared. +// +// To test this, either run on eg Raspberry Pi Model 1, or invoke the ARM +// simulator build with ARMHWCAP=vfp set. Do not set any other flags; other +// vfp/neon flags force ARMv7 to be set. + +int32_t +js::atomics_add_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return PerformAdd::operate(heap.cast<int8_t*>() + offset, value); + case Scalar::Uint8: + return PerformAdd::operate(heap.cast<uint8_t*>() + offset, value); + case Scalar::Int16: + return PerformAdd::operate(heap.cast<int16_t*>() + (offset >> 1), value); + case Scalar::Uint16: + return PerformAdd::operate(heap.cast<uint16_t*>() + (offset >> 1), value); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_sub_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return PerformSub::operate(heap.cast<int8_t*>() + offset, value); + case Scalar::Uint8: + return PerformSub::operate(heap.cast<uint8_t*>() + offset, value); + case Scalar::Int16: + return PerformSub::operate(heap.cast<int16_t*>() + (offset >> 1), value); + case Scalar::Uint16: + return PerformSub::operate(heap.cast<uint16_t*>() + (offset >> 1), value); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_and_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return PerformAnd::operate(heap.cast<int8_t*>() + offset, value); + case Scalar::Uint8: + return PerformAnd::operate(heap.cast<uint8_t*>() + offset, value); + case Scalar::Int16: + return PerformAnd::operate(heap.cast<int16_t*>() + (offset >> 1), value); + case Scalar::Uint16: + return PerformAnd::operate(heap.cast<uint16_t*>() + (offset >> 1), value); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_or_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return PerformOr::operate(heap.cast<int8_t*>() + offset, value); + case Scalar::Uint8: + return PerformOr::operate(heap.cast<uint8_t*>() + offset, value); + case Scalar::Int16: + return PerformOr::operate(heap.cast<int16_t*>() + (offset >> 1), value); + case Scalar::Uint16: + return PerformOr::operate(heap.cast<uint16_t*>() + (offset >> 1), value); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_xor_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return PerformXor::operate(heap.cast<int8_t*>() + offset, value); + case Scalar::Uint8: + return PerformXor::operate(heap.cast<uint8_t*>() + offset, value); + case Scalar::Int16: + return PerformXor::operate(heap.cast<int16_t*>() + (offset >> 1), value); + case Scalar::Uint16: + return PerformXor::operate(heap.cast<uint16_t*>() + (offset >> 1), value); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_xchg_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t value) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return ExchangeOrStore<DoExchange>(Scalar::Int8, value, heap, offset); + case Scalar::Uint8: + return ExchangeOrStore<DoExchange>(Scalar::Uint8, value, heap, offset); + case Scalar::Int16: + return ExchangeOrStore<DoExchange>(Scalar::Int16, value, heap, offset>>1); + case Scalar::Uint16: + return ExchangeOrStore<DoExchange>(Scalar::Uint16, value, heap, offset>>1); + default: + MOZ_CRASH("Invalid size"); + } +} + +int32_t +js::atomics_cmpxchg_asm_callout(wasm::Instance* instance, int32_t vt, int32_t offset, int32_t oldval, int32_t newval) +{ + SharedMem<void*> heap = instance->memoryBase().cast<void*>(); + size_t heapLength = instance->memoryLength(); + + if (size_t(offset) >= heapLength) + return 0; + + switch (Scalar::Type(vt)) { + case Scalar::Int8: + return CompareExchange(Scalar::Int8, oldval, newval, heap, offset); + case Scalar::Uint8: + return CompareExchange(Scalar::Uint8, oldval, newval, heap, offset); + case Scalar::Int16: + return CompareExchange(Scalar::Int16, oldval, newval, heap, offset>>1); + case Scalar::Uint16: + return CompareExchange(Scalar::Uint16, oldval, newval, heap, offset>>1); + default: + MOZ_CRASH("Invalid size"); + } +} + +namespace js { + +// Represents one waiting worker. +// +// The type is declared opaque in SharedArrayObject.h. Instances of +// js::FutexWaiter are stack-allocated and linked onto a list across a +// call to FutexRuntime::wait(). +// +// The 'waiters' field of the SharedArrayRawBuffer points to the highest +// priority waiter in the list, and lower priority nodes are linked through +// the 'lower_pri' field. The 'back' field goes the other direction. +// The list is circular, so the 'lower_pri' field of the lowest priority +// node points to the first node in the list. The list has no dedicated +// header node. + +class FutexWaiter +{ + public: + FutexWaiter(uint32_t offset, JSRuntime* rt) + : offset(offset), + rt(rt), + lower_pri(nullptr), + back(nullptr) + { + } + + uint32_t offset; // int32 element index within the SharedArrayBuffer + JSRuntime* rt; // The runtime of the waiter + FutexWaiter* lower_pri; // Lower priority nodes in circular doubly-linked list of waiters + FutexWaiter* back; // Other direction +}; + +class AutoLockFutexAPI +{ + // We have to wrap this in a Maybe because of the way loading + // mozilla::Atomic pointers works. + mozilla::Maybe<js::UniqueLock<js::Mutex>> unique_; + + public: + AutoLockFutexAPI() { + js::Mutex* lock = FutexRuntime::lock_; + unique_.emplace(*lock); + } + + ~AutoLockFutexAPI() { + unique_.reset(); + } + + js::UniqueLock<js::Mutex>& unique() { return *unique_; } +}; + +} // namespace js + +bool +js::atomics_wait(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue objv = args.get(0); + HandleValue idxv = args.get(1); + HandleValue valv = args.get(2); + HandleValue timeoutv = args.get(3); + MutableHandleValue r = args.rval(); + + JSRuntime* rt = cx->runtime(); + + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + if (view->type() != Scalar::Int32) + return ReportBadArrayType(cx); + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + int32_t value; + if (!ToInt32(cx, valv, &value)) + return false; + mozilla::Maybe<mozilla::TimeDuration> timeout; + if (!timeoutv.isUndefined()) { + double timeout_ms; + if (!ToNumber(cx, timeoutv, &timeout_ms)) + return false; + if (!mozilla::IsNaN(timeout_ms)) { + if (timeout_ms < 0) + timeout = mozilla::Some(mozilla::TimeDuration::FromSeconds(0.0)); + else if (!mozilla::IsInfinite(timeout_ms)) + timeout = mozilla::Some(mozilla::TimeDuration::FromMilliseconds(timeout_ms)); + } + } + + if (!rt->fx.canWait()) + return ReportCannotWait(cx); + + // This lock also protects the "waiters" field on SharedArrayRawBuffer, + // and it provides the necessary memory fence. + AutoLockFutexAPI lock; + + SharedMem<int32_t*>(addr) = view->viewDataShared().cast<int32_t*>() + offset; + if (jit::AtomicOperations::loadSafeWhenRacy(addr) != value) { + r.setString(cx->names().futexNotEqual); + return true; + } + + Rooted<SharedArrayBufferObject*> sab(cx, view->bufferShared()); + SharedArrayRawBuffer* sarb = sab->rawBufferObject(); + + FutexWaiter w(offset, rt); + if (FutexWaiter* waiters = sarb->waiters()) { + w.lower_pri = waiters; + w.back = waiters->back; + waiters->back->lower_pri = &w; + waiters->back = &w; + } else { + w.lower_pri = w.back = &w; + sarb->setWaiters(&w); + } + + FutexRuntime::WaitResult result = FutexRuntime::FutexOK; + bool retval = rt->fx.wait(cx, lock.unique(), timeout, &result); + if (retval) { + switch (result) { + case FutexRuntime::FutexOK: + r.setString(cx->names().futexOK); + break; + case FutexRuntime::FutexTimedOut: + r.setString(cx->names().futexTimedOut); + break; + } + } + + if (w.lower_pri == &w) { + sarb->setWaiters(nullptr); + } else { + w.lower_pri->back = w.back; + w.back->lower_pri = w.lower_pri; + if (sarb->waiters() == &w) + sarb->setWaiters(w.lower_pri); + } + return retval; +} + +bool +js::atomics_wake(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + HandleValue objv = args.get(0); + HandleValue idxv = args.get(1); + HandleValue countv = args.get(2); + MutableHandleValue r = args.rval(); + + Rooted<TypedArrayObject*> view(cx, nullptr); + if (!GetSharedTypedArray(cx, objv, &view)) + return false; + if (view->type() != Scalar::Int32) + return ReportBadArrayType(cx); + uint32_t offset; + if (!GetTypedArrayIndex(cx, idxv, view, &offset)) + return false; + double count; + if (countv.isUndefined()) { + count = mozilla::PositiveInfinity<double>(); + } else { + if (!ToInteger(cx, countv, &count)) + return false; + if (count < 0.0) + count = 0.0; + } + + AutoLockFutexAPI lock; + + Rooted<SharedArrayBufferObject*> sab(cx, view->bufferShared()); + SharedArrayRawBuffer* sarb = sab->rawBufferObject(); + int32_t woken = 0; + + FutexWaiter* waiters = sarb->waiters(); + if (waiters && count > 0) { + FutexWaiter* iter = waiters; + do { + FutexWaiter* c = iter; + iter = iter->lower_pri; + if (c->offset != offset || !c->rt->fx.isWaiting()) + continue; + c->rt->fx.wake(FutexRuntime::WakeExplicit); + ++woken; + --count; + } while (count > 0 && iter != waiters); + } + + r.setInt32(woken); + return true; +} + +/* static */ bool +js::FutexRuntime::initialize() +{ + MOZ_ASSERT(!lock_); + lock_ = js_new<js::Mutex>(mutexid::FutexRuntime); + return lock_ != nullptr; +} + +/* static */ void +js::FutexRuntime::destroy() +{ + if (lock_) { + js::Mutex* lock = lock_; + js_delete(lock); + lock_ = nullptr; + } +} + +/* static */ void +js::FutexRuntime::lock() +{ + // Load the atomic pointer. + js::Mutex* lock = lock_; + + lock->lock(); +} + +/* static */ mozilla::Atomic<js::Mutex*> FutexRuntime::lock_; + +/* static */ void +js::FutexRuntime::unlock() +{ + // Load the atomic pointer. + js::Mutex* lock = lock_; + + lock->unlock(); +} + +js::FutexRuntime::FutexRuntime() + : cond_(nullptr), + state_(Idle), + canWait_(false) +{ +} + +bool +js::FutexRuntime::initInstance() +{ + MOZ_ASSERT(lock_); + cond_ = js_new<js::ConditionVariable>(); + return cond_ != nullptr; +} + +void +js::FutexRuntime::destroyInstance() +{ + if (cond_) + js_delete(cond_); +} + +bool +js::FutexRuntime::isWaiting() +{ + // When a worker is awoken for an interrupt it goes into state + // WaitingNotifiedForInterrupt for a short time before it actually + // wakes up and goes into WaitingInterrupted. In those states the + // worker is still waiting, and if an explicit wake arrives the + // worker transitions to Woken. See further comments in + // FutexRuntime::wait(). + return state_ == Waiting || state_ == WaitingInterrupted || state_ == WaitingNotifiedForInterrupt; +} + +bool +js::FutexRuntime::wait(JSContext* cx, js::UniqueLock<js::Mutex>& locked, + mozilla::Maybe<mozilla::TimeDuration>& timeout, WaitResult* result) +{ + MOZ_ASSERT(&cx->runtime()->fx == this); + MOZ_ASSERT(cx->runtime()->fx.canWait()); + MOZ_ASSERT(state_ == Idle || state_ == WaitingInterrupted); + + // Disallow waiting when a runtime is processing an interrupt. + // See explanation below. + + if (state_ == WaitingInterrupted) { + UnlockGuard<Mutex> unlock(locked); + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_ATOMICS_WAIT_NOT_ALLOWED); + return false; + } + + const bool isTimed = timeout.isSome(); + + auto finalEnd = timeout.map([](mozilla::TimeDuration& timeout) { + return mozilla::TimeStamp::Now() + timeout; + }); + + + // 4000s is about the longest timeout slice that is guaranteed to + // work cross-platform. + auto maxSlice = mozilla::TimeDuration::FromSeconds(4000.0); + + bool retval = true; + + for (;;) { + // If we are doing a timed wait, calculate the end time for this wait + // slice. + auto sliceEnd = finalEnd.map([&](mozilla::TimeStamp& finalEnd) { + auto sliceEnd = mozilla::TimeStamp::Now() + maxSlice; + if (finalEnd < sliceEnd) + sliceEnd = finalEnd; + return sliceEnd; + }); + + state_ = Waiting; + + if (isTimed) { + mozilla::Unused << cond_->wait_until(locked, *sliceEnd); + } else { + cond_->wait(locked); + } + + switch (state_) { + case FutexRuntime::Waiting: + // Timeout or spurious wakeup. + if (isTimed) { + auto now = mozilla::TimeStamp::Now(); + if (now >= *finalEnd) { + *result = FutexTimedOut; + goto finished; + } + } + break; + + case FutexRuntime::Woken: + *result = FutexOK; + goto finished; + + case FutexRuntime::WaitingNotifiedForInterrupt: + // The interrupt handler may reenter the engine. In that case + // there are two complications: + // + // - The waiting thread is not actually waiting on the + // condition variable so we have to record that it + // should be woken when the interrupt handler returns. + // To that end, we flag the thread as interrupted around + // the interrupt and check state_ when the interrupt + // handler returns. A wake() call that reaches the + // runtime during the interrupt sets state_ to Woken. + // + // - It is in principle possible for wait() to be + // reentered on the same thread/runtime and waiting on the + // same location and to yet again be interrupted and enter + // the interrupt handler. In this case, it is important + // that when another agent wakes waiters, all waiters using + // the same runtime on the same location are woken in LIFO + // order; FIFO may be the required order, but FIFO would + // fail to wake up the innermost call. Interrupts are + // outside any spec anyway. Also, several such suspended + // waiters may be woken at a time. + // + // For the time being we disallow waiting from within code + // that runs from within an interrupt handler; this may + // occasionally (very rarely) be surprising but is + // expedient. Other solutions exist, see bug #1131943. The + // code that performs the check is above, at the head of + // this function. + + state_ = WaitingInterrupted; + { + UnlockGuard<Mutex> unlock(locked); + retval = cx->runtime()->handleInterrupt(cx); + } + if (!retval) + goto finished; + if (state_ == Woken) { + *result = FutexOK; + goto finished; + } + break; + + default: + MOZ_CRASH("Bad FutexState in wait()"); + } + } +finished: + state_ = Idle; + return retval; +} + +void +js::FutexRuntime::wake(WakeReason reason) +{ + MOZ_ASSERT(isWaiting()); + + if ((state_ == WaitingInterrupted || state_ == WaitingNotifiedForInterrupt) && reason == WakeExplicit) { + state_ = Woken; + return; + } + switch (reason) { + case WakeExplicit: + state_ = Woken; + break; + case WakeForJSInterrupt: + if (state_ == WaitingNotifiedForInterrupt) + return; + state_ = WaitingNotifiedForInterrupt; + break; + default: + MOZ_CRASH("bad WakeReason in FutexRuntime::wake()"); + } + cond_->notify_all(); +} + +const JSFunctionSpec AtomicsMethods[] = { + JS_INLINABLE_FN("compareExchange", atomics_compareExchange, 4,0, AtomicsCompareExchange), + JS_INLINABLE_FN("load", atomics_load, 2,0, AtomicsLoad), + JS_INLINABLE_FN("store", atomics_store, 3,0, AtomicsStore), + JS_INLINABLE_FN("exchange", atomics_exchange, 3,0, AtomicsExchange), + JS_INLINABLE_FN("add", atomics_add, 3,0, AtomicsAdd), + JS_INLINABLE_FN("sub", atomics_sub, 3,0, AtomicsSub), + JS_INLINABLE_FN("and", atomics_and, 3,0, AtomicsAnd), + JS_INLINABLE_FN("or", atomics_or, 3,0, AtomicsOr), + JS_INLINABLE_FN("xor", atomics_xor, 3,0, AtomicsXor), + JS_INLINABLE_FN("isLockFree", atomics_isLockFree, 1,0, AtomicsIsLockFree), + JS_FN("wait", atomics_wait, 4,0), + JS_FN("wake", atomics_wake, 3,0), + JS_FS_END +}; + +JSObject* +AtomicsObject::initClass(JSContext* cx, Handle<GlobalObject*> global) +{ + // Create Atomics Object. + RootedObject objProto(cx, global->getOrCreateObjectPrototype(cx)); + if (!objProto) + return nullptr; + RootedObject Atomics(cx, NewObjectWithGivenProto(cx, &AtomicsObject::class_, objProto, + SingletonObject)); + if (!Atomics) + return nullptr; + + if (!JS_DefineFunctions(cx, Atomics, AtomicsMethods)) + return nullptr; + + RootedValue AtomicsValue(cx, ObjectValue(*Atomics)); + + // Everything is set up, install Atomics on the global object. + if (!DefineProperty(cx, global, cx->names().Atomics, AtomicsValue, nullptr, nullptr, + JSPROP_RESOLVING)) + { + return nullptr; + } + + global->setConstructor(JSProto_Atomics, AtomicsValue); + return Atomics; +} + +JSObject* +js::InitAtomicsClass(JSContext* cx, HandleObject obj) +{ + MOZ_ASSERT(obj->is<GlobalObject>()); + Rooted<GlobalObject*> global(cx, &obj->as<GlobalObject>()); + return AtomicsObject::initClass(cx, global); +} + +#undef CXX11_ATOMICS +#undef GNU_ATOMICS |