path: root/devtools/server/actors/webaudio.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";

const {Cc, Ci, Cu, Cr} = require("chrome");

const Services = require("Services");

const events = require("sdk/event/core");
const promise = require("promise");
const { on: systemOn, off: systemOff } = require("sdk/system/events");
const protocol = require("devtools/shared/protocol");
const { CallWatcherActor } = require("devtools/server/actors/call-watcher");
const { CallWatcherFront } = require("devtools/shared/fronts/call-watcher");
const { createValueGrip } = require("devtools/server/actors/object");
const AutomationTimeline = require("./utils/automation-timeline");
const { on, once, off, emit } = events;
const { types, method, Arg, Option, RetVal, preEvent } = protocol;
const {
  audionodeSpec,
  webAudioSpec,
  AUTOMATION_METHODS,
  NODE_CREATION_METHODS,
  NODE_ROUTING_METHODS,
} = require("devtools/shared/specs/webaudio");
const { WebAudioFront } = require("devtools/shared/fronts/webaudio");
const AUDIO_NODE_DEFINITION = require("devtools/server/actors/utils/audionodes.json");
const ENABLE_AUTOMATION = false;
const AUTOMATION_GRANULARITY = 2000;
const AUTOMATION_GRANULARITY_MAX = 6000;

const AUDIO_GLOBALS = [
  "AudioContext", "AudioNode", "AudioParam"
];

/**
 * An Audio Node actor allowing communication to a specific audio node in the
 * Audio Context graph.
 */
var AudioNodeActor = exports.AudioNodeActor = protocol.ActorClassWithSpec(audionodeSpec, {
  form: function (detail) {
    if (detail === "actorid") {
      return this.actorID;
    }

    return {
      actor: this.actorID, // actorID is set when this is added to a pool
      type: this.type,
      source: this.source,
      bypassable: this.bypassable,
    };
  },

  /**
   * Create the Audio Node actor.
   *
   * @param DebuggerServerConnection conn
   *        The server connection.
   * @param AudioNode node
   *        The AudioNode that was created.
   */
  initialize: function (conn, node) {
    protocol.Actor.prototype.initialize.call(this, conn);

    // Store ChromeOnly property `id` to identify AudioNode,
    // rather than storing a strong reference, and store a weak
    // ref to underlying node for controlling.
    this.nativeID = node.id;
    this.node = Cu.getWeakReference(node);

    // Stores the AutomationTimelines for this node's AudioParams.
    this.automation = {};

    try {
      this.type = getConstructorName(node);
    } catch (e) {
      this.type = "";
    }

    this.source = !!AUDIO_NODE_DEFINITION[this.type].source;
    this.bypassable = !AUDIO_NODE_DEFINITION[this.type].unbypassable;

    // Create automation timelines for all AudioParams
    Object.keys(AUDIO_NODE_DEFINITION[this.type].properties || {})
      .filter(isAudioParam.bind(null, node))
      .forEach(paramName => {
        this.automation[paramName] = new AutomationTimeline(node[paramName].defaultValue);
      });
  },

  /**
   * Returns the string name of the audio type.
   *
   * DEPRECATED: Use `audionode.type` instead, left here for legacy reasons.
   */
  getType: function () {
    return this.type;
  },

  /**
   * Returns a boolean indicating if the AudioNode has been "bypassed",
   * via `AudioNodeActor#bypass` method.
   *
   * @return Boolean
   */
  isBypassed: function () {
    let node = this.node.get();
    if (node === null) {
      return false;
    }

    // Cast to boolean incase `passThrough` is undefined,
    // like for AudioDestinationNode
    return !!node.passThrough;
  },

  /**
   * Takes a boolean, either enabling or disabling the "passThrough" option
   * on an AudioNode. If a node is bypassed, an effects processing node (like gain, biquad),
   * will allow the audio stream to pass through the node, unaffected. Returns
   * the bypass state of the node.
   *
   * @param Boolean enable
   *        Whether the bypass value should be set on or off.
   * @return Boolean
   */
  bypass: function (enable) {
    let node = this.node.get();

    if (node === null) {
      return;
    }

    if (this.bypassable) {
      node.passThrough = enable;
    }

    return this.isBypassed();
  },

  /**
   * Changes a param on the audio node. Responds with either `undefined`
   * on success, or a description of the error upon param set failure.
   *
   * @param String param
   *        Name of the AudioParam to change.
   * @param String value
   *        Value to change AudioParam to.
   */
  setParam: function (param, value) {
    let node = this.node.get();

    if (node === null) {
      return CollectedAudioNodeError();
    }

    try {
      if (isAudioParam(node, param)) {
        node[param].value = value;
        this.automation[param].setValue(value);
      }
      else {
        node[param] = value;
      }
      return undefined;
    } catch (e) {
      return constructError(e);
    }
  },

  /**
   * Gets a param on the audio node.
   *
   * @param String param
   *        Name of the AudioParam to fetch.
   */
  getParam: function (param) {
    let node = this.node.get();

    if (node === null) {
      return CollectedAudioNodeError();
    }

    // Check to see if it's an AudioParam -- if so,
    // return the `value` property of the parameter.
    let value = isAudioParam(node, param) ? node[param].value : node[param];

    // Return the grip form of the value; at this time,
    // there shouldn't be any non-primitives at the moment, other than
    // AudioBuffer or Float32Array references and the like,
    // so this just formats the value to be displayed in the VariablesView,
    // without using real grips and managing via actor pools.
    let grip = createValueGrip(value, null, createObjectGrip);

    return grip;
  },

  /**
   * Get an object containing key-value pairs of additional attributes
   * to be consumed by a front end, like if a property should be read only,
   * or is a special type (Float32Array, Buffer, etc.)
   *
   * @param String param
   *        Name of the AudioParam whose flags are desired.
   */
  getParamFlags: function (param) {
    return ((AUDIO_NODE_DEFINITION[this.type] || {}).properties || {})[param];
  },

  /**
   * Get an array of objects each containing a `param` and `value` property,
   * corresponding to a property name and current value of the audio node.
   */
  getParams: function (param) {
    let props = Object.keys(AUDIO_NODE_DEFINITION[this.type].properties || {});
    return props.map(prop =>
      ({ param: prop, value: this.getParam(prop), flags: this.getParamFlags(prop) }));
  },

  /**
   * Connects this audionode to an AudioParam via `node.connect(param)`.
   */
  connectParam: function (destActor, paramName, output) {
    let srcNode = this.node.get();
    let destNode = destActor.node.get();

    if (srcNode === null || destNode === null) {
      return CollectedAudioNodeError();
    }

    try {
      // Connect via the unwrapped node, so we can call the
      // patched method that fires the webaudio actor's `connect-param` event.
      // Connect directly to the wrapped `destNode`, otherwise
      // the patched method thinks this is a new node and won't be
      // able to find it in `_nativeToActorID`.
      XPCNativeWrapper.unwrap(srcNode).connect(destNode[paramName], output);
    } catch (e) {
      return constructError(e);
    }
  },

  /**
   * Connects this audionode to another via `node.connect(dest)`.
   */
  connectNode: function (destActor, output, input) {
    let srcNode = this.node.get();
    let destNode = destActor.node.get();

    if (srcNode === null || destNode === null) {
      return CollectedAudioNodeError();
    }

    try {
      // Connect via the unwrapped node, so we can call the
      // patched method that fires the webaudio actor's `connect-node` event.
      // Connect directly to the wrapped `destNode`, otherwise
      // the patched method thinks this is a new node and won't be
      // able to find it in `_nativeToActorID`.
      XPCNativeWrapper.unwrap(srcNode).connect(destNode, output, input);
    } catch (e) {
      return constructError(e);
    }
  },

  /**
   * Disconnects this audionode from all connections via `node.disconnect()`.
   */
  disconnect: function (destActor, output) {
    let node = this.node.get();

    if (node === null) {
      return CollectedAudioNodeError();
    }

    try {
      // Disconnect via the unwrapped node, so we can call the
      // patched method that fires the webaudio actor's `disconnect` event.
      XPCNativeWrapper.unwrap(node).disconnect(output);
    } catch (e) {
      return constructError(e);
    }
  },

  getAutomationData: function (paramName) {
    let timeline = this.automation[paramName];
    if (!timeline) {
      return null;
    }

    let events = timeline.events;
    let values = [];
    let i = 0;

    if (!timeline.events.length) {
      return { events, values };
    }

    let firstEvent = events[0];
    let lastEvent = events[timeline.events.length - 1];
    // `setValueCurveAtTime` will have a duration value -- other
    // events will have duration of `0`.
    let timeDelta = (lastEvent.time + lastEvent.duration) - firstEvent.time;
    let scale = timeDelta / AUTOMATION_GRANULARITY;

    for (; i < AUTOMATION_GRANULARITY; i++) {
      let delta = firstEvent.time + (i * scale);
      let value = timeline.getValueAtTime(delta);
      values.push({ delta, value });
    }

    // If the last event is setTargetAtTime, the automation
    // doesn't actually begin until the event's time, and exponentially
    // approaches the target value. In this case, we add more values
    // until we're "close enough" to the target.
    if (lastEvent.type === "setTargetAtTime") {
      for (; i < AUTOMATION_GRANULARITY_MAX; i++) {
        let delta = firstEvent.time + (++i * scale);
        let value = timeline.getValueAtTime(delta);
        values.push({ delta, value });
      }
    }

    return { events, values };
  },

  /**
   * Called via WebAudioActor, registers an automation event
   * for the AudioParam called.
   *
   * @param String paramName
   *        Name of the AudioParam.
   * @param String eventName
   *        Name of the automation event called.
   * @param Array args
   *        Arguments passed into the automation call.
   */
  addAutomationEvent: function (paramName, eventName, args = []) {
    let node = this.node.get();
    let timeline = this.automation[paramName];

    if (node === null) {
      return CollectedAudioNodeError();
    }

    if (!timeline || !node[paramName][eventName]) {
      return InvalidCommandError();
    }

    try {
      // Using the unwrapped node and parameter, the corresponding
      // WebAudioActor event will be fired, subsequently calling
      // `_recordAutomationEvent`. Some finesse is required to handle
      // the cast of TypedArray arguments over the protocol, which is
      // taken care of below. The event will cast the argument back
      // into an array to be broadcasted from WebAudioActor, but the
      // double-casting will only occur when starting from `addAutomationEvent`,
      // which is only used in tests.
      let param = XPCNativeWrapper.unwrap(node[paramName]);
      let contentGlobal = Cu.getGlobalForObject(param);
      let contentArgs = Cu.cloneInto(args, contentGlobal);

      // If calling `setValueCurveAtTime`, the first argument
      // is a Float32Array, which won't be able to be serialized
      // over the protocol. Cast a normal array to a Float32Array here.
      if (eventName === "setValueCurveAtTime") {
        // Create a Float32Array from the content, seeding with an array
        // from the same scope.
        let curve = new contentGlobal.Float32Array(contentArgs[0]);
        contentArgs[0] = curve;
      }

      // Apply the args back from the content scope, which is necessary
      // due to the method wrapping changing in bug 1130901 to be exported
      // directly to the content scope.
      param[eventName].apply(param, contentArgs);
    } catch (e) {
      return constructError(e);
    }
  },

  /**
   * Registers the automation event in the AudioNodeActor's
   * internal timeline. Called when setting automation via
   * `addAutomationEvent`, or from the WebAudioActor's listening
   * to the event firing via content.
   *
   * @param String paramName
   *        Name of the AudioParam.
   * @param String eventName
   *        Name of the automation event called.
   * @param Array args
   *        Arguments passed into the automation call.
   */
  _recordAutomationEvent: function (paramName, eventName, args) {
    let timeline = this.automation[paramName];
    timeline[eventName].apply(timeline, args);
  }
});

/**
 * The Web Audio Actor handles simple interaction with an AudioContext
 * high-level methods. After instantiating this actor, you'll need to set it
 * up by calling setup().
 */
var WebAudioActor = exports.WebAudioActor = protocol.ActorClassWithSpec(webAudioSpec, {
  initialize: function (conn, tabActor) {
    protocol.Actor.prototype.initialize.call(this, conn);
    this.tabActor = tabActor;

    this._onContentFunctionCall = this._onContentFunctionCall.bind(this);

    // Store ChromeOnly ID (`nativeID` property on AudioNodeActor) mapped
    // to the associated actorID, so we don't have to expose `nativeID`
    // to the client in any way.
    this._nativeToActorID = new Map();

    this._onDestroyNode = this._onDestroyNode.bind(this);
    this._onGlobalDestroyed = this._onGlobalDestroyed.bind(this);
    this._onGlobalCreated = this._onGlobalCreated.bind(this);
  },

  destroy: function (conn) {
    protocol.Actor.prototype.destroy.call(this, conn);
    this.finalize();
  },

  /**
   * Returns definition of all AudioNodes, such as AudioParams, and
   * flags.
   */
  getDefinition: function () {
    return AUDIO_NODE_DEFINITION;
  },

  /**
   * Starts waiting for the current tab actor's document global to be
   * created, in order to instrument the Canvas context and become
   * aware of everything the content does with Web Audio.
   *
   * See ContentObserver and WebAudioInstrumenter for more details.
   */
  setup: function ({ reload }) {
    // Used to track when something is happening with the web audio API
    // the first time, to ultimately fire `start-context` event
    this._firstNodeCreated = false;

    // Clear out stored nativeIDs on reload as we do not want to track
    // AudioNodes that are no longer on this document.
    this._nativeToActorID.clear();

    if (this._initialized) {
      if (reload) {
        this.tabActor.window.location.reload();
      }
      return;
    }

    this._initialized = true;

    this._callWatcher = new CallWatcherActor(this.conn, this.tabActor);
    this._callWatcher.onCall = this._onContentFunctionCall;
    this._callWatcher.setup({
      tracedGlobals: AUDIO_GLOBALS,
      startRecording: true,
      performReload: reload,
      holdWeak: true,
      storeCalls: false
    });
    // Bind to `window-ready` so we can reenable recording on the
    // call watcher
    on(this.tabActor, "window-ready", this._onGlobalCreated);
    // Bind to the `window-destroyed` event so we can unbind events between
    // the global destruction and the `finalize` cleanup method on the actor.
    on(this.tabActor, "window-destroyed", this._onGlobalDestroyed);
  },

  /**
   * Invoked whenever an instrumented function is called, like an AudioContext
   * method or an AudioNode method.
   */
  _onContentFunctionCall: function (functionCall) {
    let { name } = functionCall.details;

    // All Web Audio nodes inherit from AudioNode's prototype, so
    // hook into the `connect` and `disconnect` methods
    if (WebAudioFront.NODE_ROUTING_METHODS.has(name)) {
      this._handleRoutingCall(functionCall);
    }
    else if (WebAudioFront.NODE_CREATION_METHODS.has(name)) {
      this._handleCreationCall(functionCall);
    }
    else if (ENABLE_AUTOMATION && WebAudioFront.AUTOMATION_METHODS.has(name)) {
      this._handleAutomationCall(functionCall);
    }
  },

  _handleRoutingCall: function (functionCall) {
    let { caller, args, name } = functionCall.details;
    let source = caller;
    let dest = args[0];
    let isAudioParam = dest ? getConstructorName(dest) === "AudioParam" : false;

    // audionode.connect(param)
    if (name === "connect" && isAudioParam) {
      this._onConnectParam(source, dest);
    }
    // audionode.connect(node)
    else if (name === "connect") {
      this._onConnectNode(source, dest);
    }
    // audionode.disconnect()
    else if (name === "disconnect") {
      this._onDisconnectNode(source);
    }
  },

  _handleCreationCall: function (functionCall) {
    let { caller, result } = functionCall.details;
    // Keep track of the first node created, so we can alert
    // the front end that an audio context is being used since
    // we're not hooking into the constructor itself, just its
    // instance's methods.
    if (!this._firstNodeCreated) {
      // Fire the start-up event if this is the first node created
      // and trigger a `create-node` event for the context destination
      this._onStartContext();
      this._onCreateNode(caller.destination);
      this._firstNodeCreated = true;
    }
    this._onCreateNode(result);
  },

  _handleAutomationCall: function (functionCall) {
    let { caller, name, args } = functionCall.details;
    let wrappedParam = new XPCNativeWrapper(caller);

    // Sanitize arguments, as these should all be numbers,
    // with the exception of a TypedArray, which needs
    // casted to an Array
    args = sanitizeAutomationArgs(args);

    let nodeActor = this._getActorByNativeID(wrappedParam._parentID);
    nodeActor._recordAutomationEvent(wrappedParam._paramName, name, args);

    this._onAutomationEvent({
      node: nodeActor,
      paramName: wrappedParam._paramName,
      eventName: name,
      args: args
    });
  },

  /**
   * Stops listening for document global changes and puts this actor
   * to hibernation. This method is called automatically just before the
   * actor is destroyed.
   */
  finalize: function () {
    if (!this._initialized) {
      return;
    }
    this._initialized = false;
    systemOff("webaudio-node-demise", this._onDestroyNode);

    off(this.tabActor, "window-destroyed", this._onGlobalDestroyed);
    off(this.tabActor, "window-ready", this._onGlobalCreated);
    this.tabActor = null;
    this._nativeToActorID = null;
    this._callWatcher.eraseRecording();
    this._callWatcher.finalize();
    this._callWatcher = null;
  },

  /**
   * Helper for constructing an AudioNodeActor, assigning to
   * internal weak map, and tracking via `manage` so it is assigned
   * an `actorID`.
   */
  _constructAudioNode: function (node) {
    // Ensure AudioNode is wrapped.
    node = new XPCNativeWrapper(node);

    this._instrumentParams(node);

    let actor = new AudioNodeActor(this.conn, node);
    this.manage(actor);
    this._nativeToActorID.set(node.id, actor.actorID);
    return actor;
  },

  /**
   * Takes an XrayWrapper node, and attaches the node's `nativeID`
   * to the AudioParams as `_parentID`, as well as the the type of param
   * as a string on `_paramName`.
   */
  _instrumentParams: function (node) {
    let type = getConstructorName(node);
    Object.keys(AUDIO_NODE_DEFINITION[type].properties || {})
      .filter(isAudioParam.bind(null, node))
      .forEach(paramName => {
        let param = node[paramName];
        param._parentID = node.id;
        param._paramName = paramName;
      });
  },

  /**
   * Takes an AudioNode and returns the stored actor for it.
   * In some cases, we won't have an actor stored (for example,
   * connecting to an AudioDestinationNode, since it's implicitly
   * created), so make a new actor and store that.
   */
  _getActorByNativeID: function (nativeID) {
    // Ensure we have a Number, rather than a string
    // return via notification.
    nativeID = ~~nativeID;

    let actorID = this._nativeToActorID.get(nativeID);
    let actor = actorID != null ? this.conn.getActor(actorID) : null;
    return actor;
  },

  /**
   * Called on first audio node creation, signifying audio context usage
   */
  _onStartContext: function () {
    systemOn("webaudio-node-demise", this._onDestroyNode);
    emit(this, "start-context");
  },

  /**
   * Called when one audio node is connected to another.
   */
  _onConnectNode: function (source, dest) {
    let sourceActor = this._getActorByNativeID(source.id);
    let destActor = this._getActorByNativeID(dest.id);

    emit(this, "connect-node", {
      source: sourceActor,
      dest: destActor
    });
  },

  /**
   * Called when an audio node is connected to an audio param.
   */
  _onConnectParam: function (source, param) {
    let sourceActor = this._getActorByNativeID(source.id);
    let destActor = this._getActorByNativeID(param._parentID);
    emit(this, "connect-param", {
      source: sourceActor,
      dest: destActor,
      param: param._paramName
    });
  },

  /**
   * Called when an audio node is disconnected.
   */
  _onDisconnectNode: function (node) {
    let actor = this._getActorByNativeID(node.id);
    emit(this, "disconnect-node", actor);
  },

  /**
   * Called when a parameter changes on an audio node
   */
  _onParamChange: function (node, param, value) {
    let actor = this._getActorByNativeID(node.id);
    emit(this, "param-change", {
      source: actor,
      param: param,
      value: value
    });
  },

  /**
   * Called on node creation.
   */
  _onCreateNode: function (node) {
    let actor = this._constructAudioNode(node);
    emit(this, "create-node", actor);
  },

  /** Called when `webaudio-node-demise` is triggered,
   * and emits the associated actor to the front if found.
   */
  _onDestroyNode: function ({data}) {
    // Cast to integer.
    let nativeID = ~~data;

    let actor = this._getActorByNativeID(nativeID);

    // If actorID exists, emit; in the case where we get demise
    // notifications for a document that no longer exists,
    // the mapping should not be found, so we do not emit an event.
    if (actor) {
      this._nativeToActorID.delete(nativeID);
      emit(this, "destroy-node", actor);
    }
  },

  /**
   * Ensures that the new global has recording on
   * so we can proxy the function calls.
   */
  _onGlobalCreated: function () {
    // Used to track when something is happening with the web audio API
    // the first time, to ultimately fire `start-context` event
    this._firstNodeCreated = false;

    // Clear out stored nativeIDs on reload as we do not want to track
    // AudioNodes that are no longer on this document.
    this._nativeToActorID.clear();

    this._callWatcher.resumeRecording();
  },

  /**
   * Fired when an automation event is added to an AudioNode.
   */
  _onAutomationEvent: function ({node, paramName, eventName, args}) {
    emit(this, "automation-event", {
      node: node,
      paramName: paramName,
      eventName: eventName,
      args: args
    });
  },

  /**
   * Called when the underlying ContentObserver fires `global-destroyed`
   * so we can cleanup some things between the global being destroyed and
   * when the actor's `finalize` method gets called.
   */
  _onGlobalDestroyed: function ({id}) {
    if (this._callWatcher._tracedWindowId !== id) {
      return;
    }

    if (this._nativeToActorID) {
      this._nativeToActorID.clear();
    }
    systemOff("webaudio-node-demise", this._onDestroyNode);
  }
});

/**
 * Determines whether or not property is an AudioParam.
 *
 * @param AudioNode node
 *        An AudioNode.
 * @param String prop
 *        Property of `node` to evaluate to see if it's an AudioParam.
 * @return Boolean
 */
function isAudioParam(node, prop) {
  return !!(node[prop] && /AudioParam/.test(node[prop].toString()));
}

/**
 * Takes an `Error` object and constructs a JSON-able response
 *
 * @param Error err
 *        A TypeError, RangeError, etc.
 * @return Object
 */
function constructError(err) {
  return {
    message: err.message,
    type: err.constructor.name
  };
}

/**
 * Creates and returns a JSON-able response used to indicate
 * attempt to access an AudioNode that has been GC'd.
 *
 * @return Object
 */
function CollectedAudioNodeError() {
  return {
    message: "AudioNode has been garbage collected and can no longer be reached.",
    type: "UnreachableAudioNode"
  };
}

function InvalidCommandError() {
  return {
    message: "The command on AudioNode is invalid.",
    type: "InvalidCommand"
  };
}

/**
 * Takes an object and converts it's `toString()` form, like
 * "[object OscillatorNode]" or "[object Float32Array]",
 * or XrayWrapper objects like "[object XrayWrapper [object Array]]"
 * to a string of just the constructor name, like "OscillatorNode",
 * or "Float32Array".
 */
function getConstructorName(obj) {
  return Object.prototype.toString.call(obj).match(/\[object ([^\[\]]*)\]\]?$/)[1];
}

/**
 * Create a grip-like object to pass in renderable information
 * to the front-end for things like Float32Arrays, AudioBuffers,
 * without tracking them in an actor pool.
 */
function createObjectGrip(value) {
  return {
    type: "object",
    preview: {
      kind: "ObjectWithText",
      text: ""
    },
    class: getConstructorName(value)
  };
}

/**
 * Converts all TypedArrays of the array that cannot
 * be passed over the wire into a normal Array equivilent.
 */
function sanitizeAutomationArgs(args) {
  return args.reduce((newArgs, el) => {
    newArgs.push(typeof el === "object" && getConstructorName(el) === "Float32Array" ? castToArray(el) : el);
    return newArgs;
  }, []);
}

/**
 * Casts TypedArray to a normal array via a
 * new scope.
 */
function castToArray(typedArray) {
  // The Xray machinery for TypedArrays denies indexed access on the grounds
  // that it's slow, and advises callers to do a structured clone instead.
  let global = Cu.getGlobalForObject(this);
  let safeView = Cu.cloneInto(typedArray.subarray(), global);
  return copyInto([], safeView);
}

/**
 * Copies values of an array-like `source` into
 * a similarly array-like `dest`.
 */
function copyInto(dest, source) {
  for (let i = 0; i < source.length; i++) {
    dest[i] = source[i];
  }
  return dest;
}