path: root/devtools/client/netmonitor/har/har-builder.js

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";

const { defer, all } = require("promise");
const { LocalizationHelper } = require("devtools/shared/l10n");
const Services = require("Services");
const appInfo = Services.appinfo;
const { CurlUtils } = require("devtools/client/shared/curl");
const { getFormDataSections } = require("devtools/client/netmonitor/request-utils");

loader.lazyRequireGetter(this, "NetworkHelper", "devtools/shared/webconsole/network-helper");

loader.lazyGetter(this, "L10N", () => {
  return new LocalizationHelper("devtools/client/locales/har.properties");
});

const HAR_VERSION = "1.1";

/**
 * This object is responsible for building HAR file. See HAR spec:
 * https://dvcs.w3.org/hg/webperf/raw-file/tip/specs/HAR/Overview.html
 * http://www.softwareishard.com/blog/har-12-spec/
 *
 * @param {Object} options configuration object
 *
 * The following options are supported:
 *
 * - items {Array}: List of Network requests to be exported. It is possible
 *   to use directly: NetMonitorView.RequestsMenu.items
 *
 * - id {String}: ID of the exported page.
 *
 * - title {String}: Title of the exported page.
 *
 * - includeResponseBodies {Boolean}: Set to true to include HTTP response
 *   bodies in the result data structure.
 */
var HarBuilder = function (options) {
  this._options = options;
  this._pageMap = [];
};

HarBuilder.prototype = {
  // Public API

  /**
   * This is the main method used to build the entire result HAR data.
   * The process is asynchronous since it can involve additional RDP
   * communication (e.g. resolving long strings).
   *
   * @returns {Promise} A promise that resolves to the HAR object when
   * the entire build process is done.
   */
  build: function () {
    this.promises = [];

    // Build basic structure for data.
    let log = this.buildLog();

    // Build entries.
    let items = this._options.items;
    for (let i = 0; i < items.length; i++) {
      let file = items[i].attachment;
      log.entries.push(this.buildEntry(log, file));
    }

    // Some data needs to be fetched from the backend during the
    // build process, so wait till all is done.
    let { resolve, promise } = defer();
    all(this.promises).then(results => resolve({ log: log }));

    return promise;
  },

  // Helpers

  buildLog: function () {
    return {
      version: HAR_VERSION,
      creator: {
        name: appInfo.name,
        version: appInfo.version
      },
      browser: {
        name: appInfo.name,
        version: appInfo.version
      },
      pages: [],
      entries: [],
    };
  },

  buildPage: function (file) {
    let page = {};

    // Page start time is set when the first request is processed
    // (see buildEntry)
    page.startedDateTime = 0;
    page.id = "page_" + this._options.id;
    page.title = this._options.title;

    return page;
  },

  getPage: function (log, file) {
    let id = this._options.id;
    let page = this._pageMap[id];
    if (page) {
      return page;
    }

    this._pageMap[id] = page = this.buildPage(file);
    log.pages.push(page);

    return page;
  },

  buildEntry: function (log, file) {
    let page = this.getPage(log, file);

    let entry = {};
    entry.pageref = page.id;
    entry.startedDateTime = dateToJSON(new Date(file.startedMillis));
    entry.time = file.endedMillis - file.startedMillis;

    entry.request = this.buildRequest(file);
    entry.response = this.buildResponse(file);
    entry.cache = this.buildCache(file);
    entry.timings = file.eventTimings ? file.eventTimings.timings : {};

    if (file.remoteAddress) {
      entry.serverIPAddress = file.remoteAddress;
    }

    if (file.remotePort) {
      entry.connection = file.remotePort + "";
    }

    // Compute page load start time according to the first request start time.
    if (!page.startedDateTime) {
      page.startedDateTime = entry.startedDateTime;
      page.pageTimings = this.buildPageTimings(page, file);
    }

    return entry;
  },

  buildPageTimings: function (page, file) {
    // Event timing info isn't available
    let timings = {
      onContentLoad: -1,
      onLoad: -1
    };

    return timings;
  },

  buildRequest: function (file) {
    let request = {
      bodySize: 0
    };

    request.method = file.method;
    request.url = file.url;
    request.httpVersion = file.httpVersion || "";

    request.headers = this.buildHeaders(file.requestHeaders);
    request.headers = this.appendHeadersPostData(request.headers, file);
    request.cookies = this.buildCookies(file.requestCookies);

    request.queryString = NetworkHelper.parseQueryString(
      NetworkHelper.nsIURL(file.url).query) || [];

    request.postData = this.buildPostData(file);

    request.headersSize = file.requestHeaders.headersSize;

    // Set request body size, but make sure the body is fetched
    // from the backend.
    if (file.requestPostData) {
      this.fetchData(file.requestPostData.postData.text).then(value => {
        request.bodySize = value.length;
      });
    }

    return request;
  },

  /**
   * Fetch all header values from the backend (if necessary) and
   * build the result HAR structure.
   *
   * @param {Object} input Request or response header object.
   */
  buildHeaders: function (input) {
    if (!input) {
      return [];
    }

    return this.buildNameValuePairs(input.headers);
  },

  appendHeadersPostData: function (input = [], file) {
    if (!file.requestPostData) {
      return input;
    }

    this.fetchData(file.requestPostData.postData.text).then(value => {
      let multipartHeaders = CurlUtils.getHeadersFromMultipartText(value);
      for (let header of multipartHeaders) {
        input.push(header);
      }
    });

    return input;
  },

  buildCookies: function (input) {
    if (!input) {
      return [];
    }

    return this.buildNameValuePairs(input.cookies);
  },

  buildNameValuePairs: function (entries) {
    let result = [];

    // HAR requires headers array to be presented, so always
    // return at least an empty array.
    if (!entries) {
      return result;
    }

    // Make sure header values are fully fetched from the server.
    entries.forEach(entry => {
      this.fetchData(entry.value).then(value => {
        result.push({
          name: entry.name,
          value: value
        });
      });
    });

    return result;
  },

  buildPostData: function (file) {
    let postData = {
      mimeType: findValue(file.requestHeaders.headers, "content-type"),
      params: [],
      text: ""
    };

    if (!file.requestPostData) {
      return postData;
    }

    if (file.requestPostData.postDataDiscarded) {
      postData.comment = L10N.getStr("har.requestBodyNotIncluded");
      return postData;
    }

    // Load request body from the backend.
    this.fetchData(file.requestPostData.postData.text).then(postDataText => {
      postData.text = postDataText;

      // If we are dealing with URL encoded body, parse parameters.
      let { headers } = file.requestHeaders;
      if (CurlUtils.isUrlEncodedRequest({ headers, postDataText })) {
        postData.mimeType = "application/x-www-form-urlencoded";

        // Extract form parameters and produce nice HAR array.
        getFormDataSections(
          file.requestHeaders,
          file.requestHeadersFromUploadStream,
          file.requestPostData,
          this._options.getString
        ).then(formDataSections => {
          formDataSections.forEach(section => {
            let paramsArray = NetworkHelper.parseQueryString(section);
            if (paramsArray) {
              postData.params = [...postData.params, ...paramsArray];
            }
          });
        });
      }
    });

    return postData;
  },

  buildResponse: function (file) {
    let response = {
      status: 0
    };

    // Arbitrary value if it's aborted to make sure status has a number
    if (file.status) {
      response.status = parseInt(file.status, 10);
    }

    let responseHeaders = file.responseHeaders;

    response.statusText = file.statusText || "";
    response.httpVersion = file.httpVersion || "";

    response.headers = this.buildHeaders(responseHeaders);
    response.cookies = this.buildCookies(file.responseCookies);
    response.content = this.buildContent(file);

    let headers = responseHeaders ? responseHeaders.headers : null;
    let headersSize = responseHeaders ? responseHeaders.headersSize : -1;

    response.redirectURL = findValue(headers, "Location");
    response.headersSize = headersSize;

    // 'bodySize' is size of the received response body in bytes.
    // Set to zero in case of responses coming from the cache (304).
    // Set to -1 if the info is not available.
    if (typeof file.transferredSize != "number") {
      response.bodySize = (response.status == 304) ? 0 : -1;
    } else {
      response.bodySize = file.transferredSize;
    }

    return response;
  },

  buildContent: function (file) {
    let content = {
      mimeType: file.mimeType,
      size: -1
    };

    let responseContent = file.responseContent;
    if (responseContent && responseContent.content) {
      content.size = responseContent.content.size;
      content.encoding = responseContent.content.encoding;
    }

    let includeBodies = this._options.includeResponseBodies;
    let contentDiscarded = responseContent ?
      responseContent.contentDiscarded : false;

    // The comment is appended only if the response content
    // is explicitly discarded.
    if (!includeBodies || contentDiscarded) {
      content.comment = L10N.getStr("har.responseBodyNotIncluded");
      return content;
    }

    if (responseContent) {
      let text = responseContent.content.text;
      this.fetchData(text).then(value => {
        content.text = value;
      });
    }

    return content;
  },

  buildCache: function (file) {
    let cache = {};

    if (!file.fromCache) {
      return cache;
    }

    // There is no such info yet in the Net panel.
    // cache.beforeRequest = {};

    if (file.cacheEntry) {
      cache.afterRequest = this.buildCacheEntry(file.cacheEntry);
    } else {
      cache.afterRequest = null;
    }

    return cache;
  },

  buildCacheEntry: function (cacheEntry) {
    let cache = {};

    cache.expires = findValue(cacheEntry, "Expires");
    cache.lastAccess = findValue(cacheEntry, "Last Fetched");
    cache.eTag = "";
    cache.hitCount = findValue(cacheEntry, "Fetch Count");

    return cache;
  },

  getBlockingEndTime: function (file) {
    if (file.resolveStarted && file.connectStarted) {
      return file.resolvingTime;
    }

    if (file.connectStarted) {
      return file.connectingTime;
    }

    if (file.sendStarted) {
      return file.sendingTime;
    }

    return (file.sendingTime > file.startTime) ?
      file.sendingTime : file.waitingForTime;
  },

  // RDP Helpers

  fetchData: function (string) {
    let promise = this._options.getString(string).then(value => {
      return value;
    });

    // Building HAR is asynchronous and not done till all
    // collected promises are resolved.
    this.promises.push(promise);

    return promise;
  }
};

// Helpers

/**
 * Find specified value within an array of name-value pairs
 * (used for headers, cookies and cache entries)
 */
function findValue(arr, name) {
  if (!arr) {
    return "";
  }

  name = name.toLowerCase();
  let result = arr.find(entry => entry.name.toLowerCase() == name);
  return result ? result.value : "";
}

/**
 * Generate HAR representation of a date.
 * (YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00)
 * See also HAR Schema: http://janodvarko.cz/har/viewer/
 *
 * Note: it would be great if we could utilize Date.toJSON(), but
 * it doesn't return proper time zone offset.
 *
 * An example:
 * This helper returns:    2015-05-29T16:10:30.424+02:00
 * Date.toJSON() returns:  2015-05-29T14:10:30.424Z
 *
 * @param date {Date} The date object we want to convert.
 */
function dateToJSON(date) {
  function f(n, c) {
    if (!c) {
      c = 2;
    }
    let s = new String(n);
    while (s.length < c) {
      s = "0" + s;
    }
    return s;
  }

  let result = date.getFullYear() + "-" +
    f(date.getMonth() + 1) + "-" +
    f(date.getDate()) + "T" +
    f(date.getHours()) + ":" +
    f(date.getMinutes()) + ":" +
    f(date.getSeconds()) + "." +
    f(date.getMilliseconds(), 3);

  let offset = date.getTimezoneOffset();
  let positive = offset > 0;

  // Convert to positive number before using Math.floor (see issue 5512)
  offset = Math.abs(offset);
  let offsetHours = Math.floor(offset / 60);
  let offsetMinutes = Math.floor(offset % 60);
  let prettyOffset = (positive > 0 ? "-" : "+") + f(offsetHours) +
    ":" + f(offsetMinutes);

  return result + prettyOffset;
}

// Exports from this module
exports.HarBuilder = HarBuilder;