path: root/application/basilisk/components/migration/AutoMigrate.jsm

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

"use strict";

this.EXPORTED_SYMBOLS = ["AutoMigrate"];

const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;

const kAutoMigrateEnabledPref = "browser.migrate.automigrate.enabled";
const kUndoUIEnabledPref = "browser.migrate.automigrate.ui.enabled";

const kAutoMigrateBrowserPref = "browser.migrate.automigrate.browser";
const kAutoMigrateImportedItemIds = "browser.migrate.automigrate.imported-items";

const kAutoMigrateLastUndoPromptDateMsPref = "browser.migrate.automigrate.lastUndoPromptDateMs";
const kAutoMigrateDaysToOfferUndoPref = "browser.migrate.automigrate.daysToOfferUndo";

const kAutoMigrateUndoSurveyPref = "browser.migrate.automigrate.undo-survey";
const kAutoMigrateUndoSurveyLocalePref = "browser.migrate.automigrate.undo-survey-locales";

const kNotificationId = "automigration-undo";

Cu.import("resource:///modules/MigrationUtils.jsm");
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/Task.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
                                  "resource://gre/modules/AsyncShutdown.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "LoginHelper",
                                  "resource://gre/modules/LoginHelper.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils",
                                  "resource://gre/modules/NewTabUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "OS",
                                  "resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
                                  "resource://gre/modules/PlacesUtils.jsm");

XPCOMUtils.defineLazyGetter(this, "gBrandBundle", function() {
  const kBrandBundle = "chrome://branding/locale/brand.properties";
  return Services.strings.createBundle(kBrandBundle);
});

XPCOMUtils.defineLazyGetter(this, "gHardcodedStringBundle", function() {
  const kBundleURI = "chrome://browser/content/migration/extra-migration-strings.properties";
  return Services.strings.createBundle(kBundleURI);
});

Cu.importGlobalProperties(["URL"]);

/* globals kUndoStateFullPath */
XPCOMUtils.defineLazyGetter(this, "kUndoStateFullPath", function() {
  return OS.Path.join(OS.Constants.Path.profileDir, "initialMigrationMetadata.jsonlz4");
});

const AutoMigrate = {
  get resourceTypesToUse() {
    let {BOOKMARKS, HISTORY, PASSWORDS} = Ci.nsIBrowserProfileMigrator;
    return BOOKMARKS | HISTORY | PASSWORDS;
  },

  _checkIfEnabled() {
    let pref = Preferences.get(kAutoMigrateEnabledPref, false);
    // User-set values should take precedence:
    if (Services.prefs.prefHasUserValue(kAutoMigrateEnabledPref)) {
      return pref;
    }
    // If we're using the default value, make sure the distribution.ini
    // value is taken into account even early on startup.
    try {
      let distributionFile = Services.dirsvc.get("XREAppDist", Ci.nsIFile);
      distributionFile.append("distribution.ini");
      let parser = Cc["@mozilla.org/xpcom/ini-parser-factory;1"].
                 getService(Ci.nsIINIParserFactory).
                 createINIParser(distributionFile);
      return JSON.parse(parser.getString("Preferences", kAutoMigrateEnabledPref));
    } catch (ex) { /* ignore exceptions (file doesn't exist, invalid value, etc.) */ }

    return pref;
  },

  init() {
    this.enabled = this._checkIfEnabled();
  },

  /**
   * Automatically pick a migrator and resources to migrate,
   * then migrate those and start up.
   *
   * @throws if automatically deciding on migrators/data
   *         failed for some reason.
   */
  migrate(profileStartup, migratorKey, profileToMigrate) {
    let histogram = Services.telemetry.getHistogramById(
      "FX_STARTUP_MIGRATION_AUTOMATED_IMPORT_PROCESS_SUCCESS");
    histogram.add(0);
    let {migrator, pickedKey} = this.pickMigrator(migratorKey);
    histogram.add(5);

    profileToMigrate = this.pickProfile(migrator, profileToMigrate);
    histogram.add(10);

    let resourceTypes = migrator.getMigrateData(profileToMigrate, profileStartup);
    if (!(resourceTypes & this.resourceTypesToUse)) {
      throw new Error("No usable resources were found for the selected browser!");
    }
    histogram.add(15);

    let sawErrors = false;
    let migrationObserver = (subject, topic) => {
      if (topic == "Migration:ItemError") {
        sawErrors = true;
      } else if (topic == "Migration:Ended") {
        histogram.add(25);
        if (sawErrors) {
          histogram.add(26);
        }
        Services.obs.removeObserver(migrationObserver, "Migration:Ended");
        Services.obs.removeObserver(migrationObserver, "Migration:ItemError");
        Services.prefs.setCharPref(kAutoMigrateBrowserPref, pickedKey);
        // Save the undo history and block shutdown on that save completing.
        AsyncShutdown.profileBeforeChange.addBlocker(
          "AutoMigrate Undo saving", this.saveUndoState(), () => {
            return {state: this._saveUndoStateTrackerForShutdown};
          });
      }
    };

    MigrationUtils.initializeUndoData();
    Services.obs.addObserver(migrationObserver, "Migration:Ended", false);
    Services.obs.addObserver(migrationObserver, "Migration:ItemError", false);
    migrator.migrate(this.resourceTypesToUse, profileStartup, profileToMigrate);
    histogram.add(20);
  },

  /**
   * Pick and return a migrator to use for automatically migrating.
   *
   * @param {String} migratorKey   optional, a migrator key to prefer/pick.
   * @returns {Object}             an object with the migrator to use for migrating, as
   *                               well as the key we eventually ended up using to obtain it.
   */
  pickMigrator(migratorKey) {
    if (!migratorKey) {
      let defaultKey = MigrationUtils.getMigratorKeyForDefaultBrowser();
      if (!defaultKey) {
        throw new Error("Could not determine default browser key to migrate from");
      }
      migratorKey = defaultKey;
    }
    if (migratorKey == "firefox") {
      throw new Error("Can't automatically migrate from Firefox.");
    }

    let migrator = MigrationUtils.getMigrator(migratorKey);
    if (!migrator) {
      throw new Error("Migrator specified or a default was found, but the migrator object is not available (or has no data).");
    }
    return {migrator, pickedKey: migratorKey};
  },

  /**
   * Pick a source profile (from the original browser) to use.
   *
   * @param {Migrator} migrator     the migrator object to use
   * @param {String}   suggestedId  the id of the profile to migrate, if pre-specified, or null
   * @returns                       the profile to migrate, or null if migrating
   *                                from the default profile.
   */
  pickProfile(migrator, suggestedId) {
    let profiles = migrator.sourceProfiles;
    if (profiles && !profiles.length) {
      throw new Error("No profile data found to migrate.");
    }
    if (suggestedId) {
      if (!profiles) {
        throw new Error("Profile specified but only a default profile found.");
      }
      let suggestedProfile = profiles.find(profile => profile.id == suggestedId);
      if (!suggestedProfile) {
        throw new Error("Profile specified was not found.");
      }
      return suggestedProfile;
    }
    if (profiles && profiles.length > 1) {
      throw new Error("Don't know how to pick a profile when more than 1 profile is present.");
    }
    return profiles ? profiles[0] : null;
  },

  _pendingUndoTasks: false,
  canUndo: Task.async(function* () {
    if (this._savingPromise) {
      yield this._savingPromise;
    }
    if (this._pendingUndoTasks) {
      return false;
    }
    let fileExists = false;
    try {
      fileExists = yield OS.File.exists(kUndoStateFullPath);
    } catch (ex) {
      Cu.reportError(ex);
    }
    return fileExists;
  }),

  undo: Task.async(function* () {
    let browserId = Preferences.get(kAutoMigrateBrowserPref, "unknown");
    let histogram = Services.telemetry.getHistogramById("FX_STARTUP_MIGRATION_AUTOMATED_IMPORT_UNDO");
    histogram.add(0);
    if (!(yield this.canUndo())) {
      histogram.add(5);
      throw new Error("Can't undo!");
    }

    this._pendingUndoTasks = true;
    this._removeNotificationBars();
    histogram.add(10);

    let readPromise = OS.File.read(kUndoStateFullPath, {
      encoding: "utf-8",
      compression: "lz4",
    });
    let stateData = this._dejsonifyUndoState(yield readPromise);
    histogram.add(12);

    this._errorMap = {bookmarks: 0, visits: 0, logins: 0};
    let reportErrorTelemetry = (type) => {
      let histogramId = `FX_STARTUP_MIGRATION_UNDO_${type.toUpperCase()}_ERRORCOUNT`;
      Services.telemetry.getKeyedHistogramById(histogramId).add(browserId, this._errorMap[type]);
    };

    yield this._removeUnchangedBookmarks(stateData.get("bookmarks")).catch(ex => {
      Cu.reportError("Uncaught exception when removing unchanged bookmarks!");
      Cu.reportError(ex);
    });
    reportErrorTelemetry("bookmarks");
    histogram.add(15);

    yield this._removeSomeVisits(stateData.get("visits")).catch(ex => {
      Cu.reportError("Uncaught exception when removing history visits!");
      Cu.reportError(ex);
    });
    reportErrorTelemetry("visits");
    histogram.add(20);

    yield this._removeUnchangedLogins(stateData.get("logins")).catch(ex => {
      Cu.reportError("Uncaught exception when removing unchanged logins!");
      Cu.reportError(ex);
    });
    reportErrorTelemetry("logins");
    histogram.add(25);

    // This is async, but no need to wait for it.
    NewTabUtils.links.populateCache(() => {
      NewTabUtils.allPages.update();
    }, true);

    this._purgeUndoState(this.UNDO_REMOVED_REASON_UNDO_USED);
    histogram.add(30);
  }),

  _removeNotificationBars() {
    let browserWindows = Services.wm.getEnumerator("navigator:browser");
    while (browserWindows.hasMoreElements()) {
      let win = browserWindows.getNext();
      if (!win.closed) {
        for (let browser of win.gBrowser.browsers) {
          let nb = win.gBrowser.getNotificationBox(browser);
          let notification = nb.getNotificationWithValue(kNotificationId);
          if (notification) {
            nb.removeNotification(notification);
          }
        }
      }
    }
  },

  _purgeUndoState(reason) {
    // We don't wait for the off-main-thread removal to complete. OS.File will
    // ensure it happens before shutdown.
    OS.File.remove(kUndoStateFullPath, {ignoreAbsent: true}).then(() => {
      this._pendingUndoTasks = false;
    });

    let migrationBrowser = Preferences.get(kAutoMigrateBrowserPref, "unknown");
    Services.prefs.clearUserPref(kAutoMigrateBrowserPref);

    let histogram =
      Services.telemetry.getKeyedHistogramById("FX_STARTUP_MIGRATION_UNDO_REASON");
    histogram.add(migrationBrowser, reason);
  },

  getBrowserUsedForMigration() {
    let browserId = Services.prefs.getCharPref(kAutoMigrateBrowserPref);
    if (browserId) {
      return MigrationUtils.getBrowserName(browserId);
    }
    return null;
  },

  /**
   * Show the user a notification bar indicating we automatically imported
   * their data and offering them the possibility of removing it.
   * @param target (xul:browser)
   *        The browser in which we should show the notification.
   */
  maybeShowUndoNotification: Task.async(function* (target) {
    if (!(yield this.canUndo())) {
      return;
    }

    // The tab might have navigated since we requested the undo state:
    let canUndoFromThisPage = ["about:home", "about:newtab"].includes(target.currentURI.spec);
    if (!canUndoFromThisPage ||
        !Preferences.get(kUndoUIEnabledPref, false)) {
      return;
    }

    let win = target.ownerGlobal;
    let notificationBox = win.gBrowser.getNotificationBox(target);
    if (!notificationBox || notificationBox.getNotificationWithValue(kNotificationId)) {
      return;
    }

    // At this stage we're committed to show the prompt - unless we shouldn't,
    // in which case we remove the undo prefs (which will cause canUndo() to
    // return false from now on.):
    if (!this.shouldStillShowUndoPrompt()) {
      this._purgeUndoState(this.UNDO_REMOVED_REASON_OFFER_EXPIRED);
      this._removeNotificationBars();
      return;
    }

    let browserName = this.getBrowserUsedForMigration();
    if (!browserName) {
      browserName = gHardcodedStringBundle.GetStringFromName("automigration.undo.unknownbrowser");
    }
    const kMessageId = "automigration.undo.message." +
                      Preferences.get(kAutoMigrateImportedItemIds, "all");
    const kBrandShortName = gBrandBundle.GetStringFromName("brandShortName");
    let message = gHardcodedStringBundle.formatStringFromName(kMessageId,
                                                              [browserName, kBrandShortName], 2);

    let buttons = [
      {
        label: gHardcodedStringBundle.GetStringFromName("automigration.undo.keep2.label"),
        accessKey: gHardcodedStringBundle.GetStringFromName("automigration.undo.keep2.accesskey"),
        callback: () => {
          this._purgeUndoState(this.UNDO_REMOVED_REASON_OFFER_REJECTED);
          this._removeNotificationBars();
        },
      },
      {
        label: gHardcodedStringBundle.GetStringFromName("automigration.undo.dontkeep2.label"),
        accessKey: gHardcodedStringBundle.GetStringFromName("automigration.undo.dontkeep2.accesskey"),
        callback: () => {
          this._maybeOpenUndoSurveyTab(win);
          this.undo();
        },
      },
    ];
    notificationBox.appendNotification(
      message, kNotificationId, null, notificationBox.PRIORITY_INFO_HIGH, buttons
    );
    let remainingDays = Preferences.get(kAutoMigrateDaysToOfferUndoPref, 0);
    Services.telemetry.getHistogramById("FX_STARTUP_MIGRATION_UNDO_OFFERED").add(4 - remainingDays);
  }),

  shouldStillShowUndoPrompt() {
    let today = new Date();
    // Round down to midnight:
    today = new Date(today.getFullYear(), today.getMonth(), today.getDate());
    // We store the unix timestamp corresponding to midnight on the last day
    // on which we prompted. Fetch that and compare it to today's date.
    // (NB: stored as a string because int prefs are too small for unix
    // timestamps.)
    let previousPromptDateMsStr = Preferences.get(kAutoMigrateLastUndoPromptDateMsPref, "0");
    let previousPromptDate = new Date(parseInt(previousPromptDateMsStr, 10));
    if (previousPromptDate < today) {
      let remainingDays = Preferences.get(kAutoMigrateDaysToOfferUndoPref, 4) - 1;
      Preferences.set(kAutoMigrateDaysToOfferUndoPref, remainingDays);
      Preferences.set(kAutoMigrateLastUndoPromptDateMsPref, today.valueOf().toString());
      if (remainingDays <= 0) {
        return false;
      }
    }
    return true;
  },

  UNDO_REMOVED_REASON_UNDO_USED: 0,
  UNDO_REMOVED_REASON_SYNC_SIGNIN: 1,
  UNDO_REMOVED_REASON_PASSWORD_CHANGE: 2,
  UNDO_REMOVED_REASON_BOOKMARK_CHANGE: 3,
  UNDO_REMOVED_REASON_OFFER_EXPIRED: 4,
  UNDO_REMOVED_REASON_OFFER_REJECTED: 5,

  _jsonifyUndoState(state) {
    if (!state) {
      return "null";
    }
    // Deal with date serialization.
    let bookmarks = state.get("bookmarks");
    for (let bm of bookmarks) {
      bm.lastModified = bm.lastModified.getTime();
    }
    let serializableState = {
      bookmarks,
      logins: state.get("logins"),
      visits: state.get("visits"),
    };
    return JSON.stringify(serializableState);
  },

  _dejsonifyUndoState(state) {
    state = JSON.parse(state);
    if (!state) {
      return new Map();
    }
    for (let bm of state.bookmarks) {
      bm.lastModified = new Date(bm.lastModified);
    }
    return new Map([
      ["bookmarks", state.bookmarks],
      ["logins", state.logins],
      ["visits", state.visits],
    ]);
  },

  /**
   * Store the items we've saved into a pref. We use this to be able to show
   * a detailed message to the user indicating what we've imported.
   * @param state (Map)
   *        The 'undo' state for the import, which contains info about
   *        how many items of each kind we've (tried to) import.
   */
  _setImportedItemPrefFromState(state) {
    let itemsWithData = [];
    if (state) {
      for (let itemType of state.keys()) {
        if (state.get(itemType).length) {
          itemsWithData.push(itemType);
        }
      }
    }
    if (itemsWithData.length == 3) {
      itemsWithData = "all";
    } else {
      itemsWithData = itemsWithData.sort().join(".");
    }
    if (itemsWithData) {
      Preferences.set(kAutoMigrateImportedItemIds, itemsWithData);
    }
  },

  /**
   * Used for the shutdown blocker's information field.
   */
  _saveUndoStateTrackerForShutdown: "not running",
  /**
   * Store the information required for using 'undo' of the automatic
   * migration in the user's profile.
   */
  saveUndoState: Task.async(function* () {
    let resolveSavingPromise;
    this._saveUndoStateTrackerForShutdown = "processing undo history";
    this._savingPromise = new Promise(resolve => { resolveSavingPromise = resolve });
    let state = yield MigrationUtils.stopAndRetrieveUndoData();

    if (!state || ![...state.values()].some(ary => ary.length > 0)) {
      // If we didn't import anything, abort now.
      resolveSavingPromise();
      return Promise.resolve();
    }

    this._saveUndoStateTrackerForShutdown = "saving imported item list";
    this._setImportedItemPrefFromState(state);

    this._saveUndoStateTrackerForShutdown = "writing undo history";
    this._undoSavePromise = OS.File.writeAtomic(
      kUndoStateFullPath, this._jsonifyUndoState(state), {
        encoding: "utf-8",
        compression: "lz4",
        tmpPath: kUndoStateFullPath + ".tmp",
      });
    this._undoSavePromise.then(
      rv => {
        resolveSavingPromise(rv);
        delete this._savingPromise;
      },
      e => {
        Cu.reportError("Could not write undo state for automatic migration.");
        throw e;
      });
    return this._undoSavePromise;
  }),

  _removeUnchangedBookmarks: Task.async(function* (bookmarks) {
    if (!bookmarks.length) {
      return;
    }

    let guidToLMMap = new Map(bookmarks.map(b => [b.guid, b.lastModified]));
    let bookmarksFromDB = [];
    let bmPromises = Array.from(guidToLMMap.keys()).map(guid => {
      // Ignore bookmarks where the promise doesn't resolve (ie that are missing)
      // Also check that the bookmark fetch returns isn't null before adding it.
      try {
        return PlacesUtils.bookmarks.fetch(guid).then(bm => bm && bookmarksFromDB.push(bm), () => {});
      } catch (ex) {
        // Ignore immediate exceptions, too.
      }
      return Promise.resolve();
    });
    // We can't use the result of Promise.all because that would include nulls
    // for bookmarks that no longer exist (which we're catching above).
    yield Promise.all(bmPromises);
    let unchangedBookmarks = bookmarksFromDB.filter(bm => {
      return bm.lastModified.getTime() == guidToLMMap.get(bm.guid).getTime();
    });

    // We need to remove items without children first, followed by their
    // parents, etc. In order to do this, find out how many ancestors each item
    // has that also appear in our list of things to remove, and sort the items
    // by those numbers. This ensures that children are always removed before
    // their parents.
    function determineAncestorCount(bm) {
      if (bm._ancestorCount) {
        return bm._ancestorCount;
      }
      let myCount = 0;
      let parentBM = unchangedBookmarks.find(item => item.guid == bm.parentGuid);
      if (parentBM) {
        myCount = determineAncestorCount(parentBM) + 1;
      }
      bm._ancestorCount = myCount;
      return myCount;
    }
    unchangedBookmarks.forEach(determineAncestorCount);
    unchangedBookmarks.sort((a, b) => b._ancestorCount - a._ancestorCount);
    for (let {guid} of unchangedBookmarks) {
      // Can't just use a .catch() because Bookmarks.remove() can throw (rather
      // than returning rejected promises).
      try {
        yield PlacesUtils.bookmarks.remove(guid, {preventRemovalOfNonEmptyFolders: true});
      } catch (err) {
        if (err && err.message != "Cannot remove a non-empty folder.") {
          this._errorMap.bookmarks++;
          Cu.reportError(err);
        }
      }
    }
  }),

  _removeUnchangedLogins: Task.async(function* (logins) {
    for (let login of logins) {
      let foundLogins = LoginHelper.searchLoginsWithObject({guid: login.guid});
      if (foundLogins.length) {
        let foundLogin = foundLogins[0];
        foundLogin.QueryInterface(Ci.nsILoginMetaInfo);
        if (foundLogin.timePasswordChanged == login.timePasswordChanged) {
          try {
            Services.logins.removeLogin(foundLogin);
          } catch (ex) {
            Cu.reportError("Failed to remove a login for " + foundLogins.hostname);
            Cu.reportError(ex);
            this._errorMap.logins++;
          }
        }
      }
    }
  }),

  _removeSomeVisits: Task.async(function* (visits) {
    for (let urlVisits of visits) {
      let urlObj;
      try {
        urlObj = new URL(urlVisits.url);
      } catch (ex) {
        continue;
      }
      let visitData = {
        url: urlObj,
        beginDate: PlacesUtils.toDate(urlVisits.first),
        endDate: PlacesUtils.toDate(urlVisits.last),
        limit: urlVisits.visitCount,
      };
      try {
        yield PlacesUtils.history.removeVisitsByFilter(visitData);
      } catch (ex) {
        this._errorMap.visits++;
        try {
          visitData.url = visitData.url.href;
        } catch (ignoredEx) {}
        Cu.reportError("Failed to remove a visit: " + JSON.stringify(visitData));
        Cu.reportError(ex);
      }
    }
  }),

  /**
   * Maybe open a new tab with a survey. The tab will only be opened if all of
   * the following are true:
   * - the 'browser.migrate.automigrate.undo-survey' pref is not empty.
   *   It should contain the URL of the survey to open.
   * - the 'browser.migrate.automigrate.undo-survey-locales' pref, a
   *   comma-separated list of language codes, contains the language code
   *   that is currently in use for the 'global' chrome pacakge (ie the
   *   locale in which the user is currently using Firefox).
   *   The URL will be passed through nsIURLFormatter to allow including
   *   build ids etc. The special additional formatting variable
   *   "%IMPORTEDBROWSER" is also replaced with the name of the browser
   *   from which we imported data.
   *
   * @param {Window} chromeWindow   A reference to the window in which to open a link.
   */
  _maybeOpenUndoSurveyTab(chromeWindow) {
    let canDoSurveyInLocale = false;
    try {
      let surveyLocales = Preferences.get(kAutoMigrateUndoSurveyLocalePref, "");
      surveyLocales = surveyLocales.split(",").map(str => str.trim());
      // Strip out any empty elements, so an empty pref doesn't
      // lead to a an array with 1 empty string in it.
      surveyLocales = new Set(surveyLocales.filter(str => !!str));
      let chromeRegistry = Cc["@mozilla.org/chrome/chrome-registry;1"]
                             .getService(Ci.nsIXULChromeRegistry);
      canDoSurveyInLocale =
        surveyLocales.has(chromeRegistry.getSelectedLocale("global"));
    } catch (ex) {
      /* ignore exceptions and just don't do the survey. */
    }

    let migrationBrowser = this.getBrowserUsedForMigration();
    let rawURL = Preferences.get(kAutoMigrateUndoSurveyPref, "");
    if (!canDoSurveyInLocale || !migrationBrowser || !rawURL) {
      return;
    }

    let url = Services.urlFormatter.formatURL(rawURL);
    url = url.replace("%IMPORTEDBROWSER%", encodeURIComponent(migrationBrowser));
    chromeWindow.openUILinkIn(url, "tab");
  },

  QueryInterface: XPCOMUtils.generateQI(
    [Ci.nsIObserver, Ci.nsINavBookmarkObserver, Ci.nsISupportsWeakReference]
  ),
};

AutoMigrate.init();