/* Any copyright is dedicated to the Public Domain.
   http://creativecommons.org/publicdomain/zero/1.0/ */

Cu.import("resource://services-sync/engines/clients.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/keys.js");
Cu.import("resource://services-sync/policies.js");
Cu.import("resource://services-sync/service.js");
Cu.import("resource://services-sync/status.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://testing-common/services/sync/utils.js");
Cu.import("resource://gre/modules/FileUtils.jsm");

var fakeServer = new SyncServer();
fakeServer.start();

do_register_cleanup(function() {
  return new Promise(resolve => {
    fakeServer.stop(resolve);
  });
});

var fakeServerUrl = "http://localhost:" + fakeServer.port;

const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);

const PROLONGED_ERROR_DURATION =
  (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') * 2) * 1000;

const NON_PROLONGED_ERROR_DURATION =
  (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') / 2) * 1000;

Service.engineManager.clear();

function setLastSync(lastSyncValue) {
  Svc.Prefs.set("lastSync", (new Date(Date.now() - lastSyncValue)).toString());
}

var engineManager = Service.engineManager;
engineManager.register(EHTestsCommon.CatapultEngine);

// This relies on Service/ErrorHandler being a singleton. Fixing this will take
// a lot of work.
var errorHandler = Service.errorHandler;

function run_test() {
  initTestLogging("Trace");

  Log.repository.getLogger("Sync.Service").level = Log.Level.Trace;
  Log.repository.getLogger("Sync.SyncScheduler").level = Log.Level.Trace;
  Log.repository.getLogger("Sync.ErrorHandler").level = Log.Level.Trace;

  ensureLegacyIdentityManager();

  run_next_test();
}


function clean() {
  Service.startOver();
  Status.resetSync();
  Status.resetBackoff();
  errorHandler.didReportProlongedError = false;
}

add_identity_test(this, function* test_crypto_keys_login_server_maintenance_error() {
  Status.resetSync();
  // Test crypto/keys server maintenance errors are not reported.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  // Force re-download of keys
  Service.collectionKeys.clear();

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  function onUIUpdate() {
    do_throw("Shouldn't get here!");
  }
  Svc.Obs.add("weave:ui:login:error", onUIUpdate);

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
    Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);

    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    clean();
    server.stop(deferred.resolve);
  });

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_task(function* test_sync_prolonged_server_maintenance_error() {
  // Test prolonged server maintenance errors are reported.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  const BACKOFF = 42;
  let engine = engineManager.get("catapult");
  engine.enabled = true;
  engine.exception = {status: 503,
                      headers: {"retry-after": BACKOFF}};

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    server.stop(() => {
      clean();
      deferred.resolve();
    });
  });

  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  let ping = yield sync_and_validate_telem(true);
  deepEqual(ping.status.sync, PROLONGED_SYNC_FAILURE);
  deepEqual(ping.engines.find(e => e.failureReason).failureReason,
            { name: "httperror", code: 503 });
  yield deferred.promise;
});

add_identity_test(this, function* test_info_collections_login_prolonged_server_maintenance_error(){
  // Test info/collections prolonged server maintenance errors are reported.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.info"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_identity_test(this, function* test_meta_global_login_prolonged_server_maintenance_error(){
  // Test meta/global prolonged server maintenance errors are reported.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.meta"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_identity_test(this, function* test_download_crypto_keys_login_prolonged_server_maintenance_error(){
  // Test crypto/keys prolonged server maintenance errors are reported.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";
  // Force re-download of keys
  Service.collectionKeys.clear();

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_identity_test(this, function* test_upload_crypto_keys_login_prolonged_server_maintenance_error(){
  // Test crypto/keys prolonged server maintenance errors are reported.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_identity_test(this, function* test_wipeServer_login_prolonged_server_maintenance_error(){
  // Test that we report prolonged server maintenance errors that occur whilst
  // wiping the server.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.wipe"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_true(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  Service.sync();
  yield deferred.promise;
});

add_identity_test(this, function* test_wipeRemote_prolonged_server_maintenance_error(){
  // Test that we report prolonged server maintenance errors that occur whilst
  // wiping all remote devices.
  let server = EHTestsCommon.sync_httpd_setup();

  server.registerPathHandler("/1.1/broken.wipe/storage/catapult", EHTestsCommon.service_unavailable);
  yield configureIdentity({username: "broken.wipe"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";
  EHTestsCommon.generateAndUploadKeys();

  let engine = engineManager.get("catapult");
  engine.exception = null;
  engine.enabled = true;

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
    do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
    do_check_true(errorHandler.didReportProlongedError);
    server.stop(() => {
      clean();
      deferred.resolve();
    });
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  Svc.Prefs.set("firstSync", "wipeRemote");
  setLastSync(PROLONGED_ERROR_DURATION);
  let ping = yield sync_and_validate_telem(true);
  deepEqual(ping.failureReason, { name: "httperror", code: 503 });
  yield deferred.promise;
});

add_task(function* test_sync_syncAndReportErrors_server_maintenance_error() {
  // Test server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  const BACKOFF = 42;
  let engine = engineManager.get("catapult");
  engine.enabled = true;
  engine.exception = {status: 503,
                      headers: {"retry-after": BACKOFF}};

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
    do_check_eq(Status.sync, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
  // Test info/collections server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.info"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
  // Test meta/global server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.meta"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";
  // Force re-download of keys
  Service.collectionKeys.clear();

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.wipe"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_wipeRemote_syncAndReportErrors_server_maintenance_error(){
  // Test that we report prolonged server maintenance errors that occur whilst
  // wiping all remote devices.
  let server = EHTestsCommon.sync_httpd_setup();

  yield configureIdentity({username: "broken.wipe"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";
  EHTestsCommon.generateAndUploadKeys();

  let engine = engineManager.get("catapult");
  engine.exception = null;
  engine.enabled = true;

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, SYNC_FAILED);
    do_check_eq(Status.sync, SERVER_MAINTENANCE);
    do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  Svc.Prefs.set("firstSync", "wipeRemote");
  setLastSync(NON_PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_task(function* test_sync_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test prolonged server maintenance errors are
  // reported when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  const BACKOFF = 42;
  let engine = engineManager.get("catapult");
  engine.enabled = true;
  engine.exception = {status: 503,
                      headers: {"retry-after": BACKOFF}};

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
    do_check_eq(Status.sync, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_info_collections_login_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test info/collections server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.info"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_meta_global_login_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test meta/global server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.meta"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_download_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();
  yield EHTestsCommon.setUp(server);

  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";
  // Force re-download of keys
  Service.collectionKeys.clear();

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_upload_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.keys"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_identity_test(this, function* test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
  // Test crypto/keys server maintenance errors are reported
  // when calling syncAndReportErrors.
  let server = EHTestsCommon.sync_httpd_setup();

  // Start off with an empty account, do not upload a key.
  yield configureIdentity({username: "broken.wipe"});
  Service.serverURL = server.baseURI + "/maintenance/";
  Service.clusterURL = server.baseURI + "/maintenance/";

  let backoffInterval;
  Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
    Svc.Obs.remove("weave:service:backoff:interval", observe);
    backoffInterval = subject;
  });

  let deferred = Promise.defer();
  Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
    Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
    do_check_true(Status.enforceBackoff);
    do_check_eq(backoffInterval, 42);
    do_check_eq(Status.service, LOGIN_FAILED);
    do_check_eq(Status.login, SERVER_MAINTENANCE);
    // syncAndReportErrors means dontIgnoreErrors, which means
    // didReportProlongedError not touched.
    do_check_false(errorHandler.didReportProlongedError);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_false(Status.enforceBackoff);
  do_check_eq(Status.service, STATUS_OK);

  setLastSync(PROLONGED_ERROR_DURATION);
  errorHandler.syncAndReportErrors();
  yield deferred.promise;
});

add_task(function* test_sync_engine_generic_fail() {
  let server = EHTestsCommon.sync_httpd_setup();

let engine = engineManager.get("catapult");
  engine.enabled = true;
  engine.sync = function sync() {
    Svc.Obs.notify("weave:engine:sync:error", ENGINE_UNKNOWN_FAIL, "catapult");
  };

  let log = Log.repository.getLogger("Sync.ErrorHandler");
  Svc.Prefs.set("log.appender.file.logOnError", true);

  do_check_eq(Status.engines["catapult"], undefined);

  let deferred = Promise.defer();
  // Don't wait for reset-file-log until the sync is underway.
  // This avoids us catching a delayed notification from an earlier test.
  Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() {
    Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish);

    log.info("Adding reset-file-log observer.");
    Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
      Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);

      // Put these checks here, not after sync(), so that we aren't racing the
      // log handler... which resets everything just a few lines below!
      _("Status.engines: " + JSON.stringify(Status.engines));
      do_check_eq(Status.engines["catapult"], ENGINE_UNKNOWN_FAIL);
      do_check_eq(Status.service, SYNC_FAILED_PARTIAL);

      // Test Error log was written on SYNC_FAILED_PARTIAL.
      let entries = logsdir.directoryEntries;
      do_check_true(entries.hasMoreElements());
      let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
      do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);

      clean();

      let syncErrors = sumHistogram("WEAVE_ENGINE_SYNC_ERRORS", { key: "catapult" });
      do_check_true(syncErrors, 1);

      server.stop(() => {
        clean();
        deferred.resolve();
      });
    });
  });

  do_check_true(yield EHTestsCommon.setUp(server));
  let ping = yield sync_and_validate_telem(true);
  deepEqual(ping.status.service, SYNC_FAILED_PARTIAL);
  deepEqual(ping.engines.find(e => e.status).status, ENGINE_UNKNOWN_FAIL);

  yield deferred.promise;
});

add_test(function test_logs_on_sync_error_despite_shouldReportError() {
  _("Ensure that an error is still logged when weave:service:sync:error " +
    "is notified, despite shouldReportError returning false.");

  let log = Log.repository.getLogger("Sync.ErrorHandler");
  Svc.Prefs.set("log.appender.file.logOnError", true);
  log.info("TESTING");

  // Ensure that we report no error.
  Status.login = MASTER_PASSWORD_LOCKED;
  do_check_false(errorHandler.shouldReportError());

  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);

    // Test that error log was written.
    let entries = logsdir.directoryEntries;
    do_check_true(entries.hasMoreElements());
    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
    do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);

    clean();
    run_next_test();
  });
  Svc.Obs.notify("weave:service:sync:error", {});
});

add_test(function test_logs_on_login_error_despite_shouldReportError() {
  _("Ensure that an error is still logged when weave:service:login:error " +
    "is notified, despite shouldReportError returning false.");

  let log = Log.repository.getLogger("Sync.ErrorHandler");
  Svc.Prefs.set("log.appender.file.logOnError", true);
  log.info("TESTING");

  // Ensure that we report no error.
  Status.login = MASTER_PASSWORD_LOCKED;
  do_check_false(errorHandler.shouldReportError());

  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);

    // Test that error log was written.
    let entries = logsdir.directoryEntries;
    do_check_true(entries.hasMoreElements());
    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
    do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);

    clean();
    run_next_test();
  });
  Svc.Obs.notify("weave:service:login:error", {});
});

// This test should be the last one since it monkeypatches the engine object
// and we should only have one engine object throughout the file (bug 629664).
add_task(function* test_engine_applyFailed() {
  let server = EHTestsCommon.sync_httpd_setup();

  let engine = engineManager.get("catapult");
  engine.enabled = true;
  delete engine.exception;
  engine.sync = function sync() {
    Svc.Obs.notify("weave:engine:sync:applied", {newFailed:1}, "catapult");
  };

  let log = Log.repository.getLogger("Sync.ErrorHandler");
  Svc.Prefs.set("log.appender.file.logOnError", true);

  let deferred = Promise.defer();
  Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
    Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);

    do_check_eq(Status.engines["catapult"], ENGINE_APPLY_FAIL);
    do_check_eq(Status.service, SYNC_FAILED_PARTIAL);

    // Test Error log was written on SYNC_FAILED_PARTIAL.
    let entries = logsdir.directoryEntries;
    do_check_true(entries.hasMoreElements());
    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
    do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);

    clean();
    server.stop(deferred.resolve);
  });

  do_check_eq(Status.engines["catapult"], undefined);
  do_check_true(yield EHTestsCommon.setUp(server));
  Service.sync();
  yield deferred.promise;
});