path: root/toolkit/components/url-classifier/tests/unit/test_partial.js

/**
 * DummyCompleter() lets tests easily specify the results of a partial
 * hash completion request.
 */
function DummyCompleter() {
  this.fragments = {};
  this.queries = [];
  this.tableName = "test-phish-simple";
}

DummyCompleter.prototype =
{
QueryInterface: function(iid)
{
  if (!iid.equals(Ci.nsISupports) &&
      !iid.equals(Ci.nsIUrlClassifierHashCompleter)) {
    throw Cr.NS_ERROR_NO_INTERFACE;
  }
  return this;
},

complete: function(partialHash, gethashUrl, cb)
{
  this.queries.push(partialHash);
  var fragments = this.fragments;
  var self = this;
  var doCallback = function() {
      if (self.alwaysFail) {
        cb.completionFinished(1);
        return;
      }
      var results;
      if (fragments[partialHash]) {
        for (var i = 0; i < fragments[partialHash].length; i++) {
          var chunkId = fragments[partialHash][i][0];
          var hash = fragments[partialHash][i][1];
          cb.completion(hash, self.tableName, chunkId);
        }
      }
    cb.completionFinished(0);
  }
  var timer = new Timer(0, doCallback);
},

getHash: function(fragment)
{
  var converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
  createInstance(Ci.nsIScriptableUnicodeConverter);
  converter.charset = "UTF-8";
  var data = converter.convertToByteArray(fragment);
  var ch = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
  ch.init(ch.SHA256);
  ch.update(data, data.length);
  var hash = ch.finish(false);
  return hash.slice(0, 32);
},

addFragment: function(chunkId, fragment)
{
  this.addHash(chunkId, this.getHash(fragment));
},

// This method allows the caller to generate complete hashes that match the
// prefix of a real fragment, but have different complete hashes.
addConflict: function(chunkId, fragment)
{
  var realHash = this.getHash(fragment);
  var invalidHash = this.getHash("blah blah blah blah blah");
  this.addHash(chunkId, realHash.slice(0, 4) + invalidHash.slice(4, 32));
},

addHash: function(chunkId, hash)
{
  var partial = hash.slice(0, 4);
  if (this.fragments[partial]) {
    this.fragments[partial].push([chunkId, hash]);
  } else {
    this.fragments[partial] = [[chunkId, hash]];
  }
},

compareQueries: function(fragments)
{
  var expectedQueries = [];
  for (var i = 0; i < fragments.length; i++) {
    expectedQueries.push(this.getHash(fragments[i]).slice(0, 4));
  }
  do_check_eq(this.queries.length, expectedQueries.length);
  expectedQueries.sort();
  this.queries.sort();
  for (var i = 0; i < this.queries.length; i++) {
    do_check_eq(this.queries[i], expectedQueries[i]);
  }
}
};

function setupCompleter(table, hits, conflicts)
{
  var completer = new DummyCompleter();
  completer.tableName = table;
  for (var i = 0; i < hits.length; i++) {
    var chunkId = hits[i][0];
    var fragments = hits[i][1];
    for (var j = 0; j < fragments.length; j++) {
      completer.addFragment(chunkId, fragments[j]);
    }
  }
  for (var i = 0; i < conflicts.length; i++) {
    var chunkId = conflicts[i][0];
    var fragments = conflicts[i][1];
    for (var j = 0; j < fragments.length; j++) {
      completer.addConflict(chunkId, fragments[j]);
    }
  }

  dbservice.setHashCompleter(table, completer);

  return completer;
}

function installCompleter(table, fragments, conflictFragments)
{
  return setupCompleter(table, fragments, conflictFragments);
}

function installFailingCompleter(table) {
  var completer = setupCompleter(table, [], []);
  completer.alwaysFail = true;
  return completer;
}

// Helper assertion for checking dummy completer queries
gAssertions.completerQueried = function(data, cb)
{
  var completer = data[0];
  completer.compareQueries(data[1]);
  cb();
}

function doTest(updates, assertions)
{
  doUpdateTest(updates, assertions, runNextTest, updateError);
}

// Test an add of two partial urls to a fresh database
function testPartialAdds() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);


  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };


  doTest([update], assertions);
}

function testPartialAddsWithConflicts() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);

  // Each result will have both a real match and a conflict
  var completer = installCompleter('test-phish-simple',
                                   [[1, addUrls]],
                                   [[1, addUrls]]);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);
}

// Test whether the fragmenting code does not cause duplicated completions
function testFragments() {
  var addUrls = [ "foo.com/a/b/c", "foo.net/", "foo.com/c/" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);


  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };


  doTest([update], assertions);
}

// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
// section 6.2 example 1
function testSpecFragments() {
  var probeUrls = [ "a.b.c/1/2.html?param=1" ];

  var addUrls = [ "a.b.c/1/2.html",
                  "a.b.c/",
                  "a.b.c/1/",
                  "b.c/1/2.html?param=1",
                  "b.c/1/2.html",
                  "b.c/",
                  "b.c/1/",
                  "a.b.c/1/2.html?param=1" ];

  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);


  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : probeUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);

}

// Test http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
// section 6.2 example 2
function testMoreSpecFragments() {
  var probeUrls = [ "a.b.c.d.e.f.g/1.html" ];

  var addUrls = [ "a.b.c.d.e.f.g/1.html",
                  "a.b.c.d.e.f.g/",
                  "c.d.e.f.g/1.html",
                  "c.d.e.f.g/",
                  "d.e.f.g/1.html",
                  "d.e.f.g/",
                  "e.f.g/1.html",
                  "e.f.g/",
                  "f.g/1.html",
                  "f.g/" ];

  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);

  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : probeUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);

}

function testFalsePositives() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);

  // Each result will have no matching complete hashes and a non-matching
  // conflict
  var completer = installCompleter('test-phish-simple', [], [[1, addUrls]]);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsDontExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);
}

function testEmptyCompleter() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);

  // Completer will never return full hashes
  var completer = installCompleter('test-phish-simple', [], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsDontExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);
}

function testCompleterFailure() {
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);

  // Completer will never return full hashes
  var completer = installFailingCompleter('test-phish-simple');

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsDontExist" : addUrls,
    "completerQueried" : [completer, addUrls]
  };

  doTest([update], assertions);
}

function testMixedSizesSameDomain() {
  var add1Urls = [ "foo.com/a" ];
  var add2Urls = [ "foo.com/b" ];

  var update1 = buildPhishingUpdate(
    [
      { "chunkNum" : 1,
        "urls" : add1Urls }],
    4);
  var update2 = buildPhishingUpdate(
    [
      { "chunkNum" : 2,
        "urls" : add2Urls }],
    32);

  // We should only need to complete the partial hashes
  var completer = installCompleter('test-phish-simple', [[1, add1Urls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1-2",
    // both urls should match...
    "urlsExist" : add1Urls.concat(add2Urls),
    // ... but the completer should only be queried for the partial entry
    "completerQueried" : [completer, add1Urls]
  };

  doTest([update1, update2], assertions);
}

function testMixedSizesDifferentDomains() {
  var add1Urls = [ "foo.com/a" ];
  var add2Urls = [ "bar.com/b" ];

  var update1 = buildPhishingUpdate(
    [
      { "chunkNum" : 1,
        "urls" : add1Urls }],
    4);
  var update2 = buildPhishingUpdate(
    [
      { "chunkNum" : 2,
        "urls" : add2Urls }],
    32);

  // We should only need to complete the partial hashes
  var completer = installCompleter('test-phish-simple', [[1, add1Urls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1-2",
    // both urls should match...
    "urlsExist" : add1Urls.concat(add2Urls),
    // ... but the completer should only be queried for the partial entry
    "completerQueried" : [completer, add1Urls]
  };

  doTest([update1, update2], assertions);
}

function testInvalidHashSize()
{
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
        12); // only 4 and 32 are legal hash sizes

  var addUrls2 = [ "zaz.com/a", "xyz.com/b" ];
  var update2 = buildPhishingUpdate(
        [
          { "chunkNum" : 2,
            "urls" : addUrls2
          }],
        4);

  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:2",
    "urlsDontExist" : addUrls
  };

  // A successful update will trigger an error
  doUpdateTest([update2, update], assertions, updateError, runNextTest);
}

function testWrongTable()
{
  var addUrls = [ "foo.com/a" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
        4);
  var completer = installCompleter('test-malware-simple', // wrong table
                                   [[1, addUrls]], []);

  // The above installCompleter installs the completer for test-malware-simple,
  // we want it to be used for test-phish-simple too.
  dbservice.setHashCompleter("test-phish-simple", completer);


  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    // The urls were added as phishing urls, but the completer is claiming
    // that they are malware urls, and we trust the completer in this case.
    // The result will be discarded, so we can only check for non-existence.
    "urlsDontExist" : addUrls,
    // Make sure the completer was actually queried.
    "completerQueried" : [completer, addUrls]
  };

  doUpdateTest([update], assertions,
               function() {
                 // Give the dbservice a chance to (not) cache the result.
                 var timer = new Timer(3000, function() {
                     // The miss earlier will have caused a miss to be cached.
                     // Resetting the completer does not count as an update,
                     // so we will not be probed again.
                     var newCompleter = installCompleter('test-malware-simple', [[1, addUrls]], []);                     dbservice.setHashCompleter("test-phish-simple",
                                                newCompleter);

                     var assertions = {
                       "urlsDontExist" : addUrls
                     };
                     checkAssertions(assertions, runNextTest);
                   });
               }, updateError);
}

function setupCachedResults(addUrls, part2)
{
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
        4);

  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    // Request the add url.  This should cause the completion to be cached.
    "urlsExist" : addUrls,
    // Make sure the completer was actually queried.
    "completerQueried" : [completer, addUrls]
  };

  doUpdateTest([update], assertions,
               function() {
                 // Give the dbservice a chance to cache the result.
                 var timer = new Timer(3000, part2);
               }, updateError);
}

function testCachedResults()
{
  setupCachedResults(["foo.com/a"], function(add) {
      // This is called after setupCachedResults().  Verify that
      // checking the url again does not cause a completer request.

      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter('test-phish-simple', [[1, []]], []);

      var assertions = {
        "urlsExist" : ["foo.com/a"],
        "completerQueried" : [newCompleter, []]
      };
      checkAssertions(assertions, runNextTest);
    });
}

function testCachedResultsWithSub() {
  setupCachedResults(["foo.com/a"], function() {
      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter('test-phish-simple', [[1, []]], []);

      var removeUpdate = buildPhishingUpdate(
        [ { "chunkNum" : 2,
            "chunkType" : "s",
            "urls": ["1:foo.com/a"] }],
        4);

      var assertions = {
        "urlsDontExist" : ["foo.com/a"],
        "completerQueried" : [newCompleter, []]
      }

      doTest([removeUpdate], assertions);
    });
}

function testCachedResultsWithExpire() {
  setupCachedResults(["foo.com/a"], function() {
      // install a new completer, this one should never be queried.
      var newCompleter = installCompleter('test-phish-simple', [[1, []]], []);

      var expireUpdate =
        "n:1000\n" +
        "i:test-phish-simple\n" +
        "ad:1\n";

      var assertions = {
        "urlsDontExist" : ["foo.com/a"],
        "completerQueried" : [newCompleter, []]
      }
      doTest([expireUpdate], assertions);
    });
}

function testCachedResultsUpdate()
{
  var existUrls = ["foo.com/a"];
  setupCachedResults(existUrls, function() {
    // This is called after setupCachedResults().  Verify that
    // checking the url again does not cause a completer request.

    // install a new completer, this one should never be queried.
    var newCompleter = installCompleter('test-phish-simple', [[1, []]], []);

    var assertions = {
      "urlsExist" : existUrls,
      "completerQueried" : [newCompleter, []]
    };

    var addUrls = ["foobar.org/a"];

    var update2 = buildPhishingUpdate(
        [
          { "chunkNum" : 2,
            "urls" : addUrls
          }],
        4);

    checkAssertions(assertions, function () {
      // Apply the update. The cached completes should be gone.
      doStreamUpdate(update2, function() {
        // Now the completer gets queried again.
        var newCompleter2 = installCompleter('test-phish-simple', [[1, existUrls]], []);
        var assertions2 = {
          "tableData" : "test-phish-simple;a:1-2",
          "urlsExist" : existUrls,
          "completerQueried" : [newCompleter2, existUrls]
        };
        checkAssertions(assertions2, runNextTest);
      }, updateError);
    });
  });
}

function testCachedResultsFailure()
{
  var existUrls = ["foo.com/a"];
  setupCachedResults(existUrls, function() {
    // This is called after setupCachedResults().  Verify that
    // checking the url again does not cause a completer request.

    // install a new completer, this one should never be queried.
    var newCompleter = installCompleter('test-phish-simple', [[1, []]], []);

    var assertions = {
      "urlsExist" : existUrls,
      "completerQueried" : [newCompleter, []]
    };

    var addUrls = ["foobar.org/a"];

    var update2 = buildPhishingUpdate(
        [
          { "chunkNum" : 2,
            "urls" : addUrls
          }],
        4);

    checkAssertions(assertions, function() {
      // Apply the update. The cached completes should be gone.
      doErrorUpdate("test-phish-simple,test-malware-simple", function() {
        // Now the completer gets queried again.
        var newCompleter2 = installCompleter('test-phish-simple', [[1, existUrls]], []);
        var assertions2 = {
          "tableData" : "test-phish-simple;a:1",
          "urlsExist" : existUrls,
          "completerQueried" : [newCompleter2, existUrls]
        };
        checkAssertions(assertions2, runNextTest);
      }, updateError);
    });
  });
}

function testErrorList()
{
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    4);
  // The update failure should will kill the completes, so the above
  // must be a prefix to get any hit at all past the update failure.

  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : addUrls,
    // These are complete urls, and will only be completed if the
    // list is stale.
    "completerQueried" : [completer, addUrls]
  };

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are marked
      // as fresh.  Fake an update failure to mark them stale.
      doErrorUpdate("test-phish-simple,test-malware-simple", function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, runNextTest);
        }, updateError);
    }, updateError);
}


function testStaleList()
{
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
    32);

  var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    "urlsExist" : addUrls,
    // These are complete urls, and will only be completed if the
    // list is stale.
    "completerQueried" : [completer, addUrls]
  };

  // Consider a match stale after one second.
  prefBranch.setIntPref("urlclassifier.max-complete-age", 1);

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are marked
      // as fresh.  Wait three seconds to make sure the list is marked stale.
      new Timer(3000, function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, function() {
              prefBranch.setIntPref("urlclassifier.max-complete-age", 2700);
              runNextTest();
            });
        }, updateError);
    }, updateError);
}

// Same as testStaleList, but verifies that an empty response still
// unconfirms the entry.
function testStaleListEmpty()
{
  var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : addUrls
          }],
        32);

  var completer = installCompleter('test-phish-simple', [], []);

  var assertions = {
    "tableData" : "test-phish-simple;a:1",
    // None of these should match, because they won't be completed
    "urlsDontExist" : addUrls,
    // These are complete urls, and will only be completed if the
    // list is stale.
    "completerQueried" : [completer, addUrls]
  };

  // Consider a match stale after one second.
  prefBranch.setIntPref("urlclassifier.max-complete-age", 1);

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are marked
      // as fresh.  Wait three seconds to make sure the list is marked stale.
      new Timer(3000, function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, function() {
              prefBranch.setIntPref("urlclassifier.max-complete-age", 2700);
              runNextTest();
            });
        }, updateError);
    }, updateError);
}


// Verify that different lists (test-phish-simple,
// test-malware-simple) maintain their freshness separately.
function testErrorListIndependent()
{
  var phishUrls = [ "phish.com/a" ];
  var malwareUrls = [ "attack.com/a" ];
  var update = buildPhishingUpdate(
        [
          { "chunkNum" : 1,
            "urls" : phishUrls
          }],
    4);
  // These have to persist past the update failure, so they must be prefixes,
  // not completes.

  update += buildMalwareUpdate(
        [
          { "chunkNum" : 2,
            "urls" : malwareUrls
          }],
    32);

  var completer = installCompleter('test-phish-simple', [[1, phishUrls]], []);

  var assertions = {
    "tableData" : "test-malware-simple;a:2\ntest-phish-simple;a:1",
    "urlsExist" : phishUrls,
    "malwareUrlsExist" : malwareUrls,
    // Only this phishing urls should be completed, because only the phishing
    // urls will be stale.
    "completerQueried" : [completer, phishUrls]
  };

  // Apply the update.
  doStreamUpdate(update, function() {
      // Now the test-phish-simple and test-malware-simple tables are
      // marked as fresh.  Fake an update failure to mark *just*
      // phishing data as stale.
      doErrorUpdate("test-phish-simple", function() {
          // Now the lists should be marked stale.  Check assertions.
          checkAssertions(assertions, runNextTest);
        }, updateError);
    }, updateError);
}

function run_test()
{
  runTests([
      testPartialAdds,
      testPartialAddsWithConflicts,
      testFragments,
      testSpecFragments,
      testMoreSpecFragments,
      testFalsePositives,
      testEmptyCompleter,
      testCompleterFailure,
      testMixedSizesSameDomain,
      testMixedSizesDifferentDomains,
      testInvalidHashSize,
      testWrongTable,
      testCachedResults,
      testCachedResultsWithSub,
      testCachedResultsWithExpire,
      testCachedResultsUpdate,
      testCachedResultsFailure,
      testStaleList,
      testStaleListEmpty,
      testErrorList,
      testErrorListIndependent
  ]);
}

do_test_pending();