Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

1 files changed, 2283 insertions, 0 deletions

diff --git a/mailnews/db/gloda/modules/gloda.js b/mailnews/db/gloda/modules/gloda.js
new file mode 100644
index 000000000..3a32e6041
--- /dev/null
+++ b/mailnews/db/gloda/modules/gloda.js
@@ -0,0 +1,2283 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['Gloda'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+Cu.import("resource:///modules/gloda/datastore.js");
+Cu.import("resource:///modules/gloda/datamodel.js");
+Cu.import("resource:///modules/gloda/databind.js");
+Cu.import("resource:///modules/gloda/collection.js");
+Cu.import("resource:///modules/gloda/connotent.js");
+Cu.import("resource:///modules/gloda/query.js");
+Cu.import("resource:///modules/gloda/utils.js");
+
+Cu.import("resource:///modules/iteratorUtils.jsm");
+Cu.import("resource:///modules/IOUtils.js");
+Cu.import("resource:///modules/mailServices.js");
+Cu.import("resource://gre/modules/Services.jsm");
+
+/**
+ * @see |Gloda.BadItemContentsError|
+ */
+function BadItemContentsError(aMessage) {
+  this.message = aMessage;
+}
+BadItemContentsError.prototype = {
+  toString: function BadItemContentsError_toString() {
+    return this.message;
+  }
+};
+
+
+/**
+ * Provides the user-visible (and extension visible) global database
+ *  functionality.  There is currently a dependency/ordering
+ *  problem in that the concept of 'gloda' also includes some logic that is
+ *  contributed by built-in extensions, if you will.  Those built-in extensions
+ *  (fundattr.js, explattr.js) also import this file.  To avoid a circular
+ *  dependency, those built-in extensions are loaded by everybody.js.  The
+ *  simplest/best solution is probably to move everybody.js to be gloda.js and
+ *  have it re-export only 'Gloda'.  gloda.js (this file) can then move to be
+ *  gloda_int.js (or whatever our eventual naming scheme is), which built-in
+ *  extensions can explicitly rely upon.
+ *
+ * === Concepts
+ *
+ * == Nouns
+ *
+ * Inspired by reasonable uses of triple-stores, I have tried to leverage
+ *  existing model and terminology rather than rolling out own for everything.
+ *  The idea with triple-stores is that you have a subject, a predicate, and an
+ *  object.  For example, if we are talking about a message, that is the
+ *  subject, the predicate could roughly be sent-by, and the object a person.
+ *  We can generalize this idea to say that the subject and objects are nouns.
+ * Since we want to be more flexible than only dealing with messages, we
+ *  therefore introduce the concept of nouns as an organizing principle.
+ *
+ * == Attributes
+ *
+ * Our attributes definitions are basically our predicates.  When we define
+ *  an attribute, it's a label with a bunch of meta-data.  Our attribute
+ *  instances are basically a 'triple' in a triple-store.  The attributes
+ *  are stored in database rows that imply a specific noun-type (ex: the
+ *  messageAttributes table), with an ID identifying the message which is our
+ *  subject, an attribute ID which identifies the attribute definition in use
+ *  (and therefore the predicate), plus an object ID (given context aka the
+ *  noun type by the attribute's meta-data) which identifies the 'object'.
+ *
+ * == But...
+ *
+ * Things aren't entirely as clear as they could be right now, terminology/
+ *  concept/implementation-wise.  Some work is probably still in order.
+ *
+ * === Implementation
+ *
+ * == Nouns
+ *
+ * So, we go and define the nouns that are roughly the classes in our data
+ *  model.  Every 'class' we define in datamodel.js is a noun that gets defined
+ *  here in the Gloda core.  We provide sufficient meta-data about the noun to
+ *  serialize/deserialize its representation from our database representation.
+ *  Nouns do not have to be defined in this class, but can also be contributed
+ *  by external code.
+ * We have a concept of 'first class' nouns versus non-first class nouns.  The
+ *  distinction is meant to be whether we can store meta-information about those
+ *  nouns using attributes.  Right now, only message are real first-class nouns,
+ *  but we want to expand that to include contacts and eventually events and
+ *  tasks as lightning-integration occurs.  In practice, we are stretching the
+ *  definition of first-class nouns slightly to include things we can't store
+ *  meta-data about, but want to be able to query about.  We do want to resolve
+ *  this.
+ *
+ * == Attributes
+ *
+ * Attributes are defined by "attribute providers" who are responsible for
+ *  taking an instance of a first-class noun (for which they are registered)
+ *  plus perhaps some other meta-data, and returning a list of attributes
+ *  extracted from that noun.  For now, this means messages.  Attribute
+ *  providers may create new data records as a side-effect of the indexing
+ *  process, although we have not yet fully dealt with the problem of deleting
+ *  these records should they become orphaned in the database due to the
+ *  purging of a message and its attributes.
+ * All of the 'core' gloda attributes are provided by the fundattr.js and
+ *  explattr.js providers.
+ *
+ * === (Notable) Future Work
+ *
+ * == Attributes
+ *
+ * Attribute mechanisms currently lack any support for 'overriding' attributes
+ *  provided by other attribute providers.  For example, the fundattr provider
+ *  tells us who a message is 'from' based on the e-mail address present.
+ *  However, other plugins may actually know better.  For example, the bugzilla
+ *  daemon e-mails based on bug activity although the daemon gets the credit
+ *  as the official sender.  A bugzilla plugin can easily extract the actual
+ *  person/e-mail addressed who did something on the bug to cause the
+ *  notification to be sent.  In practice, we would like that person to be
+ *  the 'sender' of the bugmail.  But we can't really do that right, yet.
+ *
+ * @namespace
+ */
+var Gloda = {
+  /**
+   * Initialize logging, the datastore (SQLite database), the core nouns and
+   *  attributes, and the contact and identities that belong to the presumed
+   *  current user (based on accounts).
+   *
+   * Additional nouns and the core attribute providers are initialized by the
+   *  everybody.js module which ensures all of those dependencies are loaded
+   *  (and initialized).
+   */
+  _init: function gloda_ns_init() {
+    this._initLogging();
+    GlodaDatastore._init(this._nounIDToDef);
+    this._initAttributes();
+    this._initMyIdentities();
+  },
+
+  _log: null,
+  /**
+   * Initialize logging; the error console window gets Warning/Error, and stdout
+   *  (via dump) gets everything.
+   */
+  _initLogging: function gloda_ns_initLogging() {
+    let formatter = new Log4Moz.BasicFormatter();
+    Log4Moz.repository.rootLogger.level = Log4Moz.Level.Debug;
+
+    let enableConsoleLogging = false;
+    let enableDumpLogging = false;
+    // should we assume there is someone else consuming our log4moz stream?
+    let enableUpstreamLogging = false;
+    let considerNetLogging = false;
+
+    let glodaLog = Log4Moz.repository.getLogger("gloda");
+    glodaLog.level = Log4Moz.Level.Warn;
+
+    try {
+      // figure out if event-driven indexing should be enabled...
+      let branch = Services.prefs.getBranch("mailnews.database.global.logging.");
+      enableConsoleLogging = branch.getBoolPref("console");
+      enableDumpLogging = branch.getBoolPref("dump");
+      enableUpstreamLogging = branch.getBoolPref("upstream");
+      considerNetLogging = branch.getBoolPref("net");
+    } catch (ex) {}
+
+    if (enableConsoleLogging) {
+      let capp = new Log4Moz.ConsoleAppender(formatter);
+      capp.level = Log4Moz.Level.Warn;
+      glodaLog.addAppender(capp);
+    }
+
+    if (enableDumpLogging) {
+      let dapp = new Log4Moz.DumpAppender(formatter);
+      dapp.level = Log4Moz.Level.All;
+      glodaLog.level = Log4Moz.Level.All;
+      glodaLog.addAppender(dapp);
+    }
+
+    if (enableUpstreamLogging) {
+      glodaLog.level = Log4Moz.Level.All;
+    }
+
+    if (considerNetLogging) {
+      let file = Services.dirsvc.get("TmpD", Ci.nsIFile);
+      file.append("chainsaw.ptr");
+      if (file.exists()) {
+        let data = IOUtils.loadFileToString(file);
+        data = data.trim();
+        let [host, port] = data.split(":");
+        let xf = new Log4Moz.XMLFormatter();
+        let sapp = new Log4Moz.SocketAppender(host, Number(port), xf);
+        sapp.level = Log4Moz.Level.All;
+        glodaLog.level = Log4Moz.Level.All;
+        glodaLog.addAppender(sapp);
+      }
+    }
+
+    this._log = Log4Moz.repository.getLogger("gloda.NS");
+    this._log.info("Logging Initialized");
+  },
+
+  /**
+   * The indexer is idle.
+   */
+  kIndexerIdle: 0,
+  /**
+   * The indexer is doing something.  We used to have other specific states, but
+   *  they have been rendered irrelevant and wiped from existence.
+   */
+  kIndexerIndexing: 1,
+
+  /**
+   * Synchronous activities performed that can be thought of as one processing
+   *  token.  Potentially yield the event-loop and re-schedule for later based
+   *  on how long we've actually taken/etc.  The goal here is that code that
+   *  is doing stuff synchronously yields with kWorkSync periodically to make
+   *  sure that it doesn't dominate the event-loop.  Unless the processing
+   *  in question is particularly intensive, it should be reasonable to apply
+   *  some decimation factor (ex: 32 or 64) with the general goal of yielding
+   *  every 3-10 milliseconds.
+   */
+  kWorkSync: 0,
+  /**
+   * Asynchronous activity performed, you need to relinquish flow control and
+   *  trust us to call callbackDriver later.
+   */
+  kWorkAsync: 1,
+  /**
+   * We are all done with our task, close us and figure out something else to do.
+   */
+  kWorkDone: 2,
+  /**
+   * We are not done with our task, but we think it's a good idea to take a
+   *  breather because we believe we have tied up the event loop for a
+   *  non-trivial amount of time.  So please re-schedule us in the future.
+   *
+   * This is currently only used internally by the indexer's batching logic;
+   *  minor changes may be required if used by actual indexers.
+   */
+  kWorkPause: 3,
+  /**
+   * We are done with our task, and have a result that we are returning.  This
+   *  should only be used by your callback handler's doneWithResult method.
+   *  Ex: you are passed aCallbackHandle, and you do
+   *  "yield aCallbackHandle.doneWithResult(myResult);".
+   */
+  kWorkDoneWithResult: 4,
+
+  /**
+   * Callers should access the unique ID for the GlodaDatastore
+   * with this getter. If the GlodaDatastore has not been
+   * initialized, this value is null.
+   *
+   * @return a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
+   */
+  get datastoreID() {
+    return GlodaDatastore._datastoreID;
+  },
+
+  /**
+   * Lookup a gloda message from an nsIMsgDBHdr, with the result returned as a
+   *  collection.  Keep in mind that the message may not be indexed, so you
+   *  may end up with an empty collection.  (Also keep in mind that this query
+   *  is asynchronous, so you will want your action-taking logic to be found
+   *  in your listener's onQueryCompleted method; the result will not be in
+   *  the collection when this method returns.)
+   *
+   * @param aMsgHdr The header of the message you want the gloda message for.
+   * @param aListener The listener that should be registered with the collection
+   * @param aData The (optional) value to set as the data attribute on the
+   *     collection.
+   *
+   * @return The collection that will receive the results.
+   *
+   * @testpoint gloda.ns.getMessageCollectionForHeader()
+   */
+  getMessageCollectionForHeader: function gloda_ns_getMessageForHeader(aMsgHdr,
+      aListener, aData) {
+    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+    query.folder(aMsgHdr.folder).messageKey(aMsgHdr.messageKey);
+    return query.getCollection(aListener, aData);
+  },
+
+  /**
+   * Given a list of message headers, return a collection containing the gloda
+   *  messages that correspond to those headers.  Keep in mind that gloda may
+   *  not have indexed all the messages, so the returned collection may not have
+   *  a message for each header you provide. (Also keep in mind that this query
+   *  is asynchronous, so you will want your action-taking logic to be found
+   *  in your listener's onQueryCompleted method; no results will be present in
+   *  the collection when this method returns.)
+   *
+   * @param aHeaders A javascript Array or and XPCOM list that fixIterator can
+   *     can handle.
+   * @param aListener The listener that should be registered with the collection
+   * @param aData The (optional) value to set as the data attribute on the
+   *     collection.
+   *
+   * @return The collection that will receive the results.
+   *
+   * @testpoint gloda.ns.getMessageCollectionForHeaders()
+   */
+  getMessageCollectionForHeaders: function gloda_ns_getMessagesForHeaders(
+      aHeaders, aListener, aData) {
+    // group the headers by the folder they are found in
+    let headersByFolder = {};
+    let iter;
+    for (let header in fixIterator(aHeaders)) {
+      let folderURI = header.folder.URI;
+      let headersForFolder = headersByFolder[folderURI];
+      if (headersForFolder === undefined)
+        headersByFolder[folderURI] = [header];
+      else
+        headersForFolder.push(header);
+    }
+
+    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+    let clause;
+    // build a query, using a separate union clause for each folder.
+    for (let folderURI in headersByFolder) {
+      let headersForFolder = headersByFolder[folderURI];
+      let folder = this.getFolderForFolder(headersForFolder[0].folder);
+      // if this is the first or clause, just use the query itself
+      if (!clause)
+        clause = query;
+      else // create a new query clause via the 'or' command
+        clause = query.or();
+
+      clause.folder(folder);
+      let messageKeys = headersForFolder.map(hdr => hdr.messageKey);
+      clause.messageKey.apply(clause, messageKeys);
+    }
+
+    return query.getCollection(aListener, aData);
+  },
+
+  /**
+   * @testpoint gloda.ns.getMessageContent
+   */
+  getMessageContent: function gloda_ns_getMessageContent(aGlodaMessage, aMimeMsg) {
+    return mimeMsgToContentAndMeta(aMimeMsg, aGlodaMessage.folderMessage.folder)[0];
+  },
+
+  getFolderForFolder: function gloda_ns_getFolderForFolder(aMsgFolder) {
+    return GlodaDatastore._mapFolder(aMsgFolder);
+  },
+
+  /**
+   * Takes one or more strings containing lists of comma-delimited e-mail
+   *  addresses with optional display names, and returns a list of sub-lists of
+   *  identities, where each sub-list corresponds to each of the strings passed
+   *  as arguments.  These identities are loaded from the database if they
+   *  already exist, or created if they do not yet exist.
+   * If the identities need to be created, they will also result in the
+   *  creation of a gloda contact.  If a display name was provided with the
+   *  e-mail address, it will become the name of the gloda contact.  If a
+   *  display name was not provided, the e-mail address will also serve as the
+   *  contact name.
+   * This method uses the indexer's callback handle mechanism, and does not
+   *  obey traditional return semantics.
+   *
+   * We normalize all e-mail addresses to be lowercase as a normative measure.
+   *
+   * @param aCallbackHandle The GlodaIndexer callback handle (or equivalent)
+   *   that you are operating under.
+   * @param ... One or more strings.  Each string can contain zero or more
+   *   e-mail addresses with display name.  If more than one address is given,
+   *   they should be comma-delimited.  For example
+   *   '"Bob Smith" <bob@example.com>' is an address with display name.  Mime
+   *   header decoding is performed, but is ignorant of any folder-level
+   *   character set overrides.
+   * @returns via the callback handle mechanism, a list containing one sub-list
+   *   for each string argument passed.  Each sub-list containts zero or more
+   *   GlodaIdentity instances corresponding to the addresses provided.
+   */
+  getOrCreateMailIdentities:
+      function* gloda_ns_getOrCreateMailIdentities(aCallbackHandle) {
+    let addresses = {};
+    let resultLists = [];
+
+    // parse the strings
+    for (let iArg = 1; iArg < arguments.length; iArg++) {
+      let aMailAddresses = arguments[iArg];
+      let parsed = GlodaUtils.parseMailAddresses(aMailAddresses);
+
+      let resultList = [];
+      resultLists.push(resultList);
+
+      let identities = [];
+      for (let iAddress = 0; iAddress < parsed.count; iAddress++) {
+        let address = parsed.addresses[iAddress].toLowerCase();
+        if (address in addresses)
+          addresses[address].push(resultList);
+        else
+          addresses[address] = [parsed.names[iAddress], resultList];
+      }
+    }
+
+    let addressList = Object.keys(addresses);
+    if (addressList.length == 0) {
+      yield aCallbackHandle.doneWithResult(resultLists);
+      // we should be stopped before we reach this point, but safety first.
+      return;
+    }
+
+    let query = this.newQuery(this.NOUN_IDENTITY);
+    query.kind("email");
+    query.value.apply(query, addressList);
+    let collection = query.getCollection(aCallbackHandle);
+    yield this.kWorkAsync;
+
+    // put the identities in the appropriate result lists
+    for (let identity of collection.items) {
+      let nameAndResultLists = addresses[identity.value];
+      this._log.debug(" found identity for '" + nameAndResultLists[0] + "' (" +
+                      identity.value + ")");
+      // index 0 is the name, skip it
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
+        nameAndResultLists[iResList].push(identity);
+      }
+      delete addresses[identity.value];
+    }
+
+    // create the identities that did not exist yet
+    for (let address in addresses) {
+      let nameAndResultLists = addresses[address];
+      let name = nameAndResultLists[0];
+
+      this._log.debug(" creating contact for '" + name + "' (" + address + ")");
+
+      // try and find an existing address book contact.
+      let card = GlodaUtils.getCardForEmail(address);
+      // XXX when we have the address book GUID stuff, we need to use that to
+      //  find existing contacts... (this will introduce a new query phase
+      //  where we batch all the GUIDs for an async query)
+      // XXX when the address book supports multiple e-mail addresses, we
+      //  should also just create identities for any that don't yet exist
+
+      // if there is no name, just use the e-mail (the ab indexer actually
+      //  processes the card's displayName for synchronization, so we don't
+      //  need to do that.)
+      if (!name)
+        name = address;
+
+      let contact = GlodaDatastore.createContact(null, null, name, 0, 0);
+
+      // we must create the identity.  use a blank description because there's
+      //  nothing to differentiate it from other identities, as this contact
+      //  only has one initially (us).
+      // XXX when we have multiple e-mails and there is a meaning associated
+      //  with each e-mail, try and use that to populate the description.
+      // XXX we are creating the identity here before we insert the contact.
+      //  conceptually it is good for us to be creating the identity before
+      //  exposing it to the address-book indexer, but we could get our id's
+      //  in a bad way from not deferring the identity insertion until after
+      //  the contact insertion.
+      let identity = GlodaDatastore.createIdentity(contact.id, contact,
+        "email", address, /* description */ "", /* relay? */ false);
+      contact._identities = [identity];
+
+      // give the address book indexer a chance if we have a card.
+      // (it will fix-up the name based on the card as appropriate)
+      if (card)
+        yield aCallbackHandle.pushAndGo(
+          Gloda.grokNounItem(contact, {card: card}, true, true,
+                             aCallbackHandle));
+      else // grokNounItem will issue the insert for us...
+        GlodaDatastore.insertContact(contact);
+
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
+        nameAndResultLists[iResList].push(identity);
+      }
+    }
+
+    yield aCallbackHandle.doneWithResult(resultLists);
+  },
+
+  /**
+   * Dictionary of the user's known identities; key is the identity id, value
+   *  is the actual identity.  This is populated by _initMyIdentities based on
+   *  the accounts defined.
+   */
+  myIdentities: {},
+  /**
+   * The contact corresponding to the current user.  We are assuming that only
+   *  a single user/human being uses the current profile.  This is known to be
+   *  a flawed assumption, but is the best first approximation available.
+   *
+   * @TODO attempt to deal with multile people using the same profile
+   */
+  myContact: null,
+  /**
+   * Populate myIdentities with all of our identities.  Currently we do this
+   *  by assuming that there is one human/user per profile, and that all of the
+   *  accounts defined in the profile belong to them.  The single contact is
+   *  stored on myContact.
+   *
+   * @TODO deal with account addition/modification/removal
+   * @TODO attempt to deal with multiple people using the same profile
+   */
+  _initMyIdentities: function gloda_ns_initMyIdentities() {
+    let myContact = null;
+    let myIdentities = {};
+    let myEmailAddresses = {}; // process each email at most once; stored here
+
+    let fullName, fallbackName;
+    let existingIdentities = [];
+    let identitiesToCreate = [];
+
+    let numIdentities = MailServices.accounts.allIdentities.length;
+
+    // nothing to do if there are no accounts/identities.
+    if (!numIdentities)
+      return;
+
+    for (let iIdentity = 0; iIdentity < numIdentities; iIdentity++) {
+      let msgIdentity =
+        MailServices.accounts.allIdentities.queryElementAt(iIdentity,
+                                                           Ci.nsIMsgIdentity);
+
+      if (!fullName)
+        fullName = msgIdentity.fullName;
+      if (!fallbackName)
+        fallbackName = msgIdentity.email;
+
+      let emailAddress = msgIdentity.email;
+      let replyTo = msgIdentity.replyTo;
+
+      // find the identities if they exist, flag to create them if they don't
+      if (emailAddress) {
+        let parsed = GlodaUtils.parseMailAddresses(emailAddress);
+        if (!(parsed.addresses[0] in myEmailAddresses)) {
+          let identity = GlodaDatastore.getIdentity("email",
+                                                    parsed.addresses[0]);
+          if (identity)
+            existingIdentities.push(identity);
+          else
+            identitiesToCreate.push(parsed.addresses[0]);
+          myEmailAddresses[parsed.addresses[0]] = true;
+        }
+      }
+      if (replyTo) {
+        let parsed = GlodaUtils.parseMailAddresses(replyTo);
+        if (!(parsed.addresses[0] in myEmailAddresses)) {
+          let identity = GlodaDatastore.getIdentity("email",
+                                                    parsed.addresses[0]);
+          if (identity)
+            existingIdentities.push(identity);
+          else
+            identitiesToCreate.push(parsed.addresses[0]);
+          myEmailAddresses[parsed.addresses[0]] = true;
+        }
+      }
+    }
+
+    // we need to establish the identity.contact portions of the relationship
+    for (let identity of existingIdentities) {
+      identity._contact = GlodaDatastore.getContactByID(identity.contactID);
+    }
+
+    if (existingIdentities.length) {
+      // just use the first guy's contact
+      myContact = existingIdentities[0].contact;
+    }
+    else {
+      // create a new contact
+      myContact = GlodaDatastore.createContact(null, null,
+                                               fullName || fallbackName,
+                                               0, 0);
+      GlodaDatastore.insertContact(myContact);
+    }
+
+    if (identitiesToCreate.length) {
+      for (let iIdentity = 0; iIdentity < identitiesToCreate.length;
+          iIdentity++) {
+        let emailAddress = identitiesToCreate[iIdentity];
+        // XXX this won't always be of type "email" as we add new account types
+        // XXX the blank string could be trying to differentiate; we do have
+        //  enough info to do it.
+        let identity = GlodaDatastore.createIdentity(myContact.id, myContact,
+                                                     "email",
+                                                     emailAddress,
+                                                     "", false);
+        existingIdentities.push(identity);
+      }
+    }
+
+    for (let iIdentity = 0; iIdentity < existingIdentities.length;
+        iIdentity++) {
+      let identity = existingIdentities[iIdentity];
+      myIdentities[identity.id] = identity;
+    }
+
+    this.myContact = myContact;
+    this.myIdentities = myIdentities;
+    myContact._identities = Object.keys(myIdentities).
+      map(id => myIdentities[id]);
+
+    // we need contacts to make these objects reachable via the collection
+    //  manager.
+    this._myContactCollection = this.explicitCollection(this.NOUN_CONTACT,
+                                                        [this.myContact]);
+    this._myIdentitiesCollection =
+      this.explicitCollection(this.NOUN_IDENTITY, this.myContact._identities);
+  },
+
+  /**
+   * An attribute that is a defining characteristic of the subject.
+   */
+  kAttrFundamental: 0,
+  /**
+   * An attribute that is an optimization derived from two or more fundamental
+   *  attributes and exists solely to improve database query performance.
+   */
+  kAttrOptimization: 1,
+  /**
+   * An attribute that is derived from the content of the subject.  For example,
+   *  a message that references a bugzilla bug could have a "derived" attribute
+   *  that captures the bugzilla reference.  This is not
+   */
+  kAttrDerived: 2,
+  /**
+   * An attribute that is the result of an explicit and intentional user action
+   *  upon the subject.  For example, a tag placed on a message by a user (or
+   *  at the user's request by a filter) is explicit.
+   */
+  kAttrExplicit: 3,
+  /**
+   * An attribute that is indirectly the result of a user's behaviour.  For
+   *  example, if a user consults a message multiple times, we may conclude that
+   *  the user finds the message interesting.  It is "implied", if you will,
+   *  that the message is interesting.
+   */
+  kAttrImplicit: 4,
+
+  /**
+   * This attribute is not 'special'; it is stored as a (thing id, attribute id,
+   *  attribute id) tuple in the database rather than on thing's row or on
+   *  thing's fulltext row.  (Where "thing" could be a message or any other
+   *  first class noun.)
+   */
+  kSpecialNotAtAll: GlodaDatastore.kSpecialNotAtAll,
+  /**
+   * This attribute is stored as a numeric column on the row for the noun.  The
+   *  attribute definition should include this value as 'special' and the
+   *  column name that stores the attribute as 'specialColumnName'.
+   */
+  kSpecialColumn: GlodaDatastore.kSpecialColumn,
+  kSpecialColumnChildren: GlodaDatastore.kSpecialColumnChildren,
+  kSpecialColumnParent: GlodaDatastore.kSpecialColumnParent,
+  /**
+   * This attribute is stored as a string column on the row for the noun.  It
+   *  differs from kSpecialColumn in that it is a string, which once had
+   *  query ramifications and one day may have them again.
+   */
+  kSpecialString: GlodaDatastore.kSpecialString,
+  /**
+   * This attribute is stored as a fulltext column on the fulltext table for
+   *  the noun.  The attribute defintion should include this value as 'special'
+   *  and the column name that stores the table as 'specialColumnName'.
+   */
+  kSpecialFulltext: GlodaDatastore.kSpecialFulltext,
+
+  /**
+   * The extensionName used for the attributes defined by core gloda plugins
+   *  such as fundattr.js and explattr.js.
+   */
+  BUILT_IN: "built-in",
+
+  /**
+   * Special sentinel value that will cause facets to skip a noun instance
+   * when an attribute has this value.
+   */
+  IGNORE_FACET: GlodaDatastore.IGNORE_FACET,
+
+  /*
+   * The following are explicit noun IDs.  While most extension-provided nouns
+   *  will have dynamically allocated id's that are looked up by name, these
+   *  id's can be relied upon to exist and be accessible via these
+   *  pseudo-constants.  It's not really clear that we need these, although it
+   *  does potentially simplify code to not have to look up all of their nouns
+   *  at initialization time.
+   */
+  /**
+   * Boolean values, expressed as 0/1 in the database and non-continuous for
+   *  constraint purposes.  Like numbers, such nouns require their attributes
+   *  to provide them with context, lacking any of their own.
+   * Having this as a noun type may be a bad idea; a change of nomenclature
+   *  (so that we are not claiming a boolean value is a noun, but still using
+   *  it in the same way) or implementation to require each boolean noun
+   *  actually be its own noun may be in order.
+   */
+  NOUN_BOOLEAN: 1,
+  /**
+   * A number, which could mean an integer or floating point values.  We treat
+   *  these as continuous, meaning that queries on them can have ranged
+   *  constraints expressed on them.  Lacking any inherent context, numbers
+   *  depend on their attributes to parameterize them as required.
+   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
+   */
+  NOUN_NUMBER: 2,
+  /**
+   * A (non-fulltext) string.
+   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
+   */
+  NOUN_STRING: 3,
+  /** A date, encoded as a PRTime, represented as a js Date object. */
+  NOUN_DATE: 10,
+  /**
+   * Fulltext search support, somewhat magical.  This is only intended to be
+   *  used for kSpecialFulltext attributes, and exclusively as a constraint
+   *  mechanism.  The values are always represented as strings.  It is presumed
+   *  that the user of this functionality knows how to generate SQLite FTS3
+   *  style MATCH queries, or is okay with us just gluing them together with
+   *  " OR " when used in an or-constraint case.  Gloda's query mechanism
+   *  currently lacks the ability to to compile Gloda-style and-constraints
+   *  into a single MATCH query, but it will turn out okay, just less
+   *  efficiently than it could.
+   */
+  NOUN_FULLTEXT: 20,
+  /**
+   * Represents a MIME Type.  We currently lack any human-intelligible
+   *  descriptions of mime types.
+   */
+  NOUN_MIME_TYPE: 40,
+  /**
+   * Captures a message tag as well as when the tag's presence was observed,
+   *  hoping to approximate when the tag was applied.  It's a somewhat dubious
+   *  attempt to not waste our opporunity to store a value along with the tag.
+   *  (The tag is actually stored as an attribute parameter on the attribute
+   *  definition, rather than a value in the attribute 'instance' for the
+   *  message.)
+   */
+  NOUN_TAG: 50,
+  /**
+   * Doesn't actually work owing to a lack of an object to represent a folder.
+   *  We do expose the folderURI and folderID of a message, but need to map that
+   *  to a good abstraction.  Probably something thin around a SteelFolder or
+   *  the like; we would contribute the functionality to easily move from a
+   *  folder to the list of gloda messages in that folder, as well as the
+   *  indexing preferences for that folder.
+   * @TODO folder noun and related abstraction
+   */
+  NOUN_FOLDER: GlodaFolder.prototype.NOUN_ID, // 100
+  /**
+   * All messages belong to a conversation.  See datamodel.js for the
+   *  definition of the GlodaConversation class.
+   */
+  NOUN_CONVERSATION: GlodaConversation.prototype.NOUN_ID, // 101
+  /**
+   * A one-to-one correspondence with underlying (indexed) nsIMsgDBHdr
+   *  instances.  See datamodel.js for the definition of the GlodaMessage class.
+   */
+  NOUN_MESSAGE: GlodaMessage.prototype.NOUN_ID, // 102
+  /**
+   * Corresponds to a human being, who may have multiple electronic identities
+   *  (a la NOUN_IDENTITY).  There is no requirement for association with an
+   *  address book contact, although when the address book contact exists,
+   *  we want to be associated with it.  See datamodel.js for the definition
+   *  of the GlodaContact class.
+   */
+  NOUN_CONTACT: GlodaContact.prototype.NOUN_ID, // 103
+  /**
+   * A single identity of a contact, who may have one or more.  E-mail accounts,
+   *  instant messaging accounts, social network site accounts, etc. are each
+   *  identities.  See datamodel.js for the definition of the GlodaIdentity
+   *  class.
+   */
+  NOUN_IDENTITY: GlodaIdentity.prototype.NOUN_ID, // 104
+  /**
+   * An attachment to a message. A message may have many different attachments.
+   */
+  NOUN_ATTACHMENT: GlodaAttachment.prototype.NOUN_ID, // 105
+  /**
+   * An account related to a message. A message can have only one account.
+   */
+  NOUN_ACCOUNT: GlodaAccount.prototype.NOUN_ID, // 106
+
+  /**
+   * Parameterized identities, for use in the from-me, to-me, cc-me optimization
+   *  cases.  Not for reuse without some thought.  These nouns use the parameter
+   *  to store the 'me' identity that we are talking about, and the value to
+   *  store the identity of the other party.  So in both the from-me and to-me
+   *  cases involving 'me' and 'foo@bar', the 'me' identity is always stored via
+   *  the attribute parameter, and the 'foo@bar' identity is always stored as
+   *  the attribute value.  See fundattr.js for more information on this, but
+   *  you probably shouldn't be touching this unless you are fundattr.
+   */
+  NOUN_PARAM_IDENTITY: 200,
+
+  /** Next Noun ID to hand out, these don't need to be persisted (for now). */
+  _nextNounID: 1000,
+
+  /**
+   * Maps noun names to noun IDs.
+   */
+  _nounNameToNounID: {},
+  /**
+   * Maps noun IDs to noun definition dictionaries.  (Noun definition
+   *  dictionaries provided to us at the time a noun was defined, plus some
+   *  additional stuff we put in there.)
+   */
+  _nounIDToDef: {},
+
+  _managedToJSON: function gloda_ns_managedToJSON(aItem) {
+    return aItem.id;
+  },
+
+  /**
+   * Define a noun.  Takes a dictionary with the following keys/values:
+   *
+   * @param aNounDef.name The name of the noun.  This is not a display name
+   *     (anything being displayed needs to be localized, after all), but simply
+   *     the canonical name for debugging purposes and for people to pass to
+   *     lookupNoun.  The suggested convention is lower-case-dash-delimited,
+   *     with names being singular (since it's a single noun we are referring
+   *     to.)
+   * @param aNounDef.class The 'class' to which an instance of the noun will
+   *     belong (aka will pass an instanceof test).  You may also provide this
+   *     as 'clazz' if the keyword makes your IDE angry.
+   * @param aNounDef.allowsArbitraryAttrs Is this a 'first class noun'/can it be
+   *     a subject, AKA can this noun have attributes stored on it that relate
+   *     it to other things?  For example, a message is first-class; we store
+   *     attributes of messages.  A date is not first-class now, nor is it
+   *     likely to be; we will not store attributes about a date, although dates
+   *     will be the objects of other subjects.  (For example: we might
+   *     associate a date with a calendar event, but the date is an attribute of
+   *     the calendar event and not vice versa.)
+   * @param aNounDef.usesParameter A boolean indicating whether this noun
+   *     requires use of the 'parameter' BLOB storage field on the attribute
+   *     bindings in the database to persist itself.  Use of parameters should
+   *     be limited to a reasonable number of values (16-32 is okay, more than
+   *     that is pushing it and 256 should be considered an absolute upper
+   *     bound) because of the database organization.  When false, your
+   *     toParamAndValue function is expected to return null for the parameter
+   *     and likewise your fromParamAndValue should expect ignore and generally
+   *     ignore the argument.
+   * @param aNounDef.toParamAndValue A function that takes an instantiated noun
+   *     instance and returns a 2-element list of [parameter, value] where
+   *     parameter may only be non-null if you passed a usesParameter of true.
+   *     Parameter may be of any type (BLOB), and value must be numeric (pass
+   *     0 if you don't need the value).
+   *
+   * @param aNounDef.isPrimitive True when the noun instance is a raw numeric
+   *     value/string/boolean.  False when the instance is an object.  When
+   *     false, it is assumed the attribute that serves as a unique identifier
+   *     for the value is "id" unless 'idAttr' is provided.
+   * @param [aNounDef.idAttr="id"] For non-primitive nouns, this is the
+   *     attribute on the object that uniquely identifies it.
+   *
+   * @param aNounDef.schema Unsupported mechanism by which you can define a
+   *     table that corresponds to this noun.  The table will be created if it
+   *     does not exist.
+   *     - name The table name; don't conflict with other things!
+   *     - columns A list of [column name, sqlite type] tuples.  You should
+   *       always include a definition like ["id", "INTEGER PRIMARY KEY"] for
+   *       now (and it should be the first column name too.)  If you care about
+   *       how the attributes are poked into your object (for example, you want
+   *       underscores used for some of them because the attributes should be
+   *       immutable), then you can include a third string that is the name of
+   *       the attribute to use.
+   *     - indices A dictionary of lists of column names, where the key name
+   *       becomes the index name.  Ex: {foo: ["bar"]} results in an index on
+   *       the column "bar" where the index is named "foo".
+  */
+  defineNoun: function gloda_ns_defineNoun(aNounDef, aNounID) {
+    this._log.info("Defining noun: " + aNounDef.name);
+    if (aNounID === undefined)
+      aNounID = this._nextNounID++;
+    aNounDef.id = aNounID;
+
+    // Let people whose editors get angry about illegal attribute names use
+    //  clazz instead of class.
+    if (aNounDef.clazz)
+      aNounDef.class = aNounDef.clazz;
+
+    if (!("idAttr" in aNounDef))
+      aNounDef.idAttr = "id";
+    if (!("comparator" in aNounDef)) {
+      aNounDef.comparator = function() {
+        throw new Error("Noun type '" + aNounDef.name +
+                        "' lacks a real comparator.");
+      };
+    }
+
+    // We allow nouns to have data tables associated with them where we do all
+    //  the legwork.  The schema attribute is the gateway to this magical world
+    //  of functionality.  Said door is officially unsupported.
+    if (aNounDef.schema) {
+      if (!aNounDef.tableName) {
+        if (aNounDef.schema.name)
+          aNounDef.tableName = "ext_" + aNounDef.schema.name;
+        else
+          aNounDef.tableName = "ext_" + aNounDef.name;
+      }
+      // this creates the data table and binder and hooks everything up
+      GlodaDatastore.createNounTable(aNounDef);
+
+      if (!aNounDef.toParamAndValue)
+        aNounDef.toParamAndValue = function (aThing) {
+          if (aThing instanceof aNounDef.class)
+            return [null, aThing.id];
+          else // assume they're just passing the id directly
+            return [null, aThing];
+        };
+    }
+
+    // if it has a table, you can query on it.  seems straight-forward.
+    if (aNounDef.tableName) {
+      [aNounDef.queryClass, aNounDef.nullQueryClass,
+       aNounDef.explicitQueryClass, aNounDef.wildcardQueryClass] =
+          GlodaQueryClassFactory(aNounDef);
+      aNounDef._dbMeta = {};
+      aNounDef.class.prototype.NOUN_ID = aNounDef.id;
+      aNounDef.class.prototype.NOUN_DEF = aNounDef;
+      aNounDef.toJSON = this._managedToJSON;
+
+      aNounDef.specialLoadAttribs = [];
+
+      // - define the 'id' constrainer
+      let idConstrainer = function() {
+        let constraint = [GlodaDatastore.kConstraintIdIn, null];
+        for (let iArg = 0; iArg < arguments.length; iArg++) {
+          constraint.push(arguments[iArg]);
+        }
+        this._constraints.push(constraint);
+        return this;
+      };
+      aNounDef.queryClass.prototype.id = idConstrainer;
+    }
+    if (aNounDef.cache) {
+      let cacheCost = aNounDef.cacheCost || 1024;
+      let cacheBudget = aNounDef.cacheBudget || 128 * 1024;
+      let cacheSize = Math.floor(cacheBudget / cacheCost);
+      if (cacheSize)
+        GlodaCollectionManager.defineCache(aNounDef, cacheSize);
+    }
+    aNounDef.attribsByBoundName = {};
+    aNounDef.domExposeAttribsByBoundName = {};
+
+    aNounDef.objectNounOfAttributes = [];
+
+    this._nounNameToNounID[aNounDef.name] = aNounID;
+    this._nounIDToDef[aNounID] = aNounDef;
+    aNounDef.actions = [];
+
+    this._attrProviderOrderByNoun[aNounDef.id] = [];
+    this._attrOptimizerOrderByNoun[aNounDef.id] = [];
+    this._attrProvidersByNoun[aNounDef.id] = {};
+
+    return aNounDef;
+  },
+
+  /**
+   * Lookup a noun (ID) suitable for passing to defineAttribute's various
+   *  noun arguments.  Throws an exception if the noun with the given name
+   *  cannot be found; the assumption is that you can't live without the noun.
+   */
+  lookupNoun: function gloda_ns_lookupNoun(aNounName) {
+    if (aNounName in this._nounNameToNounID)
+      return this._nounNameToNounID[aNounName];
+
+    throw Error("Unable to locate noun with name '" + aNounName + "', but I " +
+                "do know about: " +
+                Object.keys(this._nounNameToNounID).join(", "));
+  },
+
+  /**
+   * Lookup a noun def given a name.
+   */
+  lookupNounDef: function gloda_ns_lookupNoun(aNounName) {
+    return this._nounIDToDef[this.lookupNoun(aNounName)];
+  },
+
+
+  /**
+   * Define an action on a noun.  During the prototype stage, this was conceived
+   *  of as a way to expose all the constraints possible given a noun.  For
+   *  example, if you have an identity or a contact, you could use this to
+   *  see all the messages sent from/to a given contact.  It was likewise
+   *  thought potentially usable for future expansion.  For example, you could
+   *  also decide to send an e-mail to a contact when you have the contact
+   *  instance available.
+   * Outside of the 'expmess' checkbox-happy prototype, this functionality is
+   *  not used.  As such, this functionality should be considered in flux and
+   *  subject to changes.  Also, very open to specific suggestsions motivated
+   *  by use cases.
+   * One conceptual issue raised by this mechanism is the interaction of actions
+   *  with facts like "this message is read".  We currently implement the 'fact'
+   *  by defining an attribute with a 'boolean' noun type.  To deal with this,
+   *  in various places we pass-in the attribute as well as the noun value.
+   *  Since the relationships for booleans and integers in these cases is
+   *  standard and well-defined, this works out pretty well, but suggests we
+   *  need to think things through.
+   *
+   * @param aNounID The ID of the noun you want to define an action on.
+   * @param aActionMeta The dictionary describing the noun.  The dictionary
+   *     should have the following fields:
+   * - actionType: a string indicating the type of action.  Currently, only
+   *   "filter" is a legal value.
+   * - actionTarget: the noun ID of the noun type on which this action is
+   *   applicable.  For example,
+   *
+   * The following should be present for actionType=="filter";
+   * - shortName: The name that should be used to display this constraint.  For
+   *   example, a checkbox-heavy UI might display a checkbox for each constraint
+   *   using shortName as the label.
+   * - makeConstraint: A function that takes the attribute that is the source
+   *   of the noun and the noun instance as arguments, and returns APV-style
+   *   constraints.  Since the APV-style query mechanism is now deprecated,
+   *   this signature is deprecated.  Probably the way to update this would be
+   *   to pass in the query instance that constraints should be contributed to.
+   */
+  defineNounAction: function gloda_ns_defineNounAction(aNounID, aActionMeta) {
+    let nounDef = this._nounIDToDef[aNounID];
+    nounDef.actions.push(aActionMeta);
+  },
+
+  /**
+   * Retrieve all of the actions (as defined using defineNounAction) for the
+   *  given noun type (via noun ID) with the given action type (ex: filter).
+   */
+  getNounActions: function gloda_ns_getNounActions(aNounID, aActionType) {
+    let nounDef = this._nounIDToDef[aNounID];
+    if (!nounDef)
+      return [];
+    return nounDef.actions.
+      filter(action => !aActionType || (action.actionType == aActionType));
+  },
+
+  /** Attribute providers in the sequence to process them. */
+  _attrProviderOrderByNoun: {},
+  /** Attribute providers that provide optimizers, in the sequence to proc. */
+  _attrOptimizerOrderByNoun: {},
+  /** Maps attribute providers to the list of attributes they provide */
+  _attrProviders: {},
+  /**
+   * Maps nouns to their attribute providers to a list of the attributes they
+   *  provide for the noun.
+   */
+  _attrProvidersByNoun: {},
+
+  /**
+   * Define the core nouns (that are not defined elsewhere) and a few noun
+   *  actions.  Core nouns could be defined in other files, assuming dependency
+   *  issues are resolved via the everybody.js mechanism or something else.
+   *  Right now, noun_tag defines the tag noun.  If we broke more of these out,
+   *  we would probably want to move the 'class' code from datamodel.js, the
+   *  SQL table def and helper code from datastore.js (and this code) to their
+   *  own noun_*.js files.  There are some trade-offs to be made, and I think
+   *  we can deal with those once we start to integrate lightning/calendar and
+   *  our noun space gets large and more heterogeneous.
+   */
+  _initAttributes: function gloda_ns_initAttributes() {
+    this.defineNoun({
+      name: "bool",
+      clazz: Boolean, allowsArbitraryAttrs: false,
+      isPrimitive: true,
+      // favor true before false
+      comparator: function gloda_bool_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return b - a;
+      },
+      toParamAndValue: function(aBool) {
+        return [null, aBool ? 1 : 0];
+      }}, this.NOUN_BOOLEAN);
+    this.defineNoun({
+      name: "number",
+      clazz: Number, allowsArbitraryAttrs: false, continuous: true,
+      isPrimitive: true,
+      comparator: function gloda_number_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a - b;
+      },
+      toParamAndValue: function(aNum) {
+        return [null, aNum];
+      }}, this.NOUN_NUMBER);
+    this.defineNoun({
+      name: "string",
+      clazz: String, allowsArbitraryAttrs: false,
+      isPrimitive: true,
+      comparator: function gloda_string_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.localeCompare(b);
+      },
+      toParamAndValue: function(aString) {
+        return [null, aString];
+      }}, this.NOUN_STRING);
+    this.defineNoun({
+      name: "date",
+      clazz: Date, allowsArbitraryAttrs: false, continuous: true,
+      isPrimitive: true,
+      comparator: function gloda_data_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a - b;
+      },
+      toParamAndValue: function(aDate) {
+        return [null, aDate.valueOf() * 1000];
+      }}, this.NOUN_DATE);
+    this.defineNoun({
+      name: "fulltext",
+      clazz: String, allowsArbitraryAttrs: false, continuous: false,
+      isPrimitive: true,
+      comparator: function gloda_fulltext_comparator(a, b) {
+        throw new Error("Fulltext nouns are not comparable!");
+      },
+      // as noted on NOUN_FULLTEXT, we just pass the string around.  it never
+      //  hits the database, so it's okay.
+      toParamAndValue: function(aString) {
+        return [null, aString];
+      }}, this.NOUN_FULLTEXT);
+
+    this.defineNoun({
+      name: "folder",
+      clazz: GlodaFolder,
+      allowsArbitraryAttrs: false,
+      isPrimitive: false,
+      queryHelpers: {
+        /**
+         * Query for accounts based on the account associated with folders.  We
+         *  walk all of the folders associated with an account and put them in
+         *  the list of folders that match if gloda would index them.  This is
+         *  unsuitable for producing a persistable constraint since it does not
+         *  adapt for added/deleted folders.  However, it is sufficient for
+         *  faceting.  Also, we don't persist constraints yet.
+         *
+         * @TODO The long-term solution is to move towards using arithmetic
+         *     encoding on folder-id's like we use for MIME types and friends.
+         */
+        Account: function(aAttrDef, aArguments) {
+          let folderValues = [];
+          let seenRootFolders = {};
+          for (let iArg = 0; iArg < aArguments.length; iArg++) {
+            let givenFolder = aArguments[iArg];
+            let givenMsgFolder = givenFolder.getXPCOMFolder(
+                                   givenFolder.kActivityFolderOnlyNoData);
+            let rootFolder = givenMsgFolder.rootFolder;
+
+            // skip processing this folder if we have already processed its
+            //  root folder.
+            if (rootFolder.URI in seenRootFolders)
+              continue;
+            seenRootFolders[rootFolder.URI] = true;
+
+            let allFolders = rootFolder.descendants;
+            for (let folder in fixIterator(allFolders, Ci.nsIMsgFolder)) {
+              let folderFlags = folder.flags;
+
+              // Ignore virtual folders, non-mail folders.
+              // XXX this is derived from GlodaIndexer's shouldIndexFolder.
+              //  This should probably just use centralized code or the like.
+              if (!(folderFlags & Ci.nsMsgFolderFlags.Mail) ||
+                  (folderFlags & Ci.nsMsgFolderFlags.Virtual))
+                continue;
+              // we only index local or IMAP folders
+              if (!(folder instanceof Ci.nsIMsgLocalMailFolder) &&
+                !(folder instanceof Ci.nsIMsgImapMailFolder))
+                continue;
+
+              let glodaFolder = Gloda.getFolderForFolder(folder);
+              folderValues.push(glodaFolder);
+            }
+          }
+          return this._inConstraintHelper(aAttrDef, folderValues);
+        }
+      },
+      comparator: function gloda_folder_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.name.localeCompare(b.name);
+      },
+      toParamAndValue: function(aFolderOrGlodaFolder) {
+        if (aFolderOrGlodaFolder instanceof GlodaFolder)
+          return [null, aFolderOrGlodaFolder.id];
+        else
+          return [null, GlodaDatastore._mapFolder(aFolderOrGlodaFolder).id];
+      }}, this.NOUN_FOLDER);
+    this.defineNoun({
+      name: "account",
+      clazz: GlodaAccount,
+      allowsArbitraryAttrs: false,
+      isPrimitive: false,
+      equals: function(a, b) {
+        if (a && !b || !a && b)
+          return false;
+        if (!a && !b)
+          return true;
+        return a.id == b.id;
+      },
+      comparator: function gloda_account_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.name.localeCompare(b.name);
+      }}, this.NOUN_ACCOUNT);
+    this.defineNoun({
+      name: "conversation",
+      clazz: GlodaConversation,
+      allowsArbitraryAttrs: false,
+      isPrimitive: false,
+      cache: true, cacheCost: 512,
+      tableName: "conversations",
+      attrTableName: "messageAttributes", attrIDColumnName: "conversationID",
+      datastore: GlodaDatastore,
+      objFromRow: GlodaDatastore._conversationFromRow,
+      comparator: function gloda_conversation_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.subject.localeCompare(b.subject);
+      },
+      toParamAndValue: function(aConversation) {
+        if (aConversation instanceof GlodaConversation)
+          return [null, aConversation.id];
+        else // assume they're just passing the id directly
+          return [null, aConversation];
+      }}, this.NOUN_CONVERSATION);
+    this.defineNoun({
+      name: "message",
+      clazz: GlodaMessage,
+      allowsArbitraryAttrs: true,
+      isPrimitive: false,
+      cache: true, cacheCost: 2048,
+      tableName: "messages",
+      // we will always have a fulltext row, even for messages where we don't
+      //  have the body available.  this is because we want the subject indexed.
+      dbQueryJoinMagic:
+        " INNER JOIN messagesText ON messages.id = messagesText.rowid",
+      attrTableName: "messageAttributes", attrIDColumnName: "messageID",
+      datastore: GlodaDatastore, objFromRow: GlodaDatastore._messageFromRow,
+      dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
+      dbQueryValidityConstraintSuffix:
+        " AND +deleted = 0 AND +folderID IS NOT NULL AND +messageKey IS NOT NULL",
+      // This is what's used when we have no validity constraints, i.e. we allow
+      // for ghost messages, which do not have a row in the messagesText table.
+      dbQueryJoinMagicWithNoValidityConstraints:
+        " LEFT JOIN messagesText ON messages.id = messagesText.rowid",
+      objInsert: GlodaDatastore.insertMessage,
+      objUpdate: GlodaDatastore.updateMessage,
+      toParamAndValue: function(aMessage) {
+        if (aMessage instanceof GlodaMessage)
+          return [null, aMessage.id];
+        else // assume they're just passing the id directly
+          return [null, aMessage];
+      }}, this.NOUN_MESSAGE);
+    this.defineNoun({
+      name: "contact",
+      clazz: GlodaContact,
+      allowsArbitraryAttrs: true,
+      isPrimitive: false,
+      cache: true, cacheCost: 128,
+      tableName: "contacts",
+      attrTableName: "contactAttributes", attrIDColumnName: "contactID",
+      datastore: GlodaDatastore, objFromRow: GlodaDatastore._contactFromRow,
+      dbAttribAdjuster: GlodaDatastore.adjustAttributes,
+      objInsert: GlodaDatastore.insertContact,
+      objUpdate: GlodaDatastore.updateContact,
+      comparator: function gloda_contact_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.name.localeCompare(b.name);
+      },
+      toParamAndValue: function(aContact) {
+        if (aContact instanceof GlodaContact)
+          return [null, aContact.id];
+        else // assume they're just passing the id directly
+          return [null, aContact];
+      }}, this.NOUN_CONTACT);
+    this.defineNoun({
+      name: "identity",
+      clazz: GlodaIdentity,
+      allowsArbitraryAttrs: false,
+      isPrimitive: false,
+      cache: true, cacheCost: 128,
+      usesUniqueValue: true,
+      tableName: "identities",
+      datastore: GlodaDatastore, objFromRow: GlodaDatastore._identityFromRow,
+      /**
+       * Short string is the contact name, long string includes the identity
+       *  value too, delimited by a colon.  Not tremendously localizable.
+       */
+      userVisibleString: function(aIdentity, aLong) {
+        if (!aLong)
+          return aIdentity.contact.name;
+        if (aIdentity.contact.name == aIdentity.value)
+          return aIdentity.value;
+        return aIdentity.contact.name + " (" + aIdentity.value + ")";
+      },
+      comparator: function gloda_identity_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        return a.contact.name.localeCompare(b.contact.name);
+      },
+      toParamAndValue: function(aIdentity) {
+        if (aIdentity instanceof GlodaIdentity)
+          return [null, aIdentity.id];
+        else // assume they're just passing the id directly
+          return [null, aIdentity];
+      }}, this.NOUN_IDENTITY);
+    this.defineNoun({
+      name: "attachment-infos",
+      clazz: GlodaAttachment,
+      allowsArbitraryAttrs: false,
+      isPrimitive: false,
+      toJSON: function (x) {
+          return [
+            x._name,
+            x._contentType,
+            x._size,
+            x._part,
+            x._externalUrl,
+            x._isExternal
+          ]
+        },
+      fromJSON: function (x, aGlodaMessage) {
+          let [name, contentType, size, _part, _externalUrl, isExternal] = x;
+          return new GlodaAttachment(aGlodaMessage, name, contentType, size, _part, _externalUrl, isExternal);
+        },
+      }, this.NOUN_ATTACHMENT);
+
+    // parameterized identity is just two identities; we store the first one
+    //  (whose value set must be very constrainted, like the 'me' identities)
+    //  as the parameter, the second (which does not need to be constrained)
+    //  as the value.
+    this.defineNoun({
+      name: "parameterized-identity",
+      clazz: null,
+      allowsArbitraryAttrs: false,
+      comparator: function gloda_fulltext_comparator(a, b) {
+        if (a == null) {
+          if (b == null)
+            return 0;
+          else
+            return 1;
+        }
+        else if (b == null) {
+          return -1;
+        }
+        // First sort by the first identity in the tuple
+        // Since our general use-case is for the first guy to be "me", we only
+        //  compare the identity value, not the name.
+        let fic = a[0].value.localeCompare(b[0].value);
+        if (fic)
+          return fic;
+        // Next compare the second identity in the tuple, but use the contact
+        //  this time to be consistent with our identity comparator.
+        return a[1].contact.name.localeCompare(b[1].contact.name);
+      },
+      computeDelta: function(aCurValues, aOldValues) {
+        let oldMap = {};
+        for (let tupe of aOldValues) {
+          let [originIdentity, targetIdentity] = tupe;
+          let targets = oldMap[originIdentity];
+          if (targets === undefined)
+            targets = oldMap[originIdentity] = {};
+          targets[targetIdentity] = true;
+        }
+
+        let added = [], removed = [];
+        for (let tupe of aCurValues) {
+          let [originIdentity, targetIdentity] = tupe;
+          let targets = oldMap[originIdentity];
+          if ((targets === undefined) || !(targetIdentity in targets))
+            added.push(tupe);
+          else
+            delete targets[targetIdentity];
+        }
+
+        for (let originIdentity in oldMap) {
+          let targets = oldMap[originIdentity];
+          for (let targetIdentity in targets) {
+            removed.push([originIdentity, targetIdentity]);
+          }
+        }
+
+        return [added, removed];
+      },
+      contributeObjDependencies: function(aJsonValues, aReferencesByNounID,
+          aInverseReferencesByNounID) {
+        // nothing to do with a zero-length list
+        if (aJsonValues.length == 0)
+          return false;
+
+        let nounIdentityDef = Gloda._nounIDToDef[Gloda.NOUN_IDENTITY];
+        let references = aReferencesByNounID[nounIdentityDef.id];
+        if (references === undefined)
+          references = aReferencesByNounID[nounIdentityDef.id] = {};
+
+        for (let tupe of aJsonValues) {
+          let [originIdentityID, targetIdentityID] = tupe;
+          if (!(originIdentityID in references))
+            references[originIdentityID] = null;
+          if (!(targetIdentityID in references))
+            references[targetIdentityID] = null;
+        }
+
+        return true;
+      },
+      resolveObjDependencies: function(aJsonValues, aReferencesByNounID,
+          aInverseReferencesByNounID) {
+        let references =
+          aReferencesByNounID[Gloda.NOUN_IDENTITY];
+
+        let results = [];
+        for (let tupe of aJsonValues) {
+          let [originIdentityID, targetIdentityID] = tupe;
+          results.push([references[originIdentityID],
+                        references[targetIdentityID]]);
+        }
+
+        return results;
+      },
+      toJSON: function (aIdentityTuple) {
+        return [aIdentityTuple[0].id, aIdentityTuple[1].id];
+      },
+      toParamAndValue: function(aIdentityTuple) {
+        return [aIdentityTuple[0].id, aIdentityTuple[1].id];
+      }}, this.NOUN_PARAM_IDENTITY);
+
+    GlodaDatastore.getAllAttributes();
+  },
+
+  /**
+   * Create accessor functions to 'bind' an attribute to underlying normalized
+   *  attribute storage, as well as creating the appropriate query object
+   *  constraint helper functions.  This name is somewhat of a misnomer because
+   *  special attributes are not 'bound' (because specific/non-generic per-class
+   *  code provides the properties) but still depend on this method to
+   *  establish their constraint helper methods.
+   *
+   * @XXX potentially rename to not suggest binding is required.
+   */
+  _bindAttribute: function gloda_ns_bindAttr(aAttrDef, aSubjectNounDef) {
+    let objectNounDef = aAttrDef.objectNounDef;
+
+    // -- the query constraint helpers
+    if (aSubjectNounDef.queryClass !== undefined) {
+      let constrainer;
+      let canQuery = true;
+      if (aAttrDef.special == this.kSpecialFulltext) {
+        constrainer = function() {
+          let constraint = [GlodaDatastore.kConstraintFulltext, aAttrDef];
+          for (let iArg = 0; iArg < arguments.length; iArg++) {
+            constraint.push(arguments[iArg]);
+          }
+          this._constraints.push(constraint);
+          return this;
+        };
+      }
+      else if (aAttrDef.canQuery || aAttrDef.attributeName.startsWith("_")) {
+        constrainer = function() {
+          let constraint = [GlodaDatastore.kConstraintIn, aAttrDef];
+          for (let iArg = 0; iArg < arguments.length; iArg++) {
+            constraint.push(arguments[iArg]);
+          }
+          this._constraints.push(constraint);
+          return this;
+        };
+      } else {
+        constrainer = function() {
+          throw new Error(
+              "Cannot query on attribute "+aAttrDef.attributeName
+            + " because its canQuery parameter hasn't been set to true."
+            + " Reading the comments about Gloda.defineAttribute may be a"
+            + " sensible thing to do now.");
+        }
+        canQuery = false;
+      }
+
+      aSubjectNounDef.queryClass.prototype[aAttrDef.boundName] = constrainer;
+
+      // Don't bind extra query-able attributes if we're unable to perform a
+      // search on the attribute.
+      if (!canQuery)
+        return;
+
+      // - ranged value helper: fooRange
+      if (objectNounDef.continuous) {
+        // takes one or more tuples of [lower bound, upper bound]
+        let rangedConstrainer = function() {
+          let constraint = [GlodaDatastore.kConstraintRanges, aAttrDef];
+          for (let iArg = 0; iArg < arguments.length; iArg++ ) {
+            constraint.push(arguments[iArg]);
+          }
+          this._constraints.push(constraint);
+          return this;
+        };
+
+        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Range"] =
+          rangedConstrainer;
+      }
+
+      // - string LIKE helper for special on-row attributes: fooLike
+      // (it is impossible to store a string as an indexed attribute, which is
+      //  why we do this for on-row only.)
+      if (aAttrDef.special == this.kSpecialString) {
+        let likeConstrainer = function() {
+          let constraint = [GlodaDatastore.kConstraintStringLike, aAttrDef];
+          for (let iArg = 0; iArg < arguments.length; iArg++) {
+            constraint.push(arguments[iArg]);
+          }
+          this._constraints.push(constraint);
+          return this;
+        };
+
+        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Like"] =
+          likeConstrainer;
+      }
+
+      // - Custom helpers provided by the noun type...
+      if ("queryHelpers" in objectNounDef) {
+        for (let name in objectNounDef.queryHelpers) {
+          let helper = objectNounDef.queryHelpers[name];
+          // we need a new closure...
+          let helperFunc = helper;
+          aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + name] =
+            function() {
+              return helperFunc.call(this, aAttrDef, arguments);
+            };
+        }
+      }
+    }
+  },
+
+  /**
+   * Names of attribute-specific localized strings and the JS attribute they are
+   *  exposed as in the attribute's "strings" attribute (if the provider has a
+   *  string bundle exposed on its "strings" attribute).  They are rooted at
+   *  "gloda.SUBJECT-NOUN-NAME.attr.ATTR-NAME.*".
+   *
+   * Please consult the localization notes in gloda.properties to understand
+   *  what these are used for.
+   */
+  _ATTR_LOCALIZED_STRINGS: {
+    /* - Faceting */
+    facetNameLabel: "facetNameLabel",
+    includeLabel: "includeLabel",
+    excludeLabel: "excludeLabel",
+    remainderLabel: "remainderLabel",
+    mustMatchLabel: "mustMatchLabel",
+    cantMatchLabel: "cantMatchLabel",
+    mayMatchLabel: "mayMatchLabel",
+    mustMatchNoneLabel: "mustMatchNoneLabel",
+    mustMatchSomeLabel: "mustMatchSomeLabel",
+    mayMatchAnyLabel: "mayMatchAnyLabel",
+  },
+  /**
+   * Define an attribute and all its meta-data.  Takes a single dictionary as
+   *  its argument, with the following required properties:
+   *
+   * @param aAttrDef.provider The object instance providing a 'process' method.
+   * @param aAttrDef.extensionName The name of the extension providing these
+   *     attributes.
+   * @param aAttrDef.attributeType The type of attribute, one of the values from
+   *     the kAttr* enumeration.
+   * @param aAttrDef.attributeName The name of the attribute, which also doubles
+   *     as the bound property name if you pass 'bind' a value of true.  You are
+   *     responsible for avoiding collisions, which presumably will mean
+   *     checking/updating a wiki page in the future, or just prefixing your
+   *     attribute name with your extension name or something like that.
+   * @param aAttrDef.bind Should this attribute be 'bound' as a convenience
+   *     attribute on the subject's object (true/false)?  For example, with an
+   *     attributeName of "foo" and passing true for 'bind' with a subject noun
+   *     of NOUN_MESSAGE, GlodaMessage instances will expose a "foo" getter that
+   *     returns the value of the attribute.  If 'singular' is true, this means
+   *     an instance of the object class corresponding to the noun type or null
+   *     if the attribute does not exist.  If 'singular' is false, this means a
+   *     list of instances of the object class corresponding to the noun type,
+   *     where the list may be empty if no instances of the attribute are
+   *     present.
+   * @param aAttrDef.bindName Optional override of attributeName for purposes of
+   *     the binding property's name.
+   * @param aAttrDef.singular Is the attribute going to happen at most once
+   *     (true), or potentially multiple times (false).  This affects whether
+   *     the binding returns a list or just a single item (which is null when
+   *     the attribute is not present).
+   * @param [aAttrDef.emptySetIsSignificant=false] Should we
+   * @param aAttrDef.subjectNouns A list of object types (NOUNs) that this
+   *     attribute can be set on.  Each element in the list should be one of the
+   *     NOUN_* constants or a dynamically registered noun type.
+   * @param aAttrDef.objectNoun The object type (one of the NOUN_* constants or
+   *     a dynamically registered noun types) that is the 'object' in the
+   *     traditional RDF triple.  More pragmatically, in the database row used
+   *     to represent an attribute, we store the subject (ex: message ID),
+   *     attribute ID, and an integer which is the integer representation of the
+   *     'object' whose type you are defining right here.
+   */
+  defineAttribute: function gloda_ns_defineAttribute(aAttrDef) {
+    // ensure required properties exist on aAttrDef
+    if (!("provider" in aAttrDef) ||
+        !("extensionName" in aAttrDef) ||
+        !("attributeType" in aAttrDef) ||
+        !("attributeName" in aAttrDef) ||
+        !("singular" in aAttrDef) ||
+        !("subjectNouns" in aAttrDef) ||
+        !("objectNoun" in aAttrDef))
+      // perhaps we should have a list of required attributes, perchance with
+      //  and explanation of what it holds, and use that to be friendlier?
+      throw Error("You omitted a required attribute defining property, please" +
+                  " consult the documentation as penance.");
+
+    // -- Fill in defaults
+    if (!("emptySetIsSignificant" in aAttrDef))
+      aAttrDef.emptySetIsSignificant = false;
+
+    if (!("canQuery" in aAttrDef))
+      aAttrDef.canQuery = aAttrDef.facet ? true : false;
+
+    // return if the attribute has already been defined
+    if (aAttrDef.dbDef)
+      return aAttrDef;
+
+    // - first time we've seen a provider init logic
+    if (!(aAttrDef.provider.providerName in this._attrProviders)) {
+      this._attrProviders[aAttrDef.provider.providerName] = [];
+      if (aAttrDef.provider.contentWhittle)
+        whittlerRegistry.registerWhittler(aAttrDef.provider);
+    }
+
+    let compoundName = aAttrDef.extensionName + ":" + aAttrDef.attributeName;
+    // -- Database Definition
+    let attrDBDef;
+    if (compoundName in GlodaDatastore._attributeDBDefs) {
+      // the existence of the GlodaAttributeDBDef means that either it has
+      //  already been fully defined, or has been loaded from the database but
+      //  not yet 'bound' to a provider (and had important meta-info that
+      //  doesn't go in the db copied over)
+      attrDBDef = GlodaDatastore._attributeDBDefs[compoundName];
+    }
+    // we need to create the attribute definition in the database
+    else {
+      let attrID = null;
+      attrID = GlodaDatastore._createAttributeDef(aAttrDef.attributeType,
+                                                  aAttrDef.extensionName,
+                                                  aAttrDef.attributeName,
+                                                  null);
+
+      attrDBDef = new GlodaAttributeDBDef(GlodaDatastore, attrID, compoundName,
+        aAttrDef.attributeType, aAttrDef.extensionName, aAttrDef.attributeName);
+      GlodaDatastore._attributeDBDefs[compoundName] = attrDBDef;
+      GlodaDatastore._attributeIDToDBDefAndParam[attrID] = [attrDBDef, null];
+    }
+
+    aAttrDef.dbDef = attrDBDef;
+    attrDBDef.attrDef = aAttrDef;
+
+    aAttrDef.id = aAttrDef.dbDef.id;
+
+    if ("bindName" in aAttrDef)
+      aAttrDef.boundName = aAttrDef.bindName;
+    else
+      aAttrDef.boundName = aAttrDef.attributeName;
+
+    aAttrDef.objectNounDef = this._nounIDToDef[aAttrDef.objectNoun];
+    aAttrDef.objectNounDef.objectNounOfAttributes.push(aAttrDef);
+
+    // -- Facets
+    function normalizeFacetDef(aFacetDef) {
+      if (!("groupIdAttr" in aFacetDef))
+        aFacetDef.groupIdAttr = aAttrDef.objectNounDef.idAttr;
+      if (!("groupComparator" in aFacetDef))
+        aFacetDef.groupComparator = aAttrDef.objectNounDef.comparator;
+      if (!("filter" in aFacetDef))
+        aFacetDef.filter = null;
+    }
+    // No facet attribute means no facet desired; set an explicit null so that
+    //  code can check without doing an "in" check.
+    if (!("facet" in aAttrDef))
+      aAttrDef.facet = null;
+    // Promote "true" facet values to the defaults.  Where attributes have
+    //  specified values, make sure we fill in any missing defaults.
+    else {
+      if (aAttrDef.facet == true) {
+        aAttrDef.facet = {
+          type: "default",
+          groupIdAttr: aAttrDef.objectNounDef.idAttr,
+          groupComparator: aAttrDef.objectNounDef.comparator,
+          filter: null,
+        };
+      }
+      else {
+        normalizeFacetDef(aAttrDef.facet);
+      }
+    }
+    if ("extraFacets" in aAttrDef) {
+      for (let facetDef of aAttrDef.extraFacets) {
+        normalizeFacetDef(facetDef);
+      }
+    }
+
+    function gatherLocalizedStrings(aBundle, aPropRoot, aStickIn) {
+      for (let propName in Gloda._ATTR_LOCALIZED_STRINGS) {
+        let attrName = Gloda._ATTR_LOCALIZED_STRINGS[propName];
+        try {
+          aStickIn[attrName] = aBundle.get(aPropRoot + propName);
+        }
+        catch (ex) {
+          // do nothing.  nsIStringBundle throws exceptions because it is a
+          //  standard nsresult type of API and our helper buddy does nothing
+          //  to help us.  (StringBundle.js, that is.)
+        }
+      }
+    }
+
+    // -- L10n.
+    // If the provider has a string bundle, populate a "strings" attribute with
+    //  our standard attribute strings that can be UI exposed.
+    if (("strings" in aAttrDef.provider) && (aAttrDef.facet)) {
+      let bundle = aAttrDef.provider.strings;
+
+      // -- attribute strings
+      let attrStrings = aAttrDef.facet.strings = {};
+      // we use the first subject the attribute applies to as the basis of
+      //  where to get the string from.  Mainly because we currently don't have
+      //  any attributes with multiple subjects nor a use-case where we expose
+      //  multiple noun types via the UI.  (Just messages right now.)
+      let canonicalSubject = this._nounIDToDef[aAttrDef.subjectNouns[0]];
+      let propRoot = "gloda." + canonicalSubject.name + ".attr." +
+                       aAttrDef.attributeName + ".";
+      gatherLocalizedStrings(bundle, propRoot, attrStrings);
+
+      // -- alias strings for synthetic facets
+      if ("extraFacets" in aAttrDef) {
+        for (let facetDef of aAttrDef.extraFacets) {
+          facetDef.strings = {};
+          let aliasPropRoot = "gloda." + canonicalSubject.name + ".attr." +
+                                facetDef.alias + ".";
+          gatherLocalizedStrings(bundle, aliasPropRoot, facetDef.strings);
+        }
+      }
+    }
+
+    // -- Subject Noun Binding
+    for (let iSubject = 0; iSubject < aAttrDef.subjectNouns.length;
+           iSubject++) {
+      let subjectType = aAttrDef.subjectNouns[iSubject];
+      let subjectNounDef = this._nounIDToDef[subjectType];
+      this._bindAttribute(aAttrDef, subjectNounDef);
+
+      // update the provider maps...
+      if (this._attrProviderOrderByNoun[subjectType]
+              .indexOf(aAttrDef.provider) == -1) {
+        this._attrProviderOrderByNoun[subjectType].push(aAttrDef.provider);
+        if (aAttrDef.provider.optimize)
+          this._attrOptimizerOrderByNoun[subjectType].push(aAttrDef.provider);
+        this._attrProvidersByNoun[subjectType][aAttrDef.provider] = [];
+      }
+      this._attrProvidersByNoun[subjectType][aAttrDef.provider].push(aAttrDef);
+
+      subjectNounDef.attribsByBoundName[aAttrDef.boundName] = aAttrDef;
+      if (aAttrDef.domExpose)
+        subjectNounDef.domExposeAttribsByBoundName[aAttrDef.boundName] =
+          aAttrDef;
+
+      if (aAttrDef.special & this.kSpecialColumn)
+        subjectNounDef.specialLoadAttribs.push(aAttrDef);
+
+      // if this is a parent column attribute, make note of it so that if we
+      //  need to do an inverse references lookup, we know what column we are
+      //  issuing against.
+      if (aAttrDef.special === this.kSpecialColumnParent) {
+        subjectNounDef.parentColumnAttr = aAttrDef;
+      }
+
+      if (aAttrDef.objectNounDef.tableName ||
+          aAttrDef.objectNounDef.contributeObjDependencies) {
+        subjectNounDef.hasObjDependencies = true;
+      }
+    }
+
+    this._attrProviders[aAttrDef.provider.providerName].push(aAttrDef);
+    return aAttrDef;
+  },
+
+  /**
+   * Retrieve the attribute provided by the given extension with the given
+   *  attribute name.  The original idea was that plugins would effectively
+   *  name-space attributes, helping avoid collisions.  Since we are leaning
+   *  towards using binding heavily, this doesn't really help, as the collisions
+   *  will just occur on the attribute name instead.  Also, this can turn
+   *  extensions into liars as name changes/moves to core/etc. happen.
+   * @TODO consider removing the extension name argument parameter requirement
+   */
+  getAttrDef: function gloda_ns_getAttrDef(aPluginName, aAttrName) {
+    let compoundName = aPluginName + ":" + aAttrName;
+    return GlodaDatastore._attributeDBDefs[compoundName];
+  },
+
+  /**
+   * Create a new query instance for the given noun-type.  This provides
+   *  a generic way to provide constraint-based queries of any first-class
+   *  nouns supported by the system.
+   *
+   * The idea is that every attribute on an object can be used to express
+   *  a constraint on the query object.  Constraints implicitly 'AND' together,
+   *  but providing multiple arguments to a constraint function results in an
+   *  'OR'ing of those values.  Additionally, you can call or() on the returned
+   *  query to create an alternate query that is effectively a giant OR against
+   *  all the constraints you create on the main query object (or any other
+   *  alternate queries returned by or()).  (Note: there is no nesting of these
+   *  alternate queries. query.or().or() is equivalent to query.or())
+   * For each attribute, there is a constraint with the same name that takes
+   *  one or more arguments.  The arguments represent a set of OR values that
+   *  objects matching the query can have.  (If you want the constraint
+   *  effectively ANDed together, just invoke the constraint function
+   *  multiple times.)  For example, newQuery(NOUN_PERSON).age(25) would
+   *  constraint to all the people aged 25, while age(25, 26) would constrain
+   *  to all the people age 25 or 26.
+   * For each attribute with a 'continuous' noun, there is a constraint with the
+   *  attribute name with "Range" appended.  It takes two arguments which are an
+   *  inclusive lower bound and an inclusive lower bound for values in the
+   *  range.  If you would like an open-ended range on either side, pass null
+   *  for that argument.  If you would like to specify multiple ranges that
+   *  should be ORed together, simply pass additional (pairs of) arguments.
+   *  For example, newQuery(NOUN_PERSON).age(25,100) would constraint to all
+   *  the people who are >= 25 and <= 100.  Likewise age(25, null) would just
+   *  return all the people who are 25 or older.  And age(25,30,35,40) would
+   *  return people who are either 25-30 or 35-30.
+   * There are also full-text constraint columns.  In a nutshell, their
+   *  arguments are the strings that should be passed to the SQLite FTS3
+   *  MATCH clause.
+   *
+   * @param aNounID The (integer) noun-id of the noun you want to query on.
+   * @param aOptions an optional dictionary of query options, see the GlodaQuery
+   *     class documentation.
+   */
+  newQuery: function gloda_ns_newQuery(aNounID, aOptions) {
+    let nounDef = this._nounIDToDef[aNounID];
+    return new nounDef.queryClass(aOptions);
+  },
+
+  /**
+   * Create a collection/query for the given noun-type that only matches the
+   *  provided items.  This is to be used when you have an explicit set of items
+   *  that you would still like to receive updates for.
+   */
+  explicitCollection: function gloda_ns_explicitCollection(aNounID, aItems) {
+    let nounDef = this._nounIDToDef[aNounID];
+    let collection = new GlodaCollection(nounDef, aItems, null, null);
+    let query = new nounDef.explicitQueryClass(collection);
+    collection.query = query;
+    GlodaCollectionManager.registerCollection(collection);
+    return collection;
+  },
+
+  /**
+   * Debugging 'wildcard' collection creation support.  A wildcard collection
+   *  will 'accept' any new item instances presented to the collection manager
+   *  as new.  The result is that it allows you to be notified as new items
+   *  as they are indexed, existing items as they are loaded from the database,
+   *  etc.
+   * Because the items are added to the collection without limit, this will
+   *  result in a leak if you don't do something to clean up after the
+   *  collection.  (Forgetting about the collection will suffice, as it is still
+   *  weakly held.)
+   */
+  _wildcardCollection: function gloda_ns_wildcardCollection(aNounID, aItems) {
+    let nounDef = this._nounIDToDef[aNounID];
+    let collection = new GlodaCollection(nounDef, aItems, null, null);
+    let query = new nounDef.wildcardQueryClass(collection);
+    collection.query = query;
+    GlodaCollectionManager.registerCollection(collection);
+    return collection;
+  },
+
+  /**
+   * Attribute providers attempting to index something that experience a fatal
+   *  problem should throw one of these.  For example:
+   *  "throw new Gloda.BadItemContentsError('Message lacks an author.');".
+   *
+   * We're not really taking advantage of this yet, but it's a good idea.
+   */
+  BadItemContentsError: BadItemContentsError,
+
+  /**
+   * Populate a gloda representation of an item given the thus-far built
+   *  representation, the previous representation, and one or more raw
+   *  representations.  The attribute providers/optimizers for the given noun
+   *  type are invoked, allowing them to contribute/alter things.  Following
+   *  that, we build and persist our attribute representations.
+   *
+   * The result of the processing ends up with attributes in 3 different forms:
+   * - Database attribute rows (to be added and removed).
+   * - In-memory representation.
+   * - JSON-able representation.
+   *
+   * @param aItem The noun instance you want processed.
+   * @param aRawReps A dictionary that we pass to the attribute providers.
+   *     There is a(n implied) contract between the caller of grokNounItem for a
+   *     given noun type and the attribute providers for that noun type, and we
+   *     have nothing to do with it OTHER THAN inserting a 'trueGlodaRep'
+   *     value into it.  In the event of reindexing an existing object, the
+   *     gloda representation we pass to the indexers is actually a clone that
+   *     allows the asynchronous indexers to mutate the object without
+   *     causing visible changes in the existing representation of the gloda
+   *     object.  We patch the changes back onto the original item atomically
+   *     once indexing completes.  The 'trueGlodaRep' is then useful for
+   *     objects that hang off of the gloda instance that need a reference
+   *     back to their containing object for API convenience purposes.
+   * @param aIsConceptuallyNew Is the item "new" in the sense that it would
+   *     never have been visible from within user code?  This translates into
+   *     whether this should trigger an itemAdded notification or an
+   *     itemModified notification.
+   * @param aIsRecordNew Is the item "new" in the sense that we should INSERT
+   *     a record rather than UPDATE-ing a record.  For example, when dealing
+   *     with messages where we may have a ghost, the ghost message is not a
+   *     new record, but is conceptually new.
+   * @param aCallbackHandle The GlodaIndexer-style callback handle that is being
+   *     used to drive this processing in an async fashion.  (See
+   *     GlodaIndexer._callbackHandle).
+   * @param aDoCache Should we allow this item to be contributed to its noun
+   *     cache?
+   */
+  grokNounItem: function* gloda_ns_grokNounItem(aItem, aRawReps,
+      aIsConceptuallyNew, aIsRecordNew, aCallbackHandle, aDoCache) {
+    let itemNounDef = aItem.NOUN_DEF;
+    let attribsByBoundName = itemNounDef.attribsByBoundName;
+
+    this._log.info(" ** grokNounItem: " + itemNounDef.name);
+
+    let addDBAttribs = [];
+    let removeDBAttribs = [];
+
+    let jsonDict = {};
+
+    let aOldItem;
+    aRawReps.trueGlodaRep = aItem;
+    if (aIsConceptuallyNew) // there is no old item if we are new.
+      aOldItem = {};
+    else {
+      aOldItem = aItem;
+      // we want to create a clone of the existing item so that we can know the
+      //  deltas that happened for indexing purposes
+      aItem = aItem._clone();
+    }
+
+    // Have the attribute providers directly set properties on the aItem
+    let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
+    for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
+      this._log.info("  * provider: " + attrProviders[iProvider].providerName);
+      yield aCallbackHandle.pushAndGo(
+        attrProviders[iProvider].process(aItem, aRawReps, aIsConceptuallyNew,
+                                         aCallbackHandle));
+    }
+
+    let attrOptimizers = this._attrOptimizerOrderByNoun[itemNounDef.id];
+    for (let iProvider = 0; iProvider < attrOptimizers.length; iProvider++) {
+      this._log.info("  * optimizer: " + attrOptimizers[iProvider].providerName);
+      yield aCallbackHandle.pushAndGo(
+        attrOptimizers[iProvider].optimize(aItem, aRawReps, aIsConceptuallyNew,
+                                           aCallbackHandle));
+    }
+    this._log.info(" ** done with providers.");
+
+    // Iterate over the attributes on the item
+    for (let key of Object.keys(aItem)) {
+      let value = aItem[key];
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key.startsWith("_"))
+        continue;
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined) {
+        this._log.warn("new proc ignoring attrib: " + key);
+        continue;
+      }
+
+      let attribDB = attrib.dbDef;
+      let objectNounDef = attrib.objectNounDef;
+
+      // - translate for our JSON rep
+      if (attrib.singular) {
+        if (objectNounDef.toJSON)
+          jsonDict[attrib.id] = objectNounDef.toJSON(value);
+        else
+          jsonDict[attrib.id] = value;
+      }
+      else {
+        if (objectNounDef.toJSON) {
+          let toJSON = objectNounDef.toJSON;
+          jsonDict[attrib.id] = [];
+          for (let [, subValue] in Iterator(value)) {
+            jsonDict[attrib.id].push(toJSON(subValue));
+          }
+        }
+        else
+          jsonDict[attrib.id] = value;
+      }
+
+      let oldValue = aOldItem[key];
+
+      // the 'old' item is still the canonical one; update it
+      // do the update now, because we may skip operations on addDBAttribs and
+      //  removeDBattribs, if the attribute is not to generate entries in
+      //  messageAttributes
+      if (oldValue !== undefined || !aIsConceptuallyNew)
+        aOldItem[key] = value;
+
+      // the new canQuery property has to be set to true to generate entries
+      // in the messageAttributes table. Any other truthy value (like a non
+      // empty string), will still make the message query-able but without
+      // using the database.
+      if (attrib.canQuery !== true) {
+        continue;
+      }
+
+      // - database index attributes
+
+      // perform a delta analysis against the old value, if we have one
+      if (oldValue !== undefined) {
+        // in the singular case if they don't match, it's one add and one remove
+        if (attrib.singular) {
+          // test for identicality, failing that, see if they have explicit
+          //  equals support.
+          if ((value !== oldValue) &&
+              (!value.equals || !value.equals(oldValue))) {
+            addDBAttribs.push(attribDB.convertValuesToDBAttributes([value])[0]);
+            removeDBAttribs.push(
+              attribDB.convertValuesToDBAttributes([oldValue])[0]);
+          }
+        }
+        // in the plural case, we have to figure the deltas accounting for
+        //  possible changes in ordering (which is insignificant from an
+        //  indexing perspective)
+        // some nouns may not meet === equivalence needs, so must provide a
+        //  custom computeDelta method to help us out
+        else if (objectNounDef.computeDelta) {
+          let [valuesAdded, valuesRemoved] =
+            objectNounDef.computeDelta(value, oldValue);
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(addDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesAdded));
+          removeDBAttribs.push.apply(removeDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesRemoved));
+        }
+        else {
+          // build a map of the previous values; we will delete the values as
+          //  we see them so that we will know what old values are no longer
+          //  present in the current set of values.
+          let oldValueMap = {};
+          for (let anOldValue of oldValue) {
+            // remember, the key is just the toString'ed value, so we need to
+            //  store and use the actual value as the value!
+            oldValueMap[anOldValue] = anOldValue;
+          }
+          // traverse the current values...
+          let valuesAdded = [];
+          for (let curValue of value) {
+            if (curValue in oldValueMap)
+              delete oldValueMap[curValue];
+            else
+              valuesAdded.push(curValue);
+          }
+          // anything still on oldValueMap was removed.
+          let valuesRemoved = Object.keys(oldValueMap).
+            map(key => oldValueMap[key]);
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(addDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesAdded));
+          removeDBAttribs.push.apply(removeDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesRemoved));
+        }
+
+        // Add/remove the empty set indicator as appropriate.
+        if (attrib.emptySetIsSignificant) {
+          // if we are now non-zero but previously were zero, remove.
+          if (value.length && !oldValue.length)
+            removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+          // if we are now zero length but previously were not, add
+          else if (!value.length && oldValue.length)
+            addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+        }
+      }
+      // no old value, all values are new
+      else {
+        // add the db reps on the new values
+        if (attrib.singular)
+          value = [value];
+        addDBAttribs.push.apply(addDBAttribs,
+                                attribDB.convertValuesToDBAttributes(value));
+        // Add the empty set indicator for the attribute id if appropriate.
+        if (!value.length && attrib.emptySetIsSignificant)
+          addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+      }
+    }
+
+    // Iterate over any remaining values in old items for purge purposes.
+    for (let key of Object.keys(aOldItem)) {
+      let value = aOldItem[key];
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key.startsWith("_"))
+        continue;
+      // ignore things we saw in the new guy
+      if (key in aItem)
+        continue;
+
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined) {
+        continue;
+      }
+
+      // delete these from the old item, as the old item is canonical, and
+      //  should no longer have these values
+      delete aOldItem[key];
+
+      if (attrib.canQuery !== true) {
+        this._log.debug("Not inserting attribute "+attrib.attributeName
+            +" into the db, since we don't plan on querying on it");
+        continue;
+      }
+
+      if (attrib.singular)
+        value = [value];
+      let attribDB = attrib.dbDef;
+      removeDBAttribs.push.apply(removeDBAttribs,
+                                 attribDB.convertValuesToDBAttributes(value));
+      // remove the empty set marker if there should have been one
+      if (!value.length && attrib.emptySetIsSignificant)
+        removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+    }
+
+    aItem._jsonText = JSON.stringify(jsonDict);
+    this._log.debug("  json text: " + aItem._jsonText);
+
+    if (aIsRecordNew) {
+      this._log.debug(" inserting item");
+      itemNounDef.objInsert.call(itemNounDef.datastore, aItem);
+    }
+    else {
+      this._log.debug(" updating item");
+      itemNounDef.objUpdate.call(itemNounDef.datastore, aItem);
+    }
+
+    this._log.debug(" adjusting attributes, add: " + addDBAttribs + " rem: " +
+        removeDBAttribs);
+    itemNounDef.dbAttribAdjuster.call(itemNounDef.datastore, aItem,
+      addDBAttribs, removeDBAttribs);
+
+    if (!aIsConceptuallyNew && ("_declone" in aOldItem))
+      aOldItem._declone(aItem);
+
+    // Cache ramifications...
+    if (aDoCache === undefined || aDoCache) {
+      if (aIsConceptuallyNew)
+        GlodaCollectionManager.itemsAdded(aItem.NOUN_ID, [aItem]);
+      else
+        GlodaCollectionManager.itemsModified(aOldItem.NOUN_ID, [aOldItem]);
+    }
+
+    this._log.debug(" done grokking.");
+
+    yield this.kWorkDone;
+  },
+
+  /**
+   * Processes a list of noun instances for their score within a given context.
+   *  This is primarily intended for use by search ranking mechanisms, but could
+   *  be used elsewhere too.  (It does, however, depend on the complicity of the
+   *  score method implementations to not get confused.)
+   *
+   * @param aItems The non-empty list of items to score.
+   * @param aContext A noun-specific dictionary that we just pass to the funcs.
+   * @param aExtraScoreFuncs A list of extra scoring functions to apply.
+   * @returns A list of integer scores equal in length to aItems.
+   */
+  scoreNounItems: function gloda_ns_grokNounItem(aItems, aContext,
+                                                 aExtraScoreFuncs) {
+    let scores = [];
+    // bail if there is nothing to score
+    if (!aItems.length)
+      return scores;
+
+    let itemNounDef = aItems[0].NOUN_DEF;
+    if (aExtraScoreFuncs == null)
+      aExtraScoreFuncs = [];
+
+    for (let item of aItems) {
+      let score = 0;
+      let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
+      for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
+        let provider = attrProviders[iProvider];
+        if (provider.score)
+          score += provider.score(item);
+      }
+      for (let [, extraScoreFunc] in Iterator(aExtraScoreFuncs))
+        score += extraScoreFunc(item, aContext);
+      scores.push(score);
+    }
+
+    return scores;
+  }
+};
+
+/* and initialize the Gloda object/NS before we return... */
+try {
+  Gloda._init();
+}
+catch (ex) {
+  Gloda._log.debug("Exception during Gloda init (" + ex.fileName + ":" +
+                   ex.lineNumber + "): " + ex);
+};
+/* but don't forget that we effectively depend on everybody.js too, and
+   currently on our importer to be importing that if they need us fully armed
+   and operational. */