Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

25 files changed, 19608 insertions, 0 deletions

diff --git a/comm/mailnews/db/gloda/modules/Collection.jsm b/comm/mailnews/db/gloda/modules/Collection.jsm
new file mode 100644
index 0000000000..e229161fc9
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/Collection.jsm
@@ -0,0 +1,834 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaCollection", "GlodaCollectionManager"];
+
+var LOG = console.createInstance({
+  prefix: "gloda.collection",
+  maxLogLevel: "Warn",
+  maxLogLevelPref: "gloda.loglevel",
+});
+
+/**
+ * @namespace Central registry and logic for all collections.
+ *
+ * The collection manager is a singleton that has the following tasks:
+ * - Let views of objects (nouns) know when their objects have changed.  For
+ *   example, an attribute has changed due to user action.
+ * - Let views of objects based on queries know when new objects match their
+ *   query, or when their existing objects no longer match due to changes.
+ * - Caching/object-identity maintenance.  It is ideal if we only ever have
+ *   one instance of an object at a time.  (More specifically, only one instance
+ *   per database row 'id'.)  The collection mechanism lets us find existing
+ *   instances to this end.  Caching can be directly integrated by being treated
+ *   as a special collection.
+ */
+var GlodaCollectionManager = {
+  _collectionsByNoun: {},
+  _cachesByNoun: {},
+
+  /**
+   * Registers the existence of a collection with the collection manager.  This
+   *  is done using a weak reference so that the collection can go away if it
+   *  wants to.
+   */
+  registerCollection(aCollection) {
+    let collections;
+    let nounID = aCollection.query._nounDef.id;
+    if (!(nounID in this._collectionsByNoun)) {
+      collections = this._collectionsByNoun[nounID] = [];
+    } else {
+      // purge dead weak references while we're at it
+      collections = this._collectionsByNoun[nounID].filter(aRef => aRef.get());
+      this._collectionsByNoun[nounID] = collections;
+    }
+    collections.push(Cu.getWeakReference(aCollection));
+  },
+
+  getCollectionsForNounID(aNounID) {
+    if (!(aNounID in this._collectionsByNoun)) {
+      return [];
+    }
+
+    // generator would be nice, but I suspect get() is too expensive to use
+    //  twice (guard/predicate and value)
+    let weakCollections = this._collectionsByNoun[aNounID];
+    let collections = [];
+    for (let iColl = 0; iColl < weakCollections.length; iColl++) {
+      let collection = weakCollections[iColl].get();
+      if (collection) {
+        collections.push(collection);
+      }
+    }
+    return collections;
+  },
+
+  defineCache(aNounDef, aCacheSize) {
+    this._cachesByNoun[aNounDef.id] = new GlodaLRUCacheCollection(
+      aNounDef,
+      aCacheSize
+    );
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it wasn't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOne(aNounID, aID, aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aID in cache._idMap) {
+        let item = cache._idMap[aID];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false) {
+      cache = null;
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aID in collection._idMap) {
+        let item = collection._idMap[aID];
+        if (cache) {
+          cache.add([item]);
+        }
+        return item;
+      }
+    }
+
+    LOG.debug("cacheLookupOne:\nhit null");
+    return null;
+  },
+
+  /**
+   * Lookup multiple nouns by ID from the cache/existing collections.
+   *
+   * @param aNounID The kind of noun identified by its ID.
+   * @param aIDMap A dictionary/map whose keys must be gloda noun ids for the
+   *     given noun type and whose values are ignored.
+   * @param aTargetMap An object to hold the noun id's (key) and noun instances
+   *     (value) for the noun instances that were found available in memory
+   *     because they were cached or in existing query collections.
+   * @param [aDoCache=true] Should we add any items to the cache that we found
+   *     in collections that were in memory but not in the cache?  You would
+   *     likely want to pass false if you are only updating in-memory
+   *     representations rather than performing a new query.
+   *
+   * @returns [The number that were found, the number that were not found,
+   *          a dictionary whose keys are the ids of noun instances that
+   *          were not found.]
+   */
+  cacheLookupMany(aNounID, aIDMap, aTargetMap, aDoCache) {
+    let foundCount = 0,
+      notFoundCount = 0,
+      notFound = {};
+
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      for (let key in aIDMap) {
+        let cacheValue = cache._idMap[key];
+        if (cacheValue === undefined) {
+          notFoundCount++;
+          notFound[key] = null;
+        } else {
+          foundCount++;
+          aTargetMap[key] = cacheValue;
+          cache.hit(cacheValue);
+        }
+      }
+    }
+
+    if (aDoCache === false) {
+      cache = null;
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let key in notFound) {
+        let collValue = collection._idMap[key];
+        if (collValue !== undefined) {
+          aTargetMap[key] = collValue;
+          delete notFound[key];
+          foundCount++;
+          notFoundCount--;
+          if (cache) {
+            cache.add([collValue]);
+          }
+        }
+      }
+    }
+
+    return [foundCount, notFoundCount, notFound];
+  },
+
+  /**
+   * Friendlier version of |cacheLookupMany|; takes a list of ids and returns
+   *  an object whose keys and values are the gloda id's and instances of the
+   *  instances that were found.  We don't tell you who we didn't find.  The
+   *  assumption is this is being used for in-memory updates where we only need
+   *  to tweak what is in memory.
+   */
+  cacheLookupManyList(aNounID, aIds) {
+    let checkMap = {},
+      targetMap = {};
+    for (let id of aIds) {
+      checkMap[id] = null;
+    }
+    // do not promote found items into the cache
+    this.cacheLookupMany(aNounID, checkMap, targetMap, false);
+    return targetMap;
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it wasn't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOneByUniqueValue(aNounID, aUniqueValue, aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aUniqueValue in cache._uniqueValueMap) {
+        let item = cache._uniqueValueMap[aUniqueValue];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false) {
+      cache = null;
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aUniqueValue in collection._uniqueValueMap) {
+        let item = collection._uniqueValueMap[aUniqueValue];
+        if (cache) {
+          cache.add([item]);
+        }
+        return item;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Checks whether the provided item with the given id is actually a duplicate
+   *  of an instance that already exists in the cache/a collection.  If it is,
+   *  the pre-existing instance is returned and counts as a cache hit.  If it
+   *  is not, the passed-in instance is added to the cache and returned.
+   */
+  cacheLoadUnifyOne(aItem) {
+    let items = [aItem];
+    this.cacheLoadUnify(aItem.NOUN_ID, items);
+    return items[0];
+  },
+
+  /**
+   * Given a list of items, check if any of them already have duplicate,
+   *  canonical, instances in the cache or collections.  Items with pre-existing
+   *  instances are replaced by those instances in the provided list, and each
+   *  counts as a cache hit.  Items without pre-existing instances are added
+   *  to the cache and left intact.
+   */
+  cacheLoadUnify(aNounID, aItems, aCacheIfMissing) {
+    let cache = this._cachesByNoun[aNounID];
+    if (aCacheIfMissing === undefined) {
+      aCacheIfMissing = true;
+    }
+
+    // track the items we haven't yet found in a cache/collection (value) and
+    //  their index in aItems (key).  We're somewhat abusing the dictionary
+    //  metaphor with the intent of storing tuples here.  We also do it because
+    //  it allows random-access deletion theoretically without cost.  (Since
+    //  we delete during iteration, that may be wrong, but it sounds like the
+    //  semantics still work?)
+    let unresolvedIndexToItem = {};
+    let numUnresolved = 0;
+
+    if (cache) {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        let item = aItems[iItem];
+
+        if (item.id in cache._idMap) {
+          let realItem = cache._idMap[item.id];
+          // update the caller's array with the reference to the 'real' item
+          aItems[iItem] = realItem;
+          cache.hit(realItem);
+        } else {
+          unresolvedIndexToItem[iItem] = item;
+          numUnresolved++;
+        }
+      }
+
+      // we're done if everyone was a hit.
+      if (numUnresolved == 0) {
+        return;
+      }
+    } else {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        unresolvedIndexToItem[iItem] = aItems[iItem];
+      }
+      numUnresolved = aItems.length;
+    }
+
+    let needToCache = [];
+    // next, let's fall back to our collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let [iItem, item] of Object.entries(unresolvedIndexToItem)) {
+        if (item.id in collection._idMap) {
+          let realItem = collection._idMap[item.id];
+          // update the caller's array to now have the 'real' object
+          aItems[iItem] = realItem;
+          // flag that we need to cache this guy (we use an inclusive cache)
+          needToCache.push(realItem);
+          // we no longer need to resolve this item...
+          delete unresolvedIndexToItem[iItem];
+          // stop checking collections if we got everybody
+          if (--numUnresolved == 0) {
+            break;
+          }
+        }
+      }
+    }
+
+    // anything left in unresolvedIndexToItem should be added to the cache
+    //  unless !aCacheIfMissing.  plus, we already have 'needToCache'
+    if (cache && aCacheIfMissing) {
+      cache.add(
+        needToCache.concat(
+          Object.keys(unresolvedIndexToItem).map(
+            key => unresolvedIndexToItem[key]
+          )
+        )
+      );
+    }
+  },
+
+  cacheCommitDirty() {
+    for (let id in this._cachesByNoun) {
+      let cache = this._cachesByNoun[id];
+      cache.commitDirty();
+    }
+  },
+
+  /**
+   * Notifies the collection manager that an item has been loaded and should
+   *  be cached, assuming caching is active.
+   */
+  itemLoaded(aItem) {
+    let cache = this._cachesByNoun[aItem.NOUN_ID];
+    if (cache) {
+      cache.add([aItem]);
+    }
+  },
+
+  /**
+   * Notifies the collection manager that multiple items has been loaded and
+   *  should be cached, assuming caching is active.
+   */
+  itemsLoaded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+  },
+
+  /**
+   * This should be called when items are added to the global database.  This
+   *  should generally mean during indexing by indexers or an attribute
+   *  provider.
+   * We walk all existing collections for the given noun type and add the items
+   *  to the collection if the item meets the query that defines the collection.
+   */
+  itemsAdded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let addItems = aItems.filter(item => collection.query.test(item));
+      if (addItems.length) {
+        collection._onItemsAdded(addItems);
+      }
+    }
+  },
+  /**
+   * This should be called when items in the global database are modified.  For
+   *  example, as a result of indexing.  This should generally only be called
+   *  by indexers or by attribute providers.
+   * We walk all existing collections for the given noun type.  For items
+   *  currently included in each collection but should no longer be (per the
+   *  collection's defining query) we generate onItemsRemoved events.  For items
+   *  not currently included in the collection but should now be, we generate
+   *  onItemsAdded events.  For items included that still match the query, we
+   *  generate onItemsModified events.
+   */
+  itemsModified(aNounID, aItems) {
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let added = [],
+        modified = [],
+        removed = [];
+      for (let item of aItems) {
+        if (item.id in collection._idMap) {
+          // currently in... but should it still be there?
+          if (collection.query.test(item)) {
+            modified.push(item); // yes, keep it
+          } else if (!collection.query.frozen) {
+            // oy, so null queries really don't want any notifications, and they
+            //  sorta fit into our existing model, except for the removal bit.
+            //  so we need a specialized check for them, and we're using the
+            //  frozen attribute to this end.
+            removed.push(item); // no, bin it
+          }
+        } else if (collection.query.test(item)) {
+          // not in, should it be?
+          added.push(item); // yep, add it
+        }
+      }
+      if (added.length) {
+        collection._onItemsAdded(added);
+      }
+      if (modified.length) {
+        collection._onItemsModified(modified);
+      }
+      if (removed.length) {
+        collection._onItemsRemoved(removed);
+      }
+    }
+  },
+  /**
+   * This should be called when items in the global database are permanently-ish
+   *  deleted.  (This is distinct from concepts like message deletion which may
+   *  involved trash folders or other modified forms of existence.  Deleted
+   *  means the data is gone and if it were to come back, it would come back
+   *  via an itemsAdded event.)
+   * We walk all existing collections for the given noun type.  For items
+   *  currently in the collection, we generate onItemsRemoved events.
+   *
+   * @param aItemIds A list of item ids that are being deleted.
+   */
+  itemsDeleted(aNounID, aItemIds) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let itemId of aItemIds) {
+        if (itemId in cache._idMap) {
+          cache.deleted(cache._idMap[itemId]);
+        }
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = aItemIds
+        .filter(itemId => itemId in collection._idMap)
+        .map(itemId => collection._idMap[itemId]);
+      if (removeItems.length) {
+        collection._onItemsRemoved(removeItems);
+      }
+    }
+  },
+  /**
+   * Like |itemsDeleted| but for the case where the deletion is based on an
+   *  attribute that SQLite can more efficiently check than we can and where the
+   *  cost of scanning the in-memory items is presumably much cheaper than
+   *  trying to figure out what actually got deleted.
+   *
+   * Since we are doing an in-memory walk, this is obviously O(n) where n is the
+   *  number of noun instances of a given type in-memory.  We are assuming this
+   *  is a reasonable number of things and that this type of deletion call is
+   *  not going to happen all that frequently.  If these assumptions are wrong,
+   *  callers are advised to re-think the whole situation.
+   *
+   * @param aNounID Type of noun we are talking about here.
+   * @param aFilter A filter function that returns true when the item should be
+   *     thought of as deleted, or false if the item is still good.  Screw this
+   *     up and you will get some seriously wacky bugs, yo.
+   */
+  itemsDeletedByAttribute(aNounID, aFilter) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let id in cache._idMap) {
+        let item = cache._idMap[id];
+        if (aFilter(item)) {
+          cache.deleted(item);
+        }
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = collection.items.filter(aFilter);
+      if (removeItems.length) {
+        collection._onItemsRemoved(removeItems);
+      }
+    }
+  },
+};
+
+/**
+ * @class A current view of the set of first-class nouns meeting a given query.
+ *  Assuming a listener is present, events are
+ *  generated when new objects meet the query, existing objects no longer meet
+ *  the query, or existing objects have experienced a change in attributes that
+ *  does not affect their ability to be present (but the listener may care about
+ *  because it is exposing those attributes).
+ * @class
+ */
+function GlodaCollection(
+  aNounDef,
+  aItems,
+  aQuery,
+  aListener,
+  aMasterCollection
+) {
+  // if aNounDef is null, we are just being invoked for subclassing
+  if (aNounDef === undefined) {
+    return;
+  }
+
+  this._nounDef = aNounDef;
+  // should we also maintain a unique value mapping...
+  if (this._nounDef.usesUniqueValue) {
+    this._uniqueValueMap = {};
+  }
+
+  this.pendingItems = [];
+  this._pendingIdMap = {};
+  this.items = [];
+  this._idMap = {};
+
+  // force the listener to null for our call to _onItemsAdded; no events for
+  //  the initial load-out.
+  this._listener = null;
+  if (aItems && aItems.length) {
+    this._onItemsAdded(aItems);
+  }
+
+  this.query = aQuery || null;
+  if (this.query) {
+    this.query.collection = this;
+    if (this.query.options.stashColumns) {
+      this.stashedColumns = {};
+    }
+  }
+  this._listener = aListener || null;
+
+  this.deferredCount = 0;
+  this.resolvedCount = 0;
+
+  if (aMasterCollection) {
+    this.masterCollection = aMasterCollection.masterCollection;
+  } else {
+    this.masterCollection = this;
+    /** a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of these sub-dictionaries maps the IDs of desired noun instances to
+     * the actual instance, or null if it has not yet been loaded.
+     */
+    this.referencesByNounID = {};
+    /**
+     * a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of the sub-dictionaries maps the IDs of the _recognized parent
+     * noun_ to the list of children, or null if the list has not yet been
+     * populated.
+     *
+     * So if we have a noun definition A with ID 1 who is the recognized parent
+     *  noun of noun definition B with ID 2, AND we have an instance A(1) with
+     *  two children B(10), B(11), then an example might be: {2: {1: [10, 11]}}.
+     */
+    this.inverseReferencesByNounID = {};
+    this.subCollections = {};
+  }
+}
+
+GlodaCollection.prototype = {
+  get listener() {
+    return this._listener;
+  },
+  set listener(aListener) {
+    this._listener = aListener;
+  },
+
+  /**
+   * If this collection still has a query associated with it, drop the query
+   *  and replace it with an 'explicit query'.  This means that the Collection
+   *  Manager will not attempt to match new items indexed to the system against
+   *  our query criteria.
+   * Once you call this method, your collection's listener will no longer
+   *  receive onItemsAdded notifications that are not the result of your
+   *  initial database query.  It will, however, receive onItemsModified
+   *  notifications if items in the collection are re-indexed.
+   */
+  becomeExplicit() {
+    if (!(this.query instanceof this._nounDef.explicitQueryClass)) {
+      this.query = new this._nounDef.explicitQueryClass(this);
+    }
+  },
+
+  /**
+   * Clear the contents of this collection.  This only makes sense for explicit
+   *  collections or wildcard collections.  (Actual query-based collections
+   *  should represent the state of the query, so unless we're going to delete
+   *  all the items, clearing the collection would violate that constraint.)
+   */
+  clear() {
+    this._idMap = {};
+    if (this._uniqueValueMap) {
+      this._uniqueValueMap = {};
+    }
+    this.items = [];
+  },
+
+  _onItemsAdded(aItems) {
+    this.items.push.apply(this.items, aItems);
+    if (this._uniqueValueMap) {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+        this._uniqueValueMap[item.uniqueValue] = item;
+      }
+    } else {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+      }
+    }
+    if (this._listener) {
+      try {
+        this._listener.onItemsAdded(aItems, this);
+      } catch (ex) {
+        LOG.error(
+          "caught exception from listener in onItemsAdded: " +
+            ex.fileName +
+            ":" +
+            ex.lineNumber +
+            ": " +
+            ex
+        );
+      }
+    }
+  },
+
+  _onItemsModified(aItems) {
+    if (this._listener) {
+      try {
+        this._listener.onItemsModified(aItems, this);
+      } catch (ex) {
+        LOG.error(
+          "caught exception from listener in onItemsModified: " +
+            ex.fileName +
+            ":" +
+            ex.lineNumber +
+            ": " +
+            ex
+        );
+      }
+    }
+  },
+
+  /**
+   * Given a list of items that definitely no longer belong in this collection,
+   *  remove them from the collection and notify the listener.  The 'tricky'
+   *  part is that we need to remove the deleted items from our list of items.
+   */
+  _onItemsRemoved(aItems) {
+    // we want to avoid the O(n^2) deletion performance case, and deletion
+    //  should be rare enough that the extra cost of building the deletion map
+    //  should never be a real problem.
+    let deleteMap = {};
+    // build the delete map while also nuking from our id map/unique value map
+    for (let item of aItems) {
+      deleteMap[item.id] = true;
+      delete this._idMap[item.id];
+      if (this._uniqueValueMap) {
+        delete this._uniqueValueMap[item.uniqueValue];
+      }
+    }
+    let items = this.items;
+    // in-place filter.  probably needless optimization.
+    let iWrite = 0;
+    for (let iRead = 0; iRead < items.length; iRead++) {
+      let item = items[iRead];
+      if (!(item.id in deleteMap)) {
+        items[iWrite++] = item;
+      }
+    }
+    items.splice(iWrite);
+
+    if (this._listener) {
+      try {
+        this._listener.onItemsRemoved(aItems, this);
+      } catch (ex) {
+        LOG.error(
+          "caught exception from listener in onItemsRemoved: " +
+            ex.fileName +
+            ":" +
+            ex.lineNumber +
+            ": " +
+            ex
+        );
+      }
+    }
+  },
+
+  _onQueryCompleted() {
+    this.query.completed = true;
+    if (this._listener && this._listener.onQueryCompleted) {
+      this._listener.onQueryCompleted(this);
+    }
+  },
+};
+
+/**
+ * Create an LRU cache collection for the given noun with the given size.
+ *
+ * @class
+ */
+function GlodaLRUCacheCollection(aNounDef, aCacheSize) {
+  GlodaCollection.call(this, aNounDef, null, null, null);
+
+  this._head = null; // aka oldest!
+  this._tail = null; // aka newest!
+  this._size = 0;
+  // let's keep things sane, and simplify our logic a little...
+  if (aCacheSize < 32) {
+    aCacheSize = 32;
+  }
+  this._maxCacheSize = aCacheSize;
+}
+/**
+ * @class A LRU-discard cache.  We use a doubly linked-list for the eviction
+ *  tracking.  Since we require that there is at most one LRU-discard cache per
+ *  noun class, we simplify our lives by adding our own attributes to the
+ *  cached objects.
+ * @augments GlodaCollection
+ */
+GlodaLRUCacheCollection.prototype = new GlodaCollection();
+GlodaLRUCacheCollection.prototype.add = function (aItems) {
+  for (let item of aItems) {
+    if (item.id in this._idMap) {
+      // DEBUGME so, we're dealing with this, but it shouldn't happen.  need
+      //  trace-debuggage.
+      continue;
+    }
+    this._idMap[item.id] = item;
+    if (this._uniqueValueMap) {
+      this._uniqueValueMap[item.uniqueValue] = item;
+    }
+
+    item._lruPrev = this._tail;
+    // we do have to make sure that we will set _head the first time we insert
+    //  something
+    if (this._tail !== null) {
+      this._tail._lruNext = item;
+    } else {
+      this._head = item;
+    }
+    item._lruNext = null;
+    this._tail = item;
+
+    this._size++;
+  }
+
+  while (this._size > this._maxCacheSize) {
+    let item = this._head;
+
+    // we never have to deal with the possibility of needing to make _head/_tail
+    //  null.
+    this._head = item._lruNext;
+    this._head._lruPrev = null;
+    // (because we are nice, we will delete the properties...)
+    delete item._lruNext;
+    delete item._lruPrev;
+
+    // nuke from our id map
+    delete this._idMap[item.id];
+    if (this._uniqueValueMap) {
+      delete this._uniqueValueMap[item.uniqueValue];
+    }
+
+    // flush dirty items to disk (they may not have this attribute, in which
+    //  case, this returns false, which is fine.)
+    if (item.dirty) {
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+
+    this._size--;
+  }
+};
+
+GlodaLRUCacheCollection.prototype.hit = function (aItem) {
+  // don't do anything in the 0 or 1 items case, or if we're already
+  //  the last item
+  if (this._head === this._tail || this._tail === aItem) {
+    return aItem;
+  }
+
+  // - unlink the item
+  if (aItem._lruPrev !== null) {
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  } else {
+    this._head = aItem._lruNext;
+  }
+  // (_lruNext cannot be null)
+  aItem._lruNext._lruPrev = aItem._lruPrev;
+  // - link it in to the end
+  this._tail._lruNext = aItem;
+  aItem._lruPrev = this._tail;
+  aItem._lruNext = null;
+  // update tail tracking
+  this._tail = aItem;
+
+  return aItem;
+};
+
+GlodaLRUCacheCollection.prototype.deleted = function (aItem) {
+  // unlink the item
+  if (aItem._lruPrev !== null) {
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  } else {
+    this._head = aItem._lruNext;
+  }
+  if (aItem._lruNext !== null) {
+    aItem._lruNext._lruPrev = aItem._lruPrev;
+  } else {
+    this._tail = aItem._lruPrev;
+  }
+
+  // (because we are nice, we will delete the properties...)
+  delete aItem._lruNext;
+  delete aItem._lruPrev;
+
+  // nuke from our id map
+  delete this._idMap[aItem.id];
+  if (this._uniqueValueMap) {
+    delete this._uniqueValueMap[aItem.uniqueValue];
+  }
+
+  this._size--;
+};
+
+/**
+ * If any of the cached items are dirty, commit them, and make them no longer
+ *  dirty.
+ */
+GlodaLRUCacheCollection.prototype.commitDirty = function () {
+  // we can only do this if there is an update method available...
+  if (!this._nounDef.objUpdate) {
+    return;
+  }
+
+  for (let iItem in this._idMap) {
+    let item = this._idMap[iItem];
+    if (item.dirty) {
+      LOG.debug("flushing dirty: " + item);
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+  }
+};
diff --git a/comm/mailnews/db/gloda/modules/Everybody.jsm b/comm/mailnews/db/gloda/modules/Everybody.jsm
new file mode 100644
index 0000000000..4f33134ef9
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/Everybody.jsm
@@ -0,0 +1,23 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = [];
+
+const { GlodaFundAttr } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaFundAttr.jsm"
+);
+GlodaFundAttr.init();
+const { GlodaExplicitAttr } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaExplicitAttr.jsm"
+);
+GlodaExplicitAttr.init();
+
+ChromeUtils.import("resource:///modules/gloda/NounTag.jsm");
+ChromeUtils.import("resource:///modules/gloda/NounFreetag.jsm");
+ChromeUtils.import("resource:///modules/gloda/NounMimetype.jsm");
+ChromeUtils.import("resource:///modules/gloda/IndexMsg.jsm");
+const { GlodaABAttrs } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaMsgIndexer.jsm"
+);
+GlodaABAttrs.init();
diff --git a/comm/mailnews/db/gloda/modules/Facet.jsm b/comm/mailnews/db/gloda/modules/Facet.jsm
new file mode 100644
index 0000000000..96425b8838
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/Facet.jsm
@@ -0,0 +1,599 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides faceting logic.
+ */
+
+var EXPORTED_SYMBOLS = ["FacetDriver", "FacetUtils"];
+
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+const lazy = {};
+ChromeUtils.defineModuleGetter(
+  lazy,
+  "Gloda",
+  "resource:///modules/gloda/GlodaPublic.jsm"
+);
+
+/**
+ * Decides the appropriate faceters for the noun type and drives the faceting
+ *  process.  This class and the faceters are intended to be reusable so that
+ *  you only need one instance per faceting session.  (Although each faceting
+ *  pass is accordingly destructive to previous results.)
+ *
+ * Our strategy for faceting is to process one attribute at a time across all
+ *  the items in the provided set.  The alternative would be to iterate over
+ *  the items and then iterate over the attributes on each item.  While both
+ *  approaches have caching downsides
+ */
+function FacetDriver(aNounDef, aWindow) {
+  this.nounDef = aNounDef;
+  this._window = aWindow;
+
+  this._makeFaceters();
+}
+FacetDriver.prototype = {
+  /**
+   * Populate |this.faceters| with a set of faceters appropriate to the noun
+   *  definition associated with this instance.
+   */
+  _makeFaceters() {
+    let faceters = (this.faceters = []);
+
+    function makeFaceter(aAttrDef, aFacetDef) {
+      let facetType = aFacetDef.type;
+
+      if (aAttrDef.singular) {
+        if (facetType == "date") {
+          faceters.push(new DateFaceter(aAttrDef, aFacetDef));
+        } else {
+          faceters.push(new DiscreteFaceter(aAttrDef, aFacetDef));
+        }
+      } else if (facetType == "nonempty?") {
+        faceters.push(new NonEmptySetFaceter(aAttrDef, aFacetDef));
+      } else {
+        faceters.push(new DiscreteSetFaceter(aAttrDef, aFacetDef));
+      }
+    }
+
+    for (let key in this.nounDef.attribsByBoundName) {
+      let attrDef = this.nounDef.attribsByBoundName[key];
+      // ignore attributes that do not want to be faceted
+      if (!attrDef.facet) {
+        continue;
+      }
+
+      makeFaceter(attrDef, attrDef.facet);
+
+      if ("extraFacets" in attrDef) {
+        for (let facetDef of attrDef.extraFacets) {
+          makeFaceter(attrDef, facetDef);
+        }
+      }
+    }
+  },
+  /**
+   * Asynchronously facet the provided items, calling the provided callback when
+   *  completed.
+   */
+  go(aItems, aCallback, aCallbackThis) {
+    this.items = aItems;
+    this.callback = aCallback;
+    this.callbackThis = aCallbackThis;
+
+    this._nextFaceter = 0;
+    this._drive();
+  },
+
+  _MAX_FACETING_TIMESLICE_MS: 100,
+  _FACETING_YIELD_DURATION_MS: 0,
+  _driveWrapper(aThis) {
+    aThis._drive();
+  },
+  _drive() {
+    let start = Date.now();
+
+    while (this._nextFaceter < this.faceters.length) {
+      let faceter = this.faceters[this._nextFaceter++];
+      // for now we facet in one go, but the long-term plan allows for them to
+      //  be generators.
+      faceter.facetItems(this.items);
+
+      let delta = Date.now() - start;
+      if (delta > this._MAX_FACETING_TIMESLICE_MS) {
+        this._window.setTimeout(
+          this._driveWrapper,
+          this._FACETING_YIELD_DURATION_MS,
+          this
+        );
+        return;
+      }
+    }
+
+    // we only get here once we are done with the faceters
+    this.callback.call(this.callbackThis);
+  },
+};
+
+var FacetUtils = {
+  _groupSizeComparator(a, b) {
+    return b[1].length - a[1].length;
+  },
+
+  /**
+   * Given a list where each entry is a tuple of [group object, list of items
+   *  belonging to that group], produce a new list of the top grouped items.  We
+   *  used to also produce an "other" aggregation, but that turned out to be
+   *  conceptually difficult to deal with, so that's gone, leaving this method
+   *  with much less to do.
+   *
+   * @param aAttrDef The attribute for the facet we are working with.
+   * @param aGroups The list of groups built for the facet.
+   * @param aMaxCount The number of result rows you want back.
+   */
+  makeTopGroups(aAttrDef, aGroups, aMaxCount) {
+    let nounDef = aAttrDef.objectNounDef;
+    let realGroupsToUse = aMaxCount;
+
+    let orderedBySize = aGroups.concat();
+    orderedBySize.sort(this._groupSizeComparator);
+
+    // - get the real groups to use and order them by the attribute comparator
+    let outGroups = orderedBySize.slice(0, realGroupsToUse);
+    let comparator = nounDef.comparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    outGroups.sort(comparatorHelper);
+
+    return outGroups;
+  },
+};
+
+/**
+ * Facet discrete things like message authors, boolean values, etc.  Only
+ *  appropriate for use on singular values.  Use |DiscreteSetFaceter| for
+ *  non-singular values.
+ */
+function DiscreteFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive) {
+      return this.facetPrimitiveItems(aItems);
+    }
+    return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let filter = this.facetDef.filter;
+
+    let valStrToVal = {};
+    let groups = (this.groups = {});
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = attrKey in item ? item[attrKey] : null;
+      if (val === GlodaConstants.IGNORE_FACET) {
+        continue;
+      }
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val)) {
+        continue;
+      }
+
+      // We need to use hasOwnProperty because we cannot guarantee that the
+      //  contents of val won't collide with the attributes in Object.prototype.
+      if (groups.hasOwnProperty(val)) {
+        groups[val].push(item);
+      } else {
+        groups[val] = [item];
+        valStrToVal[val] = val;
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).map(key => [
+      valStrToVal[key],
+      groups[key],
+    ]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = (this.groups = {});
+    let groupMap = (this.groupMap = {});
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = attrKey in item ? item[attrKey] : null;
+      if (val === GlodaConstants.IGNORE_FACET) {
+        continue;
+      }
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val)) {
+        continue;
+      }
+
+      let valId = val == null ? null : val[idAttr];
+      // We need to use hasOwnProperty because tag nouns are complex objects
+      //  with id's that are non-numeric and so can collide with the contents
+      //  of Object.prototype.  (Note: the "tags" attribute is actually handled
+      //  by the DiscreteSetFaceter.)
+      if (groupMap.hasOwnProperty(valId)) {
+        groups[valId].push(item);
+      } else {
+        groupMap[valId] = val;
+        groups[valId] = [item];
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).map(key => [
+      groupMap[key],
+      groups[key],
+    ]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Facet sets of discrete items.  For example, tags applied to messages.
+ *
+ * The main differences between us and |DiscreteFaceter| are:
+ * - The empty set is notable.
+ * - Specific set configurations could be interesting, but are not low-hanging
+ *    fruit.
+ */
+function DiscreteSetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteSetFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive) {
+      return this.facetPrimitiveItems(aItems);
+    }
+    return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let filter = this.facetDef.filter;
+
+    let groups = (this.groups = {});
+    let valStrToVal = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = attrKey in item ? item[attrKey] : null;
+      if (vals === GlodaConstants.IGNORE_FACET) {
+        continue;
+      }
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val)) {
+          continue;
+        }
+
+        // We need to use hasOwnProperty because we cannot guarantee that the
+        //  contents of val won't collide with the attributes in
+        //  Object.prototype.
+        if (groups.hasOwnProperty(val)) {
+          groups[val].push(item);
+        } else {
+          groups[val] = [item];
+          valStrToVal[val] = val;
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).map(key => [
+      valStrToVal[key],
+      groups[key],
+    ]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = (this.groups = {});
+    let groupMap = (this.groupMap = {});
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = attrKey in item ? item[attrKey] : null;
+      if (vals === GlodaConstants.IGNORE_FACET) {
+        continue;
+      }
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val)) {
+          continue;
+        }
+
+        let valId = val == null ? null : val[idAttr];
+        // We need to use hasOwnProperty because tag nouns are complex objects
+        //  with id's that are non-numeric and so can collide with the contents
+        //  of Object.prototype.
+        if (groupMap.hasOwnProperty(valId)) {
+          groups[valId].push(item);
+        } else {
+          groupMap[valId] = val;
+          groups[valId] = [item];
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).map(key => [
+      groupMap[key],
+      groups[key],
+    ]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Given a non-singular attribute, facet it as if it were a boolean based on
+ *  whether there is anything in the list (set).
+ */
+function NonEmptySetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+NonEmptySetFaceter.prototype = {
+  type: "boolean",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+
+    let trueValues = [];
+    let falseValues = [];
+
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = attrKey in item ? item[attrKey] : null;
+      if (vals == null || vals.length == 0) {
+        falseValues.push(item);
+      } else {
+        trueValues.push(item);
+      }
+    }
+
+    this.orderedGroups = [];
+    if (trueValues.length) {
+      this.orderedGroups.push([true, trueValues]);
+    }
+    if (falseValues.length) {
+      this.orderedGroups.push([false, falseValues]);
+    }
+    this.groupCount = this.orderedGroups.length;
+  },
+  makeQuery(aGroupValues, aInclusive) {
+    let query = (this.query = lazy.Gloda.newQuery(GlodaConstants.NOUN_MESSAGE));
+
+    let constraintFunc = query[this.attrDef.boundName];
+    constraintFunc.call(query);
+
+    // Our query is always for non-empty lists (at this time), so we want to
+    //  invert if they're excluding 'true' or including 'false', which means !=.
+    let invert = aGroupValues[0] != aInclusive;
+
+    return [query, invert];
+  },
+};
+
+/**
+ * Facet dates.  We build a hierarchical nested structure of year, month, and
+ *  day nesting levels.  This decision was made speculatively in the hopes that
+ *  it would allow us to do clustered analysis and that there might be a benefit
+ *  for that.  For example, if you search for "Christmas", we might notice
+ *  clusters of messages around December of each year.  We could then present
+ *  these in a list as likely candidates, rather than a graphical timeline.
+ *  Alternately, it could be used to inform a non-linear visualization.  As it
+ *  stands (as of this writing), it's just a complicating factor.
+ */
+function DateFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DateFaceter.prototype = {
+  type: "date",
+  /**
+   *
+   */
+  facetItems(aItems) {
+    let attrKey = this.attrDef.boundName;
+
+    let years = (this.years = { _subCount: 0 });
+    // generally track the time range
+    let oldest = null,
+      newest = null;
+
+    this.validItems = [];
+
+    // just cheat and put us at the front...
+    this.groupCount = aItems.length ? 1000 : 0;
+    this.orderedGroups = null;
+
+    /** The number of items with a null/missing attribute. */
+    this.missing = 0;
+
+    /**
+     * The number of items with a date that is unreasonably far in the past or
+     *  in the future.  Old-wise, we are concerned about incorrectly formatted
+     *  messages (spam) that end up placed around the UNIX epoch.  New-wise,
+     *  we are concerned about messages that can't be explained by users who
+     *  don't know how to set their clocks (both the current user and people
+     *  sending them mail), mainly meaning spam.
+     * We want to avoid having our clever time-scale logic being made useless by
+     *  these unreasonable messages.
+     */
+    this.unreasonable = 0;
+    // feb 1, 1970
+    let tooOld = new Date(1970, 1, 1);
+    // 3 days from now
+    let tooNew = new Date(Date.now() + 3 * 24 * 60 * 60 * 1000);
+
+    for (let item of aItems) {
+      let val = attrKey in item ? item[attrKey] : null;
+      // -- missing
+      if (val == null) {
+        this.missing++;
+        continue;
+      }
+
+      // -- unreasonable
+      if (val < tooOld || val > tooNew) {
+        this.unreasonable++;
+        continue;
+      }
+
+      this.validItems.push(item);
+
+      // -- time range
+      if (oldest == null) {
+        oldest = newest = val;
+      } else if (val < oldest) {
+        oldest = val;
+      } else if (val > newest) {
+        newest = val;
+      }
+
+      // -- bucket
+      // - year
+      let year,
+        valYear = val.getYear();
+      if (valYear in years) {
+        year = years[valYear];
+        year._dateCount++;
+      } else {
+        year = years[valYear] = {
+          _dateCount: 1,
+          _subCount: 0,
+        };
+        years._subCount++;
+      }
+
+      // - month
+      let month,
+        valMonth = val.getMonth();
+      if (valMonth in year) {
+        month = year[valMonth];
+        month._dateCount++;
+      } else {
+        month = year[valMonth] = {
+          _dateCount: 1,
+          _subCount: 0,
+        };
+        year._subCount++;
+      }
+
+      // - day
+      let valDate = val.getDate();
+      if (valDate in month) {
+        month[valDate].push(item);
+      } else {
+        month[valDate] = [item];
+      }
+    }
+
+    this.oldest = oldest;
+    this.newest = newest;
+  },
+
+  _unionMonth(aMonthObj) {
+    let dayItemLists = [];
+    for (let key in aMonthObj) {
+      let dayItemList = aMonthObj[key];
+      if (typeof key == "string" && key.startsWith("_")) {
+        continue;
+      }
+      dayItemLists.push(dayItemList);
+    }
+    return dayItemLists;
+  },
+
+  _unionYear(aYearObj) {
+    let monthItemLists = [];
+    for (let key in aYearObj) {
+      let monthObj = aYearObj[key];
+      if (typeof key == "string" && key.startsWith("_")) {
+        continue;
+      }
+      monthItemLists.push(this._unionMonth(monthObj));
+    }
+    return monthItemLists;
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/Gloda.jsm b/comm/mailnews/db/gloda/modules/Gloda.jsm
new file mode 100644
index 0000000000..77b2288e53
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/Gloda.jsm
@@ -0,0 +1,2275 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["Gloda"];
+
+const { GlodaDatastore } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaDatastore.jsm"
+);
+const {
+  GlodaAttributeDBDef,
+  GlodaAccount,
+  GlodaConversation,
+  GlodaFolder,
+  GlodaMessage,
+  GlodaContact,
+  GlodaIdentity,
+  GlodaAttachment,
+} = ChromeUtils.import("resource:///modules/gloda/GlodaDataModel.jsm");
+const { GlodaCollection, GlodaCollectionManager } = ChromeUtils.import(
+  "resource:///modules/gloda/Collection.jsm"
+);
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { whittlerRegistry, mimeMsgToContentAndMeta } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaContent.jsm"
+);
+const { GlodaQueryClassFactory } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaQueryClassFactory.jsm"
+);
+const { GlodaUtils } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaUtils.jsm"
+);
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+
+/**
+ * @see |Gloda.BadItemContentsError|
+ */
+function BadItemContentsError(aMessage) {
+  this.message = aMessage;
+}
+BadItemContentsError.prototype = {
+  toString() {
+    return this.message;
+  },
+};
+
+/**
+ * Provides the user-visible (and extension visible) global database
+ *  functionality.  There is currently a dependency/ordering
+ *  problem in that the concept of 'gloda' also includes some logic that is
+ *  contributed by built-in extensions, if you will.  Those built-in extensions
+ *  (fundattr.js, GlodaExplicitAttr.jsm) also import this file.  To avoid a circular
+ *  dependency, those built-in extensions are loaded by Everybody.jsm.  The
+ *  simplest/best solution is probably to move Everybody.jsm to be Gloda.jsm and
+ *  have it re-export only 'Gloda'.  Gloda.jsm (this file) can then move to be
+ *  gloda_int.js (or whatever our eventual naming scheme is), which built-in
+ *  extensions can explicitly rely upon.
+ *
+ * === Concepts
+ *
+ * == Nouns
+ *
+ * Inspired by reasonable uses of triple-stores, I have tried to leverage
+ *  existing model and terminology rather than rolling out own for everything.
+ *  The idea with triple-stores is that you have a subject, a predicate, and an
+ *  object.  For example, if we are talking about a message, that is the
+ *  subject, the predicate could roughly be sent-by, and the object a person.
+ *  We can generalize this idea to say that the subject and objects are nouns.
+ * Since we want to be more flexible than only dealing with messages, we
+ *  therefore introduce the concept of nouns as an organizing principle.
+ *
+ * == Attributes
+ *
+ * Our attributes definitions are basically our predicates.  When we define
+ *  an attribute, it's a label with a bunch of meta-data.  Our attribute
+ *  instances are basically a 'triple' in a triple-store.  The attributes
+ *  are stored in database rows that imply a specific noun-type (ex: the
+ *  messageAttributes table), with an ID identifying the message which is our
+ *  subject, an attribute ID which identifies the attribute definition in use
+ *  (and therefore the predicate), plus an object ID (given context aka the
+ *  noun type by the attribute's meta-data) which identifies the 'object'.
+ *
+ * == But...
+ *
+ * Things aren't entirely as clear as they could be right now, terminology/
+ *  concept/implementation-wise.  Some work is probably still in order.
+ *
+ * === Implementation
+ *
+ * == Nouns
+ *
+ * So, we go and define the nouns that are roughly the classes in our data
+ *  model.  Every 'class' we define in GlodaDataModel.jsm is a noun that gets defined
+ *  here in the Gloda core.  We provide sufficient meta-data about the noun to
+ *  serialize/deserialize its representation from our database representation.
+ *  Nouns do not have to be defined in this class, but can also be contributed
+ *  by external code.
+ * We have a concept of 'first class' nouns versus non-first class nouns.  The
+ *  distinction is meant to be whether we can store meta-information about those
+ *  nouns using attributes.  Right now, only message are real first-class nouns,
+ *  but we want to expand that to include contacts and eventually events and
+ *  tasks as lightning-integration occurs.  In practice, we are stretching the
+ *  definition of first-class nouns slightly to include things we can't store
+ *  meta-data about, but want to be able to query about.  We do want to resolve
+ *  this.
+ *
+ * == Attributes
+ *
+ * Attributes are defined by "attribute providers" who are responsible for
+ *  taking an instance of a first-class noun (for which they are registered)
+ *  plus perhaps some other meta-data, and returning a list of attributes
+ *  extracted from that noun.  For now, this means messages.  Attribute
+ *  providers may create new data records as a side-effect of the indexing
+ *  process, although we have not yet fully dealt with the problem of deleting
+ *  these records should they become orphaned in the database due to the
+ *  purging of a message and its attributes.
+ * All of the 'core' gloda attributes are provided by the GlodaFundAttr.jsm and
+ *  GlodaExplicitAttr.jsm providers.
+ *
+ * === (Notable) Future Work
+ *
+ * == Attributes
+ *
+ * Attribute mechanisms currently lack any support for 'overriding' attributes
+ *  provided by other attribute providers.  For example, the fundattr provider
+ *  tells us who a message is 'from' based on the e-mail address present.
+ *  However, other plugins may actually know better.  For example, the bugzilla
+ *  daemon e-mails based on bug activity although the daemon gets the credit
+ *  as the official sender.  A bugzilla plugin can easily extract the actual
+ *  person/e-mail addressed who did something on the bug to cause the
+ *  notification to be sent.  In practice, we would like that person to be
+ *  the 'sender' of the bugmail.  But we can't really do that right, yet.
+ *
+ * @namespace
+ */
+var Gloda = {
+  /**
+   * Initialize logging, the datastore (SQLite database), the core nouns and
+   *  attributes, and the contact and identities that belong to the presumed
+   *  current user (based on accounts).
+   *
+   * Additional nouns and the core attribute providers are initialized by the
+   *  Everybody.jsm module which ensures all of those dependencies are loaded
+   *  (and initialized).
+   */
+  _init() {
+    this._initLogging();
+    GlodaDatastore._init(this._nounIDToDef);
+    this._initAttributes();
+    this._initMyIdentities();
+  },
+
+  _log: null,
+  /**
+   * Initialize logging; the error console window gets Warning/Error, and stdout
+   *  (via dump) gets everything.
+   */
+  _initLogging() {
+    this._log = console.createInstance({
+      prefix: "gloda",
+      maxLogLevel: "Warn",
+      maxLogLevelPref: "gloda.loglevel",
+    });
+    this._log.info("Logging Initialized");
+  },
+
+  /**
+   * Callers should access the unique ID for the GlodaDatastore
+   * with this getter. If the GlodaDatastore has not been
+   * initialized, this value is null.
+   *
+   * @returns a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
+   */
+  get datastoreID() {
+    return GlodaDatastore._datastoreID;
+  },
+
+  /**
+   * Lookup a gloda message from an nsIMsgDBHdr, with the result returned as a
+   *  collection.  Keep in mind that the message may not be indexed, so you
+   *  may end up with an empty collection.  (Also keep in mind that this query
+   *  is asynchronous, so you will want your action-taking logic to be found
+   *  in your listener's onQueryCompleted method; the result will not be in
+   *  the collection when this method returns.)
+   *
+   * @param aMsgHdr The header of the message you want the gloda message for.
+   * @param aListener The listener that should be registered with the collection
+   * @param aData The (optional) value to set as the data attribute on the
+   *     collection.
+   *
+   * @returns The collection that will receive the results.
+   *
+   * @testpoint gloda.ns.getMessageCollectionForHeader()
+   */
+  getMessageCollectionForHeader(aMsgHdr, aListener, aData) {
+    let query = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE);
+    query.folder(aMsgHdr.folder).messageKey(aMsgHdr.messageKey);
+    return query.getCollection(aListener, aData);
+  },
+
+  /**
+   * Given a list of message headers, return a collection containing the gloda
+   *  messages that correspond to those headers.  Keep in mind that gloda may
+   *  not have indexed all the messages, so the returned collection may not have
+   *  a message for each header you provide. (Also keep in mind that this query
+   *  is asynchronous, so you will want your action-taking logic to be found
+   *  in your listener's onQueryCompleted method; no results will be present in
+   *  the collection when this method returns.)
+   *
+   * @param aHeaders An array of headers
+   * @param aListener The listener that should be registered with the collection
+   * @param aData The (optional) value to set as the data attribute on the
+   *     collection.
+   *
+   * @returns The collection that will receive the results.
+   *
+   * @testpoint gloda.ns.getMessageCollectionForHeaders()
+   */
+  getMessageCollectionForHeaders(aHeaders, aListener, aData) {
+    // group the headers by the folder they are found in
+    let headersByFolder = {};
+    for (let header of aHeaders) {
+      let folderURI = header.folder.URI;
+      let headersForFolder = headersByFolder[folderURI];
+      if (headersForFolder === undefined) {
+        headersByFolder[folderURI] = [header];
+      } else {
+        headersForFolder.push(header);
+      }
+    }
+
+    let query = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE);
+    let clause;
+    // build a query, using a separate union clause for each folder.
+    for (let folderURI in headersByFolder) {
+      let headersForFolder = headersByFolder[folderURI];
+      let folder = this.getFolderForFolder(headersForFolder[0].folder);
+      // if this is the first or clause, just use the query itself
+      if (!clause) {
+        clause = query;
+      } else {
+        // Create a new query clause via the 'or' command.
+        clause = query.or();
+      }
+
+      clause.folder(folder);
+      let messageKeys = headersForFolder.map(hdr => hdr.messageKey);
+      clause.messageKey.apply(clause, messageKeys);
+    }
+
+    return query.getCollection(aListener, aData);
+  },
+
+  /**
+   * @testpoint gloda.ns.getMessageContent
+   */
+  getMessageContent(aGlodaMessage, aMimeMsg) {
+    return mimeMsgToContentAndMeta(
+      aMimeMsg,
+      aGlodaMessage.folderMessage.folder
+    )[0];
+  },
+
+  getFolderForFolder(aMsgFolder) {
+    return GlodaDatastore._mapFolder(aMsgFolder);
+  },
+
+  /**
+   * Takes one or more strings containing lists of comma-delimited e-mail
+   *  addresses with optional display names, and returns a list of sub-lists of
+   *  identities, where each sub-list corresponds to each of the strings passed
+   *  as arguments.  These identities are loaded from the database if they
+   *  already exist, or created if they do not yet exist.
+   * If the identities need to be created, they will also result in the
+   *  creation of a gloda contact.  If a display name was provided with the
+   *  e-mail address, it will become the name of the gloda contact.  If a
+   *  display name was not provided, the e-mail address will also serve as the
+   *  contact name.
+   * This method uses the indexer's callback handle mechanism, and does not
+   *  obey traditional return semantics.
+   *
+   * We normalize all e-mail addresses to be lowercase as a normative measure.
+   *
+   * @param aCallbackHandle The GlodaIndexer callback handle (or equivalent)
+   *   that you are operating under.
+   * @param aAddrGroups... One or more strings.  Each string can contain zero or more
+   *   e-mail addresses with display name.  If more than one address is given,
+   *   they should be comma-delimited.  For example
+   *   '"Bob Smith" <bob@example.com>' is an address with display name.  Mime
+   *   header decoding is performed, but is ignorant of any folder-level
+   *   character set overrides.
+   * @returns via the callback handle mechanism, a list containing one sub-list
+   *   for each string argument passed.  Each sub-list contains zero or more
+   *   GlodaIdentity instances corresponding to the addresses provided.
+   */
+  *getOrCreateMailIdentities(aCallbackHandle, ...aAddrGroups) {
+    let addresses = {};
+    let resultLists = [];
+
+    // parse the strings
+    for (let aMailAddresses of aAddrGroups) {
+      let parsed = GlodaUtils.parseMailAddresses(aMailAddresses);
+
+      let resultList = [];
+      resultLists.push(resultList);
+
+      for (let iAddress = 0; iAddress < parsed.count; iAddress++) {
+        let address = parsed.addresses[iAddress].toLowerCase();
+        if (address in addresses) {
+          addresses[address].push(resultList);
+        } else {
+          addresses[address] = [parsed.names[iAddress], resultList];
+        }
+      }
+    }
+
+    let addressList = Object.keys(addresses);
+    if (addressList.length == 0) {
+      yield aCallbackHandle.doneWithResult(resultLists);
+      // we should be stopped before we reach this point, but safety first.
+      return;
+    }
+
+    let query = this.newQuery(GlodaConstants.NOUN_IDENTITY);
+    query.kind("email");
+    query.value.apply(query, addressList);
+    let collection = query.getCollection(aCallbackHandle);
+    yield GlodaConstants.kWorkAsync;
+
+    // put the identities in the appropriate result lists
+    for (let identity of collection.items) {
+      let nameAndResultLists = addresses[identity.value];
+      this._log.debug(
+        " found identity for '" +
+          nameAndResultLists[0] +
+          "' (" +
+          identity.value +
+          ")"
+      );
+      // index 0 is the name, skip it
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
+        nameAndResultLists[iResList].push(identity);
+      }
+      delete addresses[identity.value];
+    }
+
+    // create the identities that did not exist yet
+    for (let address in addresses) {
+      let nameAndResultLists = addresses[address];
+      let name = nameAndResultLists[0];
+
+      this._log.debug(" creating contact for '" + name + "' (" + address + ")");
+
+      // try and find an existing address book contact.
+      let card = MailServices.ab.cardForEmailAddress(address);
+      // XXX when we have the address book GUID stuff, we need to use that to
+      //  find existing contacts... (this will introduce a new query phase
+      //  where we batch all the GUIDs for an async query)
+      // XXX when the address book supports multiple e-mail addresses, we
+      //  should also just create identities for any that don't yet exist
+
+      // if there is no name, just use the e-mail (the ab indexer actually
+      //  processes the card's displayName for synchronization, so we don't
+      //  need to do that.)
+      if (!name) {
+        name = address;
+      }
+
+      let contact = GlodaDatastore.createContact(null, null, name, 0, 0);
+
+      // we must create the identity.  use a blank description because there's
+      //  nothing to differentiate it from other identities, as this contact
+      //  only has one initially (us).
+      // XXX when we have multiple e-mails and there is a meaning associated
+      //  with each e-mail, try and use that to populate the description.
+      // XXX we are creating the identity here before we insert the contact.
+      //  conceptually it is good for us to be creating the identity before
+      //  exposing it to the address-book indexer, but we could get our id's
+      //  in a bad way from not deferring the identity insertion until after
+      //  the contact insertion.
+      let identity = GlodaDatastore.createIdentity(
+        contact.id,
+        contact,
+        "email",
+        address,
+        /* description */ "",
+        /* relay? */ false
+      );
+      contact._identities = [identity];
+
+      // give the address book indexer a chance if we have a card.
+      // (it will fix-up the name based on the card as appropriate)
+      if (card) {
+        yield aCallbackHandle.pushAndGo(
+          Gloda.grokNounItem(contact, { card }, true, true, aCallbackHandle)
+        );
+      } else {
+        // grokNounItem will issue the insert for us...
+        GlodaDatastore.insertContact(contact);
+      }
+
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
+        nameAndResultLists[iResList].push(identity);
+      }
+    }
+
+    yield aCallbackHandle.doneWithResult(resultLists);
+  },
+
+  /**
+   * Dictionary of the user's known identities; key is the identity id, value
+   *  is the actual identity.  This is populated by _initMyIdentities based on
+   *  the accounts defined.
+   */
+  myIdentities: {},
+  /**
+   * The contact corresponding to the current user.  We are assuming that only
+   *  a single user/human being uses the current profile.  This is known to be
+   *  a flawed assumption, but is the best first approximation available.
+   * The contact is based on the default account's default identity. The user
+   *  can change both, if desired, in Account Settings.
+   *
+   * @TODO attempt to deal with multiple people using the same profile
+   */
+  myContact: null,
+  /**
+   * Populate myIdentities with all of our identities.  Currently we do this
+   *  by assuming that there is one human/user per profile, and that all of the
+   *  accounts defined in the profile belong to them.  The single contact is
+   *  stored on myContact.
+   *
+   * @TODO deal with account addition/modification/removal
+   * @TODO attempt to deal with multiple people using the same profile
+   */
+  _initMyIdentities() {
+    let myContact = null;
+    let myIdentities = {};
+    // Process each email at most once; stored here.
+    let myEmailAddresses = new Set();
+
+    let fullName, fallbackName;
+    let existingIdentities = [];
+    let identitiesToCreate = [];
+
+    let allIdentities = MailServices.accounts.allIdentities;
+    let defaultMsgIdentity = MailServices.accounts.defaultAccount
+      ? MailServices.accounts.defaultAccount.defaultIdentity
+      : null;
+    let defaultMsgIdentityKey = defaultMsgIdentity
+      ? defaultMsgIdentity.key
+      : null;
+    let defaultIdentity;
+
+    // Nothing to do if there are no accounts/identities.
+    if (allIdentities.length == 0) {
+      return;
+    }
+
+    for (let msgIdentity of allIdentities) {
+      let emailAddress = msgIdentity.email;
+      let replyTo = msgIdentity.replyTo;
+      let msgIdentityDescription = msgIdentity.fullName || msgIdentity.email;
+      let isDefaultMsgIdentity = msgIdentity.key == defaultMsgIdentityKey;
+
+      if (!fullName || isDefaultMsgIdentity) {
+        fullName = msgIdentity.fullName;
+      }
+      if (!fallbackName || isDefaultMsgIdentity) {
+        fallbackName = msgIdentity.email;
+      }
+
+      // Find the identities if they exist, flag to create them if they don't.
+      for (let address of [emailAddress, replyTo]) {
+        if (!address) {
+          continue;
+        }
+        let parsed = GlodaUtils.parseMailAddresses(address);
+        if (myEmailAddresses.has(parsed.addresses[0])) {
+          continue;
+        }
+        let identity = GlodaDatastore.getIdentity("email", parsed.addresses[0]);
+        if (identity) {
+          if (identity.description != msgIdentityDescription) {
+            // If the user changed the identity name, update the db.
+            identity._description = msgIdentityDescription;
+            GlodaDatastore.updateIdentity(identity);
+          }
+          existingIdentities.push(identity);
+          if (isDefaultMsgIdentity) {
+            defaultIdentity = identity;
+          }
+        } else {
+          identitiesToCreate.push([
+            parsed.addresses[0],
+            msgIdentityDescription,
+          ]);
+        }
+        myEmailAddresses.add(parsed.addresses[0]);
+      }
+    }
+    // We need to establish the identity.contact portions of the relationship.
+    for (let identity of existingIdentities) {
+      identity._contact = GlodaDatastore.getContactByID(identity.contactID);
+      if (defaultIdentity && defaultIdentity.id == identity.id) {
+        if (identity.contact.name != (fullName || fallbackName)) {
+          // If the user changed the default identity, update the db.
+          identity.contact.name = fullName || fallbackName;
+          GlodaDatastore.updateContact(identity.contact);
+        }
+        defaultIdentity._contact = identity.contact;
+      }
+    }
+
+    if (defaultIdentity) {
+      // The contact is based on the default account's default identity.
+      myContact = defaultIdentity.contact;
+    } else if (existingIdentities.length) {
+      // Just use the first guy's contact.
+      myContact = existingIdentities[0].contact;
+    } else {
+      // Create a new contact.
+      myContact = GlodaDatastore.createContact(
+        null,
+        null,
+        fullName || fallbackName,
+        0,
+        0
+      );
+      GlodaDatastore.insertContact(myContact);
+    }
+
+    for (let emailAndDescription of identitiesToCreate) {
+      // XXX This won't always be of type "email" as we add new account types.
+      let identity = GlodaDatastore.createIdentity(
+        myContact.id,
+        myContact,
+        "email",
+        emailAndDescription[0],
+        emailAndDescription[1],
+        false
+      );
+      existingIdentities.push(identity);
+    }
+
+    for (let identity of existingIdentities) {
+      myIdentities[identity.id] = identity;
+    }
+
+    this.myContact = myContact;
+    this.myIdentities = myIdentities;
+    myContact._identities = Object.keys(myIdentities).map(
+      id => myIdentities[id]
+    );
+
+    // We need contacts to make these objects reachable via the collection
+    //  manager.
+    this._myContactCollection = this.explicitCollection(
+      GlodaConstants.NOUN_CONTACT,
+      [this.myContact]
+    );
+    this._myIdentitiesCollection = this.explicitCollection(
+      GlodaConstants.NOUN_IDENTITY,
+      this.myContact._identities
+    );
+  },
+
+  /** Next Noun ID to hand out, these don't need to be persisted (for now). */
+  _nextNounID: 1000,
+
+  /**
+   * Maps noun names to noun IDs.
+   */
+  _nounNameToNounID: {},
+  /**
+   * Maps noun IDs to noun definition dictionaries.  (Noun definition
+   *  dictionaries provided to us at the time a noun was defined, plus some
+   *  additional stuff we put in there.)
+   */
+  _nounIDToDef: {},
+
+  _managedToJSON(aItem) {
+    return aItem.id;
+  },
+
+  /**
+   * Define a noun.  Takes a dictionary with the following keys/values:
+   *
+   * @param aNounDef.name The name of the noun.  This is not a display name
+   *     (anything being displayed needs to be localized, after all), but simply
+   *     the canonical name for debugging purposes and for people to pass to
+   *     lookupNoun.  The suggested convention is lower-case-dash-delimited,
+   *     with names being singular (since it's a single noun we are referring
+   *     to.)
+   * @param aNounDef.class The 'class' to which an instance of the noun will
+   *     belong (aka will pass an instanceof test).  You may also provide this
+   *     as 'clazz' if the keyword makes your IDE angry.
+   * @param aNounDef.allowsArbitraryAttrs Is this a 'first class noun'/can it be
+   *     a subject, AKA can this noun have attributes stored on it that relate
+   *     it to other things?  For example, a message is first-class; we store
+   *     attributes of messages.  A date is not first-class now, nor is it
+   *     likely to be; we will not store attributes about a date, although dates
+   *     will be the objects of other subjects.  (For example: we might
+   *     associate a date with a calendar event, but the date is an attribute of
+   *     the calendar event and not vice versa.)
+   * @param aNounDef.usesParameter A boolean indicating whether this noun
+   *     requires use of the 'parameter' BLOB storage field on the attribute
+   *     bindings in the database to persist itself.  Use of parameters should
+   *     be limited to a reasonable number of values (16-32 is okay, more than
+   *     that is pushing it and 256 should be considered an absolute upper
+   *     bound) because of the database organization.  When false, your
+   *     toParamAndValue function is expected to return null for the parameter
+   *     and likewise your fromParamAndValue should expect ignore and generally
+   *     ignore the argument.
+   * @param aNounDef.toParamAndValue A function that takes an instantiated noun
+   *     instance and returns a 2-element list of [parameter, value] where
+   *     parameter may only be non-null if you passed a usesParameter of true.
+   *     Parameter may be of any type (BLOB), and value must be numeric (pass
+   *     0 if you don't need the value).
+   *
+   * @param aNounDef.isPrimitive True when the noun instance is a raw numeric
+   *     value/string/boolean.  False when the instance is an object.  When
+   *     false, it is assumed the attribute that serves as a unique identifier
+   *     for the value is "id" unless 'idAttr' is provided.
+   * @param [aNounDef.idAttr="id"] For non-primitive nouns, this is the
+   *     attribute on the object that uniquely identifies it.
+   *
+   * @param aNounDef.schema Unsupported mechanism by which you can define a
+   *     table that corresponds to this noun.  The table will be created if it
+   *     does not exist.
+   *     - name The table name; don't conflict with other things!
+   *     - columns A list of [column name, sqlite type] tuples.  You should
+   *       always include a definition like ["id", "INTEGER PRIMARY KEY"] for
+   *       now (and it should be the first column name too.)  If you care about
+   *       how the attributes are poked into your object (for example, you want
+   *       underscores used for some of them because the attributes should be
+   *       immutable), then you can include a third string that is the name of
+   *       the attribute to use.
+   *     - indices A dictionary of lists of column names, where the key name
+   *       becomes the index name.  Ex: {foo: ["bar"]} results in an index on
+   *       the column "bar" where the index is named "foo".
+   */
+  defineNoun(aNounDef, aNounID) {
+    this._log.info("Defining noun: " + aNounDef.name);
+    if (aNounID === undefined) {
+      aNounID = this._nextNounID++;
+    }
+    aNounDef.id = aNounID;
+
+    // Let people whose editors get angry about illegal attribute names use
+    //  clazz instead of class.
+    if (aNounDef.clazz) {
+      aNounDef.class = aNounDef.clazz;
+    }
+
+    if (!("idAttr" in aNounDef)) {
+      aNounDef.idAttr = "id";
+    }
+    if (!("comparator" in aNounDef)) {
+      aNounDef.comparator = function () {
+        throw new Error(
+          "Noun type '" + aNounDef.name + "' lacks a real comparator."
+        );
+      };
+    }
+
+    // We allow nouns to have data tables associated with them where we do all
+    //  the legwork.  The schema attribute is the gateway to this magical world
+    //  of functionality.  Said door is officially unsupported.
+    if (aNounDef.schema) {
+      if (!aNounDef.tableName) {
+        if (aNounDef.schema.name) {
+          aNounDef.tableName = "ext_" + aNounDef.schema.name;
+        } else {
+          aNounDef.tableName = "ext_" + aNounDef.name;
+        }
+      }
+      // this creates the data table and binder and hooks everything up
+      GlodaDatastore.createNounTable(aNounDef);
+
+      if (!aNounDef.toParamAndValue) {
+        aNounDef.toParamAndValue = function (aThing) {
+          if (aThing instanceof aNounDef.class) {
+            return [null, aThing.id];
+          }
+          // assume they're just passing the id directly
+          return [null, aThing];
+        };
+      }
+    }
+
+    // if it has a table, you can query on it.  seems straight-forward.
+    if (aNounDef.tableName) {
+      [
+        aNounDef.queryClass,
+        aNounDef.nullQueryClass,
+        aNounDef.explicitQueryClass,
+        aNounDef.wildcardQueryClass,
+      ] = GlodaQueryClassFactory(aNounDef);
+      aNounDef._dbMeta = {};
+      aNounDef.class.prototype.NOUN_ID = aNounDef.id;
+      aNounDef.class.prototype.NOUN_DEF = aNounDef;
+      aNounDef.toJSON = this._managedToJSON;
+
+      aNounDef.specialLoadAttribs = [];
+
+      // - define the 'id' constrainer
+      let idConstrainer = function (...aArgs) {
+        let constraint = [GlodaConstants.kConstraintIdIn, null, ...aArgs];
+        this._constraints.push(constraint);
+        return this;
+      };
+      aNounDef.queryClass.prototype.id = idConstrainer;
+    }
+    if (aNounDef.cache) {
+      let cacheCost = aNounDef.cacheCost || 1024;
+      let cacheBudget = aNounDef.cacheBudget || 128 * 1024;
+      let cacheSize = Math.floor(cacheBudget / cacheCost);
+      if (cacheSize) {
+        GlodaCollectionManager.defineCache(aNounDef, cacheSize);
+      }
+    }
+    aNounDef.attribsByBoundName = {};
+    aNounDef.domExposeAttribsByBoundName = {};
+
+    aNounDef.objectNounOfAttributes = [];
+
+    this._nounNameToNounID[aNounDef.name] = aNounID;
+    this._nounIDToDef[aNounID] = aNounDef;
+    aNounDef.actions = [];
+
+    this._attrProviderOrderByNoun[aNounDef.id] = [];
+    this._attrOptimizerOrderByNoun[aNounDef.id] = [];
+    this._attrProvidersByNoun[aNounDef.id] = {};
+
+    return aNounDef;
+  },
+
+  /**
+   * Lookup a noun (ID) suitable for passing to defineAttribute's various
+   *  noun arguments.  Throws an exception if the noun with the given name
+   *  cannot be found; the assumption is that you can't live without the noun.
+   */
+  lookupNoun(aNounName) {
+    if (aNounName in this._nounNameToNounID) {
+      return this._nounNameToNounID[aNounName];
+    }
+
+    throw Error(
+      "Unable to locate noun with name '" +
+        aNounName +
+        "', but I " +
+        "do know about: " +
+        Object.keys(this._nounNameToNounID).join(", ")
+    );
+  },
+
+  /**
+   * Lookup a noun def given a name.
+   */
+  lookupNounDef(aNounName) {
+    return this._nounIDToDef[this.lookupNoun(aNounName)];
+  },
+
+  /**
+   * Define an action on a noun.  During the prototype stage, this was conceived
+   *  of as a way to expose all the constraints possible given a noun.  For
+   *  example, if you have an identity or a contact, you could use this to
+   *  see all the messages sent from/to a given contact.  It was likewise
+   *  thought potentially usable for future expansion.  For example, you could
+   *  also decide to send an e-mail to a contact when you have the contact
+   *  instance available.
+   * Outside of the 'expmess' checkbox-happy prototype, this functionality is
+   *  not used.  As such, this functionality should be considered in flux and
+   *  subject to changes.  Also, very open to specific suggestsions motivated
+   *  by use cases.
+   * One conceptual issue raised by this mechanism is the interaction of actions
+   *  with facts like "this message is read".  We currently implement the 'fact'
+   *  by defining an attribute with a 'boolean' noun type.  To deal with this,
+   *  in various places we pass-in the attribute as well as the noun value.
+   *  Since the relationships for booleans and integers in these cases is
+   *  standard and well-defined, this works out pretty well, but suggests we
+   *  need to think things through.
+   *
+   * @param aNounID The ID of the noun you want to define an action on.
+   * @param aActionMeta The dictionary describing the noun.  The dictionary
+   *     should have the following fields:
+   * - actionType: a string indicating the type of action.  Currently, only
+   *   "filter" is a legal value.
+   * - actionTarget: the noun ID of the noun type on which this action is
+   *   applicable.  For example,
+   *
+   * The following should be present for actionType=="filter";
+   * - shortName: The name that should be used to display this constraint.  For
+   *   example, a checkbox-heavy UI might display a checkbox for each constraint
+   *   using shortName as the label.
+   * - makeConstraint: A function that takes the attribute that is the source
+   *   of the noun and the noun instance as arguments, and returns APV-style
+   *   constraints.  Since the APV-style query mechanism is now deprecated,
+   *   this signature is deprecated.  Probably the way to update this would be
+   *   to pass in the query instance that constraints should be contributed to.
+   */
+  defineNounAction(aNounID, aActionMeta) {
+    let nounDef = this._nounIDToDef[aNounID];
+    nounDef.actions.push(aActionMeta);
+  },
+
+  /**
+   * Retrieve all of the actions (as defined using defineNounAction) for the
+   *  given noun type (via noun ID) with the given action type (ex: filter).
+   */
+  getNounActions(aNounID, aActionType) {
+    let nounDef = this._nounIDToDef[aNounID];
+    if (!nounDef) {
+      return [];
+    }
+    return nounDef.actions.filter(
+      action => !aActionType || action.actionType == aActionType
+    );
+  },
+
+  /** Attribute providers in the sequence to process them. */
+  _attrProviderOrderByNoun: {},
+  /** Attribute providers that provide optimizers, in the sequence to proc. */
+  _attrOptimizerOrderByNoun: {},
+  /** Maps attribute providers to the list of attributes they provide */
+  _attrProviders: {},
+  /**
+   * Maps nouns to their attribute providers to a list of the attributes they
+   *  provide for the noun.
+   */
+  _attrProvidersByNoun: {},
+
+  /**
+   * Define the core nouns (that are not defined elsewhere) and a few noun
+   *  actions.  Core nouns could be defined in other files, assuming dependency
+   *  issues are resolved via the Everybody.jsm mechanism or something else.
+   *  Right now, noun_tag defines the tag noun.  If we broke more of these out,
+   *  we would probably want to move the 'class' code from GlodaDataModel.jsm, the
+   *  SQL table def and helper code from GlodaDatastore.jsm (and this code) to their
+   *  own noun_*.js files.  There are some trade-offs to be made, and I think
+   *  we can deal with those once we start to integrate lightning/calendar and
+   *  our noun space gets large and more heterogeneous.
+   */
+  _initAttributes() {
+    this.defineNoun(
+      {
+        name: "bool",
+        clazz: Boolean,
+        allowsArbitraryAttrs: false,
+        isPrimitive: true,
+        // favor true before false
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return b - a;
+        },
+        toParamAndValue(aBool) {
+          return [null, aBool ? 1 : 0];
+        },
+      },
+      GlodaConstants.NOUN_BOOLEAN
+    );
+    this.defineNoun(
+      {
+        name: "number",
+        clazz: Number,
+        allowsArbitraryAttrs: false,
+        continuous: true,
+        isPrimitive: true,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a - b;
+        },
+        toParamAndValue(aNum) {
+          return [null, aNum];
+        },
+      },
+      GlodaConstants.NOUN_NUMBER
+    );
+    this.defineNoun(
+      {
+        name: "string",
+        clazz: String,
+        allowsArbitraryAttrs: false,
+        isPrimitive: true,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.localeCompare(b);
+        },
+        toParamAndValue(aString) {
+          return [null, aString];
+        },
+      },
+      GlodaConstants.NOUN_STRING
+    );
+    this.defineNoun(
+      {
+        name: "date",
+        clazz: Date,
+        allowsArbitraryAttrs: false,
+        continuous: true,
+        isPrimitive: true,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a - b;
+        },
+        toParamAndValue(aDate) {
+          return [null, aDate.valueOf() * 1000];
+        },
+      },
+      GlodaConstants.NOUN_DATE
+    );
+    this.defineNoun(
+      {
+        name: "fulltext",
+        clazz: String,
+        allowsArbitraryAttrs: false,
+        continuous: false,
+        isPrimitive: true,
+        comparator(a, b) {
+          throw new Error("Fulltext nouns are not comparable!");
+        },
+        // as noted on NOUN_FULLTEXT, we just pass the string around.  it never
+        //  hits the database, so it's okay.
+        toParamAndValue(aString) {
+          return [null, aString];
+        },
+      },
+      GlodaConstants.NOUN_FULLTEXT
+    );
+
+    this.defineNoun(
+      {
+        name: "folder",
+        clazz: GlodaFolder,
+        allowsArbitraryAttrs: false,
+        isPrimitive: false,
+        queryHelpers: {
+          /**
+           * Query for accounts based on the account associated with folders.  We
+           *  walk all of the folders associated with an account and put them in
+           *  the list of folders that match if gloda would index them.  This is
+           *  unsuitable for producing a persistable constraint since it does not
+           *  adapt for added/deleted folders.  However, it is sufficient for
+           *  faceting.  Also, we don't persist constraints yet.
+           *
+           * @TODO The long-term solution is to move towards using arithmetic
+           *     encoding on folder-id's like we use for MIME types and friends.
+           */
+          Account(aAttrDef, aArguments) {
+            let folderValues = [];
+            let seenRootFolders = {};
+            for (let iArg = 0; iArg < aArguments.length; iArg++) {
+              let givenFolder = aArguments[iArg];
+              let givenMsgFolder = givenFolder.getXPCOMFolder(
+                givenFolder.kActivityFolderOnlyNoData
+              );
+              let rootFolder = givenMsgFolder.rootFolder;
+
+              // skip processing this folder if we have already processed its
+              //  root folder.
+              if (rootFolder.URI in seenRootFolders) {
+                continue;
+              }
+              seenRootFolders[rootFolder.URI] = true;
+
+              for (let folder of rootFolder.descendants) {
+                let folderFlags = folder.flags;
+
+                // Ignore virtual folders, non-mail folders.
+                // XXX this is derived from GlodaIndexer's shouldIndexFolder.
+                //  This should probably just use centralized code or the like.
+                if (
+                  !(folderFlags & Ci.nsMsgFolderFlags.Mail) ||
+                  folderFlags & Ci.nsMsgFolderFlags.Virtual
+                ) {
+                  continue;
+                }
+                // we only index local or IMAP folders
+                if (
+                  !(folder instanceof Ci.nsIMsgLocalMailFolder) &&
+                  !(folder instanceof Ci.nsIMsgImapMailFolder)
+                ) {
+                  continue;
+                }
+
+                let glodaFolder = Gloda.getFolderForFolder(folder);
+                folderValues.push(glodaFolder);
+              }
+            }
+            return this._inConstraintHelper(aAttrDef, folderValues);
+          },
+        },
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.name.localeCompare(b.name);
+        },
+        toParamAndValue(aFolderOrGlodaFolder) {
+          if (aFolderOrGlodaFolder instanceof GlodaFolder) {
+            return [null, aFolderOrGlodaFolder.id];
+          }
+          return [null, GlodaDatastore._mapFolder(aFolderOrGlodaFolder).id];
+        },
+      },
+      GlodaConstants.NOUN_FOLDER
+    );
+    this.defineNoun(
+      {
+        name: "account",
+        clazz: GlodaAccount,
+        allowsArbitraryAttrs: false,
+        isPrimitive: false,
+        equals(a, b) {
+          if ((a && !b) || (!a && b)) {
+            return false;
+          }
+          if (!a && !b) {
+            return true;
+          }
+          return a.id == b.id;
+        },
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.name.localeCompare(b.name);
+        },
+      },
+      GlodaConstants.NOUN_ACCOUNT
+    );
+    this.defineNoun(
+      {
+        name: "conversation",
+        clazz: GlodaConversation,
+        allowsArbitraryAttrs: false,
+        isPrimitive: false,
+        cache: true,
+        cacheCost: 512,
+        tableName: "conversations",
+        attrTableName: "messageAttributes",
+        attrIDColumnName: "conversationID",
+        datastore: GlodaDatastore,
+        objFromRow: GlodaDatastore._conversationFromRow,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.subject.localeCompare(b.subject);
+        },
+        toParamAndValue(aConversation) {
+          if (aConversation instanceof GlodaConversation) {
+            return [null, aConversation.id];
+          }
+          // assume they're just passing the id directly
+          return [null, aConversation];
+        },
+      },
+      GlodaConstants.NOUN_CONVERSATION
+    );
+    this.defineNoun(
+      {
+        name: "message",
+        clazz: GlodaMessage,
+        allowsArbitraryAttrs: true,
+        isPrimitive: false,
+        cache: true,
+        cacheCost: 2048,
+        tableName: "messages",
+        // we will always have a fulltext row, even for messages where we don't
+        //  have the body available.  this is because we want the subject indexed.
+        dbQueryJoinMagic:
+          " INNER JOIN messagesText ON messages.id = messagesText.rowid",
+        attrTableName: "messageAttributes",
+        attrIDColumnName: "messageID",
+        datastore: GlodaDatastore,
+        objFromRow: GlodaDatastore._messageFromRow,
+        dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
+        dbQueryValidityConstraintSuffix:
+          " AND +deleted = 0 AND +folderID IS NOT NULL AND +messageKey IS NOT NULL",
+        // This is what's used when we have no validity constraints, i.e. we allow
+        // for ghost messages, which do not have a row in the messagesText table.
+        dbQueryJoinMagicWithNoValidityConstraints:
+          " LEFT JOIN messagesText ON messages.id = messagesText.rowid",
+        objInsert: GlodaDatastore.insertMessage,
+        objUpdate: GlodaDatastore.updateMessage,
+        toParamAndValue(aMessage) {
+          if (aMessage instanceof GlodaMessage) {
+            return [null, aMessage.id];
+          }
+          // assume they're just passing the id directly
+          return [null, aMessage];
+        },
+      },
+      GlodaConstants.NOUN_MESSAGE
+    );
+    this.defineNoun(
+      {
+        name: "contact",
+        clazz: GlodaContact,
+        allowsArbitraryAttrs: true,
+        isPrimitive: false,
+        cache: true,
+        cacheCost: 128,
+        tableName: "contacts",
+        attrTableName: "contactAttributes",
+        attrIDColumnName: "contactID",
+        datastore: GlodaDatastore,
+        objFromRow: GlodaDatastore._contactFromRow,
+        dbAttribAdjuster: GlodaDatastore.adjustAttributes,
+        objInsert: GlodaDatastore.insertContact,
+        objUpdate: GlodaDatastore.updateContact,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.name.localeCompare(b.name);
+        },
+        toParamAndValue(aContact) {
+          if (aContact instanceof GlodaContact) {
+            return [null, aContact.id];
+          }
+          // assume they're just passing the id directly
+          return [null, aContact];
+        },
+      },
+      GlodaConstants.NOUN_CONTACT
+    );
+    this.defineNoun(
+      {
+        name: "identity",
+        clazz: GlodaIdentity,
+        allowsArbitraryAttrs: false,
+        isPrimitive: false,
+        cache: true,
+        cacheCost: 128,
+        usesUniqueValue: true,
+        tableName: "identities",
+        datastore: GlodaDatastore,
+        objFromRow: GlodaDatastore._identityFromRow,
+        /**
+         * Short string is the contact name, long string includes the identity
+         *  value too, delimited by a colon.  Not tremendously localizable.
+         */
+        userVisibleString(aIdentity, aLong) {
+          if (!aLong) {
+            return aIdentity.contact.name;
+          }
+          if (aIdentity.contact.name == aIdentity.value) {
+            return aIdentity.value;
+          }
+          return aIdentity.contact.name + " (" + aIdentity.value + ")";
+        },
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          return a.contact.name.localeCompare(b.contact.name);
+        },
+        toParamAndValue(aIdentity) {
+          if (aIdentity instanceof GlodaIdentity) {
+            return [null, aIdentity.id];
+          }
+          // assume they're just passing the id directly
+          return [null, aIdentity];
+        },
+      },
+      GlodaConstants.NOUN_IDENTITY
+    );
+    this.defineNoun(
+      {
+        name: "attachment-infos",
+        clazz: GlodaAttachment,
+        allowsArbitraryAttrs: false,
+        isPrimitive: false,
+        toJSON(x) {
+          return [
+            x._name,
+            x._contentType,
+            x._size,
+            x._part,
+            x._externalUrl,
+            x._isExternal,
+          ];
+        },
+        fromJSON(x, aGlodaMessage) {
+          let [name, contentType, size, _part, _externalUrl, isExternal] = x;
+          return new GlodaAttachment(
+            aGlodaMessage,
+            name,
+            contentType,
+            size,
+            _part,
+            _externalUrl,
+            isExternal
+          );
+        },
+      },
+      GlodaConstants.NOUN_ATTACHMENT
+    );
+
+    // parameterized identity is just two identities; we store the first one
+    //  (whose value set must be very constrainted, like the 'me' identities)
+    //  as the parameter, the second (which does not need to be constrained)
+    //  as the value.
+    this.defineNoun(
+      {
+        name: "parameterized-identity",
+        clazz: null,
+        allowsArbitraryAttrs: false,
+        comparator(a, b) {
+          if (a == null) {
+            if (b == null) {
+              return 0;
+            }
+            return 1;
+          } else if (b == null) {
+            return -1;
+          }
+          // First sort by the first identity in the tuple
+          // Since our general use-case is for the first guy to be "me", we only
+          //  compare the identity value, not the name.
+          let fic = a[0].value.localeCompare(b[0].value);
+          if (fic) {
+            return fic;
+          }
+          // Next compare the second identity in the tuple, but use the contact
+          //  this time to be consistent with our identity comparator.
+          return a[1].contact.name.localeCompare(b[1].contact.name);
+        },
+        computeDelta(aCurValues, aOldValues) {
+          let oldMap = {};
+          for (let tupe of aOldValues) {
+            let [originIdentity, targetIdentity] = tupe;
+            let targets = oldMap[originIdentity];
+            if (targets === undefined) {
+              targets = oldMap[originIdentity] = {};
+            }
+            targets[targetIdentity] = true;
+          }
+
+          let added = [],
+            removed = [];
+          for (let tupe of aCurValues) {
+            let [originIdentity, targetIdentity] = tupe;
+            let targets = oldMap[originIdentity];
+            if (targets === undefined || !(targetIdentity in targets)) {
+              added.push(tupe);
+            } else {
+              delete targets[targetIdentity];
+            }
+          }
+
+          for (let originIdentity in oldMap) {
+            let targets = oldMap[originIdentity];
+            for (let targetIdentity in targets) {
+              removed.push([originIdentity, targetIdentity]);
+            }
+          }
+
+          return [added, removed];
+        },
+        contributeObjDependencies(
+          aJsonValues,
+          aReferencesByNounID,
+          aInverseReferencesByNounID
+        ) {
+          // nothing to do with a zero-length list
+          if (aJsonValues.length == 0) {
+            return false;
+          }
+
+          let nounIdentityDef =
+            Gloda._nounIDToDef[GlodaConstants.NOUN_IDENTITY];
+          let references = aReferencesByNounID[nounIdentityDef.id];
+          if (references === undefined) {
+            references = aReferencesByNounID[nounIdentityDef.id] = {};
+          }
+
+          for (let tupe of aJsonValues) {
+            let [originIdentityID, targetIdentityID] = tupe;
+            if (!(originIdentityID in references)) {
+              references[originIdentityID] = null;
+            }
+            if (!(targetIdentityID in references)) {
+              references[targetIdentityID] = null;
+            }
+          }
+
+          return true;
+        },
+        resolveObjDependencies(
+          aJsonValues,
+          aReferencesByNounID,
+          aInverseReferencesByNounID
+        ) {
+          let references = aReferencesByNounID[GlodaConstants.NOUN_IDENTITY];
+
+          let results = [];
+          for (let tupe of aJsonValues) {
+            let [originIdentityID, targetIdentityID] = tupe;
+            results.push([
+              references[originIdentityID],
+              references[targetIdentityID],
+            ]);
+          }
+
+          return results;
+        },
+        toJSON(aIdentityTuple) {
+          return [aIdentityTuple[0].id, aIdentityTuple[1].id];
+        },
+        toParamAndValue(aIdentityTuple) {
+          return [aIdentityTuple[0].id, aIdentityTuple[1].id];
+        },
+      },
+      GlodaConstants.NOUN_PARAM_IDENTITY
+    );
+
+    GlodaDatastore.getAllAttributes();
+  },
+
+  /**
+   * Create accessor functions to 'bind' an attribute to underlying normalized
+   *  attribute storage, as well as creating the appropriate query object
+   *  constraint helper functions.  This name is somewhat of a misnomer because
+   *  special attributes are not 'bound' (because specific/non-generic per-class
+   *  code provides the properties) but still depend on this method to
+   *  establish their constraint helper methods.
+   *
+   * @XXX potentially rename to not suggest binding is required.
+   */
+  _bindAttribute(aAttrDef, aSubjectNounDef) {
+    let objectNounDef = aAttrDef.objectNounDef;
+
+    // -- the query constraint helpers
+    if (aSubjectNounDef.queryClass !== undefined) {
+      let constrainer;
+      let canQuery = true;
+      if (
+        "special" in aAttrDef &&
+        aAttrDef.special == GlodaConstants.kSpecialFulltext
+      ) {
+        constrainer = function (...aArgs) {
+          let constraint = [
+            GlodaConstants.kConstraintFulltext,
+            aAttrDef,
+            ...aArgs,
+          ];
+          this._constraints.push(constraint);
+          return this;
+        };
+      } else if (aAttrDef.canQuery || aAttrDef.attributeName.startsWith("_")) {
+        constrainer = function (...aArgs) {
+          let constraint = [GlodaConstants.kConstraintIn, aAttrDef, ...aArgs];
+          this._constraints.push(constraint);
+          return this;
+        };
+      } else {
+        constrainer = function () {
+          throw new Error(
+            "Cannot query on attribute " +
+              aAttrDef.attributeName +
+              " because its canQuery parameter hasn't been set to true." +
+              " Reading the comments about Gloda.defineAttribute may be a" +
+              " sensible thing to do now."
+          );
+        };
+        canQuery = false;
+      }
+
+      aSubjectNounDef.queryClass.prototype[aAttrDef.boundName] = constrainer;
+
+      // Don't bind extra query-able attributes if we're unable to perform a
+      // search on the attribute.
+      if (!canQuery) {
+        return;
+      }
+
+      // - ranged value helper: fooRange
+      if (objectNounDef.continuous) {
+        // takes one or more tuples of [lower bound, upper bound]
+        let rangedConstrainer = function (...aArgs) {
+          let constraint = [
+            GlodaConstants.kConstraintRanges,
+            aAttrDef,
+            ...aArgs,
+          ];
+          this._constraints.push(constraint);
+          return this;
+        };
+
+        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Range"] =
+          rangedConstrainer;
+      }
+
+      // - string LIKE helper for special on-row attributes: fooLike
+      // (it is impossible to store a string as an indexed attribute, which is
+      //  why we do this for on-row only.)
+      if (
+        "special" in aAttrDef &&
+        aAttrDef.special == GlodaConstants.kSpecialString
+      ) {
+        let likeConstrainer = function (...aArgs) {
+          let constraint = [
+            GlodaConstants.kConstraintStringLike,
+            aAttrDef,
+            ...aArgs,
+          ];
+          this._constraints.push(constraint);
+          return this;
+        };
+
+        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Like"] =
+          likeConstrainer;
+      }
+
+      // - Custom helpers provided by the noun type...
+      if ("queryHelpers" in objectNounDef) {
+        for (let name in objectNounDef.queryHelpers) {
+          let helper = objectNounDef.queryHelpers[name];
+          // we need a new closure...
+          let helperFunc = helper;
+          aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + name] =
+            function (...aArgs) {
+              return helperFunc.call(this, aAttrDef, ...aArgs);
+            };
+        }
+      }
+    }
+  },
+
+  /**
+   * Names of attribute-specific localized strings and the JS attribute they are
+   *  exposed as in the attribute's "strings" attribute (if the provider has a
+   *  string bundle exposed on its "strings" attribute).  They are rooted at
+   *  "gloda.SUBJECT-NOUN-NAME.attr.ATTR-NAME.*".
+   *
+   * Please consult the localization notes in gloda.properties to understand
+   *  what these are used for.
+   */
+  _ATTR_LOCALIZED_STRINGS: {
+    /* - Faceting */
+    facetNameLabel: "facetNameLabel",
+    noneLabel: "noneLabel",
+    includeLabel: "includeLabel",
+    excludeLabel: "excludeLabel",
+    remainderLabel: "remainderLabel",
+    mustMatchLabel: "mustMatchLabel",
+    cantMatchLabel: "cantMatchLabel",
+    mayMatchLabel: "mayMatchLabel",
+    mustMatchNoneLabel: "mustMatchNoneLabel",
+    mustMatchSomeLabel: "mustMatchSomeLabel",
+    mayMatchAnyLabel: "mayMatchAnyLabel",
+  },
+  /**
+   * Define an attribute and all its meta-data.  Takes a single dictionary as
+   *  its argument, with the following required properties:
+   *
+   * @param aAttrDef.provider The object instance providing a 'process' method.
+   * @param aAttrDef.extensionName The name of the extension providing these
+   *     attributes.
+   * @param aAttrDef.attributeType The type of attribute, one of the values from
+   *     the kAttr* enumeration.
+   * @param aAttrDef.attributeName The name of the attribute, which also doubles
+   *     as the bound property name if you pass 'bind' a value of true.  You are
+   *     responsible for avoiding collisions, which presumably will mean
+   *     checking/updating a wiki page in the future, or just prefixing your
+   *     attribute name with your extension name or something like that.
+   * @param aAttrDef.bind Should this attribute be 'bound' as a convenience
+   *     attribute on the subject's object (true/false)?  For example, with an
+   *     attributeName of "foo" and passing true for 'bind' with a subject noun
+   *     of NOUN_MESSAGE, GlodaMessage instances will expose a "foo" getter that
+   *     returns the value of the attribute.  If 'singular' is true, this means
+   *     an instance of the object class corresponding to the noun type or null
+   *     if the attribute does not exist.  If 'singular' is false, this means a
+   *     list of instances of the object class corresponding to the noun type,
+   *     where the list may be empty if no instances of the attribute are
+   *     present.
+   * @param aAttrDef.bindName Optional override of attributeName for purposes of
+   *     the binding property's name.
+   * @param aAttrDef.singular Is the attribute going to happen at most once
+   *     (true), or potentially multiple times (false).  This affects whether
+   *     the binding returns a list or just a single item (which is null when
+   *     the attribute is not present).
+   * @param [aAttrDef.emptySetIsSignificant=false] Should we
+   * @param aAttrDef.subjectNouns A list of object types (NOUNs) that this
+   *     attribute can be set on.  Each element in the list should be one of the
+   *     NOUN_* constants or a dynamically registered noun type.
+   * @param aAttrDef.objectNoun The object type (one of the NOUN_* constants or
+   *     a dynamically registered noun types) that is the 'object' in the
+   *     traditional RDF triple.  More pragmatically, in the database row used
+   *     to represent an attribute, we store the subject (ex: message ID),
+   *     attribute ID, and an integer which is the integer representation of the
+   *     'object' whose type you are defining right here.
+   */
+  defineAttribute(aAttrDef) {
+    // ensure required properties exist on aAttrDef
+    if (
+      !("provider" in aAttrDef) ||
+      !("extensionName" in aAttrDef) ||
+      !("attributeType" in aAttrDef) ||
+      !("attributeName" in aAttrDef) ||
+      !("singular" in aAttrDef) ||
+      !("subjectNouns" in aAttrDef) ||
+      !("objectNoun" in aAttrDef)
+    ) {
+      // perhaps we should have a list of required attributes, perchance with
+      //  and explanation of what it holds, and use that to be friendlier?
+      throw Error(
+        "You omitted a required attribute defining property, please" +
+          " consult the documentation as penance."
+      );
+    }
+
+    // -- Fill in defaults
+    if (!("emptySetIsSignificant" in aAttrDef)) {
+      aAttrDef.emptySetIsSignificant = false;
+    }
+
+    if (!("canQuery" in aAttrDef)) {
+      aAttrDef.canQuery = !!aAttrDef.facet;
+    }
+
+    // return if the attribute has already been defined
+    if (aAttrDef.dbDef) {
+      return aAttrDef;
+    }
+
+    // - first time we've seen a provider init logic
+    if (!(aAttrDef.provider.providerName in this._attrProviders)) {
+      this._attrProviders[aAttrDef.provider.providerName] = [];
+      if (aAttrDef.provider.contentWhittle) {
+        whittlerRegistry.registerWhittler(aAttrDef.provider);
+      }
+    }
+
+    let compoundName = aAttrDef.extensionName + ":" + aAttrDef.attributeName;
+    // -- Database Definition
+    let attrDBDef;
+    if (compoundName in GlodaDatastore._attributeDBDefs) {
+      // the existence of the GlodaAttributeDBDef means that either it has
+      //  already been fully defined, or has been loaded from the database but
+      //  not yet 'bound' to a provider (and had important meta-info that
+      //  doesn't go in the db copied over)
+      attrDBDef = GlodaDatastore._attributeDBDefs[compoundName];
+    } else {
+      // we need to create the attribute definition in the database
+      let attrID = null;
+      attrID = GlodaDatastore._createAttributeDef(
+        aAttrDef.attributeType,
+        aAttrDef.extensionName,
+        aAttrDef.attributeName,
+        null
+      );
+
+      attrDBDef = new GlodaAttributeDBDef(
+        GlodaDatastore,
+        attrID,
+        compoundName,
+        aAttrDef.attributeType,
+        aAttrDef.extensionName,
+        aAttrDef.attributeName
+      );
+      GlodaDatastore._attributeDBDefs[compoundName] = attrDBDef;
+      GlodaDatastore._attributeIDToDBDefAndParam[attrID] = [attrDBDef, null];
+    }
+
+    aAttrDef.dbDef = attrDBDef;
+    attrDBDef.attrDef = aAttrDef;
+
+    aAttrDef.id = aAttrDef.dbDef.id;
+
+    if ("bindName" in aAttrDef) {
+      aAttrDef.boundName = aAttrDef.bindName;
+    } else {
+      aAttrDef.boundName = aAttrDef.attributeName;
+    }
+
+    aAttrDef.objectNounDef = this._nounIDToDef[aAttrDef.objectNoun];
+    aAttrDef.objectNounDef.objectNounOfAttributes.push(aAttrDef);
+
+    // -- Facets
+    function normalizeFacetDef(aFacetDef) {
+      if (!("groupIdAttr" in aFacetDef)) {
+        aFacetDef.groupIdAttr = aAttrDef.objectNounDef.idAttr;
+      }
+      if (!("groupComparator" in aFacetDef)) {
+        aFacetDef.groupComparator = aAttrDef.objectNounDef.comparator;
+      }
+      if (!("filter" in aFacetDef)) {
+        aFacetDef.filter = null;
+      }
+    }
+    // No facet attribute means no facet desired; set an explicit null so that
+    //  code can check without doing an "in" check.
+    if (!("facet" in aAttrDef)) {
+      aAttrDef.facet = null;
+    } else if (aAttrDef.facet === true) {
+      // Promote "true" facet values to the defaults.  Where attributes have
+      //  specified values, make sure we fill in any missing defaults.
+      aAttrDef.facet = {
+        type: "default",
+        groupIdAttr: aAttrDef.objectNounDef.idAttr,
+        groupComparator: aAttrDef.objectNounDef.comparator,
+        filter: null,
+      };
+    } else {
+      normalizeFacetDef(aAttrDef.facet);
+    }
+    if ("extraFacets" in aAttrDef) {
+      for (let facetDef of aAttrDef.extraFacets) {
+        normalizeFacetDef(facetDef);
+      }
+    }
+
+    function gatherLocalizedStrings(aBundle, aPropRoot, aStickIn) {
+      for (let propName in Gloda._ATTR_LOCALIZED_STRINGS) {
+        let attrName = Gloda._ATTR_LOCALIZED_STRINGS[propName];
+        try {
+          aStickIn[attrName] = aBundle.GetStringFromName(aPropRoot + propName);
+        } catch (ex) {
+          // do nothing.  nsIStringBundle throws exceptions when not found
+        }
+      }
+    }
+
+    // -- L10n.
+    // If the provider has a string bundle, populate a "strings" attribute with
+    //  our standard attribute strings that can be UI exposed.
+    if ("strings" in aAttrDef.provider && aAttrDef.facet) {
+      let bundle = aAttrDef.provider.strings;
+
+      // -- attribute strings
+      let attrStrings = (aAttrDef.facet.strings = {});
+      // we use the first subject the attribute applies to as the basis of
+      //  where to get the string from.  Mainly because we currently don't have
+      //  any attributes with multiple subjects nor a use-case where we expose
+      //  multiple noun types via the UI.  (Just messages right now.)
+      let canonicalSubject = this._nounIDToDef[aAttrDef.subjectNouns[0]];
+      let propRoot =
+        "gloda." +
+        canonicalSubject.name +
+        ".attr." +
+        aAttrDef.attributeName +
+        ".";
+      gatherLocalizedStrings(bundle, propRoot, attrStrings);
+
+      // -- alias strings for synthetic facets
+      if ("extraFacets" in aAttrDef) {
+        for (let facetDef of aAttrDef.extraFacets) {
+          facetDef.strings = {};
+          let aliasPropRoot =
+            "gloda." + canonicalSubject.name + ".attr." + facetDef.alias + ".";
+          gatherLocalizedStrings(bundle, aliasPropRoot, facetDef.strings);
+        }
+      }
+    }
+
+    // -- Subject Noun Binding
+    for (
+      let iSubject = 0;
+      iSubject < aAttrDef.subjectNouns.length;
+      iSubject++
+    ) {
+      let subjectType = aAttrDef.subjectNouns[iSubject];
+      let subjectNounDef = this._nounIDToDef[subjectType];
+      this._bindAttribute(aAttrDef, subjectNounDef);
+
+      // update the provider maps...
+      if (
+        !this._attrProviderOrderByNoun[subjectType].includes(aAttrDef.provider)
+      ) {
+        this._attrProviderOrderByNoun[subjectType].push(aAttrDef.provider);
+        if (aAttrDef.provider.optimize) {
+          this._attrOptimizerOrderByNoun[subjectType].push(aAttrDef.provider);
+        }
+        this._attrProvidersByNoun[subjectType][aAttrDef.provider.providerName] =
+          [];
+      }
+      this._attrProvidersByNoun[subjectType][
+        aAttrDef.provider.providerName
+      ].push(aAttrDef);
+
+      subjectNounDef.attribsByBoundName[aAttrDef.boundName] = aAttrDef;
+      if (aAttrDef.domExpose) {
+        subjectNounDef.domExposeAttribsByBoundName[aAttrDef.boundName] =
+          aAttrDef;
+      }
+
+      if (
+        "special" in aAttrDef &&
+        aAttrDef.special & GlodaConstants.kSpecialColumn
+      ) {
+        subjectNounDef.specialLoadAttribs.push(aAttrDef);
+      }
+
+      // if this is a parent column attribute, make note of it so that if we
+      //  need to do an inverse references lookup, we know what column we are
+      //  issuing against.
+      if (
+        "special" in aAttrDef &&
+        aAttrDef.special === GlodaConstants.kSpecialColumnParent
+      ) {
+        subjectNounDef.parentColumnAttr = aAttrDef;
+      }
+
+      if (
+        aAttrDef.objectNounDef.tableName ||
+        aAttrDef.objectNounDef.contributeObjDependencies
+      ) {
+        subjectNounDef.hasObjDependencies = true;
+      }
+    }
+
+    this._attrProviders[aAttrDef.provider.providerName].push(aAttrDef);
+    return aAttrDef;
+  },
+
+  /**
+   * Retrieve the attribute provided by the given extension with the given
+   *  attribute name.  The original idea was that plugins would effectively
+   *  name-space attributes, helping avoid collisions.  Since we are leaning
+   *  towards using binding heavily, this doesn't really help, as the collisions
+   *  will just occur on the attribute name instead.  Also, this can turn
+   *  extensions into liars as name changes/moves to core/etc. happen.
+   *
+   * @TODO consider removing the extension name argument parameter requirement
+   */
+  getAttrDef(aPluginName, aAttrName) {
+    let compoundName = aPluginName + ":" + aAttrName;
+    return GlodaDatastore._attributeDBDefs[compoundName];
+  },
+
+  /**
+   * Create a new query instance for the given noun-type.  This provides
+   *  a generic way to provide constraint-based queries of any first-class
+   *  nouns supported by the system.
+   *
+   * The idea is that every attribute on an object can be used to express
+   *  a constraint on the query object.  Constraints implicitly 'AND' together,
+   *  but providing multiple arguments to a constraint function results in an
+   *  'OR'ing of those values.  Additionally, you can call or() on the returned
+   *  query to create an alternate query that is effectively a giant OR against
+   *  all the constraints you create on the main query object (or any other
+   *  alternate queries returned by or()).  (Note: there is no nesting of these
+   *  alternate queries. query.or().or() is equivalent to query.or())
+   * For each attribute, there is a constraint with the same name that takes
+   *  one or more arguments.  The arguments represent a set of OR values that
+   *  objects matching the query can have.  (If you want the constraint
+   *  effectively ANDed together, just invoke the constraint function
+   *  multiple times.)  For example, newQuery(NOUN_PERSON).age(25) would
+   *  constraint to all the people aged 25, while age(25, 26) would constrain
+   *  to all the people age 25 or 26.
+   * For each attribute with a 'continuous' noun, there is a constraint with the
+   *  attribute name with "Range" appended.  It takes two arguments which are an
+   *  inclusive lower bound and an inclusive lower bound for values in the
+   *  range.  If you would like an open-ended range on either side, pass null
+   *  for that argument.  If you would like to specify multiple ranges that
+   *  should be ORed together, simply pass additional (pairs of) arguments.
+   *  For example, newQuery(NOUN_PERSON).age(25,100) would constraint to all
+   *  the people who are >= 25 and <= 100.  Likewise age(25, null) would just
+   *  return all the people who are 25 or older.  And age(25,30,35,40) would
+   *  return people who are either 25-30 or 35-30.
+   * There are also full-text constraint columns.  In a nutshell, their
+   *  arguments are the strings that should be passed to the SQLite FTS3
+   *  MATCH clause.
+   *
+   * @param aNounID The (integer) noun-id of the noun you want to query on.
+   * @param aOptions an optional dictionary of query options, see the GlodaQuery
+   *     class documentation.
+   */
+  newQuery(aNounID, aOptions) {
+    let nounDef = this._nounIDToDef[aNounID];
+    return new nounDef.queryClass(aOptions);
+  },
+
+  /**
+   * Create a collection/query for the given noun-type that only matches the
+   *  provided items.  This is to be used when you have an explicit set of items
+   *  that you would still like to receive updates for.
+   */
+  explicitCollection(aNounID, aItems) {
+    let nounDef = this._nounIDToDef[aNounID];
+    let collection = new GlodaCollection(nounDef, aItems, null, null);
+    let query = new nounDef.explicitQueryClass(collection);
+    collection.query = query;
+    GlodaCollectionManager.registerCollection(collection);
+    return collection;
+  },
+
+  /**
+   * Debugging 'wildcard' collection creation support.  A wildcard collection
+   *  will 'accept' any new item instances presented to the collection manager
+   *  as new.  The result is that it allows you to be notified as new items
+   *  as they are indexed, existing items as they are loaded from the database,
+   *  etc.
+   * Because the items are added to the collection without limit, this will
+   *  result in a leak if you don't do something to clean up after the
+   *  collection.  (Forgetting about the collection will suffice, as it is still
+   *  weakly held.)
+   */
+  _wildcardCollection(aNounID, aItems) {
+    let nounDef = this._nounIDToDef[aNounID];
+    let collection = new GlodaCollection(nounDef, aItems, null, null);
+    let query = new nounDef.wildcardQueryClass(collection);
+    collection.query = query;
+    GlodaCollectionManager.registerCollection(collection);
+    return collection;
+  },
+
+  /**
+   * Attribute providers attempting to index something that experience a fatal
+   *  problem should throw one of these.  For example:
+   *  "throw new Gloda.BadItemContentsError('Message lacks an author.');".
+   *
+   * We're not really taking advantage of this yet, but it's a good idea.
+   */
+  BadItemContentsError,
+
+  /* eslint-disable complexity */
+  /**
+   * Populate a gloda representation of an item given the thus-far built
+   *  representation, the previous representation, and one or more raw
+   *  representations.  The attribute providers/optimizers for the given noun
+   *  type are invoked, allowing them to contribute/alter things.  Following
+   *  that, we build and persist our attribute representations.
+   *
+   * The result of the processing ends up with attributes in 3 different forms:
+   * - Database attribute rows (to be added and removed).
+   * - In-memory representation.
+   * - JSON-able representation.
+   *
+   * @param aItem The noun instance you want processed.
+   * @param aRawReps A dictionary that we pass to the attribute providers.
+   *     There is a(n implied) contract between the caller of grokNounItem for a
+   *     given noun type and the attribute providers for that noun type, and we
+   *     have nothing to do with it OTHER THAN inserting a 'trueGlodaRep'
+   *     value into it.  In the event of reindexing an existing object, the
+   *     gloda representation we pass to the indexers is actually a clone that
+   *     allows the asynchronous indexers to mutate the object without
+   *     causing visible changes in the existing representation of the gloda
+   *     object.  We patch the changes back onto the original item atomically
+   *     once indexing completes.  The 'trueGlodaRep' is then useful for
+   *     objects that hang off of the gloda instance that need a reference
+   *     back to their containing object for API convenience purposes.
+   * @param aIsConceptuallyNew Is the item "new" in the sense that it would
+   *     never have been visible from within user code?  This translates into
+   *     whether this should trigger an itemAdded notification or an
+   *     itemModified notification.
+   * @param aIsRecordNew Is the item "new" in the sense that we should INSERT
+   *     a record rather than UPDATE-ing a record.  For example, when dealing
+   *     with messages where we may have a ghost, the ghost message is not a
+   *     new record, but is conceptually new.
+   * @param aCallbackHandle The GlodaIndexer-style callback handle that is being
+   *     used to drive this processing in an async fashion.  (See
+   *     GlodaIndexer._callbackHandle).
+   * @param aDoCache Should we allow this item to be contributed to its noun
+   *     cache?
+   */
+  *grokNounItem(
+    aItem,
+    aRawReps,
+    aIsConceptuallyNew,
+    aIsRecordNew,
+    aCallbackHandle,
+    aDoCache
+  ) {
+    let itemNounDef = aItem.NOUN_DEF;
+    let attribsByBoundName = itemNounDef.attribsByBoundName;
+
+    this._log.info(" ** grokNounItem: " + itemNounDef.name);
+
+    let addDBAttribs = [];
+    let removeDBAttribs = [];
+
+    let jsonDict = {};
+
+    let aOldItem;
+    aRawReps.trueGlodaRep = aItem;
+    if (aIsConceptuallyNew) {
+      // there is no old item if we are new.
+      aOldItem = {};
+    } else {
+      aOldItem = aItem;
+      // we want to create a clone of the existing item so that we can know the
+      //  deltas that happened for indexing purposes
+      aItem = aItem._clone();
+    }
+
+    // Have the attribute providers directly set properties on the aItem
+    let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
+    for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
+      this._log.info("  * provider: " + attrProviders[iProvider].providerName);
+      yield aCallbackHandle.pushAndGo(
+        attrProviders[iProvider].process(
+          aItem,
+          aRawReps,
+          aIsConceptuallyNew,
+          aCallbackHandle
+        )
+      );
+    }
+
+    let attrOptimizers = this._attrOptimizerOrderByNoun[itemNounDef.id];
+    for (let iProvider = 0; iProvider < attrOptimizers.length; iProvider++) {
+      this._log.info(
+        "  * optimizer: " + attrOptimizers[iProvider].providerName
+      );
+      yield aCallbackHandle.pushAndGo(
+        attrOptimizers[iProvider].optimize(
+          aItem,
+          aRawReps,
+          aIsConceptuallyNew,
+          aCallbackHandle
+        )
+      );
+    }
+    this._log.info(" ** done with providers.");
+
+    // Iterate over the attributes on the item
+    for (let key of Object.keys(aItem)) {
+      let value = aItem[key];
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key.startsWith("_")) {
+        continue;
+      }
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined) {
+        this._log.warn("new proc ignoring attrib: " + key);
+        continue;
+      }
+
+      let attribDB = attrib.dbDef;
+      let objectNounDef = attrib.objectNounDef;
+
+      // - translate for our JSON rep
+      if (attrib.singular) {
+        if (objectNounDef.toJSON) {
+          jsonDict[attrib.id] = objectNounDef.toJSON(value);
+        } else {
+          jsonDict[attrib.id] = value;
+        }
+      } else if (objectNounDef.toJSON) {
+        let toJSON = objectNounDef.toJSON;
+        jsonDict[attrib.id] = [];
+        for (let subValue of value) {
+          jsonDict[attrib.id].push(toJSON(subValue));
+        }
+      } else {
+        jsonDict[attrib.id] = value;
+      }
+
+      let oldValue = aOldItem[key];
+
+      // the 'old' item is still the canonical one; update it
+      // do the update now, because we may skip operations on addDBAttribs and
+      //  removeDBattribs, if the attribute is not to generate entries in
+      //  messageAttributes
+      if (oldValue !== undefined || !aIsConceptuallyNew) {
+        aOldItem[key] = value;
+      }
+
+      // the new canQuery property has to be set to true to generate entries
+      // in the messageAttributes table. Any other truthy value (like a non
+      // empty string), will still make the message query-able but without
+      // using the database.
+      if (attrib.canQuery !== true) {
+        continue;
+      }
+
+      // - database index attributes
+
+      // perform a delta analysis against the old value, if we have one
+      if (oldValue !== undefined) {
+        // in the singular case if they don't match, it's one add and one remove
+        if (attrib.singular) {
+          // test for identicality, failing that, see if they have explicit
+          //  equals support.
+          if (
+            value !== oldValue &&
+            (!value.equals || !value.equals(oldValue))
+          ) {
+            addDBAttribs.push(attribDB.convertValuesToDBAttributes([value])[0]);
+            removeDBAttribs.push(
+              attribDB.convertValuesToDBAttributes([oldValue])[0]
+            );
+          }
+        } else if (objectNounDef.computeDelta) {
+          // in the plural case, we have to figure the deltas accounting for
+          //  possible changes in ordering (which is insignificant from an
+          //  indexing perspective)
+          // some nouns may not meet === equivalence needs, so must provide a
+          //  custom computeDelta method to help us out
+          let [valuesAdded, valuesRemoved] = objectNounDef.computeDelta(
+            value,
+            oldValue
+          );
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(
+            addDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesAdded)
+          );
+          removeDBAttribs.push.apply(
+            removeDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesRemoved)
+          );
+        } else {
+          // build a map of the previous values; we will delete the values as
+          //  we see them so that we will know what old values are no longer
+          //  present in the current set of values.
+          let oldValueMap = {};
+          for (let anOldValue of oldValue) {
+            // remember, the key is just the toString'ed value, so we need to
+            //  store and use the actual value as the value!
+            oldValueMap[anOldValue] = anOldValue;
+          }
+          // traverse the current values...
+          let valuesAdded = [];
+          for (let curValue of value) {
+            if (curValue in oldValueMap) {
+              delete oldValueMap[curValue];
+            } else {
+              valuesAdded.push(curValue);
+            }
+          }
+          // anything still on oldValueMap was removed.
+          let valuesRemoved = Object.keys(oldValueMap).map(
+            key => oldValueMap[key]
+          );
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(
+            addDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesAdded)
+          );
+          removeDBAttribs.push.apply(
+            removeDBAttribs,
+            attribDB.convertValuesToDBAttributes(valuesRemoved)
+          );
+        }
+
+        // Add/remove the empty set indicator as appropriate.
+        if (attrib.emptySetIsSignificant) {
+          // if we are now non-zero but previously were zero, remove.
+          if (value.length && !oldValue.length) {
+            removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+          } else if (!value.length && oldValue.length) {
+            // We are now zero length but previously were not, add.
+            addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+          }
+        }
+      } else {
+        // no old value, all values are new
+        // add the db reps on the new values
+        if (attrib.singular) {
+          value = [value];
+        }
+        addDBAttribs.push.apply(
+          addDBAttribs,
+          attribDB.convertValuesToDBAttributes(value)
+        );
+        // Add the empty set indicator for the attribute id if appropriate.
+        if (!value.length && attrib.emptySetIsSignificant) {
+          addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+        }
+      }
+    }
+
+    // Iterate over any remaining values in old items for purge purposes.
+    for (let key of Object.keys(aOldItem)) {
+      let value = aOldItem[key];
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key.startsWith("_")) {
+        continue;
+      }
+      // ignore things we saw in the new guy
+      if (key in aItem) {
+        continue;
+      }
+
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined) {
+        continue;
+      }
+
+      // delete these from the old item, as the old item is canonical, and
+      //  should no longer have these values
+      delete aOldItem[key];
+
+      if (attrib.canQuery !== true) {
+        this._log.debug(
+          "Not inserting attribute " +
+            attrib.attributeName +
+            " into the db, since we don't plan on querying on it"
+        );
+        continue;
+      }
+
+      if (attrib.singular) {
+        value = [value];
+      }
+      let attribDB = attrib.dbDef;
+      removeDBAttribs.push.apply(
+        removeDBAttribs,
+        attribDB.convertValuesToDBAttributes(value)
+      );
+      // remove the empty set marker if there should have been one
+      if (!value.length && attrib.emptySetIsSignificant) {
+        removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
+      }
+    }
+
+    aItem._jsonText = JSON.stringify(jsonDict);
+    this._log.debug("  json text: " + aItem._jsonText);
+
+    if (aIsRecordNew) {
+      this._log.debug(" inserting item");
+      itemNounDef.objInsert.call(itemNounDef.datastore, aItem);
+    } else {
+      this._log.debug(" updating item");
+      itemNounDef.objUpdate.call(itemNounDef.datastore, aItem);
+    }
+
+    this._log.debug(
+      " adjusting attributes, add: " + addDBAttribs + " rem: " + removeDBAttribs
+    );
+    itemNounDef.dbAttribAdjuster.call(
+      itemNounDef.datastore,
+      aItem,
+      addDBAttribs,
+      removeDBAttribs
+    );
+
+    if (!aIsConceptuallyNew && "_declone" in aOldItem) {
+      aOldItem._declone(aItem);
+    }
+
+    // Cache ramifications...
+    if (aDoCache === undefined || aDoCache) {
+      if (aIsConceptuallyNew) {
+        GlodaCollectionManager.itemsAdded(aItem.NOUN_ID, [aItem]);
+      } else {
+        GlodaCollectionManager.itemsModified(aOldItem.NOUN_ID, [aOldItem]);
+      }
+    }
+
+    this._log.debug(" done grokking.");
+
+    yield GlodaConstants.kWorkDone;
+  },
+  /* eslint-enable complexity */
+
+  /**
+   * Processes a list of noun instances for their score within a given context.
+   *  This is primarily intended for use by search ranking mechanisms, but could
+   *  be used elsewhere too.  (It does, however, depend on the complicity of the
+   *  score method implementations to not get confused.)
+   *
+   * @param aItems The non-empty list of items to score.
+   * @param aContext A noun-specific dictionary that we just pass to the funcs.
+   * @param aExtraScoreFuncs A list of extra scoring functions to apply.
+   * @returns A list of integer scores equal in length to aItems.
+   */
+  scoreNounItems(aItems, aContext, aExtraScoreFuncs) {
+    let scores = [];
+    // bail if there is nothing to score
+    if (!aItems.length) {
+      return scores;
+    }
+
+    let itemNounDef = aItems[0].NOUN_DEF;
+    if (aExtraScoreFuncs == null) {
+      aExtraScoreFuncs = [];
+    }
+
+    for (let item of aItems) {
+      let score = 0;
+      let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
+      for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
+        let provider = attrProviders[iProvider];
+        if (provider.score) {
+          score += provider.score(item);
+        }
+      }
+      for (let extraScoreFunc of aExtraScoreFuncs) {
+        score += extraScoreFunc(item, aContext);
+      }
+      scores.push(score);
+    }
+
+    return scores;
+  },
+};
+
+/* and initialize the Gloda object/NS before we return... */
+try {
+  Gloda._init();
+} catch (ex) {
+  Gloda._log.debug(
+    "Exception during Gloda init (" +
+      ex.fileName +
+      ":" +
+      ex.lineNumber +
+      "): " +
+      ex
+  );
+}
+/* but don't forget that we effectively depend on Everybody.jsm too, and
+   currently on our importer to be importing that if they need us fully armed
+   and operational. */
diff --git a/comm/mailnews/db/gloda/modules/GlodaConstants.jsm b/comm/mailnews/db/gloda/modules/GlodaConstants.jsm
new file mode 100644
index 0000000000..1e6d253f09
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaConstants.jsm
@@ -0,0 +1,250 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The constants used by Gloda files. Avoid importing anything into this file.
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaConstants"];
+
+var GlodaConstants = {
+  /**
+   * The indexer is idle.
+   */
+  kIndexerIdle: 0,
+  /**
+   * The indexer is doing something.  We used to have other specific states, but
+   *  they have been rendered irrelevant and wiped from existence.
+   */
+  kIndexerIndexing: 1,
+
+  /**
+   * Synchronous activities performed that can be thought of as one processing
+   *  token.  Potentially yield the event-loop and re-schedule for later based
+   *  on how long we've actually taken/etc.  The goal here is that code that
+   *  is doing stuff synchronously yields with kWorkSync periodically to make
+   *  sure that it doesn't dominate the event-loop.  Unless the processing
+   *  in question is particularly intensive, it should be reasonable to apply
+   *  some decimation factor (ex: 32 or 64) with the general goal of yielding
+   *  every 3-10 milliseconds.
+   */
+  kWorkSync: 0,
+  /**
+   * Asynchronous activity performed, you need to relinquish flow control and
+   *  trust us to call callbackDriver later.
+   */
+  kWorkAsync: 1,
+  /**
+   * We are all done with our task, close us and figure out something else to do.
+   */
+  kWorkDone: 2,
+  /**
+   * We are not done with our task, but we think it's a good idea to take a
+   *  breather because we believe we have tied up the event loop for a
+   *  non-trivial amount of time.  So please re-schedule us in the future.
+   *
+   * This is currently only used internally by the indexer's batching logic;
+   *  minor changes may be required if used by actual indexers.
+   */
+  kWorkPause: 3,
+  /**
+   * We are done with our task, and have a result that we are returning.  This
+   *  should only be used by your callback handler's doneWithResult method.
+   *  Ex: you are passed aCallbackHandle, and you do
+   *  "yield aCallbackHandle.doneWithResult(myResult);".
+   */
+  kWorkDoneWithResult: 4,
+
+  /**
+   * An attribute that is a defining characteristic of the subject.
+   */
+  kAttrFundamental: 0,
+  /**
+   * An attribute that is an optimization derived from two or more fundamental
+   *  attributes and exists solely to improve database query performance.
+   */
+  kAttrOptimization: 1,
+  /**
+   * An attribute that is derived from the content of the subject.  For example,
+   *  a message that references a bugzilla bug could have a "derived" attribute
+   *  that captures the bugzilla reference.  This is not
+   */
+  kAttrDerived: 2,
+  /**
+   * An attribute that is the result of an explicit and intentional user action
+   *  upon the subject.  For example, a tag placed on a message by a user (or
+   *  at the user's request by a filter) is explicit.
+   */
+  kAttrExplicit: 3,
+  /**
+   * An attribute that is indirectly the result of a user's behaviour.  For
+   *  example, if a user consults a message multiple times, we may conclude that
+   *  the user finds the message interesting.  It is "implied", if you will,
+   *  that the message is interesting.
+   */
+  kAttrImplicit: 4,
+
+  /**
+   * This attribute is not 'special'; it is stored as a (thing id, attribute id,
+   *  attribute id) tuple in the database rather than on thing's row or on
+   *  thing's fulltext row.  (Where "thing" could be a message or any other
+   *  first class noun.)
+   */
+  kSpecialNotAtAll: 0,
+  /**
+   * This attribute is stored as a numeric column on the row for the noun.  The
+   *  attribute definition should include this value as 'special' and the
+   *  column name that stores the attribute as 'specialColumnName'.
+   */
+  kSpecialColumn: 16,
+  kSpecialColumnChildren: 16 | 1,
+  kSpecialColumnParent: 16 | 2,
+  /**
+   * This attribute is stored as a string column on the row for the noun.  It
+   *  differs from kSpecialColumn in that it is a string, which once had
+   *  query ramifications and one day may have them again.
+   */
+  kSpecialString: 32,
+  /**
+   * This attribute is stored as a fulltext column on the fulltext table for
+   *  the noun.  The attribute definition should include this value as 'special'
+   *  and the column name that stores the table as 'specialColumnName'.
+   */
+  kSpecialFulltext: 64,
+
+  /**
+   * The extensionName used for the attributes defined by core gloda plugins
+   *  such as GlodaFundAttr.jsm and GlodaExplicitAttr.jsm.
+   */
+  BUILT_IN: "built-in",
+
+  /**
+   * Special sentinel value that will cause facets to skip a noun instance
+   * when an attribute has this value.
+   */
+  IGNORE_FACET: "ignore-facet",
+
+  /*
+   * The following are explicit noun IDs.  While most extension-provided nouns
+   *  will have dynamically allocated id's that are looked up by name, these
+   *  id's can be relied upon to exist and be accessible via these
+   *  pseudo-constants.  It's not really clear that we need these, although it
+   *  does potentially simplify code to not have to look up all of their nouns
+   *  at initialization time.
+   */
+  /**
+   * Boolean values, expressed as 0/1 in the database and non-continuous for
+   *  constraint purposes.  Like numbers, such nouns require their attributes
+   *  to provide them with context, lacking any of their own.
+   * Having this as a noun type may be a bad idea; a change of nomenclature
+   *  (so that we are not claiming a boolean value is a noun, but still using
+   *  it in the same way) or implementation to require each boolean noun
+   *  actually be its own noun may be in order.
+   */
+  NOUN_BOOLEAN: 1,
+  /**
+   * A number, which could mean an integer or floating point values.  We treat
+   *  these as continuous, meaning that queries on them can have ranged
+   *  constraints expressed on them.  Lacking any inherent context, numbers
+   *  depend on their attributes to parameterize them as required.
+   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
+   */
+  NOUN_NUMBER: 2,
+  /**
+   * A (non-fulltext) string.
+   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
+   */
+  NOUN_STRING: 3,
+  /** A date, encoded as a PRTime, represented as a js Date object. */
+  NOUN_DATE: 10,
+  /**
+   * Fulltext search support, somewhat magical.  This is only intended to be
+   *  used for kSpecialFulltext attributes, and exclusively as a constraint
+   *  mechanism.  The values are always represented as strings.  It is presumed
+   *  that the user of this functionality knows how to generate SQLite FTS3
+   *  style MATCH queries, or is okay with us just gluing them together with
+   *  " OR " when used in an or-constraint case.  Gloda's query mechanism
+   *  currently lacks the ability to to compile Gloda-style and-constraints
+   *  into a single MATCH query, but it will turn out okay, just less
+   *  efficiently than it could.
+   */
+  NOUN_FULLTEXT: 20,
+  /**
+   * Represents a MIME Type.  We currently lack any human-intelligible
+   *  descriptions of mime types.
+   */
+  NOUN_MIME_TYPE: 40,
+  /**
+   * Captures a message tag as well as when the tag's presence was observed,
+   *  hoping to approximate when the tag was applied.  It's a somewhat dubious
+   *  attempt to not waste our opporunity to store a value along with the tag.
+   *  (The tag is actually stored as an attribute parameter on the attribute
+   *  definition, rather than a value in the attribute 'instance' for the
+   *  message.)
+   */
+  NOUN_TAG: 50,
+  /**
+   * Doesn't actually work owing to a lack of an object to represent a folder.
+   *  We do expose the folderURI and folderID of a message, but need to map that
+   *  to a good abstraction.  Probably something thin around a SteelFolder or
+   *  the like; we would contribute the functionality to easily move from a
+   *  folder to the list of gloda messages in that folder, as well as the
+   *  indexing preferences for that folder.
+   *
+   * @TODO folder noun and related abstraction
+   */
+  NOUN_FOLDER: 100,
+  /**
+   * All messages belong to a conversation.  See GlodaDataModel.jsm for the
+   *  definition of the GlodaConversation class.
+   */
+  NOUN_CONVERSATION: 101,
+  /**
+   * A one-to-one correspondence with underlying (indexed) nsIMsgDBHdr
+   *  instances.  See GlodaDataModel.jsm for the definition of the GlodaMessage class.
+   */
+  NOUN_MESSAGE: 102,
+  /**
+   * Corresponds to a human being, who may have multiple electronic identities
+   *  (a la NOUN_IDENTITY).  There is no requirement for association with an
+   *  address book contact, although when the address book contact exists,
+   *  we want to be associated with it.  See GlodaDataModel.jsm for the definition
+   *  of the GlodaContact class.
+   */
+  NOUN_CONTACT: 103,
+  /**
+   * A single identity of a contact, who may have one or more.  E-mail accounts,
+   *  instant messaging accounts, social network site accounts, etc. are each
+   *  identities.  See GlodaDataModel.jsm for the definition of the GlodaIdentity
+   *  class.
+   */
+  NOUN_IDENTITY: 104,
+  /**
+   * An attachment to a message. A message may have many different attachments.
+   */
+  NOUN_ATTACHMENT: 105,
+  /**
+   * An account related to a message. A message can have only one account.
+   */
+  NOUN_ACCOUNT: 106,
+
+  /**
+   * Parameterized identities, for use in the from-me, to-me, cc-me optimization
+   *  cases.  Not for reuse without some thought.  These nouns use the parameter
+   *  to store the 'me' identity that we are talking about, and the value to
+   *  store the identity of the other party.  So in both the from-me and to-me
+   *  cases involving 'me' and 'foo@bar', the 'me' identity is always stored via
+   *  the attribute parameter, and the 'foo@bar' identity is always stored as
+   *  the attribute value.  See GlodaFundAttr.jsm for more information on this, but
+   *  you probably shouldn't be touching this unless you are fundattr.
+   */
+  NOUN_PARAM_IDENTITY: 200,
+
+  kConstraintIdIn: 0,
+  kConstraintIn: 1,
+  kConstraintRanges: 2,
+  kConstraintEquals: 3,
+  kConstraintStringLike: 4,
+  kConstraintFulltext: 5,
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaContent.jsm b/comm/mailnews/db/gloda/modules/GlodaContent.jsm
new file mode 100644
index 0000000000..5f1daf5e9c
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaContent.jsm
@@ -0,0 +1,285 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = [
+  "GlodaContent",
+  "whittlerRegistry",
+  "mimeMsgToContentAndMeta",
+  "mimeMsgToContentSnippetAndMeta",
+];
+
+/**
+ * Given a MimeMsg and the corresponding folder, return the GlodaContent object.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @returns an array containing the GlodaContent instance, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentAndMeta(aMimeMsg, folder) {
+  let content = new GlodaContent();
+  let meta = { subject: aMimeMsg.get("subject") };
+  let bodyLines = aMimeMsg.coerceBodyToPlaintext(folder).split(/\r?\n/);
+
+  for (let whittler of whittlerRegistry.getWhittlers()) {
+    whittler.contentWhittle(meta, bodyLines, content);
+  }
+
+  return [content, meta];
+}
+
+/**
+ * Given a MimeMsg, return the whittled content string, suitable for summarizing
+ * a message.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @param length: optional number of characters to trim the whittled content.
+ * If the actual length of the message is greater than |length|, then the return
+ * value is the first (length-1) characters with an ellipsis appended.
+ * @returns an array containing the text of the snippet, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentSnippetAndMeta(aMimeMsg, folder, length) {
+  let [content, meta] = mimeMsgToContentAndMeta(aMimeMsg, folder);
+
+  let text = content.getContentSnippet(length + 1);
+  if (length && text.length > length) {
+    text = text.substring(0, length - 1) + "\u2026"; // ellipsis
+  }
+  return [text, meta];
+}
+
+/**
+ * A registry of gloda providers that have contentWhittle() functions.
+ * used by mimeMsgToContentSnippet, but populated by the Gloda object as it's
+ * processing providers.
+ */
+function WhittlerRegistry() {
+  this._whittlers = [];
+}
+
+WhittlerRegistry.prototype = {
+  /**
+   * Add a provider as a content whittler.
+   */
+  registerWhittler(provider) {
+    this._whittlers.push(provider);
+  },
+  /**
+   * get the list of content whittlers, sorted from the most specific to
+   * the most generic
+   */
+  getWhittlers() {
+    // Use the concat() trick to avoid mutating the internal object and
+    // leaking an internal representation.
+    return this._whittlers.concat().reverse();
+  },
+};
+
+const whittlerRegistry = new WhittlerRegistry();
+
+function GlodaContent() {
+  this._contentPriority = null;
+  this._producing = false;
+  this._hunks = [];
+}
+
+GlodaContent.prototype = {
+  kPriorityBase: 0,
+  kPriorityPerfect: 100,
+
+  kHunkMeta: 1,
+  kHunkQuoted: 2,
+  kHunkContent: 3,
+
+  _resetContent() {
+    this._keysAndValues = [];
+    this._keysAndDeltaValues = [];
+    this._hunks = [];
+    this._curHunk = null;
+  },
+
+  /* ===== Consumer API ===== */
+  hasContent() {
+    return this._contentPriority != null;
+  },
+
+  /**
+   * Return content suitable for snippet display.  This means that no quoting
+   *  or meta-data should be returned.
+   *
+   * @param aMaxLength The maximum snippet length desired.
+   */
+  getContentSnippet(aMaxLength) {
+    let content = this.getContentString();
+    if (aMaxLength) {
+      content = content.substring(0, aMaxLength);
+    }
+    return content;
+  },
+
+  getContentString(aIndexingPurposes) {
+    let data = "";
+    for (let hunk of this._hunks) {
+      if (hunk.hunkType == this.kHunkContent) {
+        if (data) {
+          data += "\n" + hunk.data;
+        } else {
+          data = hunk.data;
+        }
+      }
+    }
+
+    if (aIndexingPurposes) {
+      // append the values for indexing.  we assume the keywords are cruft.
+      // this may be crazy, but things that aren't a science aren't an exact
+      // science.
+      for (let kv of this._keysAndValues) {
+        data += "\n" + kv[1];
+      }
+      for (let kon of this._keysAndValues) {
+        data += "\n" + kon[1] + "\n" + kon[2];
+      }
+    }
+
+    return data;
+  },
+
+  /* ===== Producer API ===== */
+  /**
+   * Called by a producer with the priority they believe their interpretation
+   *  of the content comes in at.
+   *
+   * @returns true if we believe the producer's interpretation will be
+   *     interesting and they should go ahead and generate events.  We return
+   *     false if we don't think they are interesting, in which case they should
+   *     probably not issue calls to us, although we don't care.  (We will
+   *     ignore their calls if we return false, this allows the simplification
+   *     of code that needs to run anyways.)
+   */
+  volunteerContent(aPriority) {
+    if (this._contentPriority === null || this._contentPriority < aPriority) {
+      this._contentPriority = aPriority;
+      this._resetContent();
+      this._producing = true;
+      return true;
+    }
+    this._producing = false;
+    return false;
+  },
+
+  keyValue(aKey, aValue) {
+    if (!this._producing) {
+      return;
+    }
+
+    this._keysAndValues.push([aKey, aValue]);
+  },
+  keyValueDelta(aKey, aOldValue, aNewValue) {
+    if (!this._producing) {
+      return;
+    }
+
+    this._keysAndDeltaValues.push([aKey, aOldValue, aNewValue]);
+  },
+
+  /**
+   * Meta lines are lines that have to do with the content but are not the
+   *  content and can generally be related to an attribute that has been derived
+   *  and stored on the item.
+   * For example, a bugzilla bug may note that an attachment was created; this
+   *  is not content and wouldn't be desired in a snippet, but is still
+   *  potentially interesting meta-data.
+   *
+   * @param aLineOrLines The line or list of lines that are meta-data.
+   * @param aAttr The attribute this meta-data is associated with.
+   * @param aIndex If the attribute is non-singular, indicate the specific
+   *     index of the item in the attribute's bound list that the meta-data
+   *     is associated with.
+   */
+  meta(aLineOrLines, aAttr, aIndex) {
+    if (!this._producing) {
+      return;
+    }
+
+    let data;
+    if (typeof aLineOrLines == "string") {
+      data = aLineOrLines;
+    } else {
+      data = aLineOrLines.join("\n");
+    }
+
+    this._curHunk = {
+      hunkType: this.kHunkMeta,
+      attr: aAttr,
+      index: aIndex,
+      data,
+    };
+    this._hunks.push(this._curHunk);
+  },
+  /**
+   * Quoted lines reference previous messages or what not.
+   *
+   * @param aLineOrLiens The line or list of lines that are quoted.
+   * @param aDepth The depth of the quoting.
+   * @param aOrigin The item that originated the original content, if known.
+   *     For example, perhaps a GlodaMessage?
+   * @param aTarget A reference to the location in the original content, if
+   *     known.  For example, the index of a line in a message or something?
+   */
+  quoted(aLineOrLines, aDepth, aOrigin, aTarget) {
+    if (!this._producing) {
+      return;
+    }
+
+    let data;
+    if (typeof aLineOrLines == "string") {
+      data = aLineOrLines;
+    } else {
+      data = aLineOrLines.join("\n");
+    }
+
+    if (
+      !this._curHunk ||
+      this._curHunk.hunkType != this.kHunkQuoted ||
+      this._curHunk.depth != aDepth ||
+      this._curHunk.origin != aOrigin ||
+      this._curHunk.target != aTarget
+    ) {
+      this._curHunk = {
+        hunkType: this.kHunkQuoted,
+        data,
+        depth: aDepth,
+        origin: aOrigin,
+        target: aTarget,
+      };
+      this._hunks.push(this._curHunk);
+    } else {
+      this._curHunk.data += "\n" + data;
+    }
+  },
+
+  content(aLineOrLines) {
+    if (!this._producing) {
+      return;
+    }
+
+    let data;
+    if (typeof aLineOrLines == "string") {
+      data = aLineOrLines;
+    } else {
+      data = aLineOrLines.join("\n");
+    }
+
+    if (!this._curHunk || this._curHunk.hunkType != this.kHunkContent) {
+      this._curHunk = { hunkType: this.kHunkContent, data };
+      this._hunks.push(this._curHunk);
+    } else {
+      this._curHunk.data += "\n" + data;
+    }
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaDataModel.jsm b/comm/mailnews/db/gloda/modules/GlodaDataModel.jsm
new file mode 100644
index 0000000000..d9361c079c
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaDataModel.jsm
@@ -0,0 +1,1020 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = [
+  "GlodaAttributeDBDef",
+  "GlodaAccount",
+  "GlodaConversation",
+  "GlodaFolder",
+  "GlodaMessage",
+  "GlodaContact",
+  "GlodaIdentity",
+  "GlodaAttachment",
+];
+
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+var LOG = console.createInstance({
+  prefix: "gloda.datamodel",
+  maxLogLevel: "Warn",
+  maxLogLevelPref: "gloda.loglevel",
+});
+
+/**
+ * @class Represents a gloda attribute definition's DB form.  This class
+ *  stores the information in the database relating to this attribute
+ *  definition.  Access its attrDef attribute to get at the really juicy data.
+ *  This main interesting thing this class does is serve as the keeper of the
+ *  mapping from parameters to attribute ids in the database if this is a
+ *  parameterized attribute.
+ */
+function GlodaAttributeDBDef(
+  aDatastore,
+  aID,
+  aCompoundName,
+  aAttrType,
+  aPluginName,
+  aAttrName
+) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._compoundName = aCompoundName;
+  this._attrType = aAttrType;
+  this._pluginName = aPluginName;
+  this._attrName = aAttrName;
+
+  this.attrDef = null;
+
+  /** Map parameter values to the underlying database id. */
+  this._parameterBindings = {};
+}
+
+GlodaAttributeDBDef.prototype = {
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() {
+    return this._id;
+  },
+  get attributeName() {
+    return this._attrName;
+  },
+
+  get parameterBindings() {
+    return this._parameterBindings;
+  },
+
+  /**
+   * Bind a parameter value to the attribute definition, allowing use of the
+   *  attribute-parameter as an attribute.
+   *
+   * @returns
+   */
+  bindParameter(aValue) {
+    // people probably shouldn't call us with null, but handle it
+    if (aValue == null) {
+      return this._id;
+    }
+    if (aValue in this._parameterBindings) {
+      return this._parameterBindings[aValue];
+    }
+    // no database entry exists if we are here, so we must create it...
+    let id = this._datastore._createAttributeDef(
+      this._attrType,
+      this._pluginName,
+      this._attrName,
+      aValue
+    );
+    this._parameterBindings[aValue] = id;
+    this._datastore.reportBinding(id, this, aValue);
+    return id;
+  },
+
+  /**
+   * Given a list of values, return a list (regardless of plurality) of
+   *  database-ready [attribute id,  value] tuples.  This is intended to be used
+   *  to directly convert the value of a property on an object that corresponds
+   *  to a bound attribute.
+   *
+   * @param {Array} aInstanceValues An array of instance values regardless of
+   *     whether or not the attribute is singular.
+   */
+  convertValuesToDBAttributes(aInstanceValues) {
+    let nounDef = this.attrDef.objectNounDef;
+    let dbAttributes = [];
+    if (nounDef.usesParameter) {
+      for (let instanceValue of aInstanceValues) {
+        let [param, dbValue] = nounDef.toParamAndValue(instanceValue);
+        dbAttributes.push([this.bindParameter(param), dbValue]);
+      }
+    } else if ("toParamAndValue" in nounDef) {
+      // Not generating any attributes is ok. This basically means the noun is
+      // just an informative property on the Gloda Message and has no real
+      // indexing purposes.
+      for (let instanceValue of aInstanceValues) {
+        dbAttributes.push([
+          this._id,
+          nounDef.toParamAndValue(instanceValue)[1],
+        ]);
+      }
+    }
+    return dbAttributes;
+  },
+
+  toString() {
+    return this._compoundName;
+  },
+};
+
+var GlodaHasAttributesMixIn = {
+  *enumerateAttributes() {
+    let nounDef = this.NOUN_DEF;
+    for (let key in this) {
+      let value = this[key];
+      let attrDef = nounDef.attribsByBoundName[key];
+      // we expect to not have attributes for underscore prefixed values (those
+      //  are managed by the instance's logic.  we also want to not explode
+      //  should someone crap other values in there, we get both birds with this
+      //  one stone.
+      if (attrDef === undefined) {
+        continue;
+      }
+      if (attrDef.singular) {
+        // ignore attributes with null values
+        if (value != null) {
+          yield [attrDef, [value]];
+        }
+      } else if (value.length) {
+        // ignore attributes with no values
+        yield [attrDef, value];
+      }
+    }
+  },
+
+  domContribute(aDomNode) {
+    let nounDef = this.NOUN_DEF;
+    for (let attrName in nounDef.domExposeAttribsByBoundName) {
+      let attr = nounDef.domExposeAttribsByBoundName[attrName];
+      if (this[attrName]) {
+        aDomNode.setAttribute(attr.domExpose, this[attrName]);
+      }
+    }
+  },
+};
+
+function MixIn(aConstructor, aMixIn) {
+  let proto = aConstructor.prototype;
+  for (let [name, func] of Object.entries(aMixIn)) {
+    if (name.startsWith("get_")) {
+      proto.__defineGetter__(name.substring(4), func);
+    } else {
+      proto[name] = func;
+    }
+  }
+}
+
+/**
+ * @class A gloda wrapper around nsIMsgIncomingServer.
+ */
+function GlodaAccount(aIncomingServer) {
+  this._incomingServer = aIncomingServer;
+}
+
+GlodaAccount.prototype = {
+  NOUN_ID: 106,
+  get id() {
+    return this._incomingServer.key;
+  },
+  get name() {
+    return this._incomingServer.prettyName;
+  },
+  get incomingServer() {
+    return this._incomingServer;
+  },
+  toString() {
+    return "Account: " + this.id;
+  },
+
+  toLocaleString() {
+    return this.name;
+  },
+};
+
+/**
+ * @class A gloda conversation (thread) exists so that messages can belong.
+ */
+function GlodaConversation(
+  aDatastore,
+  aID,
+  aSubject,
+  aOldestMessageDate,
+  aNewestMessageDate
+) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._subject = aSubject;
+  this._oldestMessageDate = aOldestMessageDate;
+  this._newestMessageDate = aNewestMessageDate;
+}
+
+GlodaConversation.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_CONVERSATION,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() {
+    return this._id;
+  },
+  get subject() {
+    return this._subject;
+  },
+  get oldestMessageDate() {
+    return this._oldestMessageDate;
+  },
+  get newestMessageDate() {
+    return this._newestMessageDate;
+  },
+
+  getMessagesCollection(aListener, aData) {
+    let query = new GlodaMessage.prototype.NOUN_DEF.queryClass();
+    query.conversation(this._id).orderBy("date");
+    return query.getCollection(aListener, aData);
+  },
+
+  toString() {
+    return "Conversation:" + this._id;
+  },
+
+  toLocaleString() {
+    return this._subject;
+  },
+};
+
+function GlodaFolder(
+  aDatastore,
+  aID,
+  aURI,
+  aDirtyStatus,
+  aPrettyName,
+  aIndexingPriority
+) {
+  // _datastore is now set by GlodaDatastore
+  this._id = aID;
+  this._uri = aURI;
+  this._dirtyStatus = aDirtyStatus;
+  this._prettyName = aPrettyName;
+  this._account = null;
+  this._activeIndexing = false;
+  this._indexingPriority = aIndexingPriority;
+  this._deleted = false;
+  this._compacting = false;
+}
+
+GlodaFolder.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_FOLDER,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  /** The folder is believed to be up-to-date */
+  kFolderClean: 0,
+  /** The folder has some un-indexed or dirty messages */
+  kFolderDirty: 1,
+  /** The folder needs to be entirely re-indexed, regardless of the flags on
+   * the messages in the folder. This state will be downgraded to dirty */
+  kFolderFilthy: 2,
+
+  _kFolderDirtyStatusMask: 0x7,
+  /**
+   * The (local) folder has been compacted and all of its message keys are
+   *  potentially incorrect.  This is not a possible state for IMAP folders
+   *  because their message keys are based on UIDs rather than offsets into
+   *  the mbox file.
+   */
+  _kFolderCompactedFlag: 0x8,
+
+  /** The folder should never be indexed. */
+  kIndexingNeverPriority: -1,
+  /** The lowest priority assigned to a folder. */
+  kIndexingLowestPriority: 0,
+  /** The highest priority assigned to a folder. */
+  kIndexingHighestPriority: 100,
+
+  /** The indexing priority for a folder if no other priority is assigned. */
+  kIndexingDefaultPriority: 20,
+  /** Folders marked check new are slightly more important I guess. */
+  kIndexingCheckNewPriority: 30,
+  /** Favorite folders are more interesting to the user, presumably. */
+  kIndexingFavoritePriority: 40,
+  /** The indexing priority for inboxes. */
+  kIndexingInboxPriority: 50,
+  /** The indexing priority for sent mail folders. */
+  kIndexingSentMailPriority: 60,
+
+  get id() {
+    return this._id;
+  },
+  get uri() {
+    return this._uri;
+  },
+  get dirtyStatus() {
+    return this._dirtyStatus & this._kFolderDirtyStatusMask;
+  },
+  /**
+   * Mark a folder as dirty if it was clean.  Do nothing if it was already dirty
+   *  or filthy.  For use by GlodaMsgIndexer only.  And maybe rkent and his
+   *  marvelous extensions.
+   */
+  _ensureFolderDirty() {
+    if (this.dirtyStatus == this.kFolderClean) {
+      this._dirtyStatus =
+        (this.kFolderDirty & this._kFolderDirtyStatusMask) |
+        (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Definitely for use only by GlodaMsgIndexer to downgrade the dirty status of
+   *  a folder.
+   */
+  _downgradeDirtyStatus(aNewStatus) {
+    if (this.dirtyStatus != aNewStatus) {
+      this._dirtyStatus =
+        (aNewStatus & this._kFolderDirtyStatusMask) |
+        (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Indicate whether this folder is currently being compacted.  The
+   *  |GlodaMsgIndexer| keeps this in-memory-only value up-to-date.
+   */
+  get compacting() {
+    return this._compacting;
+  },
+  /**
+   * Set whether this folder is currently being compacted.  This is really only
+   *  for the |GlodaMsgIndexer| to set.
+   */
+  set compacting(aCompacting) {
+    this._compacting = aCompacting;
+  },
+  /**
+   * Indicate whether this folder was compacted and has not yet been
+   *  compaction processed.
+   */
+  get compacted() {
+    return Boolean(this._dirtyStatus & this._kFolderCompactedFlag);
+  },
+  /**
+   * For use only by GlodaMsgIndexer to set/clear the compaction state of this
+   *  folder.
+   */
+  _setCompactedState(aCompacted) {
+    if (this.compacted != aCompacted) {
+      if (aCompacted) {
+        this._dirtyStatus |= this._kFolderCompactedFlag;
+      } else {
+        this._dirtyStatus &= ~this._kFolderCompactedFlag;
+      }
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+
+  get name() {
+    return this._prettyName;
+  },
+  toString() {
+    return "Folder:" + this._id;
+  },
+
+  toLocaleString() {
+    let xpcomFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+    if (!xpcomFolder) {
+      return this._prettyName;
+    }
+    return (
+      xpcomFolder.prettyName + " (" + xpcomFolder.rootFolder.prettyName + ")"
+    );
+  },
+
+  get indexingPriority() {
+    return this._indexingPriority;
+  },
+
+  /** We are going to index this folder. */
+  kActivityIndexing: 0,
+  /** Asking for the folder to perform header retrievals. */
+  kActivityHeaderRetrieval: 1,
+  /** We only want the folder for its metadata but are not going to open it. */
+  kActivityFolderOnlyNoData: 2,
+
+  /** Is this folder known to be actively used for indexing? */
+  _activeIndexing: false,
+  /** Get our indexing status. */
+  get indexing() {
+    return this._activeIndexing;
+  },
+  /**
+   * Set our indexing status.  Normally, this will be enabled through passing
+   *  an activity type of kActivityIndexing (which will set us), but we will
+   *  still need to be explicitly disabled by the indexing code.
+   * When disabling indexing, we will call forgetFolderIfUnused to take care of
+   *  shutting things down.
+   * We are not responsible for committing changes to the message database!
+   *  That is on you!
+   */
+  set indexing(aIndexing) {
+    this._activeIndexing = aIndexing;
+  },
+
+  /**
+   * Retrieve the nsIMsgFolder instance corresponding to this folder, providing
+   *  an explanation of why you are requesting it for tracking/cleanup purposes.
+   *
+   * @param aActivity One of the kActivity* constants.  If you pass
+   *     kActivityIndexing, we will set indexing for you, but you will need to
+   *     clear it when you are done.
+   * @returns The nsIMsgFolder if available, null on failure.
+   */
+  getXPCOMFolder(aActivity) {
+    switch (aActivity) {
+      case this.kActivityIndexing:
+        // mark us as indexing, but don't bother with live tracking.  we do
+        //  that independently and only for header retrieval.
+        this.indexing = true;
+        break;
+      case this.kActivityHeaderRetrieval:
+      case this.kActivityFolderOnlyNoData:
+        // we don't have to do anything here.
+        break;
+    }
+
+    return MailServices.folderLookup.getFolderForURL(this.uri);
+  },
+
+  /**
+   * Retrieve a GlodaAccount instance corresponding to this folder.
+   *
+   * @returns The GlodaAccount instance.
+   */
+  getAccount() {
+    if (!this._account) {
+      let msgFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+      this._account = new GlodaAccount(msgFolder.server);
+    }
+    return this._account;
+  },
+};
+
+/**
+ * @class A message representation.
+ */
+function GlodaMessage(
+  aDatastore,
+  aID,
+  aFolderID,
+  aMessageKey,
+  aConversationID,
+  aConversation,
+  aDate,
+  aHeaderMessageID,
+  aDeleted,
+  aJsonText,
+  aNotability,
+  aSubject,
+  aIndexedBodyText,
+  aAttachmentNames
+) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._folderID = aFolderID;
+  this._messageKey = aMessageKey;
+  this._conversationID = aConversationID;
+  this._conversation = aConversation;
+  this._date = aDate;
+  this._headerMessageID = aHeaderMessageID;
+  this._jsonText = aJsonText;
+  this._notability = aNotability;
+  this._subject = aSubject;
+  this._indexedBodyText = aIndexedBodyText;
+  this._attachmentNames = aAttachmentNames;
+
+  // only set _deleted if we're deleted, otherwise the undefined does our
+  //  speaking for us.
+  if (aDeleted) {
+    this._deleted = aDeleted;
+  }
+}
+
+GlodaMessage.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_MESSAGE,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() {
+    return this._id;
+  },
+  get folderID() {
+    return this._folderID;
+  },
+  get messageKey() {
+    return this._messageKey;
+  },
+  get conversationID() {
+    return this._conversationID;
+  },
+  // conversation is special
+  get headerMessageID() {
+    return this._headerMessageID;
+  },
+  get notability() {
+    return this._notability;
+  },
+  set notability(aNotability) {
+    this._notability = aNotability;
+  },
+
+  get subject() {
+    return this._subject;
+  },
+  get indexedBodyText() {
+    return this._indexedBodyText;
+  },
+  get attachmentNames() {
+    return this._attachmentNames;
+  },
+
+  get date() {
+    return this._date;
+  },
+  set date(aNewDate) {
+    this._date = aNewDate;
+  },
+
+  get folder() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID != null) {
+        return this._datastore._mapFolderID(this._folderID);
+      }
+    } catch (ex) {}
+    return null;
+  },
+  get folderURI() {
+    // XXX just like for folder, handle mapping failures and return null
+    try {
+      if (this._folderID != null) {
+        return this._datastore._mapFolderID(this._folderID).uri;
+      }
+    } catch (ex) {}
+    return null;
+  },
+  get account() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID == null) {
+        return null;
+      }
+      let folder = this._datastore._mapFolderID(this._folderID);
+      return folder.getAccount();
+    } catch (ex) {}
+    return null;
+  },
+  get conversation() {
+    return this._conversation;
+  },
+
+  toString() {
+    // uh, this is a tough one...
+    return "Message:" + this._id;
+  },
+
+  _clone() {
+    return new GlodaMessage(
+      /* datastore */ null,
+      this._id,
+      this._folderID,
+      this._messageKey,
+      this._conversationID,
+      this._conversation,
+      this._date,
+      this._headerMessageID,
+      "_deleted" in this ? this._deleted : undefined,
+      "_jsonText" in this ? this._jsonText : undefined,
+      this._notability,
+      this._subject,
+      this._indexedBodyText,
+      this._attachmentNames
+    );
+  },
+
+  /**
+   * Provide a means of propagating changed values on our clone back to
+   *  ourselves.  This is required because of an object identity trick gloda
+   *  does; when indexing an already existing object, all mutations happen on
+   *  a clone of the existing object so that
+   */
+  _declone(aOther) {
+    if ("_content" in aOther) {
+      this._content = aOther._content;
+    }
+
+    // The _indexedAuthor/_indexedRecipients fields don't get updated on
+    //  fulltext update so we don't need to propagate.
+    this._indexedBodyText = aOther._indexedBodyText;
+    this._attachmentNames = aOther._attachmentNames;
+  },
+
+  /**
+   * Mark this message as a ghost.  Ghosts are characterized by having no folder
+   *  id and no message key.  They also are not deleted or they would be of
+   *  absolutely no use to us.
+   *
+   * These changes are suitable for persistence.
+   */
+  _ghost() {
+    this._folderID = null;
+    this._messageKey = null;
+    if ("_deleted" in this) {
+      delete this._deleted;
+    }
+  },
+
+  /**
+   * Are we a ghost (which implies not deleted)?  We are not a ghost if we have
+   *  a definite folder location (we may not know our message key in the case
+   *  of IMAP moves not fully completed) and are not deleted.
+   */
+  get _isGhost() {
+    return this._folderID == null && !this._isDeleted;
+  },
+
+  /**
+   * If we were dead, un-dead us.
+   */
+  _ensureNotDeleted() {
+    if ("_deleted" in this) {
+      delete this._deleted;
+    }
+  },
+
+  /**
+   * Are we deleted?  This is private because deleted gloda messages are not
+   *  visible to non-core-gloda code.
+   */
+  get _isDeleted() {
+    return "_deleted" in this && this._deleted;
+  },
+
+  /**
+   * Trash this message's in-memory representation because it should no longer
+   *  be reachable by any code.  The database record is gone, it's not coming
+   *  back.
+   */
+  _objectPurgedMakeYourselfUnpleasant() {
+    this._id = null;
+    this._folderID = null;
+    this._messageKey = null;
+    this._conversationID = null;
+    this._conversation = null;
+    this.date = null;
+    this._headerMessageID = null;
+  },
+
+  /**
+   * Return the underlying nsIMsgDBHdr from the folder storage for this, or
+   *  null if the message does not exist for one reason or another.  We may log
+   *  to our logger in the failure cases.
+   *
+   * This method no longer caches the result, so if you need to hold onto it,
+   *  hold onto it.
+   *
+   * In the process of retrieving the underlying message header, we may have to
+   *  open the message header database associated with the folder.  This may
+   *  result in blocking while the load happens, so you may want to try and find
+   *  an alternate way to initiate the load before calling us.
+   * We provide hinting to the GlodaDatastore via the GlodaFolder so that it
+   *  knows when it's a good time for it to go and detach from the database.
+   *
+   * @returns The nsIMsgDBHdr associated with this message if available, null on
+   *     failure.
+   */
+  get folderMessage() {
+    if (this._folderID === null || this._messageKey === null) {
+      return null;
+    }
+
+    // XXX like for folder and folderURI, return null if we can't map the folder
+    let glodaFolder;
+    try {
+      glodaFolder = this._datastore._mapFolderID(this._folderID);
+    } catch (ex) {
+      return null;
+    }
+    let folder = glodaFolder.getXPCOMFolder(
+      glodaFolder.kActivityHeaderRetrieval
+    );
+    if (folder) {
+      let folderMessage;
+      try {
+        folderMessage = folder.GetMessageHeader(this._messageKey);
+      } catch (ex) {
+        folderMessage = null;
+      }
+      if (folderMessage !== null) {
+        // verify the message-id header matches what we expect...
+        if (folderMessage.messageId != this._headerMessageID) {
+          LOG.info(
+            "Message with message key " +
+              this._messageKey +
+              " in folder '" +
+              folder.URI +
+              "' does not match expected " +
+              "header! (" +
+              this._headerMessageID +
+              " expected, got " +
+              folderMessage.messageId +
+              ")"
+          );
+          folderMessage = null;
+        }
+      }
+      return folderMessage;
+    }
+
+    // this only gets logged if things have gone very wrong.  we used to throw
+    //  here, but it's unlikely our caller can do anything more meaningful than
+    //  treating this as a disappeared message.
+    LOG.info(
+      "Unable to locate folder message for: " +
+        this._folderID +
+        ":" +
+        this._messageKey
+    );
+    return null;
+  },
+  get folderMessageURI() {
+    let folderMessage = this.folderMessage;
+    if (folderMessage) {
+      return folderMessage.folder.getUriForMsg(folderMessage);
+    }
+    return null;
+  },
+};
+MixIn(GlodaMessage, GlodaHasAttributesMixIn);
+
+/**
+ * @class Contacts correspond to people (one per person), and may own multiple
+ *  identities (e-mail address, IM account, etc.)
+ */
+function GlodaContact(
+  aDatastore,
+  aID,
+  aDirectoryUUID,
+  aContactUUID,
+  aName,
+  aPopularity,
+  aFrecency,
+  aJsonText
+) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._directoryUUID = aDirectoryUUID;
+  this._contactUUID = aContactUUID;
+  this._name = aName;
+  this._popularity = aPopularity;
+  this._frecency = aFrecency;
+  if (aJsonText) {
+    this._jsonText = aJsonText;
+  }
+
+  this._identities = null;
+}
+
+GlodaContact.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_CONTACT,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  get id() {
+    return this._id;
+  },
+  get directoryUUID() {
+    return this._directoryUUID;
+  },
+  get contactUUID() {
+    return this._contactUUID;
+  },
+  get name() {
+    return this._name;
+  },
+  set name(aName) {
+    this._name = aName;
+  },
+
+  get popularity() {
+    return this._popularity;
+  },
+  set popularity(aPopularity) {
+    this._popularity = aPopularity;
+    this.dirty = true;
+  },
+
+  get frecency() {
+    return this._frecency;
+  },
+  set frecency(aFrecency) {
+    this._frecency = aFrecency;
+    this.dirty = true;
+  },
+
+  get identities() {
+    return this._identities;
+  },
+
+  toString() {
+    return "Contact:" + this._id;
+  },
+
+  get accessibleLabel() {
+    return "Contact: " + this._name;
+  },
+
+  _clone() {
+    return new GlodaContact(
+      /* datastore */ null,
+      this._id,
+      this._directoryUUID,
+      this._contactUUID,
+      this._name,
+      this._popularity,
+      this._frecency
+    );
+  },
+};
+MixIn(GlodaContact, GlodaHasAttributesMixIn);
+
+/**
+ * @class A specific means of communication for a contact.
+ */
+function GlodaIdentity(
+  aDatastore,
+  aID,
+  aContactID,
+  aContact,
+  aKind,
+  aValue,
+  aDescription,
+  aIsRelay
+) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._contactID = aContactID;
+  this._contact = aContact;
+  this._kind = aKind;
+  this._value = aValue;
+  this._description = aDescription;
+  this._isRelay = aIsRelay;
+  // Cached indication of whether there is an address book card for this
+  //  identity.  We keep this up-to-date via address book listener
+  //  notifications in |GlodaABIndexer|.
+  this._hasAddressBookCard = undefined;
+}
+
+GlodaIdentity.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_IDENTITY,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() {
+    return this._id;
+  },
+  get contactID() {
+    return this._contactID;
+  },
+  get contact() {
+    return this._contact;
+  },
+  get kind() {
+    return this._kind;
+  },
+  get value() {
+    return this._value;
+  },
+  get description() {
+    return this._description;
+  },
+  get isRelay() {
+    return this._isRelay;
+  },
+
+  get uniqueValue() {
+    return this._kind + "@" + this._value;
+  },
+
+  toString() {
+    return "Identity:" + this._kind + ":" + this._value;
+  },
+
+  toLocaleString() {
+    if (this.contact.name == this.value) {
+      return this.value;
+    }
+    return this.contact.name + " : " + this.value;
+  },
+
+  get abCard() {
+    // for our purposes, the address book only speaks email
+    if (this._kind != "email") {
+      return false;
+    }
+    let card = MailServices.ab.cardForEmailAddress(this._value);
+    this._hasAddressBookCard = card != null;
+    return card;
+  },
+
+  /**
+   * Indicates whether we have an address book card for this identity.  This
+   *  value is cached once looked-up and kept up-to-date by |GlodaABIndexer|
+   *  and its notifications.
+   */
+  get inAddressBook() {
+    if (this._hasAddressBookCard !== undefined) {
+      return this._hasAddressBookCard;
+    }
+    return (this.abCard && true) || false;
+  },
+};
+
+/**
+ * An attachment, with as much information as we can gather on it
+ */
+function GlodaAttachment(
+  aGlodaMessage,
+  aName,
+  aContentType,
+  aSize,
+  aPart,
+  aExternalUrl,
+  aIsExternal
+) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._glodaMessage = aGlodaMessage;
+  this._name = aName;
+  this._contentType = aContentType;
+  this._size = aSize;
+  this._part = aPart;
+  this._externalUrl = aExternalUrl;
+  this._isExternal = aIsExternal;
+}
+
+GlodaAttachment.prototype = {
+  NOUN_ID: GlodaConstants.NOUN_ATTACHMENT,
+  // set by GlodaDatastore
+  get name() {
+    return this._name;
+  },
+  get contentType() {
+    return this._contentType;
+  },
+  get size() {
+    return this._size;
+  },
+  get url() {
+    if (this.isExternal) {
+      return this._externalUrl;
+    }
+
+    let uri = this._glodaMessage.folderMessageURI;
+    if (!uri) {
+      throw new Error(
+        "The message doesn't exist anymore, unable to rebuild attachment URL"
+      );
+    }
+    let msgService = MailServices.messageServiceFromURI(uri);
+    let neckoURL = msgService.getUrlForUri(uri);
+    let url = neckoURL.spec;
+    let hasParamAlready = url.match(/\?[a-z]+=[^\/]+$/);
+    let sep = hasParamAlready ? "&" : "?";
+    return (
+      url +
+      sep +
+      "part=" +
+      this._part +
+      "&filename=" +
+      encodeURIComponent(this._name)
+    );
+  },
+  get isExternal() {
+    return this._isExternal;
+  },
+
+  toString() {
+    return "attachment: " + this._name + ":" + this._contentType;
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaDatabind.jsm b/comm/mailnews/db/gloda/modules/GlodaDatabind.jsm
new file mode 100644
index 0000000000..eda41cb91a
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaDatabind.jsm
@@ -0,0 +1,210 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaDatabind"];
+
+function GlodaDatabind(aNounDef, aDatastore) {
+  this._nounDef = aNounDef;
+  this._tableName = aNounDef.tableName;
+  this._tableDef = aNounDef.schema;
+  this._datastore = aDatastore;
+  this._log = console.createInstance({
+    prefix: `gloda.databind.${this._tableName}`,
+    maxLogLevel: "Warn",
+    maxLogLevelPref: "gloda.loglevel",
+  });
+
+  // process the column definitions and make sure they have an attribute mapping
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    // default to the other dude's thing.
+    if (coldef.length < 3) {
+      coldef[2] = coldef[0];
+    }
+    if (coldef[0] == "id") {
+      this._idAttr = coldef[2];
+    }
+    // colDef[3] is the index of us in our SQL bindings, storage-numbering
+    coldef[3] = iColDef;
+  }
+
+  // XXX This is obviously synchronous and not perfectly async.  Since we are
+  //  doing this, we don't actually need to move to ordinal binding below
+  //  since we could just as well compel creation of the name map and thereby
+  //  avoid ever acquiring the mutex after bootstrap.
+  // However, this specific check can be cleverly avoided with future work.
+  // Namely, at startup we can scan for extension-defined tables and get their
+  //  maximum id so that we don't need to do it here.  The table will either
+  //  be brand new and thus have a maximum id of 1 or we will already know it
+  //  because of that scan.
+  this._nextId = 1;
+  let stmt = this._datastore._createSyncStatement(
+    "SELECT MAX(id) FROM " + this._tableName,
+    true
+  );
+  if (stmt.executeStep()) {
+    // no chance of this SQLITE_BUSY on this call
+    this._nextId = stmt.getInt64(0) + 1;
+  }
+  stmt.finalize();
+
+  let insertColumns = [];
+  let insertValues = [];
+  let updateItems = [];
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    let column = coldef[0];
+    let placeholder = "?" + (iColDef + 1);
+    insertColumns.push(column);
+    insertValues.push(placeholder);
+    if (column != "id") {
+      updateItems.push(column + " = " + placeholder);
+    }
+  }
+
+  let insertSql =
+    "INSERT INTO " +
+    this._tableName +
+    " (" +
+    insertColumns.join(", ") +
+    ") VALUES (" +
+    insertValues.join(", ") +
+    ")";
+
+  // For the update, we want the 'id' to be a constraint and not a value
+  //  that gets set...
+  let updateSql =
+    "UPDATE " +
+    this._tableName +
+    " SET " +
+    updateItems.join(", ") +
+    " WHERE id = ?1";
+  this._insertStmt = aDatastore._createAsyncStatement(insertSql);
+  this._updateStmt = aDatastore._createAsyncStatement(updateSql);
+
+  if (this._tableDef.fulltextColumns) {
+    for (let [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      if (coldef.length < 3) {
+        coldef[2] = coldef[0];
+      }
+      // colDef[3] is the index of us in our SQL bindings, storage-numbering
+      coldef[3] = iColDef + 1;
+    }
+
+    let insertColumns = [];
+    let insertValues = [];
+    let updateItems = [];
+    for (var [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      let column = coldef[0];
+      // +2 instead of +1 because docid is implied
+      let placeholder = "?" + (iColDef + 2);
+      insertColumns.push(column);
+      insertValues.push(placeholder);
+      if (column != "id") {
+        updateItems.push(column + " = " + placeholder);
+      }
+    }
+
+    let insertFulltextSql =
+      "INSERT INTO " +
+      this._tableName +
+      "Text (docid," +
+      insertColumns.join(", ") +
+      ") VALUES (?1," +
+      insertValues.join(", ") +
+      ")";
+
+    // For the update, we want the 'id' to be a constraint and not a value
+    //  that gets set...
+    let updateFulltextSql =
+      "UPDATE " +
+      this._tableName +
+      "Text SET " +
+      updateItems.join(", ") +
+      " WHERE docid = ?1";
+
+    this._insertFulltextStmt =
+      aDatastore._createAsyncStatement(insertFulltextSql);
+    this._updateFulltextStmt =
+      aDatastore._createAsyncStatement(updateFulltextSql);
+  }
+}
+
+GlodaDatabind.prototype = {
+  /**
+   * Perform appropriate binding coercion based on the schema provided to us.
+   * Although we end up effectively coercing JS Date objects to numeric values,
+   *  we should not be provided with JS Date objects!  There is no way for us
+   *  to know to turn them back into JS Date objects on the way out.
+   *  Additionally, there is the small matter of storage's bias towards
+   *  PRTime representations which may not always be desirable.
+   */
+  bindByType(aStmt, aColDef, aValue) {
+    aStmt.bindByIndex(aColDef[3], aValue);
+  },
+
+  objFromRow(aRow) {
+    let getVariant = this._datastore._getVariant;
+    let obj = new this._nounDef.class();
+    for (let [iCol, colDef] of this._tableDef.columns.entries()) {
+      obj[colDef[2]] = getVariant(aRow, iCol);
+    }
+    return obj;
+  },
+
+  objInsert(aThing) {
+    let bindByType = this.bindByType;
+    if (!aThing[this._idAttr]) {
+      aThing[this._idAttr] = this._nextId++;
+    }
+
+    let stmt = this._insertStmt;
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._insertFulltextStmt) {
+      stmt = this._insertFulltextStmt;
+      stmt.bindByIndex(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  objUpdate(aThing) {
+    let bindByType = this.bindByType;
+    let stmt = this._updateStmt;
+    // note, we specially bound the location of 'id' for the insert, but since
+    //  we're using named bindings, there is nothing special about setting it
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._updateFulltextStmt) {
+      stmt = this._updateFulltextStmt;
+      // fulltextColumns doesn't include id/docid, need to explicitly set it
+      stmt.bindByIndex(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  adjustAttributes(...aArgs) {
+    // just proxy the call over to the datastore... we have to do this for
+    //  'this' reasons.  we don't refactor things to avoid this because it does
+    //  make some sense to have all the methods exposed from a single object,
+    //  even if the implementation does live elsewhere.
+    return this._datastore.adjustAttributes(...aArgs);
+  },
+
+  // also proxied...
+  queryFromQuery(...aArgs) {
+    return this._datastore.queryFromQuery(...aArgs);
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaDatastore.jsm b/comm/mailnews/db/gloda/modules/GlodaDatastore.jsm
new file mode 100644
index 0000000000..1391ceaaf2
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaDatastore.jsm
@@ -0,0 +1,4402 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This file looks to Myk Melez <myk@mozilla.org>'s Mozilla Labs snowl
+ * project's (https://hg.mozilla.org/labs/snowl/) modules/GlodaDatastore.jsm
+ * for inspiration and idioms (and also a name :).
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaDatastore"];
+
+const {
+  GlodaAttributeDBDef,
+  GlodaConversation,
+  GlodaFolder,
+  GlodaMessage,
+  GlodaContact,
+  GlodaIdentity,
+} = ChromeUtils.import("resource:///modules/gloda/GlodaDataModel.jsm");
+const { GlodaDatabind } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaDatabind.jsm"
+);
+const { GlodaCollection, GlodaCollectionManager } = ChromeUtils.import(
+  "resource:///modules/gloda/Collection.jsm"
+);
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+var MIN_CACHE_SIZE = 8 * 1048576;
+var MAX_CACHE_SIZE = 64 * 1048576;
+var MEMSIZE_FALLBACK_BYTES = 256 * 1048576;
+
+var PCH_LOG = console.createInstance({
+  prefix: "gloda.ds.pch",
+  maxLogLevel: "Warn",
+  maxLogLevelPref: "gloda.loglevel",
+});
+
+/**
+ * Commit async handler; hands off the notification to
+ *  |GlodaDatastore._asyncCompleted|.
+ */
+function PostCommitHandler(aCallbacks) {
+  this.callbacks = aCallbacks;
+  GlodaDatastore._pendingAsyncStatements++;
+}
+
+PostCommitHandler.prototype = {
+  handleResult(aResultSet) {},
+
+  handleError(aError) {
+    PCH_LOG.error("database error:" + aError);
+  },
+
+  handleCompletion(aReason) {
+    // just outright bail if we are shutdown
+    if (GlodaDatastore.datastoreIsShutdown) {
+      return;
+    }
+
+    if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
+      for (let callback of this.callbacks) {
+        try {
+          callback();
+        } catch (ex) {
+          PCH_LOG.error(
+            "PostCommitHandler callback (" +
+              ex.fileName +
+              ":" +
+              ex.lineNumber +
+              ") threw: " +
+              ex
+          );
+        }
+      }
+    }
+    try {
+      GlodaDatastore._asyncCompleted();
+    } catch (e) {
+      PCH_LOG.error("Exception in handleCompletion:", e);
+    }
+  },
+};
+
+var QFQ_LOG = console.createInstance({
+  prefix: "gloda.ds.qfq",
+  maxLogLevel: "Warn",
+  maxLogLevelPref: "gloda.loglevel",
+});
+
+/**
+ * Singleton collection listener used by |QueryFromQueryCallback| to assist in
+ *  the loading of referenced noun instances.  Which is to say, messages have
+ *  identities (specific e-mail addresses) associated with them via attributes.
+ *  And these identities in turn reference / are referenced by contacts (the
+ *  notion of a person).
+ *
+ * This listener is primarily concerned with fixing up the references in each
+ *  noun instance to its referenced instances once they have been loaded.  It
+ *  also deals with caching so that our identity invariant is maintained: user
+ *  code should only ever see one distinct instance of a thing at a time.
+ */
+var QueryFromQueryResolver = {
+  onItemsAdded(aIgnoredItems, aCollection, aFake) {
+    let originColl = aCollection.dataStack
+      ? aCollection.dataStack.pop()
+      : aCollection.data;
+    // QFQ_LOG.debug("QFQR: originColl: " + originColl);
+    if (aCollection.completionShifter) {
+      aCollection.completionShifter.push(originColl);
+    } else {
+      aCollection.completionShifter = [originColl];
+    }
+
+    if (!aFake) {
+      originColl.deferredCount--;
+      originColl.resolvedCount++;
+    }
+
+    // bail if we are still pending on some other load completion
+    if (originColl.deferredCount > 0) {
+      // QFQ_LOG.debug("QFQR: bailing " + originColl._nounDef.name);
+      return;
+    }
+
+    let referencesByNounID = originColl.masterCollection.referencesByNounID;
+    let inverseReferencesByNounID =
+      originColl.masterCollection.inverseReferencesByNounID;
+
+    if (originColl.pendingItems) {
+      for (let item of originColl.pendingItems) {
+        // QFQ_LOG.debug("QFQR: loading deferred " + item.NOUN_ID + ":" + item.id);
+        GlodaDatastore.loadNounDeferredDeps(
+          item,
+          referencesByNounID,
+          inverseReferencesByNounID
+        );
+      }
+
+      // we need to consider the possibility that we are racing a collection very
+      //  much like our own.  as such, this means we need to perform cache
+      //  unification as our last step.
+      GlodaCollectionManager.cacheLoadUnify(
+        originColl._nounDef.id,
+        originColl.pendingItems,
+        false
+      );
+
+      // just directly tell the collection about the items.  we know the query
+      //  matches (at least until we introduce predicates that we cannot express
+      //  in SQL.)
+      // QFQ_LOG.debug(" QFQR: about to trigger listener: " + originColl._listener +
+      //    "with collection: " + originColl._nounDef.name);
+      originColl._onItemsAdded(originColl.pendingItems);
+      delete originColl.pendingItems;
+      delete originColl._pendingIdMap;
+    }
+  },
+  onItemsModified() {},
+  onItemsRemoved() {},
+  onQueryCompleted(aCollection) {
+    let originColl = aCollection.completionShifter
+      ? aCollection.completionShifter.shift()
+      : aCollection.data;
+    // QFQ_LOG.debug(" QFQR about to trigger completion with collection: " +
+    //  originColl._nounDef.name);
+    if (originColl.deferredCount <= 0) {
+      originColl._onQueryCompleted();
+    }
+  },
+};
+
+/**
+ * Handles the results from a GlodaDatastore.queryFromQuery call in cooperation
+ *  with the |QueryFromQueryResolver| collection listener.  We do a lot of
+ *  legwork related to satisfying references to other noun instances on the
+ *  noun instances the user directly queried.  Messages reference identities
+ *  reference contacts which in turn (implicitly) reference identities again.
+ *  We have to spin up those other queries and stitch things together.
+ *
+ * While the code is generally up to the existing set of tasks it is called to
+ *  handle, I would not be surprised for it to fall down if things get more
+ *  complex.  Some of the logic here 'evolved' a bit and could benefit from
+ *  additional documentation and a fresh go-through.
+ */
+function QueryFromQueryCallback(aStatement, aNounDef, aCollection) {
+  this.statement = aStatement;
+  this.nounDef = aNounDef;
+  this.collection = aCollection;
+
+  // QFQ_LOG.debug("Creating QFQCallback for noun: " + aNounDef.name);
+
+  // the master collection holds the referencesByNounID
+  this.referencesByNounID = {};
+  this.masterReferencesByNounID =
+    this.collection.masterCollection.referencesByNounID;
+  this.inverseReferencesByNounID = {};
+  this.masterInverseReferencesByNounID =
+    this.collection.masterCollection.inverseReferencesByNounID;
+  // we need to contribute our references as we load things; we need this
+  //  because of the potential for circular dependencies and our inability to
+  //  put things into the caching layer (or collection's _idMap) until we have
+  //  fully resolved things.
+  if (this.nounDef.id in this.masterReferencesByNounID) {
+    this.selfReferences = this.masterReferencesByNounID[this.nounDef.id];
+  } else {
+    this.selfReferences = this.masterReferencesByNounID[this.nounDef.id] = {};
+  }
+  if (this.nounDef.parentColumnAttr) {
+    if (this.nounDef.id in this.masterInverseReferencesByNounID) {
+      this.selfInverseReferences =
+        this.masterInverseReferencesByNounID[this.nounDef.id];
+    } else {
+      this.selfInverseReferences = this.masterInverseReferencesByNounID[
+        this.nounDef.id
+      ] = {};
+    }
+  }
+
+  this.needsLoads = false;
+
+  GlodaDatastore._pendingAsyncStatements++;
+}
+
+QueryFromQueryCallback.prototype = {
+  handleResult(aResultSet) {
+    try {
+      // just outright bail if we are shutdown
+      if (GlodaDatastore.datastoreIsShutdown) {
+        return;
+      }
+
+      let pendingItems = this.collection.pendingItems;
+      let pendingIdMap = this.collection._pendingIdMap;
+      let row;
+      let nounDef = this.nounDef;
+      let nounID = nounDef.id;
+      while ((row = aResultSet.getNextRow())) {
+        let item = nounDef.objFromRow.call(nounDef.datastore, row);
+        if (this.collection.stashedColumns) {
+          let stashed = (this.collection.stashedColumns[item.id] = []);
+          for (let iCol of this.collection.query.options.stashColumns) {
+            stashed.push(GlodaDatastore._getVariant(row, iCol));
+          }
+        }
+        // try and replace the item with one from the cache, if we can
+        let cachedItem = GlodaCollectionManager.cacheLookupOne(
+          nounID,
+          item.id,
+          false
+        );
+
+        // if we already have a copy in the pending id map, skip it
+        if (item.id in pendingIdMap) {
+          continue;
+        }
+
+        // QFQ_LOG.debug("loading item " + nounDef.id + ":" + item.id + " existing: " +
+        //    this.selfReferences[item.id] + " cached: " + cachedItem);
+        if (cachedItem) {
+          item = cachedItem;
+        } else if (this.selfReferences[item.id] != null) {
+          // We may already have been loaded by this process.
+          item = this.selfReferences[item.id];
+        } else {
+          // Perform loading logic which may produce reference dependencies.
+          this.needsLoads =
+            GlodaDatastore.loadNounItem(
+              item,
+              this.referencesByNounID,
+              this.inverseReferencesByNounID
+            ) || this.needsLoads;
+        }
+
+        // add ourself to the references by our id
+        // QFQ_LOG.debug("saving item " + nounDef.id + ":" + item.id + " to self-refs");
+        this.selfReferences[item.id] = item;
+
+        // if we're tracking it, add ourselves to our parent's list of children
+        //  too
+        if (this.selfInverseReferences) {
+          let parentID = item[nounDef.parentColumnAttr.idStorageAttributeName];
+          let childrenList = this.selfInverseReferences[parentID];
+          if (childrenList === undefined) {
+            childrenList = this.selfInverseReferences[parentID] = [];
+          }
+          childrenList.push(item);
+        }
+
+        pendingItems.push(item);
+        pendingIdMap[item.id] = item;
+      }
+    } catch (e) {
+      GlodaDatastore._log.error("Exception in handleResult:", e);
+    }
+  },
+
+  handleError(aError) {
+    GlodaDatastore._log.error(
+      "Async queryFromQuery error: " + aError.result + ": " + aError.message
+    );
+  },
+
+  handleCompletion(aReason) {
+    try {
+      try {
+        this.statement.finalize();
+        this.statement = null;
+
+        // just outright bail if we are shutdown
+        if (GlodaDatastore.datastoreIsShutdown) {
+          return;
+        }
+
+        // QFQ_LOG.debug("handleCompletion: " + this.collection._nounDef.name);
+
+        if (this.needsLoads) {
+          for (let nounID in this.referencesByNounID) {
+            let references = this.referencesByNounID[nounID];
+            if (nounID == this.nounDef.id) {
+              continue;
+            }
+            let nounDef = GlodaDatastore._nounIDToDef[nounID];
+            // QFQ_LOG.debug("  have references for noun: " + nounDef.name);
+            // try and load them out of the cache/existing collections.  items in the
+            //  cache will be fully formed, which is nice for us.
+            // XXX this mechanism will get dubious when we have multiple paths to a
+            //  single noun-type.  For example, a -> b -> c, a-> c; two paths to c
+            //  and we're looking at issuing two requests to c, the latter of which
+            //  will be a superset of the first one.  This does not currently pose
+            //  a problem because we only have a -> b -> c -> b, and sequential
+            //  processing means no alarms and no surprises.
+            let masterReferences = this.masterReferencesByNounID[nounID];
+            if (masterReferences === undefined) {
+              masterReferences = this.masterReferencesByNounID[nounID] = {};
+            }
+            let outReferences;
+            if (nounDef.parentColumnAttr) {
+              outReferences = {};
+            } else {
+              outReferences = masterReferences;
+            }
+            let [, notFoundCount, notFound] =
+              GlodaCollectionManager.cacheLookupMany(
+                nounDef.id,
+                references,
+                outReferences
+              );
+
+            if (nounDef.parentColumnAttr) {
+              let inverseReferences;
+              if (nounDef.id in this.masterInverseReferencesByNounID) {
+                inverseReferences =
+                  this.masterInverseReferencesByNounID[nounDef.id];
+              } else {
+                inverseReferences = this.masterInverseReferencesByNounID[
+                  nounDef.id
+                ] = {};
+              }
+
+              for (let key in outReferences) {
+                let item = outReferences[key];
+                masterReferences[item.id] = item;
+                let parentID =
+                  item[nounDef.parentColumnAttr.idStorageAttributeName];
+                let childrenList = inverseReferences[parentID];
+                if (childrenList === undefined) {
+                  childrenList = inverseReferences[parentID] = [];
+                }
+                childrenList.push(item);
+              }
+            }
+
+            // QFQ_LOG.debug("  found: " + foundCount + " not found: " + notFoundCount);
+            if (notFoundCount === 0) {
+              this.collection.resolvedCount++;
+            } else {
+              this.collection.deferredCount++;
+              let query = new nounDef.queryClass();
+              query.id.apply(query, Object.keys(notFound));
+
+              // we fully expect/allow for there being no such subcollection yet.
+              let subCollection =
+                nounDef.id in this.collection.masterCollection.subCollections
+                  ? this.collection.masterCollection.subCollections[nounDef.id]
+                  : undefined;
+              this.collection.masterCollection.subCollections[nounDef.id] =
+                GlodaDatastore.queryFromQuery(
+                  query,
+                  QueryFromQueryResolver,
+                  this.collection,
+                  subCollection,
+                  this.collection.masterCollection,
+                  { becomeExplicit: true }
+                );
+            }
+          }
+
+          for (let nounID in this.inverseReferencesByNounID) {
+            let inverseReferences = this.inverseReferencesByNounID[nounID];
+            this.collection.deferredCount++;
+            let nounDef = GlodaDatastore._nounIDToDef[nounID];
+
+            // QFQ_LOG.debug("Want to load inverse via " + nounDef.parentColumnAttr.boundName);
+
+            let query = new nounDef.queryClass();
+            // we want to constrain using the parent column
+            let queryConstrainer = query[nounDef.parentColumnAttr.boundName];
+            queryConstrainer.apply(query, Object.keys(inverseReferences));
+            // we fully expect/allow for there being no such subcollection yet.
+            let subCollection =
+              nounDef.id in this.collection.masterCollection.subCollections
+                ? this.collection.masterCollection.subCollections[nounDef.id]
+                : undefined;
+            this.collection.masterCollection.subCollections[nounDef.id] =
+              GlodaDatastore.queryFromQuery(
+                query,
+                QueryFromQueryResolver,
+                this.collection,
+                subCollection,
+                this.collection.masterCollection,
+                { becomeExplicit: true }
+              );
+          }
+        } else {
+          this.collection.deferredCount--;
+          this.collection.resolvedCount++;
+        }
+
+        // QFQ_LOG.debug("  defer: " + this.collection.deferredCount +
+        //              " resolved: " + this.collection.resolvedCount);
+
+        // process immediately and kick-up to the master collection...
+        if (this.collection.deferredCount <= 0) {
+          // this guy will resolve everyone using referencesByNounID and issue the
+          //  call to this.collection._onItemsAdded to propagate things to the
+          //  next concerned subCollection or the actual listener if this is the
+          //  master collection.  (Also, call _onQueryCompleted).
+          QueryFromQueryResolver.onItemsAdded(
+            null,
+            { data: this.collection },
+            true
+          );
+          QueryFromQueryResolver.onQueryCompleted({ data: this.collection });
+        }
+      } catch (e) {
+        console.error(e);
+        QFQ_LOG.error("Exception:", e);
+      }
+    } finally {
+      GlodaDatastore._asyncCompleted();
+    }
+  },
+};
+
+/**
+ * Used by |GlodaDatastore.folderCompactionPassBlockFetch| to accumulate the
+ *  results and pass them back in to the compaction process in
+ *  |GlodaMsgIndexer._worker_folderCompactionPass|.
+ */
+function CompactionBlockFetcherHandler(aCallback) {
+  this.callback = aCallback;
+  this.idsAndMessageKeys = [];
+  GlodaDatastore._pendingAsyncStatements++;
+}
+CompactionBlockFetcherHandler.prototype = {
+  handleResult(aResultSet) {
+    let row;
+    while ((row = aResultSet.getNextRow())) {
+      this.idsAndMessageKeys.push([
+        row.getInt64(0), // id
+        row.getInt64(1), // messageKey
+        row.getString(2), // headerMessageID
+      ]);
+    }
+  },
+  handleError(aError) {
+    GlodaDatastore._log.error(
+      "CompactionBlockFetcherHandler error: " +
+        aError.result +
+        ": " +
+        aError.message
+    );
+  },
+  handleCompletion(aReason) {
+    GlodaDatastore._asyncCompleted();
+    this.callback(this.idsAndMessageKeys);
+  },
+};
+
+/**
+ * Use this as the callback handler when you have a SQL query that returns a
+ *  single row with a single integer column value, like a COUNT() query.
+ */
+function SingletonResultValueHandler(aCallback) {
+  this.callback = aCallback;
+  this.result = null;
+  GlodaDatastore._pendingAsyncStatements++;
+}
+SingletonResultValueHandler.prototype = {
+  handleResult(aResultSet) {
+    let row;
+    while ((row = aResultSet.getNextRow())) {
+      this.result = row.getInt64(0);
+    }
+  },
+  handleError(aError) {
+    GlodaDatastore._log.error(
+      "SingletonResultValueHandler error: " +
+        aError.result +
+        ": " +
+        aError.message
+    );
+  },
+  handleCompletion(aReason) {
+    GlodaDatastore._asyncCompleted();
+    this.callback(this.result);
+  },
+};
+
+/**
+ * Wrapper that duplicates actions taken on a real statement to an explain
+ *  statement.  Currently only fires an explain statement once.
+ */
+function ExplainedStatementWrapper(
+  aRealStatement,
+  aExplainStatement,
+  aSQLString,
+  aExplainHandler
+) {
+  this.real = aRealStatement;
+  this.explain = aExplainStatement;
+  this.sqlString = aSQLString;
+  this.explainHandler = aExplainHandler;
+  this.done = false;
+}
+ExplainedStatementWrapper.prototype = {
+  bindByIndex(aColIndex, aValue) {
+    this.real.bindByIndex(aColIndex, aValue);
+    if (!this.done) {
+      this.explain.bindByIndex(aColIndex, aValue);
+    }
+  },
+  executeAsync(aCallback) {
+    if (!this.done) {
+      this.explainHandler.sqlEnRoute(this.sqlString);
+      this.explain.executeAsync(this.explainHandler);
+      this.explain.finalize();
+      this.done = true;
+    }
+    return this.real.executeAsync(aCallback);
+  },
+  finalize() {
+    if (!this.done) {
+      this.explain.finalize();
+    }
+    this.real.finalize();
+  },
+};
+
+/**
+ * Writes a single JSON document to the provide file path in a streaming
+ *  fashion.  At startup we open an array to place the queries in and at
+ *  shutdown we close it.
+ */
+function ExplainedStatementProcessor(aDumpPath) {
+  Services.obs.addObserver(this, "quit-application");
+
+  this._sqlStack = [];
+  this._curOps = [];
+  this._objsWritten = 0;
+
+  let filePath = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+  filePath.initWithPath(aDumpPath);
+
+  this._ostream = Cc[
+    "@mozilla.org/network/file-output-stream;1"
+  ].createInstance(Ci.nsIFileOutputStream);
+  this._ostream.init(filePath, -1, -1, 0);
+
+  let s = '{"queries": [';
+  this._ostream.write(s, s.length);
+}
+ExplainedStatementProcessor.prototype = {
+  sqlEnRoute(aSQLString) {
+    this._sqlStack.push(aSQLString);
+  },
+  handleResult(aResultSet) {
+    let row;
+    // addr  opcode (s)      p1    p2    p3    p4 (s)   p5   comment (s)
+    while ((row = aResultSet.getNextRow())) {
+      this._curOps.push([
+        row.getInt64(0), // addr
+        row.getString(1), // opcode
+        row.getInt64(2), // p1
+        row.getInt64(3), // p2
+        row.getInt64(4), // p3
+        row.getString(5), // p4
+        row.getString(6), // p5
+        row.getString(7), // comment
+      ]);
+    }
+  },
+  handleError(aError) {
+    console.error("Unexpected error in EXPLAIN handler: " + aError);
+  },
+  handleCompletion(aReason) {
+    let obj = {
+      sql: this._sqlStack.shift(),
+      operations: this._curOps,
+    };
+    let s = (this._objsWritten++ ? ", " : "") + JSON.stringify(obj, null, 2);
+    this._ostream.write(s, s.length);
+
+    this._curOps = [];
+  },
+
+  observe(aSubject, aTopic, aData) {
+    if (aTopic == "quit-application") {
+      this.shutdown();
+    }
+  },
+
+  shutdown() {
+    let s = "]}";
+    this._ostream.write(s, s.length);
+    this._ostream.close();
+
+    Services.obs.removeObserver(this, "quit-application");
+  },
+};
+
+// See the documentation on GlodaDatastore._schemaVersion to understand these:
+var DB_SCHEMA_ACCEPT_LEAVE_LOW = 31,
+  DB_SCHEMA_ACCEPT_LEAVE_HIGH = 34,
+  DB_SCHEMA_ACCEPT_DOWNGRADE_LOW = 35,
+  DB_SCHEMA_ACCEPT_DOWNGRADE_HIGH = 39,
+  DB_SCHEMA_DOWNGRADE_DELTA = 5;
+
+/**
+ * Database abstraction layer.  Contains explicit SQL schemas for our
+ *  fundamental representations (core 'nouns', if you will) as well as
+ *  specialized functions for then dealing with each type of object.  At the
+ *  same time, we are beginning to support extension-provided tables, which
+ *  call into question whether we really need our hand-rolled code, or could
+ *  simply improve the extension-provided table case to work for most of our
+ *  hand-rolled cases.
+ * For now, the argument can probably be made that our explicit schemas and code
+ *  is readable/intuitive (not magic) and efficient (although generic stuff
+ *  could also be made efficient, if slightly evil through use of eval or some
+ *  other code generation mechanism.)
+ *
+ * === Data Model Interaction / Dependencies
+ *
+ * Dependent on and assumes limited knowledge of the GlodaDataModel.jsm
+ *  implementations.  GlodaDataModel.jsm actually has an implicit dependency on
+ *  our implementation, reaching back into the datastore via the _datastore
+ *  attribute which we pass into every instance we create.
+ * We pass a reference to ourself as we create the GlodaDataModel.jsm instances (and
+ *  they store it as _datastore) because of a half-implemented attempt to make
+ *  it possible to live in a world where we have multiple datastores.  This
+ *  would be desirable in the cases where we are dealing with multiple SQLite
+ *  databases.  This could be because of per-account global databases or
+ *  some other segmentation.  This was abandoned when the importance of
+ *  per-account databases was diminished following public discussion, at least
+ *  for the short-term, but no attempted was made to excise the feature or
+ *  preclude it.  (Merely a recognition that it's too much to try and implement
+ *  correct right now, especially because our solution might just be another
+ *  (aggregating) layer on top of things, rather than complicating the lower
+ *  levels.)
+ *
+ * === Object Identity / Caching
+ *
+ * The issue of object identity is handled by integration with the Collection.jsm
+ *  provided GlodaCollectionManager.  By "Object Identity", I mean that we only
+ *  should ever have one object instance alive at a time that corresponds to
+ *  an underlying database row in the database.  Where possible we avoid
+ *  performing database look-ups when we can check if the object is already
+ *  present in memory; in practice, this means when we are asking for an object
+ *  by ID.  When we cannot avoid a database query, we attempt to make sure that
+ *  we do not return a duplicate object instance, instead replacing it with the
+ *  'live' copy of the object.  (Ideally, we would avoid any redundant
+ *  construction costs, but that is not currently the case.)
+ * Although you should consult the GlodaCollectionManager for details, the
+ *  general idea is that we have 'collections' which represent views of the
+ *  database (based on a query) which use a single mechanism for double duty.
+ *  The collections are registered with the collection manager via weak
+ *  reference.  The first 'duty' is that since the collections may be desired
+ *  to be 'live views' of the data, we want them to update as changes occur.
+ *  The weak reference allows the collection manager to track the 'live'
+ *  collections and update them.  The second 'duty' is the caching/object
+ *  identity duty.  In theory, every live item should be referenced by at least
+ *  one collection, making it reachable for object identity/caching purposes.
+ * There is also an explicit (inclusive) caching layer present to both try and
+ *  avoid poor performance from some of the costs of this strategy, as well as
+ *  to try and keep track of objects that are being worked with that are not
+ *  (yet) tracked by a collection.  Using a size-bounded cache is clearly not
+ *  a guarantee of correctness for this, but is suspected will work quite well.
+ *  (Well enough to be dangerous because the inevitable failure case will not be
+ *  expected.)
+ *
+ * The current strategy may not be the optimal one, feel free to propose and/or
+ *  implement better ones, especially if you have numbers.
+ * The current strategy is not fully implemented in this file, but the common
+ *  cases are believed to be covered.  (Namely, we fail to purge items from the
+ *  cache as they are purged from the database.)
+ *
+ * === Things That May Not Be Obvious (Gotchas)
+ *
+ * Although the schema includes "triggers", they are currently not used
+ *  and were added when thinking about implementing the feature.  We will
+ *  probably implement this feature at some point, which is why they are still
+ *  in there.
+ *
+ * We, and the layers above us, are not sufficiently thorough at cleaning out
+ *  data from the database, and may potentially orphan it _as new functionality
+ *  is added in the future at layers above us_.  That is, currently we should
+ *  not be leaking database rows, but we may in the future.  This is because
+ *  we/the layers above us lack a mechanism to track dependencies based on
+ *  attributes.  Say a plugin exists that extracts recipes from messages and
+ *  relates them via an attribute.  To do so, it must create new recipe rows
+ *  in its own table as new recipes are discovered.  No automatic mechanism
+ *  will purge recipes as their source messages are purged, nor does any
+ *  event-driven mechanism explicitly inform the plugin.  (It could infer
+ *  such an event from the indexing/attribute-providing process, or poll the
+ *  states of attributes to accomplish this, but that is not desirable.)  This
+ *  needs to be addressed, and may be best addressed at layers above
+ *  GlodaDatastore.jsm.
+ *
+ * @namespace
+ */
+var GlodaDatastore = {
+  _log: null,
+
+  /* ******************* SCHEMA ******************* */
+
+  /**
+   * Schema version policy. IMPORTANT!  We expect the following potential things
+   *  to happen in the life of gloda that can impact our schema and the ability
+   *  to move between different versions of Thunderbird:
+   *
+   * - Fundamental changes to the schema so that two versions of Thunderbird
+   *    cannot use the same global database.  To wit, Thunderbird N+1 needs to
+   *    blow away the database of Thunderbird N and reindex from scratch.
+   *    Likewise, Thunderbird N will need to blow away Thunderbird N+1's
+   *    database because it can't understand it.  And we can't simply use a
+   *    different file because there would be fatal bookkeeping losses.
+   *
+   * - Bidirectional minor schema changes (rare).
+   *    Thunderbird N+1 does something that does not affect Thunderbird N's use
+   *    of the database, and a user switching back to Thunderbird N will not be
+   *    negatively impacted.  It will also be fine when they go back to N+1 and
+   *    N+1 will not be missing any vital data.  The historic example of this is
+   *    when we added a missing index that was important for performance.  In
+   *    that case, Thunderbird N could have potentially left the schema revision
+   *    intact (if there was a safe revision), rather than swapping it on the
+   *    downgrade, compelling N+1 to redo the transform on upgrade.
+   *
+   * - Backwards compatible, upgrade-transition minor schema changes.
+   *    Thunderbird N+1 does something that does not require nuking the
+   *    database / a full re-index, but does require processing on upgrade from
+   *    a version of the database previously used by Thunderbird.  These changes
+   *    do not impact N's ability to use the database.  For example, adding a
+   *    new indexed attribute that affects a small number of messages could be
+   *    handled by issuing a query on upgrade to dirty/index those messages.
+   *    However, if the user goes back to N from N+1, when they upgrade to N+1
+   *    again, we need to re-index.  In this case N would need to have downgrade
+   *    the schema revision.
+   *
+   * - Backwards incompatible, minor schema changes.
+   *    Thunderbird N+1 does something that does not require nuking the database
+   *    but will break Thunderbird N's ability to use the database.
+   *
+   * - Regression fixes.  Sometimes we may land something that screws up
+   *    databases, or the platform changes in a way that breaks our code and we
+   *    had insufficient unit test coverage and so don't detect it until some
+   *    databases have gotten messed up.
+   *
+   * Accordingly, every version of Thunderbird has a concept of potential schema
+   *  versions with associated semantics to prepare for the minor schema upgrade
+   *  cases were inter-op is possible.  These ranges and their semantics are:
+   * - accepts and leaves intact.  Covers:
+   *    - regression fixes that no longer exist with the landing of the upgrade
+   *       code as long as users never go back a build in the given channel.
+   *    - bidirectional minor schema changes.
+   * - accepts but downgrades version to self.  Covers:
+   *    - backwards compatible, upgrade-transition minor schema changes.
+   * - nuke range (anything beyond a specific revision needs to be nuked):
+   *    - backwards incompatible, minor scheme changes
+   *    - fundamental changes
+   *
+   *
+   * SO, YOU WANT TO CHANGE THE SCHEMA?
+   *
+   * Use the ranges below for Thunderbird 11 as a guide, bumping things as little
+   *  as possible.  If we start to use up the "accepts and leaves intact" range
+   *  without majorly changing things up, re-do the numbering acceptance range
+   *  to give us additional runway.
+   *
+   * Also, if we keep needing non-nuking upgrades, consider adding an additional
+   *  table to the database that can tell older versions of Thunderbird what to
+   *  do when confronted with a newer database and where it can set flags to tell
+   *  the newer Thunderbird what the older Thunderbird got up to.  For example,
+   *  it would be much easier if we just tell Thunderbird N what to do when it's
+   *  confronted with the database.
+   *
+   *
+   * CURRENT STATE OF THE MIGRATION LOGIC:
+   *
+   * Thunderbird 11: uses 30 (regression fix from 26)
+   * - accepts and leaves intact: 31-34
+   * - accepts and downgrades by 5: 35-39
+   * - nukes: 40+
+   */
+  _schemaVersion: 30,
+  // what is the schema in the database right now?
+  _actualSchemaVersion: 0,
+  _schema: {
+    tables: {
+      // ----- Messages
+      folderLocations: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["folderURI", "TEXT NOT NULL"],
+          ["dirtyStatus", "INTEGER NOT NULL"],
+          ["name", "TEXT NOT NULL"],
+          ["indexingPriority", "INTEGER NOT NULL"],
+        ],
+
+        triggers: {
+          delete: "DELETE from messages WHERE folderID = OLD.id",
+        },
+      },
+
+      conversations: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["subject", "TEXT NOT NULL"],
+          ["oldestMessageDate", "INTEGER"],
+          ["newestMessageDate", "INTEGER"],
+        ],
+
+        indices: {
+          subject: ["subject"],
+          oldestMessageDate: ["oldestMessageDate"],
+          newestMessageDate: ["newestMessageDate"],
+        },
+
+        fulltextColumns: [["subject", "TEXT"]],
+
+        triggers: {
+          delete: "DELETE from messages WHERE conversationID = OLD.id",
+        },
+      },
+
+      /**
+       * A message record correspond to an actual message stored in a folder
+       *  somewhere, or is a ghost record indicating a message that we know
+       *  should exist, but which we have not seen (and which we may never see).
+       *  We represent these ghost messages by storing NULL values in the
+       *  folderID and messageKey fields; this may need to change to other
+       *  sentinel values if this somehow impacts performance.
+       */
+      messages: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["folderID", "INTEGER"],
+          ["messageKey", "INTEGER"],
+          // conversationID used to have a REFERENCES but I'm losing it for
+          //  presumed performance reasons and it doesn't do anything for us.
+          ["conversationID", "INTEGER NOT NULL"],
+          ["date", "INTEGER"],
+          // we used to have the parentID, but because of the very real
+          //  possibility of multiple copies of a message with a given
+          //  message-id, the parentID concept is unreliable.
+          ["headerMessageID", "TEXT"],
+          ["deleted", "INTEGER NOT NULL default 0"],
+          ["jsonAttributes", "TEXT"],
+          // Notability attempts to capture the static 'interestingness' of a
+          //  message as a result of being starred/flagged, labeled, read
+          //  multiple times, authored by someone in your address book or that
+          //  you converse with a lot, etc.
+          ["notability", "INTEGER NOT NULL default 0"],
+        ],
+
+        indices: {
+          messageLocation: ["folderID", "messageKey"],
+          headerMessageID: ["headerMessageID"],
+          conversationID: ["conversationID"],
+          date: ["date"],
+          deleted: ["deleted"],
+        },
+
+        // note: if reordering the columns, you need to change this file's
+        //  row-loading logic, GlodaMsgSearcher.jsm's ranking usages and also the
+        //  column saturations in nsGlodaRankerFunction
+        fulltextColumns: [
+          ["body", "TEXT"],
+          ["subject", "TEXT"],
+          ["attachmentNames", "TEXT"],
+          ["author", "TEXT"],
+          ["recipients", "TEXT"],
+        ],
+
+        triggers: {
+          delete: "DELETE FROM messageAttributes WHERE messageID = OLD.id",
+        },
+      },
+
+      // ----- Attributes
+      attributeDefinitions: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["attributeType", "INTEGER NOT NULL"],
+          ["extensionName", "TEXT NOT NULL"],
+          ["name", "TEXT NOT NULL"],
+          ["parameter", "BLOB"],
+        ],
+
+        triggers: {
+          delete: "DELETE FROM messageAttributes WHERE attributeID = OLD.id",
+        },
+      },
+
+      messageAttributes: {
+        columns: [
+          // conversationID and messageID used to have REFERENCES back to their
+          //  appropriate types.  I removed it when removing attributeID for
+          //  better reasons and because the code is not capable of violating
+          //  this constraint, so the check is just added cost.  (And we have
+          //  unit tests that sanity check my assertions.)
+          ["conversationID", "INTEGER NOT NULL"],
+          ["messageID", "INTEGER NOT NULL"],
+          // This used to be REFERENCES attributeDefinitions(id) but then we
+          //  introduced sentinel values and it's hard to justify the effort
+          //  to compel injection of the record or the overhead to do the
+          //  references checking.
+          ["attributeID", "INTEGER NOT NULL"],
+          ["value", "NUMERIC"],
+        ],
+
+        indices: {
+          attribQuery: [
+            "attributeID",
+            "value",
+            /* covering: */ "conversationID",
+            "messageID",
+          ],
+          // This is required for deletion of a message's attributes to be
+          // performant.  We could optimize this index away if we changed our
+          // deletion logic to issue specific attribute deletions based on the
+          // information it already has available in the message's JSON blob.
+          // The rub there is that if we screwed up we could end up leaking
+          // attributes and there is a non-trivial performance overhead to
+          // the many requests it would cause (which can also be reduced in
+          // the future by changing our SQL dispatch code.)
+          messageAttribFastDeletion: ["messageID"],
+        },
+      },
+
+      // ----- Contacts / Identities
+
+      /**
+       * Corresponds to a human being and roughly to an address book entry.
+       *  Contrast with an identity, which is a specific e-mail address, IRC
+       *  nick, etc.  Identities belong to contacts, and this relationship is
+       *  expressed on the identityAttributes table.
+       */
+      contacts: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["directoryUUID", "TEXT"],
+          ["contactUUID", "TEXT"],
+          ["popularity", "INTEGER"],
+          ["frecency", "INTEGER"],
+          ["name", "TEXT"],
+          ["jsonAttributes", "TEXT"],
+        ],
+        indices: {
+          popularity: ["popularity"],
+          frecency: ["frecency"],
+        },
+      },
+
+      contactAttributes: {
+        columns: [
+          ["contactID", "INTEGER NOT NULL"],
+          ["attributeID", "INTEGER NOT NULL"],
+          ["value", "NUMERIC"],
+        ],
+        indices: {
+          contactAttribQuery: [
+            "attributeID",
+            "value",
+            /* covering: */ "contactID",
+          ],
+        },
+      },
+
+      /**
+       * Identities correspond to specific e-mail addresses, IRC nicks, etc.
+       */
+      identities: {
+        columns: [
+          ["id", "INTEGER PRIMARY KEY"],
+          ["contactID", "INTEGER NOT NULL"],
+          ["kind", "TEXT NOT NULL"], // ex: email, irc, etc.
+          ["value", "TEXT NOT NULL"], // ex: e-mail address, irc nick/handle...
+          ["description", "NOT NULL"], // what makes this identity different
+          // from the others? (ex: home, work, etc.)
+          ["relay", "INTEGER NOT NULL"], // is the identity just a relay
+          // mechanism? (ex: mailing list, twitter 'bouncer', IRC gateway, etc.)
+        ],
+
+        indices: {
+          contactQuery: ["contactID"],
+          valueQuery: ["kind", "value"],
+        },
+      },
+    },
+  },
+
+  /* ******************* LOGIC ******************* */
+  /**
+   * We only have one connection; this name exists for legacy reasons but helps
+   *  track when we are intentionally doing synchronous things during startup.
+   *  We do nothing synchronous once our setup has completed.
+   */
+  syncConnection: null,
+  /**
+   * We only have one connection and we only do asynchronous things after setup;
+   *  this name still exists mainly for legacy reasons.
+   */
+  asyncConnection: null,
+
+  /**
+   * Our "mailnews.database.global.datastore." preferences branch for debug
+   * notification handling.  We register as an observer against this.
+   */
+  _prefBranch: null,
+
+  /**
+   * The unique ID assigned to an index when it has been built. This value
+   * changes once the index has been rebuilt.
+   */
+  _datastoreID: null,
+
+  /**
+   * Initialize logging, create the database if it doesn't exist, "upgrade" it
+   *  if it does and it's not up-to-date, fill our authoritative folder uri/id
+   *  mapping.
+   */
+  _init(aNounIDToDef) {
+    this._log = console.createInstance({
+      prefix: "gloda.datastore",
+      maxLogLevel: "Warn",
+      maxLogLevelPref: "gloda.loglevel",
+    });
+    this._log.debug("Beginning datastore initialization.");
+
+    this._nounIDToDef = aNounIDToDef;
+
+    let branch = Services.prefs.getBranch(
+      "mailnews.database.global.datastore."
+    );
+    this._prefBranch = branch;
+
+    // Not sure the weak reference really makes a difference given that we are a
+    // GC root.
+    branch.addObserver("", this);
+    // claim the pref changed so we can centralize our logic there.
+    this.observe(null, "nsPref:changed", "explainToPath");
+
+    // Get the path to our global database
+    var dbFile = Services.dirsvc.get("ProfD", Ci.nsIFile);
+    dbFile.append("global-messages-db.sqlite");
+
+    var dbConnection;
+
+    // Report about the size of the database through telemetry (if there's a
+    // database, naturally).
+    if (dbFile.exists()) {
+      try {
+        let h = Services.telemetry.getHistogramById(
+          "THUNDERBIRD_GLODA_SIZE_MB"
+        );
+        h.add(dbFile.fileSize / 1048576);
+      } catch (e) {
+        this._log.warn("Couldn't report telemetry", e);
+      }
+    }
+
+    // Create the file if it does not exist
+    if (!dbFile.exists()) {
+      this._log.debug("Creating database because it doesn't exist.");
+      dbConnection = this._createDB(dbFile);
+    } else {
+      // It does exist, but we (someday) might need to upgrade the schema
+      // (Exceptions may be thrown if the database is corrupt)
+      try {
+        dbConnection = Services.storage.openUnsharedDatabase(dbFile);
+        let cacheSize = this._determineCachePages(dbConnection);
+        // see _createDB...
+        dbConnection.executeSimpleSQL("PRAGMA cache_size = " + cacheSize);
+        dbConnection.executeSimpleSQL("PRAGMA synchronous = FULL");
+
+        // Register custom tokenizer to index all language text
+        var tokenizer = Cc["@mozilla.org/messenger/fts3tokenizer;1"].getService(
+          Ci.nsIFts3Tokenizer
+        );
+        tokenizer.registerTokenizer(dbConnection);
+
+        // -- database schema changes
+        let dbSchemaVersion = (this._actualSchemaVersion =
+          dbConnection.schemaVersion);
+        // - database from the future!
+        if (dbSchemaVersion > this._schemaVersion) {
+          if (
+            dbSchemaVersion >= DB_SCHEMA_ACCEPT_LEAVE_LOW &&
+            dbSchemaVersion <= DB_SCHEMA_ACCEPT_LEAVE_HIGH
+          ) {
+            this._log.debug(
+              "db from the future in acceptable range; leaving " +
+                "version at: " +
+                dbSchemaVersion
+            );
+          } else if (
+            dbSchemaVersion >= DB_SCHEMA_ACCEPT_DOWNGRADE_LOW &&
+            dbSchemaVersion <= DB_SCHEMA_ACCEPT_DOWNGRADE_HIGH
+          ) {
+            let newVersion = dbSchemaVersion - DB_SCHEMA_DOWNGRADE_DELTA;
+            this._log.debug(
+              "db from the future in downgrade range; setting " +
+                "version to " +
+                newVersion +
+                " down from " +
+                dbSchemaVersion
+            );
+            dbConnection.schemaVersion = this._actualSchemaVersion = newVersion;
+          } else {
+            // too far from the future, nuke it.
+            dbConnection = this._nukeMigration(dbFile, dbConnection);
+          }
+        } else if (dbSchemaVersion < this._schemaVersion) {
+          // - database from the past!  migrate it, possibly.
+          this._log.debug(
+            "Need to migrate database.  (DB version: " +
+              this._actualSchemaVersion +
+              " desired version: " +
+              this._schemaVersion
+          );
+          dbConnection = this._migrate(
+            dbFile,
+            dbConnection,
+            this._actualSchemaVersion,
+            this._schemaVersion
+          );
+          this._log.debug("Migration call completed.");
+        }
+        // else: this database is juuust right.
+
+        // If we never had a datastore ID, make sure to create one now.
+        if (!this._prefBranch.prefHasUserValue("id")) {
+          this._datastoreID = this._generateDatastoreID();
+          this._prefBranch.setCharPref("id", this._datastoreID);
+        } else {
+          this._datastoreID = this._prefBranch.getCharPref("id");
+        }
+      } catch (ex) {
+        // Handle corrupt databases, other oddities
+        if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) {
+          this._log.warn("Database was corrupt, removing the old one.");
+          dbFile.remove(false);
+          this._log.warn("Removed old database, creating a new one.");
+          dbConnection = this._createDB(dbFile);
+        } else {
+          this._log.error(
+            "Unexpected error when trying to open the database:",
+            ex
+          );
+          throw ex;
+        }
+      }
+    }
+
+    this.syncConnection = dbConnection;
+    this.asyncConnection = dbConnection;
+
+    this._log.debug("Initializing folder mappings.");
+    this._getAllFolderMappings();
+    // we need to figure out the next id's for all of the tables where we
+    //  manage that.
+    this._log.debug("Populating managed id counters.");
+    this._populateAttributeDefManagedId();
+    this._populateConversationManagedId();
+    this._populateMessageManagedId();
+    this._populateContactManagedId();
+    this._populateIdentityManagedId();
+
+    this._log.debug("Completed datastore initialization.");
+  },
+
+  observe(aSubject, aTopic, aData) {
+    if (aTopic != "nsPref:changed") {
+      return;
+    }
+
+    if (aData == "explainToPath") {
+      let explainToPath = null;
+      try {
+        explainToPath = this._prefBranch.getCharPref("explainToPath");
+        if (explainToPath.trim() == "") {
+          explainToPath = null;
+        }
+      } catch (ex) {
+        // don't care if the pref is not there.
+      }
+
+      // It is conceivable that the name is changing and this isn't a boolean
+      // toggle, so always clean out the explain processor.
+      if (this._explainProcessor) {
+        this._explainProcessor.shutdown();
+        this._explainProcessor = null;
+      }
+
+      if (explainToPath) {
+        this._createAsyncStatement = this._createExplainedAsyncStatement;
+        this._explainProcessor = new ExplainedStatementProcessor(explainToPath);
+      } else {
+        this._createAsyncStatement = this._realCreateAsyncStatement;
+      }
+    }
+  },
+
+  datastoreIsShutdown: false,
+
+  /**
+   * Perform datastore shutdown.
+   */
+  shutdown() {
+    // Clear out any pending transaction by committing it.
+    // The indexer has been shutdown by this point; it no longer has any active
+    //  indexing logic and it no longer has active event listeners capable of
+    //  generating new activity.
+    // Semantic consistency of the database is guaranteed by the indexer's
+    //  strategy of only yielding control at coherent times.  Although it takes
+    //  multiple calls and multiple SQL operations to update the state of our
+    //  database representations, the generator does not yield until it has
+    //  issued all the database statements required for said update.  As such,
+    //  this commit will leave us in a good way (and the commit will happen
+    //  because closing the connection will drain the async execution queue.)
+    while (this._transactionDepth) {
+      this._log.info("Closing pending transaction out for shutdown.");
+      // just schedule this function to be run again once the transaction has
+      //  been closed out.
+      this._commitTransaction();
+    }
+
+    this.datastoreIsShutdown = true;
+
+    this._log.info("Closing db connection");
+
+    // we do not expect exceptions, but it's a good idea to avoid having our
+    //  shutdown process explode.
+    try {
+      this._cleanupAsyncStatements();
+      this._cleanupSyncStatements();
+    } catch (ex) {
+      this._log.debug("Unexpected exception during statement cleanup: " + ex);
+    }
+
+    // it's conceivable we might get a spurious exception here, but we really
+    //  shouldn't get one.  again, we want to ensure shutdown runs to completion
+    //  and doesn't break our caller.
+    try {
+      // This currently causes all pending asynchronous operations to be run to
+      //  completion.  this simplifies things from a correctness perspective,
+      //  and, honestly, is a lot easier than us tracking all of the async
+      //  event tasks so that we can explicitly cancel them.
+      // This is a reasonable thing to do because we don't actually ever have
+      //  a huge number of statements outstanding.  The indexing process needs
+      //  to issue async requests periodically, so the most we have in-flight
+      //  from a write perspective is strictly less than the work required to
+      //  update the database state for a single message.
+      // However, the potential for multiple pending expensive queries does
+      //  exist, and it may be advisable to attempt to track and cancel those.
+      //  For simplicity we don't currently do this, and I expect this should
+      //  not pose a major problem, but those are famous last words.
+      // Note: asyncClose does not spin a nested event loop, but the thread
+      //  manager shutdown code will spin the async thread's event loop, so it
+      //  nets out to be the same.
+      this.asyncConnection.asyncClose();
+    } catch (ex) {
+      this._log.debug(
+        "Potentially expected exception during connection closure: " + ex
+      );
+    }
+
+    this.asyncConnection = null;
+    this.syncConnection = null;
+  },
+
+  /**
+   * Generates and returns a UUID.
+   *
+   * @returns a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
+   */
+  _generateDatastoreID() {
+    let uuid = Services.uuid.generateUUID().toString();
+    // We snip off the { and } from each end of the UUID.
+    return uuid.substring(1, uuid.length - 2);
+  },
+
+  _determineCachePages(aDBConn) {
+    try {
+      // For the details of the computations, one should read
+      //  nsNavHistory::InitDB. We're slightly diverging from them in the sense
+      //  that we won't allow gloda to use insane amounts of memory cache, and
+      //  we start with 1% instead of 6% like them.
+      let pageStmt = aDBConn.createStatement("PRAGMA page_size");
+      pageStmt.executeStep();
+      let pageSize = pageStmt.row.page_size;
+      pageStmt.finalize();
+      let cachePermillage = this._prefBranch.getIntPref(
+        "cache_to_memory_permillage"
+      );
+      cachePermillage = Math.min(cachePermillage, 50);
+      cachePermillage = Math.max(cachePermillage, 0);
+      let physMem = Services.sysinfo.getPropertyAsInt64("memsize");
+      if (physMem == 0) {
+        physMem = MEMSIZE_FALLBACK_BYTES;
+      }
+      let cacheSize = Math.round((physMem * cachePermillage) / 1000);
+      cacheSize = Math.max(cacheSize, MIN_CACHE_SIZE);
+      cacheSize = Math.min(cacheSize, MAX_CACHE_SIZE);
+      let cachePages = Math.round(cacheSize / pageSize);
+      return cachePages;
+    } catch (ex) {
+      this._log.warn("Error determining cache size: " + ex);
+      // A little bit lower than on my personal machine, will result in ~40M.
+      return 1000;
+    }
+  },
+
+  /**
+   * Create our database; basically a wrapper around _createSchema.
+   */
+  _createDB(aDBFile) {
+    var dbConnection = Services.storage.openUnsharedDatabase(aDBFile);
+    // We now follow the Firefox strategy for places, which mainly consists in
+    //  picking a default 32k page size, and then figuring out the amount of
+    //  cache accordingly. The default 32k come from mozilla/toolkit/storage,
+    //  but let's get it directly from sqlite in case they change it.
+    let cachePages = this._determineCachePages(dbConnection);
+    // This is a maximum number of pages to be used.  If the database does not
+    //  get this large, then the memory does not get used.
+    // Do not forget to update the code in _init if you change this value.
+    dbConnection.executeSimpleSQL("PRAGMA cache_size = " + cachePages);
+    // The mozStorage default is NORMAL which shaves off some fsyncs in the
+    //  interest of performance.  Since everything we do after bootstrap is
+    //  async, we do not care about the performance, but we really want the
+    //  correctness.  Bug reports and support avenues indicate a non-zero number
+    //  of corrupt databases.  Note that this may not fix everything; OS X
+    //  also supports an F_FULLSYNC flag enabled by PRAGMA fullfsync that we are
+    //  not enabling that is much more comprehensive.  We can think about
+    //  turning that on after we've seen how this reduces our corruption count.
+    dbConnection.executeSimpleSQL("PRAGMA synchronous = FULL");
+    // Register custom tokenizer to index all language text
+    var tokenizer = Cc["@mozilla.org/messenger/fts3tokenizer;1"].getService(
+      Ci.nsIFts3Tokenizer
+    );
+    tokenizer.registerTokenizer(dbConnection);
+
+    // We're creating a new database, so let's generate a new ID for this
+    // version of the datastore. This way, indexers can know when the index
+    // has been rebuilt in the event that they need to rebuild dependent data.
+    this._datastoreID = this._generateDatastoreID();
+    this._prefBranch.setCharPref("id", this._datastoreID);
+
+    dbConnection.beginTransaction();
+    try {
+      this._createSchema(dbConnection);
+      dbConnection.commitTransaction();
+    } catch (ex) {
+      dbConnection.rollbackTransaction();
+      throw ex;
+    }
+
+    return dbConnection;
+  },
+
+  _createTableSchema(aDBConnection, aTableName, aTableDef) {
+    // - Create the table
+    this._log.info("Creating table: " + aTableName);
+    let columnDefs = [];
+    for (let [column, type] of aTableDef.columns) {
+      columnDefs.push(column + " " + type);
+    }
+    aDBConnection.createTable(aTableName, columnDefs.join(", "));
+
+    // - Create the fulltext table if applicable
+    if (aTableDef.fulltextColumns) {
+      let columnDefs = [];
+      for (let [column, type] of aTableDef.fulltextColumns) {
+        columnDefs.push(column + " " + type);
+      }
+      let createFulltextSQL =
+        "CREATE VIRTUAL TABLE " +
+        aTableName +
+        "Text" +
+        " USING fts3(tokenize mozporter, " +
+        columnDefs.join(", ") +
+        ")";
+      this._log.info("Creating fulltext table: " + createFulltextSQL);
+      aDBConnection.executeSimpleSQL(createFulltextSQL);
+    }
+
+    // - Create its indices
+    if (aTableDef.indices) {
+      for (let indexName in aTableDef.indices) {
+        let indexColumns = aTableDef.indices[indexName];
+        aDBConnection.executeSimpleSQL(
+          "CREATE INDEX " +
+            indexName +
+            " ON " +
+            aTableName +
+            "(" +
+            indexColumns.join(", ") +
+            ")"
+        );
+      }
+    }
+
+    // - Create the attributes table if applicable
+    if (aTableDef.genericAttributes) {
+      aTableDef.genericAttributes = {
+        columns: [
+          ["nounID", "INTEGER NOT NULL"],
+          ["attributeID", "INTEGER NOT NULL"],
+          ["value", "NUMERIC"],
+        ],
+        indices: {},
+      };
+      aTableDef.genericAttributes.indices[aTableName + "AttribQuery"] = [
+        "attributeID",
+        "value",
+        /* covering: */ "nounID",
+      ];
+      // let's use this very function!  (since we created genericAttributes,
+      //  explodey recursion is avoided.)
+      this._createTableSchema(
+        aDBConnection,
+        aTableName + "Attributes",
+        aTableDef.genericAttributes
+      );
+    }
+  },
+
+  /**
+   * Create our database schema assuming a newly created database.  This
+   *  comes down to creating normal tables, their full-text variants (if
+   *  applicable), and their indices.
+   */
+  _createSchema(aDBConnection) {
+    // -- For each table...
+    for (let tableName in this._schema.tables) {
+      let tableDef = this._schema.tables[tableName];
+      this._createTableSchema(aDBConnection, tableName, tableDef);
+    }
+
+    aDBConnection.schemaVersion = this._actualSchemaVersion =
+      this._schemaVersion;
+  },
+
+  /**
+   * Create a table for a noun, replete with data binding.
+   */
+  createNounTable(aNounDef) {
+    // give it a _jsonText attribute if appropriate...
+    if (aNounDef.allowsArbitraryAttrs) {
+      aNounDef.schema.columns.push(["jsonAttributes", "STRING", "_jsonText"]);
+    }
+    // check if the table exists
+    if (!this.asyncConnection.tableExists(aNounDef.tableName)) {
+      // it doesn't! create it (and its potentially many variants)
+      try {
+        this._createTableSchema(
+          this.asyncConnection,
+          aNounDef.tableName,
+          aNounDef.schema
+        );
+      } catch (ex) {
+        this._log.error(
+          "Problem creating table " +
+            aNounDef.tableName +
+            " " +
+            "because: " +
+            ex +
+            " at " +
+            ex.fileName +
+            ":" +
+            ex.lineNumber
+        );
+        return;
+      }
+    }
+
+    aNounDef._dataBinder = new GlodaDatabind(aNounDef, this);
+    aNounDef.datastore = aNounDef._dataBinder;
+    aNounDef.objFromRow = aNounDef._dataBinder.objFromRow;
+    aNounDef.objInsert = aNounDef._dataBinder.objInsert;
+    aNounDef.objUpdate = aNounDef._dataBinder.objUpdate;
+    aNounDef.dbAttribAdjuster = aNounDef._dataBinder.adjustAttributes;
+
+    if (aNounDef.schema.genericAttributes) {
+      aNounDef.attrTableName = aNounDef.tableName + "Attributes";
+      aNounDef.attrIDColumnName = "nounID";
+    }
+  },
+
+  _nukeMigration(aDBFile, aDBConnection) {
+    aDBConnection.close();
+    aDBFile.remove(false);
+    this._log.warn(
+      "Global database has been purged due to schema change.  " +
+        "old version was " +
+        this._actualSchemaVersion +
+        ", new version is: " +
+        this._schemaVersion
+    );
+    return this._createDB(aDBFile);
+  },
+
+  /**
+   * Migrate the database _to the latest version_ from an older version.  We
+   *  only keep enough logic around to get us to the recent version.  This code
+   *  is not a time machine!  If we need to blow away the database to get to the
+   *  most recent version, then that's the sum total of the migration!
+   */
+  _migrate(aDBFile, aDBConnection, aCurVersion, aNewVersion) {
+    // version 12:
+    // - notability column added
+    // version 13:
+    // - we are adding a new fulltext index column. blow away!
+    // - note that I screwed up and failed to mark the schema change; apparently
+    //   no database will claim to be version 13...
+    // version 14ish, still labeled 13?:
+    // - new attributes: forwarded, repliedTo, bcc, recipients
+    // - altered fromMeTo and fromMeCc to fromMe
+    // - altered toMe and ccMe to just be toMe
+    // - exposes bcc to cc-related attributes
+    // - MIME type DB schema overhaul
+    // version 15ish, still labeled 13:
+    // - change tokenizer to mozporter to support CJK
+    // (We are slip-streaming this so that only people who want to test CJK
+    //  have to test it.  We will properly bump the schema revision when the
+    //  gloda correctness patch lands.)
+    // version 16ish, labeled 14 and now 16
+    // - gloda message id's start from 32 now
+    // - all kinds of correctness changes (blow away)
+    // version 17
+    // - more correctness fixes. (blow away)
+    // version 18
+    // - significant empty set support (blow away)
+    // version 19
+    // - there was a typo that was resulting in deleted getting set to the
+    //  numeric value of the javascript undefined value.  (migrate-able)
+    // version 20
+    // - tokenizer changes to provide for case/accent-folding. (blow away)
+    // version 21
+    // - add the messagesAttribFastDeletion index we thought was already covered
+    //  by an index we removed a while ago (migrate-able)
+    // version 26
+    // - bump page size and also cache size (blow away)
+    // version 30
+    // - recover from bug 732372 that affected TB 11 beta / TB 12 alpha / TB 13
+    //    trunk.  The fix is bug 734507.  The revision bump happens
+    //    asynchronously. (migrate-able)
+
+    // nuke if prior to 26
+    if (aCurVersion < 26) {
+      return this._nukeMigration(aDBFile, aDBConnection);
+    }
+
+    // They must be desiring our "a.contact is undefined" fix!
+    // This fix runs asynchronously as the first indexing job the indexer ever
+    //  performs.  It is scheduled by the enabling of the message indexer and
+    //  it is the one that updates the schema version when done.
+
+    // return the same DB connection since we didn't create a new one or do
+    //  anything.
+    return aDBConnection;
+  },
+
+  /**
+   * Asynchronously update the schema version; only for use by in-tree callers
+   *  who asynchronously perform migration work triggered by their initial
+   *  indexing sweep and who have properly updated the schema version in all
+   *  the appropriate locations in this file.
+   *
+   * This is done without doing anything about the current transaction state,
+   *  which is desired.
+   */
+  _updateSchemaVersion(newSchemaVersion) {
+    this._actualSchemaVersion = newSchemaVersion;
+    let stmt = this._createAsyncStatement(
+      // we need to concat; pragmas don't like "?1" binds
+      "PRAGMA user_version = " + newSchemaVersion,
+      true
+    );
+    stmt.executeAsync(this.trackAsync());
+    stmt.finalize();
+  },
+
+  _outstandingAsyncStatements: [],
+
+  /**
+   * Unless debugging, this is just _realCreateAsyncStatement, but in some
+   *  debugging modes this is instead the helpful wrapper
+   *  _createExplainedAsyncStatement.
+   */
+  _createAsyncStatement: null,
+
+  _realCreateAsyncStatement(aSQLString, aWillFinalize) {
+    let statement = null;
+    try {
+      statement = this.asyncConnection.createAsyncStatement(aSQLString);
+    } catch (ex) {
+      throw new Error(
+        "error creating async statement " +
+          aSQLString +
+          " - " +
+          this.asyncConnection.lastError +
+          ": " +
+          this.asyncConnection.lastErrorString +
+          " - " +
+          ex
+      );
+    }
+
+    if (!aWillFinalize) {
+      this._outstandingAsyncStatements.push(statement);
+    }
+
+    return statement;
+  },
+
+  /**
+   * The ExplainedStatementProcessor instance used by
+   *  _createExplainedAsyncStatement.  This will be null if
+   *  _createExplainedAsyncStatement is not being used as _createAsyncStatement.
+   */
+  _explainProcessor: null,
+
+  /**
+   * Wrapped version of _createAsyncStatement that EXPLAINs the statement.  When
+   *  used this decorates _createAsyncStatement, in which case we are found at
+   *  that name and the original is at _orig_createAsyncStatement.  This is
+   *  controlled by the explainToPath preference (see |_init|).
+   */
+  _createExplainedAsyncStatement(aSQLString, aWillFinalize) {
+    let realStatement = this._realCreateAsyncStatement(
+      aSQLString,
+      aWillFinalize
+    );
+    // don't wrap transaction control statements.
+    if (
+      aSQLString == "COMMIT" ||
+      aSQLString == "BEGIN TRANSACTION" ||
+      aSQLString == "ROLLBACK"
+    ) {
+      return realStatement;
+    }
+
+    let explainSQL = "EXPLAIN " + aSQLString;
+    let explainStatement = this._realCreateAsyncStatement(explainSQL);
+
+    return new ExplainedStatementWrapper(
+      realStatement,
+      explainStatement,
+      aSQLString,
+      this._explainProcessor
+    );
+  },
+
+  _cleanupAsyncStatements() {
+    this._outstandingAsyncStatements.forEach(stmt => stmt.finalize());
+  },
+
+  _outstandingSyncStatements: [],
+
+  _createSyncStatement(aSQLString, aWillFinalize) {
+    let statement = null;
+    try {
+      statement = this.syncConnection.createStatement(aSQLString);
+    } catch (ex) {
+      throw new Error(
+        "error creating sync statement " +
+          aSQLString +
+          " - " +
+          this.syncConnection.lastError +
+          ": " +
+          this.syncConnection.lastErrorString +
+          " - " +
+          ex
+      );
+    }
+
+    if (!aWillFinalize) {
+      this._outstandingSyncStatements.push(statement);
+    }
+
+    return statement;
+  },
+
+  _cleanupSyncStatements() {
+    this._outstandingSyncStatements.forEach(stmt => stmt.finalize());
+  },
+
+  /**
+   * Perform a synchronous executeStep on the statement, handling any
+   *  SQLITE_BUSY fallout that could conceivably happen from a collision on our
+   *  read with the async writes.
+   * Basically we keep trying until we succeed or run out of tries.
+   * We believe this to be a reasonable course of action because we don't
+   *  expect this to happen much.
+   */
+  _syncStep(aStatement) {
+    let tries = 0;
+    while (tries < 32000) {
+      try {
+        return aStatement.executeStep();
+      } catch (e) {
+        // SQLITE_BUSY becomes NS_ERROR_FAILURE
+        if (e.result == Cr.NS_ERROR_FAILURE) {
+          tries++;
+          // we really need to delay here, somehow.  unfortunately, we can't
+          //  allow event processing to happen, and most of the things we could
+          //  do to delay ourselves result in event processing happening.  (Use
+          //  of a timer, a synchronous dispatch, etc.)
+          // in theory, nsIThreadEventFilter could allow us to stop other events
+          //  that aren't our timer from happening, but it seems slightly
+          //  dangerous and 'notxpcom' suggests it ain't happening anyways...
+          // so, let's just be dumb and hope that the underlying file I/O going
+          //  on makes us more likely to yield to the other thread so it can
+          //  finish what it is doing...
+        } else {
+          throw e;
+        }
+      }
+    }
+    this._log.error("Synchronous step gave up after " + tries + " tries.");
+    return false;
+  },
+
+  _bindVariant(aStatement, aIndex, aVariant) {
+    aStatement.bindByIndex(aIndex, aVariant);
+  },
+
+  /**
+   * Helper that uses the appropriate getter given the data type; should be
+   *  mooted once we move to 1.9.2 and can use built-in variant support.
+   */
+  _getVariant(aRow, aIndex) {
+    let typeOfIndex = aRow.getTypeOfIndex(aIndex);
+    if (typeOfIndex == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      // XPConnect would just end up going through an intermediary double stage
+      // for the int64 case anyways...
+      return null;
+    }
+    if (
+      typeOfIndex == Ci.mozIStorageValueArray.VALUE_TYPE_INTEGER ||
+      typeOfIndex == Ci.mozIStorageValueArray.VALUE_TYPE_DOUBLE
+    ) {
+      return aRow.getDouble(aIndex);
+    }
+    // typeOfIndex == Ci.mozIStorageValueArray.VALUE_TYPE_TEXT
+    return aRow.getString(aIndex);
+  },
+
+  /** Simple nested transaction support as a performance optimization. */
+  _transactionDepth: 0,
+  _transactionGood: false,
+
+  /**
+   * Self-memoizing BEGIN TRANSACTION statement.
+   */
+  get _beginTransactionStatement() {
+    let statement = this._createAsyncStatement("BEGIN TRANSACTION");
+    this.__defineGetter__("_beginTransactionStatement", () => statement);
+    return this._beginTransactionStatement;
+  },
+
+  /**
+   * Self-memoizing COMMIT statement.
+   */
+  get _commitTransactionStatement() {
+    let statement = this._createAsyncStatement("COMMIT");
+    this.__defineGetter__("_commitTransactionStatement", () => statement);
+    return this._commitTransactionStatement;
+  },
+
+  /**
+   * Self-memoizing ROLLBACK statement.
+   */
+  get _rollbackTransactionStatement() {
+    let statement = this._createAsyncStatement("ROLLBACK");
+    this.__defineGetter__("_rollbackTransactionStatement", () => statement);
+    return this._rollbackTransactionStatement;
+  },
+
+  _pendingPostCommitCallbacks: null,
+  /**
+   * Register a callback to be invoked when the current transaction's commit
+   *  completes.
+   */
+  runPostCommit(aCallback) {
+    this._pendingPostCommitCallbacks.push(aCallback);
+  },
+
+  /**
+   * Begin a potentially nested transaction; only the outermost transaction gets
+   *  to be an actual transaction, and the failure of any nested transaction
+   *  results in a rollback of the entire outer transaction.  If you really
+   *  need an atomic transaction
+   */
+  _beginTransaction() {
+    if (this._transactionDepth == 0) {
+      this._pendingPostCommitCallbacks = [];
+      this._beginTransactionStatement.executeAsync(this.trackAsync());
+      this._transactionGood = true;
+    }
+    this._transactionDepth++;
+  },
+  /**
+   * Commit a potentially nested transaction; if we are the outer-most
+   *  transaction and no sub-transaction issues a rollback
+   *  (via _rollbackTransaction) then we commit, otherwise we rollback.
+   */
+  _commitTransaction() {
+    this._transactionDepth--;
+    if (this._transactionDepth == 0) {
+      try {
+        if (this._transactionGood) {
+          this._commitTransactionStatement.executeAsync(
+            new PostCommitHandler(this._pendingPostCommitCallbacks)
+          );
+        } else {
+          this._rollbackTransactionStatement.executeAsync(this.trackAsync());
+        }
+      } catch (ex) {
+        this._log.error("Commit problem:", ex);
+      }
+      this._pendingPostCommitCallbacks = [];
+    }
+  },
+  /**
+   * Abort the commit of the potentially nested transaction.  If we are not the
+   *  outermost transaction, we set a flag that tells the outermost transaction
+   *  that it must roll back.
+   */
+  _rollbackTransaction() {
+    this._transactionDepth--;
+    this._transactionGood = false;
+    if (this._transactionDepth == 0) {
+      try {
+        this._rollbackTransactionStatement.executeAsync(this.trackAsync());
+      } catch (ex) {
+        this._log.error("Rollback problem:", ex);
+      }
+    }
+  },
+
+  _pendingAsyncStatements: 0,
+  /**
+   * The function to call, if any, when we hit 0 pending async statements.
+   */
+  _pendingAsyncCompletedListener: null,
+  _asyncCompleted() {
+    if (--this._pendingAsyncStatements == 0) {
+      if (this._pendingAsyncCompletedListener !== null) {
+        this._pendingAsyncCompletedListener();
+        this._pendingAsyncCompletedListener = null;
+      }
+    }
+  },
+  _asyncTrackerListener: {
+    handleResult() {},
+    handleError(aError) {
+      GlodaDatastore._log.error(
+        "got error in _asyncTrackerListener.handleError(): " +
+          aError.result +
+          ": " +
+          aError.message
+      );
+    },
+    handleCompletion() {
+      try {
+        // the helper method exists because the other classes need to call it too
+        GlodaDatastore._asyncCompleted();
+      } catch (e) {
+        this._log.error("Exception in handleCompletion:", e);
+      }
+    },
+  },
+  /**
+   * Increments _pendingAsyncStatements and returns a listener that will
+   *  decrement the value when the statement completes.
+   */
+  trackAsync() {
+    this._pendingAsyncStatements++;
+    return this._asyncTrackerListener;
+  },
+
+  /* ********** Attribute Definitions ********** */
+  /** Maps (attribute def) compound names to the GlodaAttributeDBDef objects. */
+  _attributeDBDefs: {},
+  /** Map attribute ID to the definition and parameter value that produce it. */
+  _attributeIDToDBDefAndParam: {},
+
+  /**
+   * This attribute id indicates that we are encoding that a non-singular
+   *  attribute has an empty set.  The value payload that goes with this should
+   *  the attribute id of the attribute we are talking about.
+   */
+  kEmptySetAttrId: 1,
+
+  /**
+   * We maintain the attributeDefinitions next id counter mainly because we can.
+   *  Since we mediate the access, there's no real risk to doing so, and it
+   *  allows us to keep the writes on the async connection without having to
+   *  wait for a completion notification.
+   *
+   * Start from 32 so we can have a number of sentinel values.
+   */
+  _nextAttributeId: 32,
+
+  _populateAttributeDefManagedId() {
+    let stmt = this._createSyncStatement(
+      "SELECT MAX(id) FROM attributeDefinitions",
+      true
+    );
+    if (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      // 0 gets returned even if there are no messages...
+      let highestSeen = stmt.getInt64(0);
+      if (highestSeen != 0) {
+        this._nextAttributeId = highestSeen + 1;
+      }
+    }
+    stmt.finalize();
+  },
+
+  get _insertAttributeDefStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO attributeDefinitions (id, attributeType, extensionName, \
+                                  name, parameter) \
+              VALUES (?1, ?2, ?3, ?4, ?5)"
+    );
+    this.__defineGetter__("_insertAttributeDefStatement", () => statement);
+    return this._insertAttributeDefStatement;
+  },
+
+  /**
+   * Create an attribute definition and return the row ID.  Special/atypical
+   *  in that it doesn't directly return a GlodaAttributeDBDef; we leave that up
+   *  to the caller since they know much more than actually needs to go in the
+   *  database.
+   *
+   * @returns The attribute id allocated to this attribute.
+   */
+  _createAttributeDef(aAttrType, aExtensionName, aAttrName, aParameter) {
+    let attributeId = this._nextAttributeId++;
+
+    let iads = this._insertAttributeDefStatement;
+    iads.bindByIndex(0, attributeId);
+    iads.bindByIndex(1, aAttrType);
+    iads.bindByIndex(2, aExtensionName);
+    iads.bindByIndex(3, aAttrName);
+    this._bindVariant(iads, 4, aParameter);
+
+    iads.executeAsync(this.trackAsync());
+
+    return attributeId;
+  },
+
+  /**
+   * Sync-ly look-up all the attribute definitions, populating our authoritative
+   *  _attributeDBDefss and _attributeIDToDBDefAndParam maps.  (In other words,
+   *  once this method is called, those maps should always be in sync with the
+   *  underlying database.)
+   */
+  getAllAttributes() {
+    let stmt = this._createSyncStatement(
+      "SELECT id, attributeType, extensionName, name, parameter \
+         FROM attributeDefinitions",
+      true
+    );
+
+    // map compound name to the attribute
+    let attribs = {};
+    // map the attribute id to [attribute, parameter] where parameter is null
+    //  in cases where parameter is unused.
+    let idToAttribAndParam = {};
+
+    this._log.info("loading all attribute defs");
+
+    while (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      let rowId = stmt.getInt64(0);
+      let rowAttributeType = stmt.getInt64(1);
+      let rowExtensionName = stmt.getString(2);
+      let rowName = stmt.getString(3);
+      let rowParameter = this._getVariant(stmt, 4);
+
+      let compoundName = rowExtensionName + ":" + rowName;
+
+      let attrib;
+      if (compoundName in attribs) {
+        attrib = attribs[compoundName];
+      } else {
+        attrib = new GlodaAttributeDBDef(
+          this,
+          /* aID */ null,
+          compoundName,
+          rowAttributeType,
+          rowExtensionName,
+          rowName
+        );
+        attribs[compoundName] = attrib;
+      }
+      // if the parameter is null, the id goes on the attribute def, otherwise
+      //  it is a parameter binding and goes in the binding map.
+      if (rowParameter == null) {
+        this._log.debug(compoundName + " primary: " + rowId);
+        attrib._id = rowId;
+        idToAttribAndParam[rowId] = [attrib, null];
+      } else {
+        this._log.debug(
+          compoundName + " binding: " + rowParameter + " = " + rowId
+        );
+        attrib._parameterBindings[rowParameter] = rowId;
+        idToAttribAndParam[rowId] = [attrib, rowParameter];
+      }
+    }
+    stmt.finalize();
+
+    this._log.info("done loading all attribute defs");
+
+    this._attributeDBDefs = attribs;
+    this._attributeIDToDBDefAndParam = idToAttribAndParam;
+  },
+
+  /**
+   * Helper method for GlodaAttributeDBDef to tell us when their bindParameter
+   *  method is called and they have created a new binding (using
+   *  GlodaDatastore._createAttributeDef).  In theory, that method could take
+   *  an additional argument and obviate the need for this method.
+   */
+  reportBinding(aID, aAttrDef, aParamValue) {
+    this._attributeIDToDBDefAndParam[aID] = [aAttrDef, aParamValue];
+  },
+
+  /* ********** Folders ********** */
+  /** next folder (row) id to issue, populated by _getAllFolderMappings. */
+  _nextFolderId: 1,
+
+  get _insertFolderLocationStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO folderLocations (id, folderURI, dirtyStatus, name, \
+                                    indexingPriority) VALUES \
+        (?1, ?2, ?3, ?4, ?5)"
+    );
+    this.__defineGetter__("_insertFolderLocationStatement", () => statement);
+    return this._insertFolderLocationStatement;
+  },
+
+  /**
+   * Authoritative map from folder URI to folder ID.  (Authoritative in the
+   *  sense that this map exactly represents the state of the underlying
+   *  database.  If it does not, it's a bug in updating the database.)
+   */
+  _folderByURI: {},
+  /** Authoritative map from folder ID to folder URI */
+  _folderByID: {},
+
+  /** Initialize our _folderByURI/_folderByID mappings, called by _init(). */
+  _getAllFolderMappings() {
+    let stmt = this._createSyncStatement(
+      "SELECT id, folderURI, dirtyStatus, name, indexingPriority \
+        FROM folderLocations",
+      true
+    );
+
+    while (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      let folderID = stmt.getInt64(0);
+      let folderURI = stmt.getString(1);
+      let dirtyStatus = stmt.getInt32(2);
+      let folderName = stmt.getString(3);
+      let indexingPriority = stmt.getInt32(4);
+
+      let folder = new GlodaFolder(
+        this,
+        folderID,
+        folderURI,
+        dirtyStatus,
+        folderName,
+        indexingPriority
+      );
+
+      this._folderByURI[folderURI] = folder;
+      this._folderByID[folderID] = folder;
+
+      if (folderID >= this._nextFolderId) {
+        this._nextFolderId = folderID + 1;
+      }
+    }
+    stmt.finalize();
+  },
+
+  _folderKnown(aFolder) {
+    let folderURI = aFolder.URI;
+    return folderURI in this._folderByURI;
+  },
+
+  _folderIdKnown(aFolderID) {
+    return aFolderID in this._folderByID;
+  },
+
+  /**
+   * Return the default messaging priority for a folder of this type, based
+   * on the folder's flags. If aAllowSpecialFolderIndexing is true, then
+   * folders suchs as Trash and Junk will be indexed.
+   *
+   * @param {nsIMsgFolder} aFolder
+   * @param {boolean} aAllowSpecialFolderIndexing
+   * @returns {number}
+   */
+  getDefaultIndexingPriority(aFolder, aAllowSpecialFolderIndexing) {
+    let indexingPriority = GlodaFolder.prototype.kIndexingDefaultPriority;
+    // Do not walk into trash/junk folders, unless the user is explicitly
+    //  telling us to do so.
+    let specialFolderFlags =
+      Ci.nsMsgFolderFlags.Trash | Ci.nsMsgFolderFlags.Junk;
+    if (aFolder.isSpecialFolder(specialFolderFlags, true)) {
+      indexingPriority = aAllowSpecialFolderIndexing
+        ? GlodaFolder.prototype.kIndexingDefaultPriority
+        : GlodaFolder.prototype.kIndexingNeverPriority;
+    } else if (
+      aFolder.flags &
+      (Ci.nsMsgFolderFlags.Queue | Ci.nsMsgFolderFlags.Newsgroup)
+      // In unit testing at least folders can be
+      // confusingly labeled ImapPublic when they
+      // should not be.  Or at least I don't think they
+      // should be.  So they're legit for now.
+      // | Ci.nsMsgFolderFlags.ImapPublic
+      // | Ci.nsMsgFolderFlags.ImapOtherUser
+    ) {
+      // Queue folders should always be ignored just because messages should not
+      // spend much time in there.
+      // We hate newsgroups, and public IMAP folders are similar.
+      // Other user IMAP folders should be ignored because it's not this user's
+      // mail.
+      indexingPriority = GlodaFolder.prototype.kIndexingNeverPriority;
+    } else if (aFolder.flags & Ci.nsMsgFolderFlags.Inbox) {
+      indexingPriority = GlodaFolder.prototype.kIndexingInboxPriority;
+    } else if (aFolder.flags & Ci.nsMsgFolderFlags.SentMail) {
+      indexingPriority = GlodaFolder.prototype.kIndexingSentMailPriority;
+    } else if (aFolder.flags & Ci.nsMsgFolderFlags.Favorite) {
+      indexingPriority = GlodaFolder.prototype.kIndexingFavoritePriority;
+    } else if (aFolder.flags & Ci.nsMsgFolderFlags.CheckNew) {
+      indexingPriority = GlodaFolder.prototype.kIndexingCheckNewPriority;
+    }
+
+    return indexingPriority;
+  },
+
+  /**
+   * Map a folder URI to a GlodaFolder instance, creating the mapping if it does
+   *  not yet exist.
+   *
+   * @param aFolder The nsIMsgFolder instance you would like the GlodaFolder
+   *     instance for.
+   * @returns The existing or newly created GlodaFolder instance.
+   */
+  _mapFolder(aFolder) {
+    let folderURI = aFolder.URI;
+    if (folderURI in this._folderByURI) {
+      return this._folderByURI[folderURI];
+    }
+
+    let folderID = this._nextFolderId++;
+
+    // If there's an indexingPriority stored on the folder, just use that.
+    // Otherwise, fall back to the default for folders of this type.
+    let indexingPriority = NaN;
+    try {
+      let pri = aFolder.getStringProperty("indexingPriority"); // Might throw.
+      indexingPriority = parseInt(pri); // Might return NaN.
+    } catch (ex) {}
+    if (isNaN(indexingPriority)) {
+      indexingPriority = this.getDefaultIndexingPriority(aFolder);
+    }
+
+    // If there are messages in the folder, it is filthy.  If there are no
+    //  messages, it can be clean.
+    let dirtyStatus = aFolder.getTotalMessages(false)
+      ? GlodaFolder.prototype.kFolderFilthy
+      : GlodaFolder.prototype.kFolderClean;
+    let folder = new GlodaFolder(
+      this,
+      folderID,
+      folderURI,
+      dirtyStatus,
+      aFolder.prettyName,
+      indexingPriority
+    );
+
+    this._insertFolderLocationStatement.bindByIndex(0, folder.id);
+    this._insertFolderLocationStatement.bindByIndex(1, folder.uri);
+    this._insertFolderLocationStatement.bindByIndex(2, folder.dirtyStatus);
+    this._insertFolderLocationStatement.bindByIndex(3, folder.name);
+    this._insertFolderLocationStatement.bindByIndex(4, folder.indexingPriority);
+    this._insertFolderLocationStatement.executeAsync(this.trackAsync());
+
+    this._folderByURI[folderURI] = folder;
+    this._folderByID[folderID] = folder;
+    this._log.debug("!! mapped " + folder.id + " from " + folderURI);
+    return folder;
+  },
+
+  /**
+   * Map an integer gloda folder ID to the corresponding GlodaFolder instance.
+   *
+   * @param aFolderID The known valid gloda folder ID for which you would like
+   *     a GlodaFolder instance.
+   * @returns The GlodaFolder instance with the given id.  If no such instance
+   *     exists, we will throw an exception.
+   */
+  _mapFolderID(aFolderID) {
+    if (aFolderID === null) {
+      return null;
+    }
+    if (aFolderID in this._folderByID) {
+      return this._folderByID[aFolderID];
+    }
+    throw new Error("Got impossible folder ID: " + aFolderID);
+  },
+
+  /**
+   * Mark the gloda folder as deleted for any outstanding references to it and
+   *  remove it from our tables so we don't hand out any new references.  The
+   *  latter is especially important in the case a folder with the same name
+   *  is created afterwards; we don't want to confuse the new one with the old
+   *  one!
+   */
+  _killGlodaFolderIntoTombstone(aGlodaFolder) {
+    aGlodaFolder._deleted = true;
+    delete this._folderByURI[aGlodaFolder.uri];
+    delete this._folderByID[aGlodaFolder.id];
+  },
+
+  get _updateFolderDirtyStatusStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE folderLocations SET dirtyStatus = ?1 \
+              WHERE id = ?2"
+    );
+    this.__defineGetter__("_updateFolderDirtyStatusStatement", () => statement);
+    return this._updateFolderDirtyStatusStatement;
+  },
+
+  updateFolderDirtyStatus(aFolder) {
+    let ufds = this._updateFolderDirtyStatusStatement;
+    ufds.bindByIndex(1, aFolder.id);
+    ufds.bindByIndex(0, aFolder.dirtyStatus);
+    ufds.executeAsync(this.trackAsync());
+  },
+
+  get _updateFolderIndexingPriorityStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE folderLocations SET indexingPriority = ?1 \
+              WHERE id = ?2"
+    );
+    this.__defineGetter__(
+      "_updateFolderIndexingPriorityStatement",
+      () => statement
+    );
+    return this._updateFolderIndexingPriorityStatement;
+  },
+
+  updateFolderIndexingPriority(aFolder) {
+    let ufip = this._updateFolderIndexingPriorityStatement;
+    ufip.bindByIndex(1, aFolder.id);
+    ufip.bindByIndex(0, aFolder.indexingPriority);
+    ufip.executeAsync(this.trackAsync());
+  },
+
+  get _updateFolderLocationStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE folderLocations SET folderURI = ?1 \
+              WHERE id = ?2"
+    );
+    this.__defineGetter__("_updateFolderLocationStatement", () => statement);
+    return this._updateFolderLocationStatement;
+  },
+
+  /**
+   * Non-recursive asynchronous folder renaming based on the URI.
+   *
+   * @TODO provide a mechanism for recursive folder renames or have a higher
+   *     layer deal with it and remove this note.
+   */
+  renameFolder(aOldFolder, aNewURI) {
+    if (!(aOldFolder.URI in this._folderByURI)) {
+      return;
+    }
+    let folder = this._mapFolder(aOldFolder); // ensure the folder is mapped
+    let oldURI = folder.uri;
+    this._folderByURI[aNewURI] = folder;
+    folder._uri = aNewURI;
+    this._log.info("renaming folder URI " + oldURI + " to " + aNewURI);
+    this._updateFolderLocationStatement.bindByIndex(1, folder.id);
+    this._updateFolderLocationStatement.bindByIndex(0, aNewURI);
+    this._updateFolderLocationStatement.executeAsync(this.trackAsync());
+
+    delete this._folderByURI[oldURI];
+  },
+
+  get _deleteFolderByIDStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM folderLocations WHERE id = ?1"
+    );
+    this.__defineGetter__("_deleteFolderByIDStatement", () => statement);
+    return this._deleteFolderByIDStatement;
+  },
+
+  deleteFolderByID(aFolderID) {
+    let dfbis = this._deleteFolderByIDStatement;
+    dfbis.bindByIndex(0, aFolderID);
+    dfbis.executeAsync(this.trackAsync());
+  },
+
+  /* ********** Conversation ********** */
+  /** The next conversation id to allocate.  Initialize at startup. */
+  _nextConversationId: 1,
+
+  _populateConversationManagedId() {
+    let stmt = this._createSyncStatement(
+      "SELECT MAX(id) FROM conversations",
+      true
+    );
+    if (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      this._nextConversationId = stmt.getInt64(0) + 1;
+    }
+    stmt.finalize();
+  },
+
+  get _insertConversationStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO conversations (id, subject, oldestMessageDate, \
+                                  newestMessageDate) \
+              VALUES (?1, ?2, ?3, ?4)"
+    );
+    this.__defineGetter__("_insertConversationStatement", () => statement);
+    return this._insertConversationStatement;
+  },
+
+  get _insertConversationTextStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO conversationsText (docid, subject) \
+              VALUES (?1, ?2)"
+    );
+    this.__defineGetter__("_insertConversationTextStatement", () => statement);
+    return this._insertConversationTextStatement;
+  },
+
+  /**
+   * Asynchronously create a conversation.
+   */
+  createConversation(aSubject, aOldestMessageDate, aNewestMessageDate) {
+    // create the data row
+    let conversationID = this._nextConversationId++;
+    let ics = this._insertConversationStatement;
+    ics.bindByIndex(0, conversationID);
+    ics.bindByIndex(1, aSubject);
+    if (aOldestMessageDate == null) {
+      ics.bindByIndex(2, null);
+    } else {
+      ics.bindByIndex(2, aOldestMessageDate);
+    }
+    if (aNewestMessageDate == null) {
+      ics.bindByIndex(3, null);
+    } else {
+      ics.bindByIndex(3, aNewestMessageDate);
+    }
+    ics.executeAsync(this.trackAsync());
+
+    // create the fulltext row, using the same rowid/docid
+    let icts = this._insertConversationTextStatement;
+    icts.bindByIndex(0, conversationID);
+    icts.bindByIndex(1, aSubject);
+    icts.executeAsync(this.trackAsync());
+
+    // create it
+    let conversation = new GlodaConversation(
+      this,
+      conversationID,
+      aSubject,
+      aOldestMessageDate,
+      aNewestMessageDate
+    );
+    // it's new! let the collection manager know about it.
+    GlodaCollectionManager.itemsAdded(conversation.NOUN_ID, [conversation]);
+    // return it
+    return conversation;
+  },
+
+  get _deleteConversationByIDStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM conversations WHERE id = ?1"
+    );
+    this.__defineGetter__("_deleteConversationByIDStatement", () => statement);
+    return this._deleteConversationByIDStatement;
+  },
+
+  /**
+   * Asynchronously delete a conversation given its ID.
+   */
+  deleteConversationByID(aConversationID) {
+    let dcbids = this._deleteConversationByIDStatement;
+    dcbids.bindByIndex(0, aConversationID);
+    dcbids.executeAsync(this.trackAsync());
+
+    GlodaCollectionManager.itemsDeleted(GlodaConversation.prototype.NOUN_ID, [
+      aConversationID,
+    ]);
+  },
+
+  _conversationFromRow(aStmt) {
+    let oldestMessageDate, newestMessageDate;
+    if (aStmt.getTypeOfIndex(2) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      oldestMessageDate = null;
+    } else {
+      oldestMessageDate = aStmt.getInt64(2);
+    }
+    if (aStmt.getTypeOfIndex(3) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      newestMessageDate = null;
+    } else {
+      newestMessageDate = aStmt.getInt64(3);
+    }
+    return new GlodaConversation(
+      this,
+      aStmt.getInt64(0),
+      aStmt.getString(1),
+      oldestMessageDate,
+      newestMessageDate
+    );
+  },
+
+  /* ********** Message ********** */
+  /**
+   * Next message id, managed because of our use of asynchronous inserts.
+   * Initialized by _populateMessageManagedId called by _init.
+   *
+   * Start from 32 to leave us all kinds of magical sentinel values at the
+   *  bottom.
+   */
+  _nextMessageId: 32,
+
+  _populateMessageManagedId() {
+    let stmt = this._createSyncStatement("SELECT MAX(id) FROM messages", true);
+    if (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      // 0 gets returned even if there are no messages...
+      let highestSeen = stmt.getInt64(0);
+      if (highestSeen != 0) {
+        this._nextMessageId = highestSeen + 1;
+      }
+    }
+    stmt.finalize();
+  },
+
+  get _insertMessageStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO messages (id, folderID, messageKey, conversationID, date, \
+                             headerMessageID, jsonAttributes, notability) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"
+    );
+    this.__defineGetter__("_insertMessageStatement", () => statement);
+    return this._insertMessageStatement;
+  },
+
+  get _insertMessageTextStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO messagesText (docid, subject, body, attachmentNames, \
+                                 author, recipients) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
+    );
+    this.__defineGetter__("_insertMessageTextStatement", () => statement);
+    return this._insertMessageTextStatement;
+  },
+
+  /**
+   * Create a GlodaMessage with the given properties.  Because this is only half
+   *  of the process of creating a message (the attributes still need to be
+   *  completed), it's on the caller's head to call GlodaCollectionManager's
+   *  itemAdded method once the message is fully created.
+   *
+   * This method uses the async connection, any downstream logic that depends on
+   *  this message actually existing in the database must be done using an
+   *  async query.
+   */
+  createMessage(
+    aFolder,
+    aMessageKey,
+    aConversationID,
+    aDatePRTime,
+    aHeaderMessageID
+  ) {
+    let folderID;
+    if (aFolder != null) {
+      folderID = this._mapFolder(aFolder).id;
+    } else {
+      folderID = null;
+    }
+
+    let messageID = this._nextMessageId++;
+
+    let message = new GlodaMessage(
+      this,
+      messageID,
+      folderID,
+      aMessageKey,
+      aConversationID,
+      /* conversation */ null,
+      aDatePRTime ? new Date(aDatePRTime / 1000) : null,
+      aHeaderMessageID,
+      /* deleted */ false,
+      /* jsonText */ undefined,
+      /* notability*/ 0
+    );
+
+    // We would love to notify the collection manager about the message at this
+    //  point (at least if it's not a ghost), but we can't yet.  We need to wait
+    //  until the attributes have been indexed, which means it's out of our
+    //  hands.  (Gloda.processMessage does it.)
+
+    return message;
+  },
+
+  insertMessage(aMessage) {
+    this._log.debug("insertMessage " + aMessage);
+    let ims = this._insertMessageStatement;
+    ims.bindByIndex(0, aMessage.id);
+    if (aMessage.folderID == null) {
+      ims.bindByIndex(1, null);
+    } else {
+      ims.bindByIndex(1, aMessage.folderID);
+    }
+    if (aMessage.messageKey == null) {
+      ims.bindByIndex(2, null);
+    } else {
+      ims.bindByIndex(2, aMessage.messageKey);
+    }
+    ims.bindByIndex(3, aMessage.conversationID);
+    if (aMessage.date == null) {
+      ims.bindByIndex(4, null);
+    } else {
+      ims.bindByIndex(4, aMessage.date * 1000);
+    }
+    ims.bindByIndex(5, aMessage.headerMessageID);
+    if (aMessage._jsonText) {
+      ims.bindByIndex(6, aMessage._jsonText);
+    } else {
+      ims.bindByIndex(6, null);
+    }
+    ims.bindByIndex(7, aMessage.notability);
+
+    try {
+      ims.executeAsync(this.trackAsync());
+    } catch (ex) {
+      throw new Error(
+        "error executing statement... " +
+          this.asyncConnection.lastError +
+          ": " +
+          this.asyncConnection.lastErrorString +
+          " - " +
+          ex
+      );
+    }
+
+    // we create the full-text row for any message that isn't a ghost,
+    // whether we have the body or not
+    if (aMessage.folderID !== null) {
+      this._insertMessageText(aMessage);
+    }
+  },
+
+  /**
+   * Inserts a full-text row. This should only be called if you're sure you want
+   * to insert a row into the table.
+   */
+  _insertMessageText(aMessage) {
+    if (aMessage._content && aMessage._content.hasContent()) {
+      aMessage._indexedBodyText = aMessage._content.getContentString(true);
+    } else if (aMessage._bodyLines) {
+      aMessage._indexedBodyText = aMessage._bodyLines.join("\n");
+    } else {
+      aMessage._indexedBodyText = null;
+    }
+
+    let imts = this._insertMessageTextStatement;
+    imts.bindByIndex(0, aMessage.id);
+    imts.bindByIndex(1, aMessage._subject);
+    if (aMessage._indexedBodyText == null) {
+      imts.bindByIndex(2, null);
+    } else {
+      imts.bindByIndex(2, aMessage._indexedBodyText);
+    }
+    if (aMessage._attachmentNames === null) {
+      imts.bindByIndex(3, null);
+    } else {
+      imts.bindByIndex(3, aMessage._attachmentNames.join("\n"));
+    }
+
+    // if (aMessage._indexAuthor)
+    imts.bindByIndex(4, aMessage._indexAuthor);
+    // if (aMessage._indexRecipients)
+    imts.bindByIndex(5, aMessage._indexRecipients);
+
+    try {
+      imts.executeAsync(this.trackAsync());
+    } catch (ex) {
+      throw new Error(
+        "error executing fulltext statement... " +
+          this.asyncConnection.lastError +
+          ": " +
+          this.asyncConnection.lastErrorString +
+          " - " +
+          ex
+      );
+    }
+  },
+
+  get _updateMessageStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE messages SET folderID = ?1, \
+                           messageKey = ?2, \
+                           conversationID = ?3, \
+                           date = ?4, \
+                           headerMessageID = ?5, \
+                           jsonAttributes = ?6, \
+                           notability = ?7, \
+                           deleted = ?8 \
+              WHERE id = ?9"
+    );
+    this.__defineGetter__("_updateMessageStatement", () => statement);
+    return this._updateMessageStatement;
+  },
+
+  get _updateMessageTextStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE messagesText SET body = ?1, \
+                               attachmentNames = ?2 \
+              WHERE docid = ?3"
+    );
+
+    this.__defineGetter__("_updateMessageTextStatement", () => statement);
+    return this._updateMessageTextStatement;
+  },
+
+  /**
+   * Update the database row associated with the message. If the message is
+   * not a ghost and has _isNew defined, messagesText is affected.
+   *
+   * aMessage._isNew is currently equivalent to the fact that there is no
+   * full-text row associated with this message, and we work with this
+   * assumption here. Note that if aMessage._isNew is not defined, then
+   * we don't do anything.
+   */
+  updateMessage(aMessage) {
+    this._log.debug("updateMessage " + aMessage);
+    let ums = this._updateMessageStatement;
+    ums.bindByIndex(8, aMessage.id);
+    if (aMessage.folderID === null) {
+      ums.bindByIndex(0, null);
+    } else {
+      ums.bindByIndex(0, aMessage.folderID);
+    }
+    if (aMessage.messageKey === null) {
+      ums.bindByIndex(1, null);
+    } else {
+      ums.bindByIndex(1, aMessage.messageKey);
+    }
+    ums.bindByIndex(2, aMessage.conversationID);
+    if (aMessage.date === null) {
+      ums.bindByIndex(3, null);
+    } else {
+      ums.bindByIndex(3, aMessage.date * 1000);
+    }
+    ums.bindByIndex(4, aMessage.headerMessageID);
+    if (aMessage._jsonText) {
+      ums.bindByIndex(5, aMessage._jsonText);
+    } else {
+      ums.bindByIndex(5, null);
+    }
+    ums.bindByIndex(6, aMessage.notability);
+    ums.bindByIndex(7, aMessage._isDeleted ? 1 : 0);
+
+    ums.executeAsync(this.trackAsync());
+
+    if (aMessage.folderID !== null) {
+      if ("_isNew" in aMessage && aMessage._isNew === true) {
+        this._insertMessageText(aMessage);
+      } else {
+        this._updateMessageText(aMessage);
+      }
+    }
+  },
+
+  /**
+   * Updates the full-text row associated with this message. This only performs
+   * the UPDATE query if the indexed body text has changed, which means that if
+   * the body hasn't changed but the attachments have, we don't update.
+   */
+  _updateMessageText(aMessage) {
+    let newIndexedBodyText;
+    if (aMessage._content && aMessage._content.hasContent()) {
+      newIndexedBodyText = aMessage._content.getContentString(true);
+    } else if (aMessage._bodyLines) {
+      newIndexedBodyText = aMessage._bodyLines.join("\n");
+    } else {
+      newIndexedBodyText = null;
+    }
+
+    // If the body text matches, don't perform an update
+    if (newIndexedBodyText == aMessage._indexedBodyText) {
+      this._log.debug(
+        "in _updateMessageText, skipping update because body matches"
+      );
+      return;
+    }
+
+    aMessage._indexedBodyText = newIndexedBodyText;
+    let umts = this._updateMessageTextStatement;
+    umts.bindByIndex(2, aMessage.id);
+
+    if (aMessage._indexedBodyText == null) {
+      umts.bindByIndex(0, null);
+    } else {
+      umts.bindByIndex(0, aMessage._indexedBodyText);
+    }
+
+    if (aMessage._attachmentNames == null) {
+      umts.bindByIndex(1, null);
+    } else {
+      umts.bindByIndex(1, aMessage._attachmentNames.join("\n"));
+    }
+
+    try {
+      umts.executeAsync(this.trackAsync());
+    } catch (ex) {
+      throw new Error(
+        "error executing fulltext statement... " +
+          this.asyncConnection.lastError +
+          ": " +
+          this.asyncConnection.lastErrorString +
+          " - " +
+          ex
+      );
+    }
+  },
+
+  get _updateMessageLocationStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE messages SET folderID = ?1, messageKey = ?2 WHERE id = ?3"
+    );
+    this.__defineGetter__("_updateMessageLocationStatement", () => statement);
+    return this._updateMessageLocationStatement;
+  },
+
+  /**
+   * Given a list of gloda message ids, and a list of their new message keys in
+   *  the given new folder location, asynchronously update the message's
+   *  database locations.  Also, update the in-memory representations.
+   */
+  updateMessageLocations(
+    aMessageIds,
+    aNewMessageKeys,
+    aDestFolder,
+    aDoNotNotify
+  ) {
+    this._log.debug(
+      "updateMessageLocations:\n" +
+        "ids: " +
+        aMessageIds +
+        "\n" +
+        "keys: " +
+        aNewMessageKeys +
+        "\n" +
+        "dest folder: " +
+        aDestFolder +
+        "\n" +
+        "do not notify?" +
+        aDoNotNotify +
+        "\n"
+    );
+    let statement = this._updateMessageLocationStatement;
+    let destFolderID =
+      typeof aDestFolder == "number"
+        ? aDestFolder
+        : this._mapFolder(aDestFolder).id;
+
+    // map gloda id to the new message key for in-memory rep transform below
+    let cacheLookupMap = {};
+
+    for (let iMsg = 0; iMsg < aMessageIds.length; iMsg++) {
+      let id = aMessageIds[iMsg],
+        msgKey = aNewMessageKeys[iMsg];
+      statement.bindByIndex(0, destFolderID);
+      statement.bindByIndex(1, msgKey);
+      statement.bindByIndex(2, id);
+      statement.executeAsync(this.trackAsync());
+
+      cacheLookupMap[id] = msgKey;
+    }
+
+    // - perform the cache lookup so we can update in-memory representations
+    // found in memory items, and converted to list form for notification
+    let inMemoryItems = {},
+      modifiedItems = [];
+    GlodaCollectionManager.cacheLookupMany(
+      GlodaMessage.prototype.NOUN_ID,
+      cacheLookupMap,
+      inMemoryItems,
+      /* do not cache */ false
+    );
+    for (let glodaId in inMemoryItems) {
+      let glodaMsg = inMemoryItems[glodaId];
+      glodaMsg._folderID = destFolderID;
+      glodaMsg._messageKey = cacheLookupMap[glodaId];
+      modifiedItems.push(glodaMsg);
+    }
+
+    // tell the collection manager about the modified messages so it can update
+    //  any existing views...
+    if (!aDoNotNotify && modifiedItems.length) {
+      GlodaCollectionManager.itemsModified(
+        GlodaMessage.prototype.NOUN_ID,
+        modifiedItems
+      );
+    }
+  },
+
+  get _updateMessageKeyStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE messages SET messageKey = ?1 WHERE id = ?2"
+    );
+    this.__defineGetter__("_updateMessageKeyStatement", () => statement);
+    return this._updateMessageKeyStatement;
+  },
+
+  /**
+   * Update the message keys for the gloda messages with the given id's.  This
+   *  is to be used in response to msgKeyChanged notifications and is similar to
+   *  `updateMessageLocations` except that we do not update the folder and we
+   *  do not perform itemsModified notifications (because message keys are not
+   *  intended to be relevant to the gloda message abstraction).
+   */
+  updateMessageKeys(aMessageIds, aNewMessageKeys) {
+    this._log.debug(
+      "updateMessageKeys:\n" +
+        "ids: " +
+        aMessageIds +
+        "\n" +
+        "keys:" +
+        aNewMessageKeys +
+        "\n"
+    );
+    let statement = this._updateMessageKeyStatement;
+
+    // map gloda id to the new message key for in-memory rep transform below
+    let cacheLookupMap = {};
+
+    for (let iMsg = 0; iMsg < aMessageIds.length; iMsg++) {
+      let id = aMessageIds[iMsg],
+        msgKey = aNewMessageKeys[iMsg];
+      statement.bindByIndex(0, msgKey);
+      statement.bindByIndex(1, id);
+      statement.executeAsync(this.trackAsync());
+
+      cacheLookupMap[id] = msgKey;
+    }
+
+    // - perform the cache lookup so we can update in-memory representations
+    let inMemoryItems = {};
+    GlodaCollectionManager.cacheLookupMany(
+      GlodaMessage.prototype.NOUN_ID,
+      cacheLookupMap,
+      inMemoryItems,
+      /* do not cache */ false
+    );
+    for (let glodaId in inMemoryItems) {
+      let glodaMsg = inMemoryItems[glodaId];
+      glodaMsg._messageKey = cacheLookupMap[glodaId];
+    }
+  },
+
+  /**
+   * Asynchronously mutate message folder id/message keys for the given
+   *  messages, indicating that we are moving them to the target folder, but
+   *  don't yet know their target message keys.
+   *
+   * Updates in-memory representations too.
+   */
+  updateMessageFoldersByKeyPurging(aGlodaIds, aDestFolder) {
+    let destFolderID = this._mapFolder(aDestFolder).id;
+
+    let sqlStr =
+      "UPDATE messages SET folderID = ?1, \
+                                      messageKey = ?2 \
+                   WHERE id IN (" +
+      aGlodaIds.join(", ") +
+      ")";
+    let statement = this._createAsyncStatement(sqlStr, true);
+    statement.bindByIndex(0, destFolderID);
+    statement.bindByIndex(1, null);
+    statement.executeAsync(this.trackAsync());
+    statement.finalize();
+
+    let cached = GlodaCollectionManager.cacheLookupManyList(
+      GlodaMessage.prototype.NOUN_ID,
+      aGlodaIds
+    );
+    for (let id in cached) {
+      let glodaMsg = cached[id];
+      glodaMsg._folderID = destFolderID;
+      glodaMsg._messageKey = null;
+    }
+  },
+
+  _messageFromRow(aRow) {
+    this._log.debug("_messageFromRow " + aRow);
+    let folderId,
+      messageKey,
+      date,
+      jsonText,
+      subject,
+      indexedBodyText,
+      attachmentNames;
+    if (aRow.getTypeOfIndex(1) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      folderId = null;
+    } else {
+      folderId = aRow.getInt64(1);
+    }
+    if (aRow.getTypeOfIndex(2) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      messageKey = null;
+    } else {
+      messageKey = aRow.getInt64(2);
+    }
+    if (aRow.getTypeOfIndex(4) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      date = null;
+    } else {
+      date = new Date(aRow.getInt64(4) / 1000);
+    }
+    if (aRow.getTypeOfIndex(7) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      jsonText = undefined;
+    } else {
+      jsonText = aRow.getString(7);
+    }
+    // only queryFromQuery queries will have these columns
+    if (aRow.numEntries >= 14) {
+      if (aRow.getTypeOfIndex(10) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+        subject = undefined;
+      } else {
+        subject = aRow.getString(10);
+      }
+      if (aRow.getTypeOfIndex(9) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+        indexedBodyText = undefined;
+      } else {
+        indexedBodyText = aRow.getString(9);
+      }
+      if (aRow.getTypeOfIndex(11) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+        attachmentNames = null;
+      } else {
+        attachmentNames = aRow.getString(11);
+        if (attachmentNames) {
+          attachmentNames = attachmentNames.split("\n");
+        } else {
+          attachmentNames = null;
+        }
+      }
+      // we ignore 12, author
+      // we ignore 13, recipients
+    }
+    return new GlodaMessage(
+      this,
+      aRow.getInt64(0),
+      folderId,
+      messageKey,
+      aRow.getInt64(3),
+      null,
+      date,
+      aRow.getString(5),
+      aRow.getInt64(6),
+      jsonText,
+      aRow.getInt64(8),
+      subject,
+      indexedBodyText,
+      attachmentNames
+    );
+  },
+
+  get _updateMessagesMarkDeletedByFolderID() {
+    // When marking deleted clear the folderID and messageKey so that the
+    //  indexing process can reuse it without any location constraints.
+    let statement = this._createAsyncStatement(
+      "UPDATE messages SET folderID = NULL, messageKey = NULL, \
+              deleted = 1 WHERE folderID = ?1"
+    );
+    this.__defineGetter__(
+      "_updateMessagesMarkDeletedByFolderID",
+      () => statement
+    );
+    return this._updateMessagesMarkDeletedByFolderID;
+  },
+
+  /**
+   * Efficiently mark all the messages in a folder as deleted.  Unfortunately,
+   *  we obviously do not know the id's of the messages affected by this which
+   *  complicates in-memory updates.  The options are sending out to the SQL
+   *  database for a list of the message id's or some form of in-memory
+   *  traversal.  I/O costs being what they are, users having a propensity to
+   *  have folders with tens of thousands of messages, and the unlikeliness
+   *  of all of those messages being gloda-memory-resident, we go with the
+   *  in-memory traversal.
+   */
+  markMessagesDeletedByFolderID(aFolderID) {
+    let statement = this._updateMessagesMarkDeletedByFolderID;
+    statement.bindByIndex(0, aFolderID);
+    statement.executeAsync(this.trackAsync());
+
+    // Have the collection manager generate itemsRemoved events for any
+    //  in-memory messages in that folder.
+    GlodaCollectionManager.itemsDeletedByAttribute(
+      GlodaMessage.prototype.NOUN_ID,
+      aMsg => aMsg._folderID == aFolderID
+    );
+  },
+
+  /**
+   * Mark all the gloda messages as deleted blind-fire.  Check if any of the
+   *  messages are known to the collection manager and update them to be deleted
+   *  along with the requisite collection notifications.
+   */
+  markMessagesDeletedByIDs(aMessageIDs) {
+    // When marking deleted clear the folderID and messageKey so that the
+    //  indexing process can reuse it without any location constraints.
+    let sqlString =
+      "UPDATE messages SET folderID = NULL, messageKey = NULL, " +
+      "deleted = 1 WHERE id IN (" +
+      aMessageIDs.join(",") +
+      ")";
+
+    let statement = this._createAsyncStatement(sqlString, true);
+    statement.executeAsync(this.trackAsync());
+    statement.finalize();
+
+    GlodaCollectionManager.itemsDeleted(
+      GlodaMessage.prototype.NOUN_ID,
+      aMessageIDs
+    );
+  },
+
+  get _countDeletedMessagesStatement() {
+    let statement = this._createAsyncStatement(
+      "SELECT COUNT(*) FROM messages WHERE deleted = 1"
+    );
+    this.__defineGetter__("_countDeletedMessagesStatement", () => statement);
+    return this._countDeletedMessagesStatement;
+  },
+
+  /**
+   * Count how many messages are currently marked as deleted in the database.
+   */
+  countDeletedMessages(aCallback) {
+    let cms = this._countDeletedMessagesStatement;
+    cms.executeAsync(new SingletonResultValueHandler(aCallback));
+  },
+
+  get _deleteMessageByIDStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM messages WHERE id = ?1"
+    );
+    this.__defineGetter__("_deleteMessageByIDStatement", () => statement);
+    return this._deleteMessageByIDStatement;
+  },
+
+  get _deleteMessageTextByIDStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM messagesText WHERE docid = ?1"
+    );
+    this.__defineGetter__("_deleteMessageTextByIDStatement", () => statement);
+    return this._deleteMessageTextByIDStatement;
+  },
+
+  /**
+   * Delete a message and its fulltext from the database.  It is assumed that
+   *  the message was already marked as deleted and so is not visible to the
+   *  collection manager and so nothing needs to be done about that.
+   */
+  deleteMessageByID(aMessageID) {
+    let dmbids = this._deleteMessageByIDStatement;
+    dmbids.bindByIndex(0, aMessageID);
+    dmbids.executeAsync(this.trackAsync());
+
+    this.deleteMessageTextByID(aMessageID);
+  },
+
+  deleteMessageTextByID(aMessageID) {
+    let dmt = this._deleteMessageTextByIDStatement;
+    dmt.bindByIndex(0, aMessageID);
+    dmt.executeAsync(this.trackAsync());
+  },
+
+  get _folderCompactionStatement() {
+    let statement = this._createAsyncStatement(
+      "SELECT id, messageKey, headerMessageID FROM messages \
+        WHERE folderID = ?1 AND \
+          messageKey >= ?2 AND +deleted = 0 ORDER BY messageKey LIMIT ?3"
+    );
+    this.__defineGetter__("_folderCompactionStatement", () => statement);
+    return this._folderCompactionStatement;
+  },
+
+  folderCompactionPassBlockFetch(
+    aFolderID,
+    aStartingMessageKey,
+    aLimit,
+    aCallback
+  ) {
+    let fcs = this._folderCompactionStatement;
+    fcs.bindByIndex(0, aFolderID);
+    fcs.bindByIndex(1, aStartingMessageKey);
+    fcs.bindByIndex(2, aLimit);
+    fcs.executeAsync(new CompactionBlockFetcherHandler(aCallback));
+  },
+
+  /* ********** Message Attributes ********** */
+  get _insertMessageAttributeStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO messageAttributes (conversationID, messageID, attributeID, \
+                             value) \
+              VALUES (?1, ?2, ?3, ?4)"
+    );
+    this.__defineGetter__("_insertMessageAttributeStatement", () => statement);
+    return this._insertMessageAttributeStatement;
+  },
+
+  get _deleteMessageAttributeStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM messageAttributes WHERE attributeID = ?1 AND value = ?2 \
+         AND conversationID = ?3 AND messageID = ?4"
+    );
+    this.__defineGetter__("_deleteMessageAttributeStatement", () => statement);
+    return this._deleteMessageAttributeStatement;
+  },
+
+  /**
+   * Insert and remove attributes relating to a GlodaMessage.  This is performed
+   *  inside a pseudo-transaction (we create one if we aren't in one, using
+   *  our _beginTransaction wrapper, but if we are in one, no additional
+   *  meaningful semantics are added).
+   * No attempt is made to verify uniqueness of inserted attributes, either
+   *  against the current database or within the provided list of attributes.
+   *  The caller is responsible for ensuring that unwanted duplicates are
+   *  avoided.
+   *
+   * @param aMessage The GlodaMessage the attributes belong to.  This is used
+   *     to provide the message id and conversation id.
+   * @param aAddDBAttributes A list of attribute tuples to add, where each tuple
+   *     contains an attribute ID and a value.  Lest you forget, an attribute ID
+   *     corresponds to a row in the attribute definition table.  The attribute
+   *     definition table stores the 'parameter' for the attribute, if any.
+   *     (Which is to say, our frequent Attribute-Parameter-Value triple has
+   *     the Attribute-Parameter part distilled to a single attribute id.)
+   * @param aRemoveDBAttributes A list of attribute tuples to remove.
+   */
+  adjustMessageAttributes(aMessage, aAddDBAttributes, aRemoveDBAttributes) {
+    let imas = this._insertMessageAttributeStatement;
+    let dmas = this._deleteMessageAttributeStatement;
+    this._beginTransaction();
+    try {
+      for (let iAttrib = 0; iAttrib < aAddDBAttributes.length; iAttrib++) {
+        let attribValueTuple = aAddDBAttributes[iAttrib];
+
+        imas.bindByIndex(0, aMessage.conversationID);
+        imas.bindByIndex(1, aMessage.id);
+        imas.bindByIndex(2, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null) {
+          imas.bindByIndex(3, 0);
+        } else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1]) {
+          imas.bindByIndex(3, attribValueTuple[1]);
+        } else {
+          imas.bindByIndex(3, attribValueTuple[1]);
+        }
+        imas.executeAsync(this.trackAsync());
+      }
+
+      for (let iAttrib = 0; iAttrib < aRemoveDBAttributes.length; iAttrib++) {
+        let attribValueTuple = aRemoveDBAttributes[iAttrib];
+
+        dmas.bindByIndex(0, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null) {
+          dmas.bindByIndex(1, 0);
+        } else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1]) {
+          dmas.bindByIndex(1, attribValueTuple[1]);
+        } else {
+          dmas.bindByIndex(1, attribValueTuple[1]);
+        }
+        dmas.bindByIndex(2, aMessage.conversationID);
+        dmas.bindByIndex(3, aMessage.id);
+        dmas.executeAsync(this.trackAsync());
+      }
+
+      this._commitTransaction();
+    } catch (ex) {
+      this._log.error("adjustMessageAttributes:", ex);
+      this._rollbackTransaction();
+      throw ex;
+    }
+  },
+
+  get _deleteMessageAttributesByMessageIDStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM messageAttributes WHERE messageID = ?1"
+    );
+    this.__defineGetter__(
+      "_deleteMessageAttributesByMessageIDStatement",
+      () => statement
+    );
+    return this._deleteMessageAttributesByMessageIDStatement;
+  },
+
+  /**
+   * Clear all the message attributes for a given GlodaMessage.  No changes
+   *  are made to the in-memory representation of the message; it is up to the
+   *  caller to ensure that it handles things correctly.
+   *
+   * @param aMessage The GlodaMessage whose database attributes should be
+   *     purged.
+   */
+  clearMessageAttributes(aMessage) {
+    if (aMessage.id != null) {
+      this._deleteMessageAttributesByMessageIDStatement.bindByIndex(
+        0,
+        aMessage.id
+      );
+      this._deleteMessageAttributesByMessageIDStatement.executeAsync(
+        this.trackAsync()
+      );
+    }
+  },
+
+  _stringSQLQuoter(aString) {
+    return "'" + aString.replace(/\'/g, "''") + "'";
+  },
+  _numberQuoter(aNum) {
+    return aNum;
+  },
+
+  /* ===== Generic Attribute Support ===== */
+  adjustAttributes(aItem, aAddDBAttributes, aRemoveDBAttributes) {
+    let nounDef = aItem.NOUN_DEF;
+    let dbMeta = nounDef._dbMeta;
+    if (dbMeta.insertAttrStatement === undefined) {
+      dbMeta.insertAttrStatement = this._createAsyncStatement(
+        "INSERT INTO " +
+          nounDef.attrTableName +
+          " (" +
+          nounDef.attrIDColumnName +
+          ", attributeID, value) " +
+          " VALUES (?1, ?2, ?3)"
+      );
+      // we always create this at the same time (right here), no need to check
+      dbMeta.deleteAttrStatement = this._createAsyncStatement(
+        "DELETE FROM " +
+          nounDef.attrTableName +
+          " WHERE " +
+          " attributeID = ?1 AND value = ?2 AND " +
+          nounDef.attrIDColumnName +
+          " = ?3"
+      );
+    }
+
+    let ias = dbMeta.insertAttrStatement;
+    let das = dbMeta.deleteAttrStatement;
+    this._beginTransaction();
+    try {
+      for (let iAttr = 0; iAttr < aAddDBAttributes.length; iAttr++) {
+        let attribValueTuple = aAddDBAttributes[iAttr];
+
+        ias.bindByIndex(0, aItem.id);
+        ias.bindByIndex(1, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null) {
+          ias.bindByIndex(2, 0);
+        } else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1]) {
+          ias.bindByIndex(2, attribValueTuple[1]);
+        } else {
+          ias.bindByIndex(2, attribValueTuple[1]);
+        }
+        ias.executeAsync(this.trackAsync());
+      }
+
+      for (let iAttr = 0; iAttr < aRemoveDBAttributes.length; iAttr++) {
+        let attribValueTuple = aRemoveDBAttributes[iAttr];
+
+        das.bindByIndex(0, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null) {
+          das.bindByIndex(1, 0);
+        } else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1]) {
+          das.bindByIndex(1, attribValueTuple[1]);
+        } else {
+          das.bindByIndex(1, attribValueTuple[1]);
+        }
+        das.bindByIndex(2, aItem.id);
+        das.executeAsync(this.trackAsync());
+      }
+
+      this._commitTransaction();
+    } catch (ex) {
+      this._log.error("adjustAttributes:", ex);
+      this._rollbackTransaction();
+      throw ex;
+    }
+  },
+
+  clearAttributes(aItem) {
+    let nounDef = aItem.NOUN_DEF;
+    let dbMeta = nounDef._dbMeta;
+    if (dbMeta.clearAttrStatement === undefined) {
+      dbMeta.clearAttrStatement = this._createAsyncStatement(
+        "DELETE FROM " +
+          nounDef.attrTableName +
+          " WHERE " +
+          nounDef.attrIDColumnName +
+          " = ?1"
+      );
+    }
+
+    if (aItem.id != null) {
+      dbMeta.clearAttrstatement.bindByIndex(0, aItem.id);
+      dbMeta.clearAttrStatement.executeAsync(this.trackAsync());
+    }
+  },
+
+  /**
+   * escapeStringForLIKE is only available on statements, and sometimes we want
+   *  to use it before we create our statement, so we create a statement just
+   *  for this reason.
+   */
+  get _escapeLikeStatement() {
+    let statement = this._createAsyncStatement("SELECT 0");
+    this.__defineGetter__("_escapeLikeStatement", () => statement);
+    return this._escapeLikeStatement;
+  },
+
+  *_convertToDBValuesAndGroupByAttributeID(aAttrDef, aValues) {
+    let objectNounDef = aAttrDef.objectNounDef;
+    if (!objectNounDef.usesParameter) {
+      let dbValues = [];
+      for (let iValue = 0; iValue < aValues.length; iValue++) {
+        let value = aValues[iValue];
+        // If the empty set is significant and it's an empty signifier, emit
+        //  the appropriate dbvalue.
+        if (value == null && aAttrDef.emptySetIsSignificant) {
+          yield [this.kEmptySetAttrId, [aAttrDef.id]];
+          // Bail if the only value was us; we don't want to add a
+          //  value-posessing wildcard into the mix.
+          if (aValues.length == 1) {
+            return;
+          }
+          continue;
+        }
+        let dbValue = objectNounDef.toParamAndValue(value)[1];
+        if (dbValue != null) {
+          dbValues.push(dbValue);
+        }
+      }
+      yield [aAttrDef.special ? undefined : aAttrDef.id, dbValues];
+      return;
+    }
+
+    let curParam, attrID, dbValues;
+    let attrDBDef = aAttrDef.dbDef;
+    for (let iValue = 0; iValue < aValues.length; iValue++) {
+      let value = aValues[iValue];
+      // If the empty set is significant and it's an empty signifier, emit
+      //  the appropriate dbvalue.
+      if (value == null && aAttrDef.emptySetIsSignificant) {
+        yield [this.kEmptySetAttrId, [aAttrDef.id]];
+        // Bail if the only value was us; we don't want to add a
+        //  value-posessing wildcard into the mix.
+        if (aValues.length == 1) {
+          return;
+        }
+        continue;
+      }
+      let [dbParam, dbValue] = objectNounDef.toParamAndValue(value);
+      if (curParam === undefined) {
+        curParam = dbParam;
+        attrID = attrDBDef.bindParameter(curParam);
+        if (dbValue != null) {
+          dbValues = [dbValue];
+        } else {
+          dbValues = [];
+        }
+      } else if (curParam == dbParam) {
+        if (dbValue != null) {
+          dbValues.push(dbValue);
+        }
+      } else {
+        yield [attrID, dbValues];
+        curParam = dbParam;
+        attrID = attrDBDef.bindParameter(curParam);
+        if (dbValue != null) {
+          dbValues = [dbValue];
+        } else {
+          dbValues = [];
+        }
+      }
+    }
+    if (dbValues !== undefined) {
+      yield [attrID, dbValues];
+    }
+  },
+
+  *_convertRangesToDBStringsAndGroupByAttributeID(
+    aAttrDef,
+    aValues,
+    aValueColumnName
+  ) {
+    let objectNounDef = aAttrDef.objectNounDef;
+    if (!objectNounDef.usesParameter) {
+      let dbStrings = [];
+      for (let iValue = 0; iValue < aValues.length; iValue++) {
+        let [lowerVal, upperVal] = aValues[iValue];
+        // they both can't be null.  that is the law.
+        if (lowerVal == null) {
+          dbStrings.push(
+            aValueColumnName +
+              " <= " +
+              objectNounDef.toParamAndValue(upperVal)[1]
+          );
+        } else if (upperVal == null) {
+          dbStrings.push(
+            aValueColumnName +
+              " >= " +
+              objectNounDef.toParamAndValue(lowerVal)[1]
+          );
+        } else {
+          // No one is null!
+          dbStrings.push(
+            aValueColumnName +
+              " BETWEEN " +
+              objectNounDef.toParamAndValue(lowerVal)[1] +
+              " AND " +
+              objectNounDef.toParamAndValue(upperVal)[1]
+          );
+        }
+      }
+      yield [aAttrDef.special ? undefined : aAttrDef.id, dbStrings];
+      return;
+    }
+
+    let curParam, attrID, dbStrings;
+    let attrDBDef = aAttrDef.dbDef;
+    for (let iValue = 0; iValue < aValues.length; iValue++) {
+      let [lowerVal, upperVal] = aValues[iValue];
+
+      let dbString, dbParam, lowerDBVal, upperDBVal;
+      // they both can't be null.  that is the law.
+      if (lowerVal == null) {
+        [dbParam, upperDBVal] = objectNounDef.toParamAndValue(upperVal);
+        dbString = aValueColumnName + " <= " + upperDBVal;
+      } else if (upperVal == null) {
+        [dbParam, lowerDBVal] = objectNounDef.toParamAndValue(lowerVal);
+        dbString = aValueColumnName + " >= " + lowerDBVal;
+      } else {
+        // no one is null!
+        [dbParam, lowerDBVal] = objectNounDef.toParamAndValue(lowerVal);
+        dbString =
+          aValueColumnName +
+          " BETWEEN " +
+          lowerDBVal +
+          " AND " +
+          objectNounDef.toParamAndValue(upperVal)[1];
+      }
+
+      if (curParam === undefined) {
+        curParam = dbParam;
+        attrID = attrDBDef.bindParameter(curParam);
+        dbStrings = [dbString];
+      } else if (curParam === dbParam) {
+        dbStrings.push(dbString);
+      } else {
+        yield [attrID, dbStrings];
+        curParam = dbParam;
+        attrID = attrDBDef.bindParameter(curParam);
+        dbStrings = [dbString];
+      }
+    }
+    if (dbStrings !== undefined) {
+      yield [attrID, dbStrings];
+    }
+  },
+
+  /* eslint-disable complexity */
+  /**
+   * Perform a database query given a GlodaQueryClass instance that specifies
+   *  a set of constraints relating to the noun type associated with the query.
+   *  A GlodaCollection is returned containing the results of the look-up.
+   *  By default the collection is "live", and will mutate (generating events to
+   *  its listener) as the state of the database changes.
+   * This functionality is made user/extension visible by the Query's
+   *  getCollection (asynchronous).
+   *
+   * @param [aArgs] See |GlodaQuery.getCollection| for info.
+   */
+  queryFromQuery(
+    aQuery,
+    aListener,
+    aListenerData,
+    aExistingCollection,
+    aMasterCollection,
+    aArgs
+  ) {
+    // when changing this method, be sure that GlodaQuery's testMatch function
+    //  likewise has its changes made.
+    let nounDef = aQuery._nounDef;
+
+    let whereClauses = [];
+    let unionQueries = [aQuery].concat(aQuery._unions);
+    let boundArgs = [];
+
+    // Use the dbQueryValidityConstraintSuffix to provide constraints that
+    //  filter items down to those that are valid for the query mechanism to
+    //  return.  For example, in the case of messages, deleted or ghost
+    //  messages should not be returned by this query layer.  We require
+    //  hand-rolled SQL to do that for now.
+    let validityConstraintSuffix;
+    if (
+      nounDef.dbQueryValidityConstraintSuffix &&
+      !aQuery.options.noDbQueryValidityConstraints
+    ) {
+      validityConstraintSuffix = nounDef.dbQueryValidityConstraintSuffix;
+    } else {
+      validityConstraintSuffix = "";
+    }
+
+    for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
+      let curQuery = unionQueries[iUnion];
+      let selects = [];
+
+      let lastConstraintWasSpecial = false;
+      let curConstraintIsSpecial;
+
+      for (
+        let iConstraint = 0;
+        iConstraint < curQuery._constraints.length;
+        iConstraint++
+      ) {
+        let constraint = curQuery._constraints[iConstraint];
+        let [constraintType, attrDef] = constraint;
+        let constraintValues = constraint.slice(2);
+
+        let tableName, idColumnName, valueColumnName;
+        if (constraintType == GlodaConstants.kConstraintIdIn) {
+          // we don't need any of the next cases' setup code, and we especially
+          //  would prefer that attrDef isn't accessed since it's null for us.
+        } else if (attrDef.special) {
+          tableName = nounDef.tableName;
+          idColumnName = "id"; // canonical id for a table is "id".
+          valueColumnName = attrDef.specialColumnName;
+          curConstraintIsSpecial = true;
+        } else {
+          tableName = nounDef.attrTableName;
+          idColumnName = nounDef.attrIDColumnName;
+          valueColumnName = "value";
+          curConstraintIsSpecial = false;
+        }
+
+        let select = null,
+          test = null;
+        if (constraintType === GlodaConstants.kConstraintIdIn) {
+          // this is somewhat of a trick.  this does mean that this can be the
+          //  only constraint.  Namely, our idiom is:
+          // SELECT * FROM blah WHERE id IN (a INTERSECT b INTERSECT c)
+          //  but if we only have 'a', then that becomes "...IN (a)", and if
+          //  'a' is not a select but a list of id's... tricky, no?
+          select = constraintValues.join(",");
+        } else if (constraintType === GlodaConstants.kConstraintIn) {
+          // @testpoint gloda.datastore.sqlgen.kConstraintIn
+          let clauses = [];
+          for (let [
+            attrID,
+            values,
+          ] of this._convertToDBValuesAndGroupByAttributeID(
+            attrDef,
+            constraintValues
+          )) {
+            let clausePart;
+            if (attrID !== undefined) {
+              clausePart =
+                "(attributeID = " + attrID + (values.length ? " AND " : "");
+            } else {
+              clausePart = "(";
+            }
+            if (values.length) {
+              // strings need to be escaped, we would use ? binding, except
+              //  that gets mad if we have too many strings... so we use our
+              //  own escaping logic.  correctly escaping is easy, but it still
+              //  feels wrong to do it. (just double the quote character...)
+              if (
+                "special" in attrDef &&
+                attrDef.special == GlodaConstants.kSpecialString
+              ) {
+                clausePart +=
+                  valueColumnName +
+                  " IN (" +
+                  values
+                    .map(v => "'" + v.replace(/\'/g, "''") + "'")
+                    .join(",") +
+                  "))";
+              } else {
+                clausePart +=
+                  valueColumnName + " IN (" + values.join(",") + "))";
+              }
+            } else {
+              clausePart += ")";
+            }
+            clauses.push(clausePart);
+          }
+          test = clauses.join(" OR ");
+        } else if (constraintType === GlodaConstants.kConstraintRanges) {
+          // @testpoint gloda.datastore.sqlgen.kConstraintRanges
+          let clauses = [];
+          for (let [
+            attrID,
+            dbStrings,
+          ] of this._convertRangesToDBStringsAndGroupByAttributeID(
+            attrDef,
+            constraintValues,
+            valueColumnName
+          )) {
+            if (attrID !== undefined) {
+              clauses.push(
+                "(attributeID = " +
+                  attrID +
+                  " AND (" +
+                  dbStrings.join(" OR ") +
+                  "))"
+              );
+            } else {
+              clauses.push("(" + dbStrings.join(" OR ") + ")");
+            }
+          }
+          test = clauses.join(" OR ");
+        } else if (constraintType === GlodaConstants.kConstraintEquals) {
+          // @testpoint gloda.datastore.sqlgen.kConstraintEquals
+          let clauses = [];
+          for (let [
+            attrID,
+            values,
+          ] of this._convertToDBValuesAndGroupByAttributeID(
+            attrDef,
+            constraintValues
+          )) {
+            if (attrID !== undefined) {
+              clauses.push(
+                "(attributeID = " +
+                  attrID +
+                  " AND (" +
+                  values.map(_ => valueColumnName + " = ?").join(" OR ") +
+                  "))"
+              );
+            } else {
+              clauses.push(
+                "(" +
+                  values.map(_ => valueColumnName + " = ?").join(" OR ") +
+                  ")"
+              );
+            }
+            boundArgs.push.apply(boundArgs, values);
+          }
+          test = clauses.join(" OR ");
+        } else if (constraintType === GlodaConstants.kConstraintStringLike) {
+          // @testpoint gloda.datastore.sqlgen.kConstraintStringLike
+          let likePayload = "";
+          for (let valuePart of constraintValues) {
+            if (typeof valuePart == "string") {
+              likePayload += this._escapeLikeStatement.escapeStringForLIKE(
+                valuePart,
+                "/"
+              );
+            } else {
+              likePayload += "%";
+            }
+          }
+          test = valueColumnName + " LIKE ? ESCAPE '/'";
+          boundArgs.push(likePayload);
+        } else if (constraintType === GlodaConstants.kConstraintFulltext) {
+          // @testpoint gloda.datastore.sqlgen.kConstraintFulltext
+          let matchStr = constraintValues[0];
+          select =
+            "SELECT docid FROM " +
+            nounDef.tableName +
+            "Text" +
+            " WHERE " +
+            attrDef.specialColumnName +
+            " MATCH ?";
+          boundArgs.push(matchStr);
+        }
+
+        if (curConstraintIsSpecial && lastConstraintWasSpecial && test) {
+          selects[selects.length - 1] += " AND " + test;
+        } else if (select) {
+          selects.push(select);
+        } else if (test) {
+          select =
+            "SELECT " + idColumnName + " FROM " + tableName + " WHERE " + test;
+          selects.push(select);
+        } else {
+          this._log.warn(
+            "Unable to translate constraint of type " +
+              constraintType +
+              " on attribute bound as " +
+              nounDef.name
+          );
+        }
+
+        lastConstraintWasSpecial = curConstraintIsSpecial;
+      }
+
+      if (selects.length) {
+        whereClauses.push(
+          "id IN (" +
+            selects.join(" INTERSECT ") +
+            ")" +
+            validityConstraintSuffix
+        );
+      }
+    }
+
+    let sqlString = "SELECT * FROM " + nounDef.tableName;
+    if (!aQuery.options.noMagic) {
+      if (
+        aQuery.options.noDbQueryValidityConstraints &&
+        nounDef.dbQueryJoinMagicWithNoValidityConstraints
+      ) {
+        sqlString += nounDef.dbQueryJoinMagicWithNoValidityConstraints;
+      } else if (nounDef.dbQueryJoinMagic) {
+        sqlString += nounDef.dbQueryJoinMagic;
+      }
+    }
+
+    if (whereClauses.length) {
+      sqlString += " WHERE (" + whereClauses.join(") OR (") + ")";
+    }
+
+    if (aQuery.options.explicitSQL) {
+      sqlString = aQuery.options.explicitSQL;
+    }
+
+    if (aQuery.options.outerWrapColumns) {
+      sqlString =
+        "SELECT *, " +
+        aQuery.options.outerWrapColumns.join(", ") +
+        " FROM (" +
+        sqlString +
+        ")";
+    }
+
+    if (aQuery._order.length) {
+      let orderClauses = [];
+      for (let colName of aQuery._order) {
+        if (colName.startsWith("-")) {
+          orderClauses.push(colName.substring(1) + " DESC");
+        } else {
+          orderClauses.push(colName + " ASC");
+        }
+      }
+      sqlString += " ORDER BY " + orderClauses.join(", ");
+    }
+
+    if (aQuery._limit) {
+      if (!("limitClauseAlreadyIncluded" in aQuery.options)) {
+        sqlString += " LIMIT ?";
+      }
+      boundArgs.push(aQuery._limit);
+    }
+
+    this._log.debug("QUERY FROM QUERY: " + sqlString + " ARGS: " + boundArgs);
+
+    // if we want to become explicit, replace the query (which has already
+    //  provided our actual SQL query) with an explicit query.  This will be
+    //  what gets attached to the collection in the event we create a new
+    //  collection.  If we are reusing one, we assume that the explicitness,
+    //  if desired, already happened.
+    // (we do not need to pass an argument to the explicitQueryClass constructor
+    //  because it will be passed in to the collection's constructor, which will
+    //  ensure that the collection attribute gets set.)
+    if (aArgs && "becomeExplicit" in aArgs && aArgs.becomeExplicit) {
+      aQuery = new nounDef.explicitQueryClass();
+    } else if (aArgs && "becomeNull" in aArgs && aArgs.becomeNull) {
+      aQuery = new nounDef.nullQueryClass();
+    }
+
+    return this._queryFromSQLString(
+      sqlString,
+      boundArgs,
+      nounDef,
+      aQuery,
+      aListener,
+      aListenerData,
+      aExistingCollection,
+      aMasterCollection
+    );
+  },
+  /* eslint-enable complexity */
+
+  _queryFromSQLString(
+    aSqlString,
+    aBoundArgs,
+    aNounDef,
+    aQuery,
+    aListener,
+    aListenerData,
+    aExistingCollection,
+    aMasterCollection
+  ) {
+    let statement = this._createAsyncStatement(aSqlString, true);
+    for (let [iBinding, bindingValue] of aBoundArgs.entries()) {
+      this._bindVariant(statement, iBinding, bindingValue);
+    }
+
+    let collection;
+    if (aExistingCollection) {
+      collection = aExistingCollection;
+    } else {
+      collection = new GlodaCollection(
+        aNounDef,
+        [],
+        aQuery,
+        aListener,
+        aMasterCollection
+      );
+      GlodaCollectionManager.registerCollection(collection);
+      // we don't want to overwrite the existing listener or its data, but this
+      //  does raise the question about what should happen if we get passed in
+      //  a different listener and/or data.
+      if (aListenerData !== undefined) {
+        collection.data = aListenerData;
+      }
+    }
+    if (aListenerData) {
+      if (collection.dataStack) {
+        collection.dataStack.push(aListenerData);
+      } else {
+        collection.dataStack = [aListenerData];
+      }
+    }
+
+    statement.executeAsync(
+      new QueryFromQueryCallback(statement, aNounDef, collection)
+    );
+    statement.finalize();
+    return collection;
+  },
+
+  /* eslint-disable complexity */
+  loadNounItem(aItem, aReferencesByNounID, aInverseReferencesByNounID) {
+    let attribIDToDBDefAndParam = this._attributeIDToDBDefAndParam;
+
+    let hadDeps = aItem._deps != null;
+    let deps = aItem._deps || {};
+    let hasDeps = false;
+
+    for (let attrib of aItem.NOUN_DEF.specialLoadAttribs) {
+      let objectNounDef = attrib.objectNounDef;
+
+      if (
+        "special" in attrib &&
+        attrib.special === GlodaConstants.kSpecialColumnChildren
+      ) {
+        let invReferences = aInverseReferencesByNounID[objectNounDef.id];
+        if (invReferences === undefined) {
+          invReferences = aInverseReferencesByNounID[objectNounDef.id] = {};
+        }
+        // only contribute if it's not already pending or there
+        if (
+          !(attrib.id in deps) &&
+          aItem[attrib.storageAttributeName] == null
+        ) {
+          // this._log.debug("   Adding inv ref for: " + aItem.id);
+          if (!(aItem.id in invReferences)) {
+            invReferences[aItem.id] = null;
+          }
+          deps[attrib.id] = null;
+          hasDeps = true;
+        }
+      } else if (
+        "special" in attrib &&
+        attrib.special === GlodaConstants.kSpecialColumnParent
+      ) {
+        let references = aReferencesByNounID[objectNounDef.id];
+        if (references === undefined) {
+          references = aReferencesByNounID[objectNounDef.id] = {};
+        }
+        // nothing to contribute if it's already there
+        if (
+          !(attrib.id in deps) &&
+          aItem[attrib.valueStorageAttributeName] == null
+        ) {
+          let parentID = aItem[attrib.idStorageAttributeName];
+          if (!(parentID in references)) {
+            references[parentID] = null;
+          }
+          // this._log.debug("   Adding parent ref for: " +
+          //  aItem[attrib.idStorageAttributeName]);
+          deps[attrib.id] = null;
+          hasDeps = true;
+        } else {
+          this._log.debug(
+            "  paranoia value storage: " +
+              aItem[attrib.valueStorageAttributeName]
+          );
+        }
+      }
+    }
+
+    // bail here if arbitrary values are not allowed, there just is no
+    //  encoded json, or we already had dependencies for this guy, implying
+    //  the json pass has already been performed
+    if (!aItem.NOUN_DEF.allowsArbitraryAttrs || !aItem._jsonText || hadDeps) {
+      if (hasDeps) {
+        aItem._deps = deps;
+      }
+      return hasDeps;
+    }
+
+    // this._log.debug(" load json: " + aItem._jsonText);
+    let jsonDict = JSON.parse(aItem._jsonText);
+    delete aItem._jsonText;
+
+    // Iterate over the attributes on the item
+    for (let attribId in jsonDict) {
+      let jsonValue = jsonDict[attribId];
+      // It is technically impossible for attribute ids to go away at this
+      //  point in time.  This would require someone to monkey around with
+      //  our schema.  But we will introduce this functionality one day, so
+      //  prepare for it now.
+      if (!(attribId in attribIDToDBDefAndParam)) {
+        continue;
+      }
+      // find the attribute definition that corresponds to this key
+      let dbAttrib = attribIDToDBDefAndParam[attribId][0];
+
+      let attrib = dbAttrib.attrDef;
+      // The attribute definition will fail to exist if no one defines the
+      //  attribute anymore.  This can happen for many reasons: an extension
+      //  was uninstalled, an extension was changed and no longer defines the
+      //  attribute, or patches are being applied/unapplied.  Ignore this
+      //  attribute if missing.
+      if (attrib == null) {
+        continue;
+      }
+      let objectNounDef = attrib.objectNounDef;
+
+      // If it has a tableName member but no fromJSON, then it's a persistent
+      //  object that needs to be loaded, which also means we need to hold it in
+      //  a collection owned by our collection.
+      // (If it has a fromJSON method, then it's a special case like
+      //  MimeTypeNoun where it is authoritatively backed by a table but caches
+      //  everything into memory.  There is no case where fromJSON would be
+      //  implemented but we should still be doing database lookups.)
+      if (objectNounDef.tableName && !objectNounDef.fromJSON) {
+        let references = aReferencesByNounID[objectNounDef.id];
+        if (references === undefined) {
+          references = aReferencesByNounID[objectNounDef.id] = {};
+        }
+
+        if (attrib.singular) {
+          if (!(jsonValue in references)) {
+            references[jsonValue] = null;
+          }
+        } else {
+          for (let key in jsonValue) {
+            let anID = jsonValue[key];
+            if (!(anID in references)) {
+              references[anID] = null;
+            }
+          }
+        }
+
+        deps[attribId] = jsonValue;
+        hasDeps = true;
+      } else if (objectNounDef.contributeObjDependencies) {
+        /* if it has custom contribution logic, use it */
+        if (
+          objectNounDef.contributeObjDependencies(
+            jsonValue,
+            aReferencesByNounID,
+            aInverseReferencesByNounID
+          )
+        ) {
+          deps[attribId] = jsonValue;
+          hasDeps = true;
+        } else {
+          // just propagate the value, it's some form of simple sentinel
+          aItem[attrib.boundName] = jsonValue;
+        }
+      } else if (objectNounDef.fromJSON) {
+        // otherwise, the value just needs to be de-persisted, or...
+        if (attrib.singular) {
+          // For consistency with the non-singular case, we don't assign the
+          //  attribute if undefined is returned.
+          let deserialized = objectNounDef.fromJSON(jsonValue, aItem);
+          if (deserialized !== undefined) {
+            aItem[attrib.boundName] = deserialized;
+          }
+        } else {
+          // Convert all the entries in the list filtering out any undefined
+          //  values. (TagNoun will do this if the tag is now dead.)
+          let outList = [];
+          for (let key in jsonValue) {
+            let val = jsonValue[key];
+            let deserialized = objectNounDef.fromJSON(val, aItem);
+            if (deserialized !== undefined) {
+              outList.push(deserialized);
+            }
+          }
+          // Note: It's possible if we filtered things out that this is an empty
+          //  list.  This is acceptable because this is somewhat of an unusual
+          //  case and I don't think we want to further complicate our
+          //  semantics.
+          aItem[attrib.boundName] = outList;
+        }
+      } else {
+        // it's fine as is
+        aItem[attrib.boundName] = jsonValue;
+      }
+    }
+
+    if (hasDeps) {
+      aItem._deps = deps;
+    }
+    return hasDeps;
+  },
+  /* eslint-enable complexity */
+
+  loadNounDeferredDeps(aItem, aReferencesByNounID, aInverseReferencesByNounID) {
+    if (aItem._deps === undefined) {
+      return;
+    }
+
+    let attribIDToDBDefAndParam = this._attributeIDToDBDefAndParam;
+
+    for (let [attribId, jsonValue] of Object.entries(aItem._deps)) {
+      let dbAttrib = attribIDToDBDefAndParam[attribId][0];
+      let attrib = dbAttrib.attrDef;
+
+      let objectNounDef = attrib.objectNounDef;
+      let references = aReferencesByNounID[objectNounDef.id];
+      if (attrib.special) {
+        if (attrib.special === GlodaConstants.kSpecialColumnChildren) {
+          let inverseReferences = aInverseReferencesByNounID[objectNounDef.id];
+          // this._log.info("inverse assignment: " + objectNounDef.id +
+          //    " of " + aItem.id)
+          aItem[attrib.storageAttributeName] = inverseReferences[aItem.id];
+        } else if (attrib.special === GlodaConstants.kSpecialColumnParent) {
+          // this._log.info("parent column load: " + objectNounDef.id +
+          //    " storage value: " + aItem[attrib.idStorageAttributeName]);
+          aItem[attrib.valueStorageAttributeName] =
+            references[aItem[attrib.idStorageAttributeName]];
+        }
+      } else if (objectNounDef.tableName) {
+        if (attrib.singular) {
+          aItem[attrib.boundName] = references[jsonValue];
+        } else {
+          aItem[attrib.boundName] = Object.keys(jsonValue).map(
+            key => references[jsonValue[key]]
+          );
+        }
+      } else if (objectNounDef.contributeObjDependencies) {
+        aItem[attrib.boundName] = objectNounDef.resolveObjDependencies(
+          jsonValue,
+          aReferencesByNounID,
+          aInverseReferencesByNounID
+        );
+      }
+      // there is no other case
+    }
+
+    delete aItem._deps;
+  },
+
+  /* ********** Contact ********** */
+  _nextContactId: 1,
+
+  _populateContactManagedId() {
+    let stmt = this._createSyncStatement("SELECT MAX(id) FROM contacts", true);
+    if (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      this._nextContactId = stmt.getInt64(0) + 1;
+    }
+    stmt.finalize();
+  },
+
+  get _insertContactStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO contacts (id, directoryUUID, contactUUID, name, popularity,\
+                             frecency, jsonAttributes) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"
+    );
+    this.__defineGetter__("_insertContactStatement", () => statement);
+    return this._insertContactStatement;
+  },
+
+  createContact(aDirectoryUUID, aContactUUID, aName, aPopularity, aFrecency) {
+    let contactID = this._nextContactId++;
+    let contact = new GlodaContact(
+      this,
+      contactID,
+      aDirectoryUUID,
+      aContactUUID,
+      aName,
+      aPopularity,
+      aFrecency
+    );
+    return contact;
+  },
+
+  insertContact(aContact) {
+    let ics = this._insertContactStatement;
+    ics.bindByIndex(0, aContact.id);
+    if (aContact.directoryUUID == null) {
+      ics.bindByIndex(1, null);
+    } else {
+      ics.bindByIndex(1, aContact.directoryUUID);
+    }
+    if (aContact.contactUUID == null) {
+      ics.bindByIndex(2, null);
+    } else {
+      ics.bindByIndex(2, aContact.contactUUID);
+    }
+    ics.bindByIndex(3, aContact.name);
+    ics.bindByIndex(4, aContact.popularity);
+    ics.bindByIndex(5, aContact.frecency);
+    if (aContact._jsonText) {
+      ics.bindByIndex(6, aContact._jsonText);
+    } else {
+      ics.bindByIndex(6, null);
+    }
+
+    ics.executeAsync(this.trackAsync());
+
+    return aContact;
+  },
+
+  get _updateContactStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE contacts SET directoryUUID = ?1, \
+                           contactUUID = ?2, \
+                           name = ?3, \
+                           popularity = ?4, \
+                           frecency = ?5, \
+                           jsonAttributes = ?6 \
+                       WHERE id = ?7"
+    );
+    this.__defineGetter__("_updateContactStatement", () => statement);
+    return this._updateContactStatement;
+  },
+
+  updateContact(aContact) {
+    let ucs = this._updateContactStatement;
+    ucs.bindByIndex(6, aContact.id);
+    ucs.bindByIndex(0, aContact.directoryUUID);
+    ucs.bindByIndex(1, aContact.contactUUID);
+    ucs.bindByIndex(2, aContact.name);
+    ucs.bindByIndex(3, aContact.popularity);
+    ucs.bindByIndex(4, aContact.frecency);
+    if (aContact._jsonText) {
+      ucs.bindByIndex(5, aContact._jsonText);
+    } else {
+      ucs.bindByIndex(5, null);
+    }
+
+    ucs.executeAsync(this.trackAsync());
+  },
+
+  _contactFromRow(aRow) {
+    let directoryUUID, contactUUID, jsonText;
+    if (aRow.getTypeOfIndex(1) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      directoryUUID = null;
+    } else {
+      directoryUUID = aRow.getString(1);
+    }
+    if (aRow.getTypeOfIndex(2) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      contactUUID = null;
+    } else {
+      contactUUID = aRow.getString(2);
+    }
+    if (aRow.getTypeOfIndex(6) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL) {
+      jsonText = undefined;
+    } else {
+      jsonText = aRow.getString(6);
+    }
+
+    return new GlodaContact(
+      this,
+      aRow.getInt64(0),
+      directoryUUID,
+      contactUUID,
+      aRow.getString(5),
+      aRow.getInt64(3),
+      aRow.getInt64(4),
+      jsonText
+    );
+  },
+
+  get _selectContactByIDStatement() {
+    let statement = this._createSyncStatement(
+      "SELECT * FROM contacts WHERE id = ?1"
+    );
+    this.__defineGetter__("_selectContactByIDStatement", () => statement);
+    return this._selectContactByIDStatement;
+  },
+
+  /**
+   * Synchronous contact lookup currently only for use by gloda's creation
+   *  of the concept of "me".  It is okay for it to be doing synchronous work
+   *  because it is part of the startup process before any user code could
+   *  have gotten a reference to Gloda, but no one else should do this.
+   */
+  getContactByID(aContactID) {
+    let contact = GlodaCollectionManager.cacheLookupOne(
+      GlodaContact.prototype.NOUN_ID,
+      aContactID
+    );
+
+    if (contact === null) {
+      let scbi = this._selectContactByIDStatement;
+      scbi.bindByIndex(0, aContactID);
+      if (this._syncStep(scbi)) {
+        contact = this._contactFromRow(scbi);
+        GlodaCollectionManager.itemLoaded(contact);
+      }
+      scbi.reset();
+    }
+
+    return contact;
+  },
+
+  /* ********** Identity ********** */
+  /** next identity id, managed for async use reasons. */
+  _nextIdentityId: 1,
+  _populateIdentityManagedId() {
+    let stmt = this._createSyncStatement(
+      "SELECT MAX(id) FROM identities",
+      true
+    );
+    if (stmt.executeStep()) {
+      // no chance of this SQLITE_BUSY on this call
+      this._nextIdentityId = stmt.getInt64(0) + 1;
+    }
+    stmt.finalize();
+  },
+
+  get _insertIdentityStatement() {
+    let statement = this._createAsyncStatement(
+      "INSERT INTO identities (id, contactID, kind, value, description, relay) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
+    );
+    this.__defineGetter__("_insertIdentityStatement", () => statement);
+    return this._insertIdentityStatement;
+  },
+
+  createIdentity(aContactID, aContact, aKind, aValue, aDescription, aIsRelay) {
+    let identityID = this._nextIdentityId++;
+    let iis = this._insertIdentityStatement;
+    iis.bindByIndex(0, identityID);
+    iis.bindByIndex(1, aContactID);
+    iis.bindByIndex(2, aKind);
+    iis.bindByIndex(3, aValue);
+    iis.bindByIndex(4, aDescription);
+    iis.bindByIndex(5, aIsRelay ? 1 : 0);
+    iis.executeAsync(this.trackAsync());
+
+    let identity = new GlodaIdentity(
+      this,
+      identityID,
+      aContactID,
+      aContact,
+      aKind,
+      aValue,
+      aDescription,
+      aIsRelay
+    );
+    GlodaCollectionManager.itemsAdded(identity.NOUN_ID, [identity]);
+    return identity;
+  },
+
+  get _updateIdentityStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE identities SET contactID = ?1, \
+                             kind = ?2, \
+                             value = ?3, \
+                             description = ?4, \
+                             relay = ?5 \
+                         WHERE id = ?6"
+    );
+    this.__defineGetter__("_updateIdentityStatement", () => statement);
+    return this._updateIdentityStatement;
+  },
+
+  updateIdentity(aIdentity) {
+    let ucs = this._updateIdentityStatement;
+    ucs.bindByIndex(5, aIdentity.id);
+    ucs.bindByIndex(0, aIdentity.contactID);
+    ucs.bindByIndex(1, aIdentity.kind);
+    ucs.bindByIndex(2, aIdentity.value);
+    ucs.bindByIndex(3, aIdentity.description);
+    ucs.bindByIndex(4, aIdentity.relay ? 1 : 0);
+
+    ucs.executeAsync(this.trackAsync());
+  },
+
+  _identityFromRow(aRow) {
+    return new GlodaIdentity(
+      this,
+      aRow.getInt64(0),
+      aRow.getInt64(1),
+      null,
+      aRow.getString(2),
+      aRow.getString(3),
+      aRow.getString(4),
+      !!aRow.getInt32(5)
+    );
+  },
+
+  get _selectIdentityByKindValueStatement() {
+    let statement = this._createSyncStatement(
+      "SELECT * FROM identities WHERE kind = ?1 AND value = ?2"
+    );
+    this.__defineGetter__(
+      "_selectIdentityByKindValueStatement",
+      () => statement
+    );
+    return this._selectIdentityByKindValueStatement;
+  },
+
+  /**
+   * Synchronous lookup of an identity by kind and value, only for use by
+   *  the legacy gloda core code that creates a concept of "me".
+   *  Ex: (email, foo@example.com)
+   */
+  getIdentity(aKind, aValue) {
+    let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+      GlodaIdentity.prototype.NOUN_ID,
+      aKind + "@" + aValue
+    );
+
+    let ibkv = this._selectIdentityByKindValueStatement;
+    ibkv.bindByIndex(0, aKind);
+    ibkv.bindByIndex(1, aValue);
+    if (this._syncStep(ibkv)) {
+      identity = this._identityFromRow(ibkv);
+      GlodaCollectionManager.itemLoaded(identity);
+    }
+    ibkv.reset();
+
+    return identity;
+  },
+};
+GlodaAttributeDBDef.prototype._datastore = GlodaDatastore;
+GlodaConversation.prototype._datastore = GlodaDatastore;
+GlodaFolder.prototype._datastore = GlodaDatastore;
+GlodaMessage.prototype._datastore = GlodaDatastore;
+GlodaContact.prototype._datastore = GlodaDatastore;
+GlodaIdentity.prototype._datastore = GlodaDatastore;
diff --git a/comm/mailnews/db/gloda/modules/GlodaExplicitAttr.jsm b/comm/mailnews/db/gloda/modules/GlodaExplicitAttr.jsm
new file mode 100644
index 0000000000..7a10b4112e
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaExplicitAttr.jsm
@@ -0,0 +1,188 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides the "explicit attribute" provider for messages.  It is
+ *  concerned with attributes that are the result of user actions.  For example,
+ *  whether a message is starred (flagged), message tags, whether it is
+ *  read/unread, etc.
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaExplicitAttr"];
+
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { TagNoun } = ChromeUtils.import("resource:///modules/gloda/NounTag.jsm");
+
+/**
+ * @namespace Explicit attribute provider.  Indexes/defines attributes that are
+ *  explicitly a result of user action.  This dubiously includes marking a
+ *  message as read.
+ */
+var GlodaExplicitAttr = {
+  providerName: "gloda.explattr",
+  strings: Services.strings.createBundle(
+    "chrome://messenger/locale/gloda.properties"
+  ),
+  _log: null,
+
+  init() {
+    this._log = console.createInstance({
+      prefix: "gloda.explattr",
+      maxLogLevel: "Warn",
+      maxLogLevelPref: "gloda.loglevel",
+    });
+
+    try {
+      this.defineAttributes();
+    } catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  /** Boost for starred messages. */
+  NOTABILITY_STARRED: 16,
+  /** Boost for tagged messages, first tag. */
+  NOTABILITY_TAGGED_FIRST: 8,
+  /** Boost for tagged messages, each additional tag. */
+  NOTABILITY_TAGGED_ADDL: 1,
+
+  defineAttributes() {
+    // Tag
+    this._attrTag = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "tag",
+      bindName: "tags",
+      singular: false,
+      emptySetIsSignificant: true,
+      facet: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_TAG,
+      parameterNoun: null,
+      // Property change notifications that we care about:
+      propertyChanges: ["keywords"],
+    }); // not-tested
+
+    // Star
+    this._attrStar = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "star",
+      bindName: "starred",
+      singular: true,
+      facet: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+    // Read/Unread
+    this._attrRead = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "read",
+      // Make the message query-able but without using the database.
+      canQuery: "truthy-but-not-true",
+      singular: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this message been replied to by the user.
+     */
+    this._attrRepliedTo = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "repliedTo",
+      singular: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this user forwarded this message to someone.
+     */
+    this._attrForwarded = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "forwarded",
+      singular: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+  },
+
+  *process(aGlodaMessage, aRawReps, aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+
+    aGlodaMessage.starred = aMsgHdr.isFlagged;
+    if (aGlodaMessage.starred) {
+      aGlodaMessage.notability += this.NOTABILITY_STARRED;
+    }
+
+    aGlodaMessage.read = aMsgHdr.isRead;
+
+    let flags = aMsgHdr.flags;
+    aGlodaMessage.repliedTo = Boolean(flags & Ci.nsMsgMessageFlags.Replied);
+    aGlodaMessage.forwarded = Boolean(flags & Ci.nsMsgMessageFlags.Forwarded);
+
+    let tags = (aGlodaMessage.tags = []);
+
+    // -- Tag
+    // build a map of the keywords
+    let keywords = aMsgHdr.getStringProperty("keywords");
+    let keywordList = keywords.split(" ");
+    let keywordMap = {};
+    for (let iKeyword = 0; iKeyword < keywordList.length; iKeyword++) {
+      let keyword = keywordList[iKeyword];
+      keywordMap[keyword] = true;
+    }
+
+    let tagArray = TagNoun.getAllTags();
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      if (tag.key in keywordMap) {
+        tags.push(tag);
+      }
+    }
+
+    if (tags.length) {
+      aGlodaMessage.notability +=
+        this.NOTABILITY_TAGGED_FIRST +
+        (tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from process().  Arguably process should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score(aMessage, aContext) {
+    let score = 0;
+    if (aMessage.starred) {
+      score += this.NOTABILITY_STARRED;
+    }
+    if (aMessage.tags.length) {
+      score +=
+        this.NOTABILITY_TAGGED_FIRST +
+        (aMessage.tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+    }
+    return score;
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaFundAttr.jsm b/comm/mailnews/db/gloda/modules/GlodaFundAttr.jsm
new file mode 100644
index 0000000000..364ea61bb0
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaFundAttr.jsm
@@ -0,0 +1,947 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaFundAttr"];
+
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+const { GlodaUtils } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaUtils.jsm"
+);
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaAttachment } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaDataModel.jsm"
+);
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { MimeTypeNoun } = ChromeUtils.import(
+  "resource:///modules/gloda/NounMimetype.jsm"
+);
+const { GlodaContent } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaContent.jsm"
+);
+
+/**
+ * @namespace The Gloda Fundamental Attribute provider is a special attribute
+ *  provider; it provides attributes that the rest of the providers should be
+ *  able to assume exist.  Also, it may end up accessing things at a lower level
+ *  than most extension providers should do.  In summary, don't mimic this code
+ *  unless you won't complain when your code breaks.
+ */
+var GlodaFundAttr = {
+  providerName: "gloda.fundattr",
+  strings: Services.strings.createBundle(
+    "chrome://messenger/locale/gloda.properties"
+  ),
+  _log: null,
+
+  init() {
+    this._log = console.createInstance({
+      prefix: "gloda.fundattr",
+      maxLogLevel: "Warn",
+      maxLogLevelPref: "gloda.loglevel",
+    });
+
+    try {
+      this.defineAttributes();
+    } catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  POPULARITY_FROM_ME_TO: 10,
+  POPULARITY_FROM_ME_CC: 4,
+  POPULARITY_FROM_ME_BCC: 3,
+  POPULARITY_TO_ME: 5,
+  POPULARITY_CC_ME: 1,
+  POPULARITY_BCC_ME: 1,
+
+  /** Boost for messages 'I' sent */
+  NOTABILITY_FROM_ME: 10,
+  /** Boost for messages involving 'me'. */
+  NOTABILITY_INVOLVING_ME: 1,
+  /** Boost for message from someone in 'my' address book. */
+  NOTABILITY_FROM_IN_ADDR_BOOK: 10,
+  /** Boost for the first person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_FIRST: 8,
+  /** Boost for each additional person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_ADDL: 2,
+
+  defineAttributes() {
+    /* ***** Conversations ***** */
+    // conversation: subjectMatches
+    this._attrConvSubject = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [GlodaConstants.NOUN_CONVERSATION],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    /* ***** Messages ***** */
+    // folder
+    this._attrFolder = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "folder",
+      singular: true,
+      facet: true,
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "folderID",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FOLDER,
+    }); // tested-by: test_attributes_fundamental
+    this._attrAccount = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "account",
+      canQuery: "memory",
+      singular: true,
+      facet: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_ACCOUNT,
+    });
+    this._attrMessageKey = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "messageKey",
+      singular: true,
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "messageKey",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_NUMBER,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+
+    // We need to surface the deleted attribute for querying, but there is no
+    //  reason for user code, so let's call it "_deleted" rather than deleted.
+    // (In fact, our validity constraints require a special query formulation
+    //  that user code should have no clue exists.  That's right user code,
+    //  that's a dare.)
+    Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "_deleted",
+      singular: true,
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "deleted",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_NUMBER,
+    });
+
+    // -- fulltext search helpers
+    // fulltextMatches.  Match over message subject, body, and attachments
+    // @testpoint gloda.noun.message.attr.fulltextMatches
+    this._attrFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "fulltextMatches",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "messagesText",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // subjectMatches.  Fulltext match on subject
+    // @testpoint gloda.noun.message.attr.subjectMatches
+    this._attrSubjectText = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // bodyMatches. super-synthetic full-text matching...
+    // @testpoint gloda.noun.message.attr.bodyMatches
+    this._attrBody = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "bodyMatches",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "body",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // attachmentNamesMatch
+    // @testpoint gloda.noun.message.attr.attachmentNamesMatch
+    this._attrAttachmentNames = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "attachmentNamesMatch",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "attachmentNames",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // @testpoint gloda.noun.message.attr.authorMatches
+    this._attrAuthorFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "authorMatches",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "author",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // @testpoint gloda.noun.message.attr.recipientsMatch
+    this._attrRecipientsFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "recipientsMatch",
+      singular: true,
+      special: GlodaConstants.kSpecialFulltext,
+      specialColumnName: "recipients",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_FULLTEXT,
+    });
+
+    // --- synthetic stuff for some reason
+    // conversation
+    // @testpoint gloda.noun.message.attr.conversation
+    this._attrConversation = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "conversation",
+      singular: true,
+      special: GlodaConstants.kSpecialColumnParent,
+      specialColumnName: "conversationID",
+      idStorageAttributeName: "_conversationID",
+      valueStorageAttributeName: "_conversation",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_CONVERSATION,
+      canQuery: true,
+    });
+
+    // --- Fundamental
+    // From
+    this._attrFrom = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "from",
+      singular: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // tested-by: test_attributes_fundamental
+    // To
+    this._attrTo = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "to",
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // tested-by: test_attributes_fundamental
+    // Cc
+    this._attrCc = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "cc",
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // not-tested
+    /**
+     * Bcc'ed recipients; only makes sense for sent messages.
+     */
+    this._attrBcc = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "bcc",
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // not-tested
+
+    // Date.  now lives on the row.
+    this._attrDate = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "date",
+      singular: true,
+      facet: {
+        type: "date",
+      },
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "date",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_DATE,
+    }); // tested-by: test_attributes_fundamental
+
+    // Header message ID.
+    this._attrHeaderMessageID = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "headerMessageID",
+      singular: true,
+      special: GlodaConstants.kSpecialString,
+      specialColumnName: "headerMessageID",
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_STRING,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+
+    // Attachment MIME Types
+    this._attrAttachmentTypes = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "attachmentTypes",
+      singular: false,
+      emptySetIsSignificant: true,
+      facet: {
+        type: "default",
+        // This will group the MIME types by their category.
+        groupIdAttr: "category",
+        queryHelper: "Category",
+      },
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_MIME_TYPE,
+    });
+
+    // Attachment infos
+    this._attrIsEncrypted = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "isEncrypted",
+      singular: true,
+      emptySetIsSignificant: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_NUMBER,
+    });
+
+    // Attachment infos
+    this._attrAttachmentInfos = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "attachmentInfos",
+      singular: false,
+      emptySetIsSignificant: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_ATTACHMENT,
+    });
+
+    // --- Optimization
+    /**
+     * Involves means any of from/to/cc/bcc.  The queries get ugly enough
+     *  without this that it seems to justify the cost, especially given the
+     *  frequent use case.  (In fact, post-filtering for the specific from/to/cc
+     *  is probably justifiable rather than losing this attribute...)
+     */
+    this._attrInvolves = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrOptimization,
+      attributeName: "involves",
+      singular: false,
+      facet: {
+        type: "default",
+        /**
+         * Filter out 'me', as we have other facets that deal with that, and the
+         *  'me' identities are so likely that they distort things.
+         *
+         * @returns true if the identity is not one of my identities, false if it
+         *   is.
+         */
+        filter(aItem) {
+          return !(aItem.id in Gloda.myIdentities);
+        },
+      },
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // not-tested
+
+    /**
+     * Any of to/cc/bcc.
+     */
+    this._attrRecipients = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrOptimization,
+      attributeName: "recipients",
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // not-tested
+
+    // From Me (To/Cc/Bcc)
+    this._attrFromMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrOptimization,
+      attributeName: "fromMe",
+      singular: false,
+      // The interesting thing to a facet is whether the message is from me.
+      facet: {
+        type: "nonempty?",
+      },
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_PARAM_IDENTITY,
+    }); // not-tested
+    // To/Cc/Bcc Me
+    this._attrToMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "toMe",
+      // The interesting thing to a facet is whether the message is to me.
+      facet: {
+        type: "nonempty?",
+      },
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_PARAM_IDENTITY,
+    }); // not-tested
+
+    // -- Mailing List
+    // Non-singular, but a hard call.  Namely, it is obvious that a message can
+    //  be addressed to multiple mailing lists.  However, I don't see how you
+    //  could receive a message with more than one set of List-* headers,
+    //  since each list-serve would each send you a copy.  Based on our current
+    //  decision to treat each physical message as separate, it almost seems
+    //  right to limit the list attribute to the copy that originated at the
+    //  list.  That may sound entirely wrong, but keep in mind that until we
+    //  have seen a message from the list with the List headers, we can't
+    //  definitely know it's a mailing list (although heuristics could take us
+    //  pretty far).  As such, the quasi-singular thing is appealing.
+    // Of course, the reality is that we really want to know if a message was
+    //  sent to multiple mailing lists and be able to query on that.
+    //  Additionally, our implicit-to logic needs to work on messages that
+    //  weren't relayed by the list-serve, especially messages sent to the list
+    //  by the user.
+    this._attrList = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "mailing-list",
+      bindName: "mailingLists",
+      singular: false,
+      emptySetIsSignificant: true,
+      facet: true,
+      subjectNouns: [GlodaConstants.NOUN_MESSAGE],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // not-tested, not-implemented
+  },
+
+  RE_LIST_POST: /<mailto:([^>]+)>/,
+
+  /**
+   *
+   * Specializations:
+   * - Mailing Lists.  Replies to a message on a mailing list frequently only
+   *   have the list-serve as the 'to', so we try to generate a synthetic 'to'
+   *   based on the author of the parent message when possible.  (The 'possible'
+   *   part is that we may not have a copy of the parent message at the time of
+   *   processing.)
+   * - Newsgroups.  Same deal as mailing lists.
+   */
+  *process(aGlodaMessage, aRawReps, aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+    let aMimeMsg = aRawReps.mime;
+
+    // -- From
+    // Let's use replyTo if available.
+    // er, since we are just dealing with mailing lists for now, forget the
+    //  reply-to...
+    // TODO: deal with default charset issues
+    let author = null;
+    /*
+    try {
+      author = aMsgHdr.getStringProperty("replyTo");
+    }
+    catch (ex) {
+    }
+    */
+    if (author == null || author == "") {
+      author = aMsgHdr.author;
+    }
+
+    let normalizedListPost = "";
+    if (aMimeMsg && aMimeMsg.has("list-post")) {
+      let match = this.RE_LIST_POST.exec(aMimeMsg.get("list-post"));
+      if (match) {
+        normalizedListPost = "<" + match[1] + ">";
+      }
+    }
+
+    // Do not use the MIME decoded variants of any of the email addresses
+    //  because if name is encoded and has a comma in it, it will break the
+    //  address parser (which already knows how to do the decoding anyways).
+    let [
+      authorIdentities,
+      toIdentities,
+      ccIdentities,
+      bccIdentities,
+      listIdentities,
+    ] = yield aCallbackHandle.pushAndGo(
+      Gloda.getOrCreateMailIdentities(
+        aCallbackHandle,
+        author,
+        aMsgHdr.recipients,
+        aMsgHdr.ccList,
+        aMsgHdr.bccList,
+        normalizedListPost
+      )
+    );
+
+    if (authorIdentities.length != 1) {
+      throw new Gloda.BadItemContentsError(
+        "Message with subject '" +
+          aMsgHdr.mime2DecodedSubject +
+          "' somehow lacks a valid author.  Bailing."
+      );
+    }
+    let authorIdentity = authorIdentities[0];
+    aGlodaMessage.from = authorIdentity;
+
+    // -- To, Cc, Bcc
+    aGlodaMessage.to = toIdentities;
+    aGlodaMessage.cc = ccIdentities;
+    aGlodaMessage.bcc = bccIdentities;
+
+    // -- Mailing List
+    if (listIdentities.length) {
+      aGlodaMessage.mailingLists = listIdentities;
+    }
+
+    let findIsEncrypted = x =>
+      x.isEncrypted || (x.parts ? x.parts.some(findIsEncrypted) : false);
+
+    // -- Encryption
+    aGlodaMessage.isEncrypted = false;
+    if (aMimeMsg) {
+      aGlodaMessage.isEncrypted = findIsEncrypted(aMimeMsg);
+    }
+
+    // -- Attachments
+    if (aMimeMsg) {
+      // nsParseMailbox.cpp puts the attachment flag on msgHdrs as soon as it
+      // finds a multipart/mixed part. This is a good heuristic, but if it turns
+      // out the part has no filename, then we don't treat it as an attachment.
+      // We just streamed the message, and we have all the information to figure
+      // that out, so now is a good place to clear the flag if needed.
+      let attachmentTypes = new Set();
+      for (let attachment of aMimeMsg.allAttachments) {
+        // getMimeType expects the content type to contain at least a "/".
+        if (!attachment.contentType.includes("/")) {
+          continue;
+        }
+        attachmentTypes.add(MimeTypeNoun.getMimeType(attachment.contentType));
+      }
+      if (attachmentTypes.size) {
+        aGlodaMessage.attachmentTypes = Array.from(attachmentTypes);
+      }
+
+      let aMsgHdr = aRawReps.header;
+      let wasStreamed =
+        aMsgHdr &&
+        !aGlodaMessage.isEncrypted &&
+        (aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline ||
+          aMsgHdr.folder instanceof Ci.nsIMsgLocalMailFolder);
+
+      // Clear the flag if it turns out there's no attachment after all and we
+      // streamed completely the message (if we didn't, then we have no
+      // knowledge of attachments, unless bug 673370 is fixed).
+      if (wasStreamed && !aMimeMsg.allAttachments.length) {
+        aMsgHdr.markHasAttachments(false);
+      }
+
+      // This is not the same kind of attachments as above. Now, we want to
+      // provide convenience attributes to Gloda consumers, so that they can run
+      // through the list of attachments of a given message, to possibly build a
+      // visualization on top of it. We still reject bogus mime types, which
+      // means yencode won't be supported. Oh, I feel really bad.
+      let attachmentInfos = [];
+      for (let att of aMimeMsg.allUserAttachments) {
+        attachmentInfos.push(
+          this.glodaAttFromMimeAtt(aRawReps.trueGlodaRep, att)
+        );
+      }
+      aGlodaMessage.attachmentInfos = attachmentInfos;
+    }
+
+    // TODO: deal with mailing lists, including implicit-to.  this will require
+    //  convincing the indexer to pass us in the previous message if it is
+    //  available.  (which we'll simply pass to everyone... it can help body
+    //  logic for quoting purposes, etc. too.)
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  glodaAttFromMimeAtt(aGlodaMessage, aAtt) {
+    // So we don't want to store the URL because it can change over time if
+    // the message is moved. What we do is store the full URL if it's a
+    // detached attachment, otherwise just keep the part information, and
+    // rebuild the URL according to where the message is sitting.
+    let part, externalUrl;
+    if (aAtt.isExternal) {
+      externalUrl = aAtt.url;
+    } else {
+      let matches = aAtt.url.match(GlodaUtils.PART_RE);
+      if (matches && matches.length) {
+        part = matches[1];
+      } else {
+        this._log.error("Error processing attachment: " + aAtt.url);
+      }
+    }
+    return new GlodaAttachment(
+      aGlodaMessage,
+      aAtt.name,
+      aAtt.contentType,
+      aAtt.size,
+      part,
+      externalUrl,
+      aAtt.isExternal
+    );
+  },
+
+  *optimize(aGlodaMessage, aRawReps, aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+
+    // for simplicity this is used for both involves and recipients
+    let involvesIdentities = {};
+    let involves = aGlodaMessage.involves || [];
+    let recipients = aGlodaMessage.recipients || [];
+
+    // 'me' specialization optimizations
+    let toMe = aGlodaMessage.toMe || [];
+    let fromMe = aGlodaMessage.fromMe || [];
+
+    let myIdentities = Gloda.myIdentities; // needless optimization?
+    let authorIdentity = aGlodaMessage.from;
+    let isFromMe = authorIdentity.id in myIdentities;
+
+    // The fulltext search column for the author.  We want to have in here:
+    // - The e-mail address and display name as enclosed on the message.
+    // - The name per the address book card for this e-mail address, if we have
+    //   one.
+    aGlodaMessage._indexAuthor = aMsgHdr.mime2DecodedAuthor;
+    // The fulltext search column for the recipients. (same deal)
+    aGlodaMessage._indexRecipients = aMsgHdr.mime2DecodedRecipients;
+
+    if (isFromMe) {
+      aGlodaMessage.notability += this.NOTABILITY_FROM_ME;
+    } else {
+      let authorDisplayName = MailServices.ab.cardForEmailAddress(
+        authorIdentity.value
+      )?.displayName;
+      if (authorDisplayName !== null) {
+        aGlodaMessage.notability += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+        // @testpoint gloda.noun.message.attr.authorMatches
+        aGlodaMessage._indexAuthor += " " + authorDisplayName;
+      }
+    }
+
+    involves.push(authorIdentity);
+    involvesIdentities[authorIdentity.id] = true;
+
+    let involvedAddrBookCount = 0;
+
+    for (let toIdentity of aGlodaMessage.to) {
+      if (!(toIdentity.id in involvesIdentities)) {
+        involves.push(toIdentity);
+        recipients.push(toIdentity);
+        involvesIdentities[toIdentity.id] = true;
+        let toDisplayName = MailServices.ab.cardForEmailAddress(
+          toIdentity.value
+        )?.displayName;
+        if (toDisplayName !== null) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += " " + toDisplayName;
+        }
+      }
+
+      // optimization attribute to-me ('I' am the parameter)
+      if (toIdentity.id in myIdentities) {
+        toMe.push([toIdentity, authorIdentity]);
+        if (aIsNew) {
+          authorIdentity.contact.popularity += this.POPULARITY_TO_ME;
+        }
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, toIdentity]);
+        // also, popularity
+        if (aIsNew) {
+          toIdentity.contact.popularity += this.POPULARITY_FROM_ME_TO;
+        }
+      }
+    }
+    for (let ccIdentity of aGlodaMessage.cc) {
+      if (!(ccIdentity.id in involvesIdentities)) {
+        involves.push(ccIdentity);
+        recipients.push(ccIdentity);
+        involvesIdentities[ccIdentity.id] = true;
+        let ccDisplayName = MailServices.ab.cardForEmailAddress(
+          ccIdentity.value
+        )?.displayName;
+        if (ccDisplayName !== null) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += " " + ccDisplayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (ccIdentity.id in myIdentities) {
+        toMe.push([ccIdentity, authorIdentity]);
+        if (aIsNew) {
+          authorIdentity.contact.popularity += this.POPULARITY_CC_ME;
+        }
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, ccIdentity]);
+        // also, popularity
+        if (aIsNew) {
+          ccIdentity.contact.popularity += this.POPULARITY_FROM_ME_CC;
+        }
+      }
+    }
+    // just treat bcc like cc; the intent is the same although the exact
+    //  semantics differ.
+    for (let bccIdentity of aGlodaMessage.bcc) {
+      if (!(bccIdentity.id in involvesIdentities)) {
+        involves.push(bccIdentity);
+        recipients.push(bccIdentity);
+        involvesIdentities[bccIdentity.id] = true;
+        let bccDisplayName = MailServices.ab.cardForEmailAddress(
+          bccIdentity.value
+        )?.displayName;
+        if (bccDisplayName !== null) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += " " + bccDisplayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (bccIdentity.id in myIdentities) {
+        toMe.push([bccIdentity, authorIdentity]);
+        if (aIsNew) {
+          authorIdentity.contact.popularity += this.POPULARITY_BCC_ME;
+        }
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, bccIdentity]);
+        // also, popularity
+        if (aIsNew) {
+          bccIdentity.contact.popularity += this.POPULARITY_FROM_ME_BCC;
+        }
+      }
+    }
+
+    if (involvedAddrBookCount) {
+      aGlodaMessage.notability +=
+        this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+    }
+
+    aGlodaMessage.involves = involves;
+    aGlodaMessage.recipients = recipients;
+    if (toMe.length) {
+      aGlodaMessage.toMe = toMe;
+      aGlodaMessage.notability += this.NOTABILITY_INVOLVING_ME;
+    }
+    if (fromMe.length) {
+      aGlodaMessage.fromMe = fromMe;
+    }
+
+    // Content
+    if (aRawReps.bodyLines) {
+      aGlodaMessage._content = aRawReps.content = new GlodaContent();
+      if (this.contentWhittle({}, aRawReps.bodyLines, aGlodaMessage._content)) {
+        // we were going to do something here?
+      }
+    } else {
+      aRawReps.content = null;
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from optimize().  Arguably optimize should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score(aMessage, aContext) {
+    let score = 0;
+
+    let authorIdentity = aMessage.from;
+    if (authorIdentity.id in Gloda.myIdentities) {
+      score += this.NOTABILITY_FROM_ME;
+    } else if (authorIdentity.inAddressBook) {
+      score += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+    }
+    if (aMessage.toMe) {
+      score += this.NOTABILITY_INVOLVING_ME;
+    }
+
+    let involvedAddrBookCount = 0;
+    for (let identity of aMessage.to) {
+      if (identity.inAddressBook) {
+        involvedAddrBookCount++;
+      }
+    }
+    for (let identity of aMessage.cc) {
+      if (identity.inAddressBook) {
+        involvedAddrBookCount++;
+      }
+    }
+    if (involvedAddrBookCount) {
+      score +=
+        this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+    }
+    return score;
+  },
+
+  _countQuoteDepthAndNormalize(aLine) {
+    let count = 0;
+    let lastStartOffset = 0;
+
+    for (let i = 0; i < aLine.length; i++) {
+      let c = aLine[i];
+      if (c == ">") {
+        count++;
+        lastStartOffset = i + 1;
+      } else if (c != " ") {
+        return [
+          count,
+          lastStartOffset ? aLine.substring(lastStartOffset) : aLine,
+        ];
+      }
+    }
+
+    return [count, lastStartOffset ? aLine.substring(lastStartOffset) : aLine];
+  },
+
+  /**
+   * Attempt to understand simple quoting constructs that use ">" with
+   * obvious phrases to enter the quoting block.  No support for other types
+   * of quoting at this time.  Also no support for piercing the wrapper of
+   * forwarded messages to actually be the content of the forwarded message.
+   */
+  contentWhittle(aMeta, aBodyLines, aContent) {
+    if (!aContent.volunteerContent(aContent.kPriorityBase)) {
+      return false;
+    }
+
+    // duplicate the list; we mutate somewhat...
+    let bodyLines = aBodyLines.concat();
+
+    // lastNonBlankLine originally was just for detecting quoting idioms where
+    //  the "wrote" line was separated from the quoted block by a blank line.
+    // Now we also use it for whitespace suppression at the boundaries of
+    //  quoted and un-quoted text.  (We keep blank lines within the same
+    //  'block' of quoted or non-quoted text.)
+    // Because we now have two goals for it, and we still want to suppress blank
+    //  lines when there is a 'wrote' line involved, we introduce...
+    //  prevLastNonBlankLine!  This arguably suggests refactoring should be the
+    //  next step, but things work for now.
+    let rangeStart = 0,
+      lastNonBlankLine = null,
+      prevLastNonBlankLine = null;
+    let inQuoteDepth = 0;
+    for (let [iLine, line] of bodyLines.entries()) {
+      if (!line || line == "\xa0") {
+        /* unicode non breaking space */
+        continue;
+      }
+
+      if (line.startsWith(">")) {
+        if (!inQuoteDepth) {
+          let rangeEnd = iLine - 1;
+          let quoteRangeStart = iLine;
+          // see if the last non-blank-line was a lead-in...
+          if (lastNonBlankLine != null) {
+            // TODO: localize quote range start detection
+            if (aBodyLines[lastNonBlankLine].includes("wrote")) {
+              quoteRangeStart = lastNonBlankLine;
+              rangeEnd = lastNonBlankLine - 1;
+              // we 'used up' lastNonBlankLine, let's promote the prev guy to
+              //  be the new lastNonBlankLine for the next logic block
+              lastNonBlankLine = prevLastNonBlankLine;
+            }
+            // eat the trailing whitespace...
+            if (lastNonBlankLine != null) {
+              rangeEnd = Math.min(rangeEnd, lastNonBlankLine);
+            }
+          }
+          if (rangeEnd >= rangeStart) {
+            aContent.content(aBodyLines.slice(rangeStart, rangeEnd + 1));
+          }
+
+          [inQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+          rangeStart = quoteRangeStart;
+        } else {
+          let curQuoteDepth;
+          [curQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+
+          if (curQuoteDepth != inQuoteDepth) {
+            // we could do some "wrote" compensation here, but it's not really
+            //  as important.  let's wait for a more clever algorithm.
+            aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+            inQuoteDepth = curQuoteDepth;
+            rangeStart = iLine;
+          }
+        }
+      } else if (inQuoteDepth) {
+        aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+        inQuoteDepth = 0;
+        rangeStart = iLine;
+      }
+
+      prevLastNonBlankLine = lastNonBlankLine;
+      lastNonBlankLine = iLine;
+    }
+
+    if (inQuoteDepth) {
+      aContent.quoted(aBodyLines.slice(rangeStart), inQuoteDepth);
+    } else {
+      aContent.content(aBodyLines.slice(rangeStart, lastNonBlankLine + 1));
+    }
+
+    return true;
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaIndexer.jsm b/comm/mailnews/db/gloda/modules/GlodaIndexer.jsm
new file mode 100644
index 0000000000..05919e4d67
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaIndexer.jsm
@@ -0,0 +1,1491 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file currently contains a fairly general implementation of asynchronous
+ *  indexing with a very explicit message indexing implementation.  As gloda
+ *  will eventually want to index more than just messages, the message-specific
+ *  things should ideally lose their special hold on this file.  This will
+ *  benefit readability/size as well.
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaIndexer", "IndexingJob"];
+
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+const lazy = {};
+ChromeUtils.defineModuleGetter(
+  lazy,
+  "GlodaCollectionManager",
+  "resource:///modules/gloda/Collection.jsm"
+);
+ChromeUtils.defineModuleGetter(
+  lazy,
+  "GlodaDatastore",
+  "resource:///modules/gloda/GlodaDatastore.jsm"
+);
+
+/**
+ * @class Capture the indexing batch concept explicitly.
+ *
+ * @param aJobType The type of thing we are indexing.  Current choices are:
+ *   "folder" and "message".  Previous choices included "account".  The indexer
+ *   currently knows too much about these; they should be de-coupled.
+ * @param aID Specific to the job type, but for now only used to hold folder
+ *     IDs.
+ *
+ * @ivar items The list of items to process during this job/batch.  (For
+ *     example, if this is a "messages" job, this would be the list of messages
+ *     to process, although the specific representation is determined by the
+ *     job.)  The list will only be mutated through the addition of extra items.
+ * @ivar offset The current offset into the 'items' list (if used), updated as
+ *     processing occurs.  If 'items' is not used, the processing code can also
+ *     update this in a similar fashion.  This is used by the status
+ *     notification code in conjunction with goal.
+ * @ivar goal The total number of items to index/actions to perform in this job.
+ *     This number may increase during the life of the job, but should not
+ *     decrease.  This is used by the status notification code in conjunction
+ *     with the goal.
+ */
+function IndexingJob(aJobType, aID, aItems) {
+  this.jobType = aJobType;
+  this.id = aID;
+  this.items = aItems != null ? aItems : [];
+  this.offset = 0;
+  this.goal = null;
+  this.callback = null;
+  this.callbackThis = null;
+}
+IndexingJob.prototype = {
+  /**
+   * Invoke the callback associated with this job, passing through all arguments
+   *  received by this function to the callback function.
+   */
+  safelyInvokeCallback(...aArgs) {
+    if (!this.callback) {
+      return;
+    }
+    try {
+      this.callback.apply(this.callbackThis, aArgs);
+    } catch (ex) {
+      GlodaIndexer._log.warn("job callback invocation problem:", ex);
+    }
+  },
+  toString() {
+    return (
+      "[job:" +
+      this.jobType +
+      " id:" +
+      this.id +
+      " items:" +
+      (this.items ? this.items.length : "no") +
+      " offset:" +
+      this.offset +
+      " goal:" +
+      this.goal +
+      "]"
+    );
+  },
+};
+
+/**
+ * @namespace Core indexing logic, plus message-specific indexing logic.
+ *
+ * === Indexing Goals
+ * We have the following goals:
+ *
+ * Responsiveness
+ * - When the user wants to quit, we should be able to stop and quit in a timely
+ *   fashion.
+ * - We should not interfere with the user's thunderbird usage.
+ *
+ * Correctness
+ * - Quitting should not result in any information loss; we should (eventually)
+ *   end up at the same indexed state regardless of whether a user lets
+ *   indexing run to completion or restarts thunderbird in the middle of the
+ *   process.  (It is okay to take slightly longer in the latter case.)
+ *
+ * Worst Case Scenario Avoidance
+ * - We should try to be O(1) memory-wise regardless of what notifications
+ *   are thrown at us.
+ *
+ * ===  Indexing Throttling
+ *
+ * Adaptive Indexing
+ * - The indexer tries to stay out of the way of other running code in
+ *   Thunderbird (autosync) and other code on the system.  We try and target
+ *   some number of milliseconds of activity between intentional inactive
+ *   periods.  The number of milliseconds of activity varies based on whether we
+ *   believe the user to be actively using the computer or idle.  We use our
+ *   inactive periods as a way to measure system load; if we receive our
+ *   notification promptly at the end of our inactive period, we believe the
+ *   system is not heavily loaded.  If we do not get notified promptly, we
+ *   assume there is other stuff going on and back off.
+ *
+ */
+var GlodaIndexer = {
+  /**
+   * A partial attempt to generalize to support multiple databases.  Each
+   *  database would have its own datastore would have its own indexer.  But
+   *  we rather inter-mingle our use of this field with the singleton global
+   *  GlodaDatastore.
+   */
+  _log: console.createInstance({
+    prefix: "gloda.indexer",
+    maxLogLevel: "Warn",
+    maxLogLevelPref: "gloda.loglevel",
+  }),
+  /**
+   * Our nsITimer that we use to schedule ourselves on the main thread
+   *  intermittently.  The timer always exists but may not always be active.
+   */
+  _timer: null,
+  /**
+   * Our nsITimer that we use to schedule events in the "far" future.  For now,
+   *  this means not compelling an initial indexing sweep until some number of
+   *  seconds after startup.
+   */
+  _longTimer: null,
+
+  /**
+   * Periodic performance adjustment parameters:  The overall goal is to adjust
+   *  our rate of work so that we don't interfere with the user's activities
+   *  when they are around (non-idle), and the system in general (when idle).
+   *  Being nice when idle isn't quite as important, but is a good idea so that
+   *  when the user un-idles we are able to back off nicely.  Also, we give
+   *  other processes on the system a chance to do something.
+   *
+   * We do this by organizing our work into discrete "tokens" of activity,
+   *  then processing the number of tokens that we have determined will
+   *  not impact the UI. Then we pause to give other activities a chance to get
+   *  some work done, and we measure whether anything happened during our pause.
+   *  If something else is going on in our application during that pause, we
+   *  give it priority (up to a point) by delaying further indexing.
+   *
+   * Keep in mind that many of our operations are actually asynchronous, so we
+   *  aren't entirely starving the event queue.  However, a lot of the async
+   *  stuff can end up not having any actual delay between events. For
+   *  example, we only index offline message bodies, so there's no network
+   *  latency involved, just disk IO; the only meaningful latency will be the
+   *  initial disk seek (if there is one... pre-fetching may seriously be our
+   *  friend).
+   *
+   * In order to maintain responsiveness, I assert that we want to minimize the
+   *  length of the time we are dominating the event queue.  This suggests
+   *  that we want break up our blocks of work frequently.  But not so
+   *  frequently that there is a lot of waste.  Accordingly our algorithm is
+   *  basically:
+   *
+   * - Estimate the time that it takes to process a token, and schedule the
+   *   number of tokens that should fit into that time.
+   * - Detect user activity, and back off immediately if found.
+   * - Try to delay commits and garbage collection until the user is inactive,
+   *   as these tend to cause a brief pause in the UI.
+   */
+
+  /**
+   * The number of milliseconds before we declare the user idle and step up our
+   *  indexing.
+   */
+  _INDEX_IDLE_ADJUSTMENT_TIME: 5000,
+
+  /**
+   * The time delay in milliseconds before we should schedule our initial sweep.
+   */
+  _INITIAL_SWEEP_DELAY: 10000,
+
+  /**
+   * How many milliseconds in the future should we schedule indexing to start
+   *  when turning on indexing (and it was not previously active).
+   */
+  _INDEX_KICKOFF_DELAY: 200,
+
+  /**
+   * The time interval, in milliseconds, of pause between indexing batches.  The
+   *  maximum processor consumption is determined by this constant and the
+   *  active |_cpuTargetIndexTime|.
+   *
+   * For current constants, that puts us at 50% while the user is active and 83%
+   *  when idle.
+   */
+  _INDEX_INTERVAL: 32,
+
+  /**
+   * Number of indexing 'tokens' we are allowed to consume before yielding for
+   *  each incremental pass.  Consider a single token equal to indexing a single
+   *  medium-sized message.  This may be altered by user session (in)activity.
+   * Because we fetch message bodies, which is potentially asynchronous, this
+   *  is not a precise knob to twiddle.
+   */
+  _indexTokens: 2,
+
+  /**
+   * Stopwatches used to measure performance during indexing, and during
+   * pauses between indexing. These help us adapt our indexing constants so
+   *  as to not explode your computer.  Kind of us, no?
+   */
+  _perfIndexStopwatch: null,
+  _perfPauseStopwatch: null,
+  /**
+   * Do we have an uncommitted indexer transaction that idle callback should commit?
+   */
+  _idleToCommit: false,
+  /**
+   * Target CPU time per batch of tokens, current value (milliseconds).
+   */
+  _cpuTargetIndexTime: 32,
+  /**
+   * Target CPU time per batch of tokens, during non-idle (milliseconds).
+   */
+  _CPU_TARGET_INDEX_TIME_ACTIVE: 32,
+  /**
+   * Target CPU time per batch of tokens, during idle (milliseconds).
+   */
+  _CPU_TARGET_INDEX_TIME_IDLE: 160,
+  /**
+   * Average CPU time per processed token (milliseconds).
+   */
+  _cpuAverageTimePerToken: 16,
+  /**
+   * Damping factor for _cpuAverageTimePerToken, as an approximate
+   * number of tokens to include in the average time.
+   */
+  _CPU_AVERAGE_TIME_DAMPING: 200,
+  /**
+   * Maximum tokens per batch. This is normally just a sanity check.
+   */
+  _CPU_MAX_TOKENS_PER_BATCH: 100,
+  /**
+   * CPU usage during a pause to declare that system was busy (milliseconds).
+   * This is typically set as 1.5 times the minimum resolution of the cpu
+   * usage clock, which is 16 milliseconds on Windows systems, and (I think)
+   * smaller on other systems, so we take the worst case.
+   */
+  _CPU_IS_BUSY_TIME: 24,
+  /**
+   * Time that return from pause may be late before the system is declared
+   * busy, in milliseconds. (Same issues as _CPU_IS_BUSY_TIME).
+   */
+  _PAUSE_LATE_IS_BUSY_TIME: 24,
+  /**
+   * Number of times that we will repeat a pause while waiting for a
+   * free CPU.
+   */
+  _PAUSE_REPEAT_LIMIT: 10,
+  /**
+   * Minimum time delay between commits, in milliseconds.
+   */
+  _MINIMUM_COMMIT_TIME: 5000,
+  /**
+   * Maximum time delay between commits, in milliseconds.
+   */
+  _MAXIMUM_COMMIT_TIME: 20000,
+
+  /**
+   * Unit testing hook to get us to emit additional logging that verges on
+   *  inane for general usage but is helpful in unit test output to get a lay
+   *  of the land and for paranoia reasons.
+   */
+  _unitTestSuperVerbose: false,
+  /**
+   * Unit test vector to get notified when a worker has a problem and it has
+   *  a recover helper associated.  This gets called with an argument
+   *  indicating whether the recovery helper indicates recovery was possible.
+   */
+  _unitTestHookRecover: null,
+  /**
+   * Unit test vector to get notified when a worker runs into an exceptional
+   *  situation (an exception propagates or gets explicitly killed) and needs
+   *  to be cleaned up.  This gets called with an argument indicating if there
+   *  was a helper that was used or if we just did the default cleanup thing.
+   */
+  _unitTestHookCleanup: null,
+
+  /**
+   * Last commit time.  Tracked to try and only commit at reasonable intervals.
+   */
+  _lastCommitTime: Date.now(),
+
+  _inited: false,
+  /**
+   * Initialize the indexer.
+   */
+  _init() {
+    if (this._inited) {
+      return;
+    }
+
+    this._inited = true;
+
+    this._callbackHandle.init();
+
+    if (Services.io.offline) {
+      this._suppressIndexing = true;
+    }
+
+    // create the timer that drives our intermittent indexing
+    this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+    // create the timer for larger offsets independent of indexing
+    this._longTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+
+    this._idleService = Cc["@mozilla.org/widget/useridleservice;1"].getService(
+      Ci.nsIUserIdleService
+    );
+
+    // create our performance stopwatches
+    try {
+      this._perfIndexStopwatch = Cc["@mozilla.org/stopwatch;1"].createInstance(
+        Ci.nsIStopwatch
+      );
+      this._perfPauseStopwatch = Cc["@mozilla.org/stopwatch;1"].createInstance(
+        Ci.nsIStopwatch
+      );
+    } catch (ex) {
+      this._log.error("problem creating stopwatch!: " + ex);
+    }
+
+    // register for shutdown notifications
+    Services.obs.addObserver(this, "quit-application");
+
+    // figure out if event-driven indexing should be enabled...
+    let branch = Services.prefs.getBranch("mailnews.database.global.indexer.");
+    let eventDrivenEnabled = branch.getBoolPref("enabled", false);
+    let performInitialSweep = branch.getBoolPref("perform_initial_sweep", true);
+    // pretend we have already performed an initial sweep...
+    if (!performInitialSweep) {
+      this._initialSweepPerformed = true;
+    }
+
+    this.enabled = eventDrivenEnabled;
+  },
+
+  /**
+   * When shutdown, indexing immediately ceases and no further progress should
+   *  be made.  This flag goes true once, and never returns to false.  Being
+   *  in this state is a destructive thing from whence we cannot recover.
+   */
+  _indexerIsShutdown: false,
+
+  /**
+   * Shutdown the indexing process and datastore as quickly as possible in
+   *  a synchronous fashion.
+   */
+  _shutdown() {
+    // no more timer events, please
+    try {
+      this._timer.cancel();
+    } catch (ex) {}
+    this._timer = null;
+    try {
+      this._longTimer.cancel();
+    } catch (ex) {}
+    this._longTimer = null;
+
+    this._perfIndexStopwatch = null;
+    this._perfPauseStopwatch = null;
+
+    // Remove listeners to avoid reference cycles on the off chance one of them
+    // holds a reference to the indexer object.
+    this._indexListeners = [];
+
+    this._indexerIsShutdown = true;
+
+    if (this.enabled) {
+      this._log.info("Shutting Down");
+    }
+
+    // don't let anything try and convince us to start indexing again
+    this.suppressIndexing = true;
+
+    // If there is an active job and it has a cleanup handler, run it.
+    if (this._curIndexingJob) {
+      let workerDef = this._curIndexingJob._workerDef;
+      try {
+        if (workerDef.cleanup) {
+          workerDef.cleanup.call(workerDef.indexer, this._curIndexingJob);
+        }
+      } catch (ex) {
+        this._log.error("problem during worker cleanup during shutdown.");
+      }
+    }
+    // Definitely clean out the async call stack and any associated data
+    this._callbackHandle.cleanup();
+    this._workBatchData = undefined;
+
+    // disable ourselves and all of the specific indexers
+    this.enabled = false;
+
+    lazy.GlodaDatastore.shutdown();
+  },
+
+  /**
+   * The list of indexers registered with us.  If you are a core gloda indexer
+   *  (you ship with gloda), then you can import this file directly and should
+   *  make sure your indexer is imported in 'Everybody.jsm' in the right order.
+   *  If you are not core gloda, then you should import 'GlodaPublic.jsm' and only
+   *  then should you import 'GlodaIndexer.jsm' to get at GlodaIndexer.
+   */
+  _indexers: [],
+  /**
+   * Register an indexer with the Gloda indexing mechanism.
+   *
+   * @param aIndexer.name The name of your indexer.
+   * @param aIndexer.enable Your enable function.  This will be called during
+   *     the call to registerIndexer if Gloda indexing is already enabled.  If
+   *     indexing is not yet enabled, you will be called
+   * @param aIndexer.disable Your disable function.  This will be called when
+   *     indexing is disabled or we are shutting down.  This will only be called
+   *     if enable has already been called.
+   * @param aIndexer.workers A list of tuples of the form [worker type code,
+   *     worker generator function, optional scheduling trigger function].  The
+   *     type code is the string used to uniquely identify the job type.  If you
+   *     are not core gloda, your job type must start with your extension's name
+   *     and a colon; you can collow that with anything you want.  The worker
+   *     generator is not easily explained in here.  The trigger function is
+   *     invoked immediately prior to calling the generator to create it.  The
+   *     trigger function takes the job as an argument and should perform any
+   *     finalization required on the job.  Most workers should not need to use
+   *     the trigger function.
+   * @param aIndexer.initialSweep We call this to tell each indexer when it is
+   *     its turn to run its indexing sweep.  The idea of the indexing sweep is
+   *     that this is when you traverse things eligible for indexing to make
+   *     sure they are indexed.  Right now we just call everyone at the same
+   *     time and hope that their jobs don't fight too much.
+   */
+  registerIndexer(aIndexer) {
+    this._log.info("Registering indexer: " + aIndexer.name);
+    this._indexers.push(aIndexer);
+
+    try {
+      for (let workerInfo of aIndexer.workers) {
+        let workerCode = workerInfo[0];
+        let workerDef = workerInfo[1];
+        workerDef.name = workerCode;
+        workerDef.indexer = aIndexer;
+        this._indexerWorkerDefs[workerCode] = workerDef;
+        if (!("recover" in workerDef)) {
+          workerDef.recover = null;
+        }
+        if (!("cleanup" in workerDef)) {
+          workerDef.cleanup = null;
+        }
+        if (!("onSchedule" in workerDef)) {
+          workerDef.onSchedule = null;
+        }
+        if (!("jobCanceled" in workerDef)) {
+          workerDef.jobCanceled = null;
+        }
+      }
+    } catch (ex) {
+      this._log.warn("Helper indexer threw exception on worker enum.");
+    }
+
+    if (this._enabled) {
+      try {
+        aIndexer.enable();
+      } catch (ex) {
+        this._log.warn("Helper indexer threw exception on enable: " + ex);
+      }
+    }
+  },
+
+  /**
+   * Are we enabled, read: are we processing change events?
+   */
+  _enabled: false,
+  get enabled() {
+    return this._enabled;
+  },
+  set enabled(aEnable) {
+    if (!this._enabled && aEnable) {
+      // register for offline notifications
+      Services.obs.addObserver(this, "network:offline-status-changed");
+
+      // register for idle notification
+      this._idleService.addIdleObserver(this, this._indexIdleThresholdSecs);
+
+      this._enabled = true;
+
+      for (let indexer of this._indexers) {
+        try {
+          indexer.enable();
+        } catch (ex) {
+          this._log.warn("Helper indexer threw exception on enable: " + ex);
+        }
+      }
+
+      // if we have an accumulated desire to index things, kick it off again.
+      if (this._indexingDesired) {
+        this._indexingDesired = false; // it's edge-triggered for now
+        this.indexing = true;
+      }
+
+      // if we have not done an initial sweep, schedule scheduling one.
+      if (!this._initialSweepPerformed) {
+        this._longTimer.initWithCallback(
+          this._scheduleInitialSweep,
+          this._INITIAL_SWEEP_DELAY,
+          Ci.nsITimer.TYPE_ONE_SHOT
+        );
+      }
+    } else if (this._enabled && !aEnable) {
+      for (let indexer of this._indexers) {
+        try {
+          indexer.disable();
+        } catch (ex) {
+          this._log.warn("Helper indexer threw exception on disable: " + ex);
+        }
+      }
+
+      // remove offline observer
+      Services.obs.removeObserver(this, "network:offline-status-changed");
+
+      // remove idle
+      this._idleService.removeIdleObserver(this, this._indexIdleThresholdSecs);
+
+      this._enabled = false;
+    }
+  },
+
+  /** Track whether indexing is desired (we have jobs to prosecute). */
+  _indexingDesired: false,
+  /**
+   * Track whether we have an actively pending callback or timer event.  We do
+   *  this so we don't experience a transient suppression and accidentally
+   *  get multiple event-chains driving indexing at the same time (which the
+   *  code will not handle correctly).
+   */
+  _indexingActive: false,
+  /**
+   * Indicates whether indexing is currently ongoing.  This may return false
+   *  while indexing activities are still active, but they will quiesce shortly.
+   */
+  get indexing() {
+    return this._indexingDesired && !this._suppressIndexing;
+  },
+  /** Indicates whether indexing is desired. */
+  get indexingDesired() {
+    return this._indexingDesired;
+  },
+  /**
+   * Set this to true to indicate there is indexing work to perform.  This does
+   *  not mean indexing will begin immediately (if it wasn't active), however.
+   *  If suppressIndexing has been set, we won't do anything until indexing is
+   *  no longer suppressed.
+   */
+  set indexing(aShouldIndex) {
+    if (!this._indexingDesired && aShouldIndex) {
+      this._indexingDesired = true;
+      if (this.enabled && !this._indexingActive && !this._suppressIndexing) {
+        this._log.info("+++ Indexing Queue Processing Commencing");
+        this._indexingActive = true;
+        this._timer.initWithCallback(
+          this._timerCallbackDriver,
+          this._INDEX_KICKOFF_DELAY,
+          Ci.nsITimer.TYPE_ONE_SHOT
+        );
+      }
+    }
+  },
+
+  _suppressIndexing: false,
+  /**
+   * Set whether or not indexing should be suppressed.  This is to allow us to
+   *  avoid running down a laptop's battery when it is not on AC.  Only code
+   *  in charge of regulating that tracking should be setting this variable; if
+   *  other factors want to contribute to such a decision, this logic needs to
+   *  be changed to track that, since last-write currently wins.
+   */
+  set suppressIndexing(aShouldSuppress) {
+    this._suppressIndexing = aShouldSuppress;
+
+    // re-start processing if we are no longer suppressing, there is work yet
+    //  to do, and the indexing process had actually stopped.
+    if (
+      !this._suppressIndexing &&
+      this._indexingDesired &&
+      !this._indexingActive
+    ) {
+      this._log.info("+++ Indexing Queue Processing Resuming");
+      this._indexingActive = true;
+      this._timer.initWithCallback(
+        this._timerCallbackDriver,
+        this._INDEX_KICKOFF_DELAY,
+        Ci.nsITimer.TYPE_ONE_SHOT
+      );
+    }
+  },
+
+  /**
+   * Track whether an initial sweep has been performed.  This mainly exists so
+   *  that unit testing can stop us from performing an initial sweep.
+   */
+  _initialSweepPerformed: false,
+  /**
+   * Our timer-driven callback to schedule our first initial indexing sweep.
+   *  Because it is invoked by an nsITimer it operates without the benefit of
+   *  a 'this' context and must use GlodaIndexer instead of this.
+   * Since an initial sweep could have been performed before we get invoked,
+   *  we need to check whether an initial sweep is still desired before trying
+   *  to schedule one.  We don't need to worry about whether one is active
+   *  because the indexingSweepNeeded takes care of that.
+   */
+  _scheduleInitialSweep() {
+    if (GlodaIndexer._initialSweepPerformed) {
+      return;
+    }
+    GlodaIndexer._initialSweepPerformed = true;
+    for (let indexer of GlodaIndexer._indexers) {
+      indexer.initialSweep();
+    }
+  },
+
+  /**
+   * Our current job number.  Meaningless value that increments with every job
+   *  we process that resets to 0 when we run out of jobs.  Currently used by
+   *  the activity manager's gloda listener to tell when we have changed jobs.
+   * We really need a better listener mechanism.
+   */
+  _indexingJobCount: 0,
+
+  /**
+   * A list of IndexingJob instances to process.
+   */
+  _indexQueue: [],
+
+  /**
+   * The current indexing job.
+   */
+  _curIndexingJob: null,
+
+  /**
+   * The number of seconds before we declare the user idle and commit if
+   *  needed.
+   */
+  _indexIdleThresholdSecs: 3,
+
+  _indexListeners: [],
+  /**
+   * Add an indexing progress listener.  The listener will be notified of at
+   *  least all major status changes (idle -> indexing, indexing -> idle), plus
+   *  arbitrary progress updates during the indexing process.
+   * If indexing is not active when the listener is added, a synthetic idle
+   *  notification will be generated.
+   *
+   * @param aListener A listener function, taking arguments: status (Gloda.
+   *     kIndexer*), the folder name if a folder is involved (string or null),
+   *     current zero-based job number (int),
+   *     current item number being indexed in this job (int), total number
+   *     of items in this job to be indexed (int).
+   *
+   * @TODO should probably allow for a 'this' value to be provided
+   * @TODO generalize to not be folder/message specific.  use nouns!
+   */
+  addListener(aListener) {
+    // should we weakify?
+    if (!this._indexListeners.includes(aListener)) {
+      this._indexListeners.push(aListener);
+    }
+    // if we aren't indexing, give them an idle indicator, otherwise they can
+    //  just be happy when we hit the next actual status point.
+    if (!this.indexing) {
+      aListener(GlodaConstants.kIndexerIdle, null, 0, 0, 1);
+    }
+    return aListener;
+  },
+  /**
+   * Remove the given listener so that it no longer receives indexing progress
+   *  updates.
+   */
+  removeListener(aListener) {
+    let index = this._indexListeners.indexOf(aListener);
+    if (index != -1) {
+      this._indexListeners.splice(index, 1);
+    }
+  },
+  /**
+   * Helper method to tell listeners what we're up to.  For code simplicity,
+   *  the caller is just deciding when to send this update (preferably at
+   *  reasonable intervals), and doesn't need to provide any indication of
+   *  state... we figure that out ourselves.
+   *
+   * This was not pretty but got ugly once we moved the message indexing out
+   *  to its own indexer.  Some generalization is required but will likely
+   *  require string hooks.
+   */
+  _notifyListeners() {
+    let status, prettyName, jobIndex, jobItemIndex, jobItemGoal, jobType;
+
+    if (this.indexing && this._curIndexingJob) {
+      let job = this._curIndexingJob;
+      status = GlodaConstants.kIndexerIndexing;
+
+      let indexer = this._indexerWorkerDefs[job.jobType].indexer;
+      if ("_indexingFolder" in indexer) {
+        prettyName =
+          indexer._indexingFolder != null
+            ? indexer._indexingFolder.prettyName
+            : null;
+      } else {
+        prettyName = null;
+      }
+
+      jobIndex = this._indexingJobCount - 1;
+      jobItemIndex = job.offset;
+      jobItemGoal = job.goal;
+      jobType = job.jobType;
+    } else {
+      status = GlodaConstants.kIndexerIdle;
+      prettyName = null;
+      jobIndex = 0;
+      jobItemIndex = 0;
+      jobItemGoal = 1;
+      jobType = null;
+    }
+
+    // Some people ascribe to the belief that the most you can give is 100%.
+    // We know better, but let's humor them.
+    if (jobItemIndex > jobItemGoal) {
+      jobItemGoal = jobItemIndex;
+    }
+
+    for (
+      let iListener = this._indexListeners.length - 1;
+      iListener >= 0;
+      iListener--
+    ) {
+      let listener = this._indexListeners[iListener];
+      try {
+        listener(
+          status,
+          prettyName,
+          jobIndex,
+          jobItemIndex,
+          jobItemGoal,
+          jobType
+        );
+      } catch (ex) {
+        this._log.error(ex);
+      }
+    }
+  },
+
+  /**
+   * A wrapped callback driver intended to be used by timers that provide
+   *  arguments we really do not care about.
+   */
+  _timerCallbackDriver() {
+    GlodaIndexer.callbackDriver();
+  },
+
+  /**
+   * A simple callback driver wrapper to provide 'this'.
+   */
+  _wrapCallbackDriver(...aArgs) {
+    GlodaIndexer.callbackDriver(...aArgs);
+  },
+
+  /**
+   * The current processing 'batch' generator, produced by a call to workBatch()
+   *  and used by callbackDriver to drive execution.
+   */
+  _batch: null,
+  _inCallback: false,
+  _savedCallbackArgs: null,
+  /**
+   * The root work-driver.  callbackDriver creates workBatch generator instances
+   *  (stored in _batch) which run until they are done (kWorkDone) or they
+   *  (really the embedded activeIterator) encounter something asynchronous.
+   *  The convention is that all the callback handlers end up calling us,
+   *  ensuring that control-flow properly resumes.  If the batch completes,
+   *  we re-schedule ourselves after a time delay (controlled by _INDEX_INTERVAL)
+   *  and return.  (We use one-shot timers because repeating-slack does not
+   *  know enough to deal with our (current) asynchronous nature.)
+   */
+  callbackDriver(...aArgs) {
+    // just bail if we are shutdown
+    if (this._indexerIsShutdown) {
+      return;
+    }
+
+    // it is conceivable that someone we call will call something that in some
+    //  cases might be asynchronous, and in other cases immediately generate
+    //  events without returning.  In the interest of (stack-depth) sanity,
+    //  let's handle this by performing a minimal time-delay callback.
+    // this is also now a good thing sequencing-wise.  if we get our callback
+    //  with data before the underlying function has yielded, we obviously can't
+    //  cram the data in yet.  Our options in this case are to either mark the
+    //  fact that the callback has already happened and immediately return to
+    //  the iterator when it does bubble up the kWorkAsync, or we can do as we
+    //  have been doing, but save the
+    if (this._inCallback) {
+      this._savedCallbackArgs = aArgs;
+      this._timer.initWithCallback(
+        this._timerCallbackDriver,
+        0,
+        Ci.nsITimer.TYPE_ONE_SHOT
+      );
+      return;
+    }
+    this._inCallback = true;
+
+    try {
+      if (this._batch === null) {
+        this._batch = this.workBatch();
+      }
+
+      // kWorkAsync, kWorkDone, kWorkPause are allowed out; kWorkSync is not
+      // On kWorkDone, we want to schedule another timer to fire on us if we are
+      //  not done indexing.  (On kWorkAsync, we don't care what happens, because
+      //  someone else will be receiving the callback, and they will call us when
+      //  they are done doing their thing.
+      let args;
+      if (this._savedCallbackArgs != null) {
+        args = this._savedCallbackArgs;
+        this._savedCallbackArgs = null;
+      } else {
+        args = aArgs;
+      }
+
+      let result;
+      if (args.length == 0) {
+        result = this._batch.next().value;
+      } else if (args.length == 1) {
+        result = this._batch.next(args[0]).value;
+      } else {
+        // Arguments works with destructuring assignment.
+        result = this._batch.next(args).value;
+      }
+      switch (result) {
+        // job's done, close the batch and re-schedule ourselves if there's more
+        //  to do.
+        case GlodaConstants.kWorkDone:
+          this._batch.return();
+          this._batch = null;
+        // the batch wants to get re-scheduled, do so.
+        // (intentional fall-through to re-scheduling logic)
+        case GlodaConstants.kWorkPause:
+          if (this.indexing) {
+            this._timer.initWithCallback(
+              this._timerCallbackDriver,
+              this._INDEX_INTERVAL,
+              Ci.nsITimer.TYPE_ONE_SHOT
+            );
+          } else {
+            // it's important to indicate no more callbacks are in flight
+            this._indexingActive = false;
+          }
+          break;
+        case GlodaConstants.kWorkAsync:
+          // there is nothing to do.  some other code is now responsible for
+          //  calling us.
+          break;
+      }
+    } finally {
+      this._inCallback = false;
+    }
+  },
+
+  _callbackHandle: {
+    init() {
+      this.wrappedCallback = GlodaIndexer._wrapCallbackDriver;
+      this.callbackThis = GlodaIndexer;
+      this.callback = GlodaIndexer.callbackDriver;
+    },
+    /**
+     * The stack of generators we are processing.  The (numerically) last one is
+     *  also the |activeIterator|.
+     */
+    activeStack: [],
+    /**
+     * The generator at the top of the |activeStack| and that we will call next
+     *  or send on next if nothing changes.
+     */
+    activeIterator: null,
+    /**
+     * Meta-information about the generators at each level of the stack.
+     */
+    contextStack: [],
+    /**
+     * Push a new generator onto the stack.  It becomes the active generator.
+     */
+    push(aIterator, aContext) {
+      this.activeStack.push(aIterator);
+      this.contextStack.push(aContext);
+      this.activeIterator = aIterator;
+    },
+    /**
+     * For use by generators that want to call another asynchronous process
+     *  implemented as a generator.  They should do
+     *  "yield aCallbackHandle.pushAndGo(someGenerator(arg1, arg2));".
+     *
+     * @public
+     */
+    pushAndGo(aIterator, aContext) {
+      this.push(aIterator, aContext);
+      return GlodaConstants.kWorkSync;
+    },
+    /**
+     * Pop the active generator off the stack.
+     */
+    pop() {
+      this.activeIterator.return();
+      this.activeStack.pop();
+      this.contextStack.pop();
+      if (this.activeStack.length) {
+        this.activeIterator = this.activeStack[this.activeStack.length - 1];
+      } else {
+        this.activeIterator = null;
+      }
+    },
+    /**
+     * Someone propagated an exception and we need to clean-up all the active
+     *  logic as best we can.  Which is not really all that well.
+     *
+     * @param [aOptionalStopAtDepth=0] The length the stack should be when this
+     *     method completes.  Pass 0 or omit for us to clear everything out.
+     *     Pass 1 to leave just the top-level generator intact.
+     */
+    cleanup(aOptionalStopAtDepth) {
+      if (aOptionalStopAtDepth === undefined) {
+        aOptionalStopAtDepth = 0;
+      }
+      while (this.activeStack.length > aOptionalStopAtDepth) {
+        this.pop();
+      }
+    },
+    /**
+     * For use when a generator finishes up by calling |doneWithResult| on us;
+     *  the async driver calls this to pop that generator off the stack
+     *  and get the result it passed in to its call to |doneWithResult|.
+     *
+     * @protected
+     */
+    popWithResult() {
+      this.pop();
+      let result = this._result;
+      this._result = null;
+      return result;
+    },
+    _result: null,
+    /**
+     * For use by generators that want to return a result to the calling
+     *  asynchronous generator.  Specifically, they should do
+     *  "yield aCallbackHandle.doneWithResult(RESULT);".
+     *
+     * @public
+     */
+    doneWithResult(aResult) {
+      this._result = aResult;
+      return GlodaConstants.kWorkDoneWithResult;
+    },
+
+    /* be able to serve as a collection listener, resuming the active iterator's
+       last yield kWorkAsync */
+    onItemsAdded() {},
+    onItemsModified() {},
+    onItemsRemoved() {},
+    onQueryCompleted(aCollection) {
+      GlodaIndexer.callbackDriver();
+    },
+  },
+  _workBatchData: undefined,
+  /* eslint-disable complexity */
+  /**
+   * The workBatch generator handles a single 'batch' of processing, managing
+   *  the database transaction and keeping track of "tokens".  It drives the
+   *  activeIterator generator which is doing the work.
+   * workBatch will only produce kWorkAsync, kWorkPause, and kWorkDone
+   *  notifications.  If activeIterator returns kWorkSync and there are still
+   *  tokens available, workBatch will keep driving the activeIterator until it
+   *  encounters a kWorkAsync (which workBatch will yield to callbackDriver), or
+   *  it runs out of tokens and yields a kWorkPause or kWorkDone.
+   */
+  *workBatch() {
+    // Do we still have an open transaction? If not, start a new one.
+    if (!this._idleToCommit) {
+      lazy.GlodaDatastore._beginTransaction();
+    } else {
+      // We'll manage commit ourself while this routine is active.
+      this._idleToCommit = false;
+    }
+
+    this._perfIndexStopwatch.start();
+    let batchCount;
+    let haveMoreWork = true;
+    let transactionToCommit = true;
+    let inIdle;
+
+    let notifyDecimator = 0;
+
+    while (haveMoreWork) {
+      // Both explicit work activity points (sync + async) and transfer of
+      //  control return (via kWorkDone*) results in a token being eaten.  The
+      //  idea now is to make tokens less precious so that the adaptive logic
+      //  can adjust them with less impact.  (Before this change, doing 1
+      //  token's work per cycle ended up being an entire non-idle time-slice's
+      //  work.)
+      // During this loop we track the clock real-time used even though we
+      //  frequently yield to asynchronous operations.  These asynchronous
+      //  operations are either database queries or message streaming requests.
+      //  Both may involve disk I/O but no network I/O (since we only stream
+      //  messages that are already available offline), but in an ideal
+      //  situation will come from cache and so the work this function kicks off
+      //  will dominate.
+      // We do not use the CPU time to this end because...
+      //  1) Our timer granularity on linux is worse for CPU than for wall time.
+      //  2) That can fail to account for our I/O cost.
+      //  3) If something with a high priority / low latency need (like playing
+      //     a video) is fighting us, although using CPU time will accurately
+      //     express how much time we are actually spending to index, our goal
+      //     is to control the duration of our time slices, not be "right" about
+      //     the actual CPU cost.  In that case, if we attempted to take on more
+      //     work, we would likely interfere with the higher priority process or
+      //     make ourselves less responsive by drawing out the period of time we
+      //     are dominating the main thread.
+      this._perfIndexStopwatch.start();
+      // For telemetry purposes, we want to know how many messages we've been
+      //  processing during that batch, and how long it took, pauses included.
+      let t0 = Date.now();
+      this._indexedMessageCount = 0;
+      batchCount = 0;
+      while (batchCount < this._indexTokens) {
+        if (
+          this._callbackHandle.activeIterator === null &&
+          !this._hireJobWorker()
+        ) {
+          haveMoreWork = false;
+          break;
+        }
+        batchCount++;
+
+        // XXX for performance, we may want to move the try outside the for loop
+        //  with a quasi-redundant outer loop that shunts control back inside
+        //  if we left the loop due to an exception (without consuming all the
+        //  tokens.)
+        try {
+          switch (
+            this._callbackHandle.activeIterator.next(this._workBatchData).value
+          ) {
+            case GlodaConstants.kWorkSync:
+              this._workBatchData = undefined;
+              break;
+            case GlodaConstants.kWorkAsync:
+              this._workBatchData = yield GlodaConstants.kWorkAsync;
+              break;
+            case GlodaConstants.kWorkDone:
+              this._callbackHandle.pop();
+              this._workBatchData = undefined;
+              break;
+            case GlodaConstants.kWorkDoneWithResult:
+              this._workBatchData = this._callbackHandle.popWithResult();
+              break;
+            default:
+              break;
+          }
+        } catch (ex) {
+          this._log.debug("Exception in batch processing:", ex);
+          let workerDef = this._curIndexingJob._workerDef;
+          if (workerDef.recover) {
+            let recoverToDepth;
+            try {
+              recoverToDepth = workerDef.recover.call(
+                workerDef.indexer,
+                this._curIndexingJob,
+                this._callbackHandle.contextStack,
+                ex
+              );
+            } catch (ex2) {
+              this._log.error(
+                "Worker '" +
+                  workerDef.name +
+                  "' recovery function itself failed:",
+                ex2
+              );
+            }
+            if (this._unitTestHookRecover) {
+              this._unitTestHookRecover(
+                recoverToDepth,
+                ex,
+                this._curIndexingJob,
+                this._callbackHandle
+              );
+            }
+
+            if (recoverToDepth) {
+              this._callbackHandle.cleanup(recoverToDepth);
+              continue;
+            }
+          }
+          // (we either did not have a recover handler or it couldn't recover)
+          // call the cleanup helper if there is one
+          if (workerDef.cleanup) {
+            try {
+              workerDef.cleanup.call(workerDef.indexer, this._curIndexingJob);
+            } catch (ex2) {
+              this._log.error(
+                "Worker '" +
+                  workerDef.name +
+                  "' cleanup function itself failed:",
+                ex2
+              );
+            }
+            if (this._unitTestHookCleanup) {
+              this._unitTestHookCleanup(
+                true,
+                ex,
+                this._curIndexingJob,
+                this._callbackHandle
+              );
+            }
+          } else if (this._unitTestHookCleanup) {
+            this._unitTestHookCleanup(
+              false,
+              ex,
+              this._curIndexingJob,
+              this._callbackHandle
+            );
+          }
+
+          // Clean out everything on the async stack, warn about the job, kill.
+          // We do not log this warning lightly; it will break unit tests and
+          //  be visible to users.  Anything expected should likely have a
+          //  recovery function or the cleanup logic should be extended to
+          //  indicate that the failure is acceptable.
+          this._callbackHandle.cleanup();
+          this._log.warn(
+            "Problem during " + this._curIndexingJob + ", bailing:",
+            ex
+          );
+          this._curIndexingJob = null;
+          // the data must now be invalid
+          this._workBatchData = undefined;
+        }
+      }
+      this._perfIndexStopwatch.stop();
+
+      // idleTime can throw if there is no idle-provider available, such as an
+      //  X session without the relevant extensions available.  In this case
+      //  we assume that the user is never idle.
+      try {
+        // We want to stop ASAP when leaving idle, so we can't rely on the
+        // standard polled callback. We do the polling ourselves.
+        if (this._idleService.idleTime < this._INDEX_IDLE_ADJUSTMENT_TIME) {
+          inIdle = false;
+          this._cpuTargetIndexTime = this._CPU_TARGET_INDEX_TIME_ACTIVE;
+        } else {
+          inIdle = true;
+          this._cpuTargetIndexTime = this._CPU_TARGET_INDEX_TIME_IDLE;
+        }
+      } catch (ex) {
+        inIdle = false;
+      }
+
+      // take a breather by having the caller re-schedule us sometime in the
+      //  future, but only if we're going to perform another loop iteration.
+      if (haveMoreWork) {
+        notifyDecimator = (notifyDecimator + 1) % 32;
+        if (!notifyDecimator) {
+          this._notifyListeners();
+        }
+
+        for (
+          let pauseCount = 0;
+          pauseCount < this._PAUSE_REPEAT_LIMIT;
+          pauseCount++
+        ) {
+          this._perfPauseStopwatch.start();
+
+          yield GlodaConstants.kWorkPause;
+
+          this._perfPauseStopwatch.stop();
+          // We repeat the pause if the pause was longer than
+          //  we expected, or if it used a significant amount
+          //  of cpu, either of which indicate significant other
+          //  activity.
+          if (
+            this._perfPauseStopwatch.cpuTimeSeconds * 1000 <
+              this._CPU_IS_BUSY_TIME &&
+            this._perfPauseStopwatch.realTimeSeconds * 1000 -
+              this._INDEX_INTERVAL <
+              this._PAUSE_LATE_IS_BUSY_TIME
+          ) {
+            break;
+          }
+        }
+      }
+
+      // All pauses have been taken, how effective were we? Report!
+      // XXX: there's possibly a lot of fluctuation since we go through here
+      // every 5 messages or even less
+      if (this._indexedMessageCount > 0) {
+        let delta = (Date.now() - t0) / 1000; // in seconds
+        let v = Math.round(this._indexedMessageCount / delta);
+        try {
+          let h = Services.telemetry.getHistogramById(
+            "THUNDERBIRD_INDEXING_RATE_MSG_PER_S"
+          );
+          h.add(v);
+        } catch (e) {
+          this._log.warn("Couldn't report telemetry", e, v);
+        }
+      }
+
+      if (batchCount > 0) {
+        let totalTime = this._perfIndexStopwatch.realTimeSeconds * 1000;
+        let timePerToken = totalTime / batchCount;
+        // Damp the average time since it is a rough estimate only.
+        this._cpuAverageTimePerToken =
+          (totalTime +
+            this._CPU_AVERAGE_TIME_DAMPING * this._cpuAverageTimePerToken) /
+          (batchCount + this._CPU_AVERAGE_TIME_DAMPING);
+        // We use the larger of the recent or the average time per token, so
+        //  that we can respond quickly to slow down indexing if there
+        //  is a sudden increase in time per token.
+        let bestTimePerToken = Math.max(
+          timePerToken,
+          this._cpuAverageTimePerToken
+        );
+        // Always index at least one token!
+        this._indexTokens = Math.max(
+          1,
+          this._cpuTargetIndexTime / bestTimePerToken
+        );
+        // But no more than the a maximum limit, just for sanity's sake.
+        this._indexTokens = Math.min(
+          this._CPU_MAX_TOKENS_PER_BATCH,
+          this._indexTokens
+        );
+        this._indexTokens = Math.ceil(this._indexTokens);
+      }
+
+      // Should we try to commit now?
+      let elapsed = Date.now() - this._lastCommitTime;
+      // Commit tends to cause a brief UI pause, so we try to delay it (but not
+      //  forever) if the user is active. If we're done and idling, we'll also
+      //  commit, otherwise we'll let the idle callback do it.
+      let doCommit =
+        transactionToCommit &&
+        (elapsed > this._MAXIMUM_COMMIT_TIME ||
+          (inIdle && (elapsed > this._MINIMUM_COMMIT_TIME || !haveMoreWork)));
+      if (doCommit) {
+        lazy.GlodaCollectionManager.cacheCommitDirty();
+        // Set up an async notification to happen after the commit completes so
+        //  that we can avoid the indexer doing something with the database that
+        //  causes the main thread to block against the completion of the commit
+        //  (which can be a while) on 1.9.1.
+        lazy.GlodaDatastore.runPostCommit(this._callbackHandle.wrappedCallback);
+        // kick off the commit
+        lazy.GlodaDatastore._commitTransaction();
+        yield GlodaConstants.kWorkAsync;
+        this._lastCommitTime = Date.now();
+        // Restart the transaction if we still have work.
+        if (haveMoreWork) {
+          lazy.GlodaDatastore._beginTransaction();
+        } else {
+          transactionToCommit = false;
+        }
+      }
+    }
+
+    this._notifyListeners();
+
+    // If we still have a transaction to commit, tell idle to do the commit
+    //  when it gets around to it.
+    if (transactionToCommit) {
+      this._idleToCommit = true;
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+  /* eslint-enable complexity */
+
+  /**
+   * Maps indexing job type names to a worker definition.
+   * The worker definition is an object with the following attributes where
+   *  only worker is required:
+   * - worker:
+   * - onSchedule: A function to be invoked when the worker is scheduled.  The
+   *    job is passed as an argument.
+   * - recover:
+   * - cleanup:
+   */
+  _indexerWorkerDefs: {},
+  /**
+   * Perform the initialization step and return a generator if there is any
+   *  steady-state processing to be had.
+   */
+  _hireJobWorker() {
+    // In no circumstances should there be data bouncing around from previous
+    //  calls if we are here.  |killActiveJob| depends on this.
+    this._workBatchData = undefined;
+
+    if (this._indexQueue.length == 0) {
+      this._log.info("--- Done indexing, disabling timer renewal.");
+
+      this._curIndexingJob = null;
+      this._indexingDesired = false;
+      this._indexingJobCount = 0;
+      return false;
+    }
+
+    let job = (this._curIndexingJob = this._indexQueue.shift());
+    this._indexingJobCount++;
+
+    let generator = null;
+
+    if (job.jobType in this._indexerWorkerDefs) {
+      let workerDef = this._indexerWorkerDefs[job.jobType];
+      job._workerDef = workerDef;
+
+      // Prior to creating the worker, call the scheduling trigger function
+      //  if there is one.  This is so that jobs can be finalized.  The
+      //  initial use case is event-driven message indexing that accumulates
+      //  a list of messages to index but wants it locked down once we start
+      //  processing the list.
+      if (workerDef.onSchedule) {
+        workerDef.onSchedule.call(workerDef.indexer, job);
+      }
+
+      generator = workerDef.worker.call(
+        workerDef.indexer,
+        job,
+        this._callbackHandle
+      );
+    } else {
+      // Nothing we can do about this.  Be loud about it and try to schedule
+      //  something else.
+      this._log.error("Unknown job type: " + job.jobType);
+      return this._hireJobWorker();
+    }
+
+    if (this._unitTestSuperVerbose) {
+      this._log.debug("Hired job of type: " + job.jobType);
+    }
+
+    this._notifyListeners();
+
+    if (generator) {
+      this._callbackHandle.push(generator);
+      return true;
+    }
+    return false;
+  },
+
+  /**
+   * Schedule a job for indexing.
+   */
+  indexJob(aJob) {
+    this._log.info("Queue-ing job for indexing: " + aJob.jobType);
+
+    this._indexQueue.push(aJob);
+    this.indexing = true;
+  },
+
+  /**
+   * Kill the active job.  This means a few things:
+   * - Kill all the generators in the callbackHandle stack.
+   * - If we are currently waiting on an async return, we need to make sure it
+   *    does not screw us up.
+   * - Make sure the job's cleanup function gets called if appropriate.
+   *
+   * The async return case is actually not too troublesome.  Since there is an
+   *  active indexing job and we are not (by fiat) in that call stack, we know
+   *  that the callback driver is guaranteed to get triggered again somehow.
+   *  The only issue is to make sure that _workBatchData does not end up with
+   *  the data.  We compel |_hireJobWorker| to erase it to this end.
+   *
+   * @note You MUST NOT call this function from inside a job or an async function
+   *    on the callbackHandle's stack of generators.  If you are in that
+   *    situation, you should just throw an exception.  At the very least,
+   *    use a timeout to trigger us.
+   */
+  killActiveJob() {
+    // There is nothing to do if we have no job
+    if (!this._curIndexingJob) {
+      return;
+    }
+
+    // -- Blow away the stack with cleanup.
+    let workerDef = this._curIndexingJob._workerDef;
+    if (this._unitTestSuperVerbose) {
+      this._log.debug("Killing job of type: " + this._curIndexingJob.jobType);
+    }
+    if (this._unitTestHookCleanup) {
+      this._unitTestHookCleanup(
+        !!workerDef.cleanup,
+        "no exception, this was killActiveJob",
+        this._curIndexingJob,
+        this._callbackHandle
+      );
+    }
+    this._callbackHandle.cleanup();
+    if (workerDef.cleanup) {
+      workerDef.cleanup.call(workerDef.indexer, this._curIndexingJob);
+    }
+
+    // Eliminate the job.
+    this._curIndexingJob = null;
+  },
+
+  /**
+   * Purge all jobs that the filter function returns true for.  This does not
+   *  kill the active job, use |killActiveJob| to do that.
+   *
+   * Make sure to call this function before killActiveJob
+   *
+   * @param aFilterElimFunc A filter function that takes an |IndexingJob| and
+   *     returns true if the job should be purged, false if it should not be.
+   *     The filter sees the jobs in the order they are scheduled.
+   */
+  purgeJobsUsingFilter(aFilterElimFunc) {
+    for (let iJob = 0; iJob < this._indexQueue.length; iJob++) {
+      let job = this._indexQueue[iJob];
+
+      // If the filter says to, splice the job out of existence (and make sure
+      //  to fixup iJob to compensate.)
+      if (aFilterElimFunc(job)) {
+        if (this._unitTestSuperVerbose) {
+          this._log.debug("Purging job of type: " + job.jobType);
+        }
+        this._indexQueue.splice(iJob--, 1);
+        let workerDef = this._indexerWorkerDefs[job.jobType];
+        if (workerDef.jobCanceled) {
+          workerDef.jobCanceled.call(workerDef.indexer, job);
+        }
+      }
+    }
+  },
+
+  /* *********** Event Processing *********** */
+  observe(aSubject, aTopic, aData) {
+    // idle
+    if (aTopic == "idle") {
+      // Do we need to commit an indexer transaction?
+      if (this._idleToCommit) {
+        this._idleToCommit = false;
+        lazy.GlodaCollectionManager.cacheCommitDirty();
+        lazy.GlodaDatastore._commitTransaction();
+        this._lastCommitTime = Date.now();
+        this._notifyListeners();
+      }
+    } else if (aTopic == "network:offline-status-changed") {
+      // offline status
+      if (aData == "offline") {
+        this.suppressIndexing = true;
+      } else {
+        // online
+        this.suppressIndexing = false;
+      }
+    } else if (aTopic == "quit-application") {
+      // shutdown fallback
+      this._shutdown();
+    }
+  },
+};
+// we used to initialize here; now we have GlodaPublic.jsm do it for us after the
+//  indexers register themselves so we know about all our built-in indexers
+//  at init-time.
diff --git a/comm/mailnews/db/gloda/modules/GlodaMsgIndexer.jsm b/comm/mailnews/db/gloda/modules/GlodaMsgIndexer.jsm
new file mode 100644
index 0000000000..54ceacb59a
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaMsgIndexer.jsm
@@ -0,0 +1,310 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaABIndexer", "GlodaABAttrs"];
+
+const { GlodaCollectionManager } = ChromeUtils.import(
+  "resource:///modules/gloda/Collection.jsm"
+);
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { GlodaIndexer, IndexingJob } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaIndexer.jsm"
+);
+const { FreeTagNoun } = ChromeUtils.import(
+  "resource:///modules/gloda/NounFreetag.jsm"
+);
+
+var GlodaABIndexer = {
+  _log: null,
+  _notifications: [
+    "addrbook-contact-created",
+    "addrbook-contact-updated",
+    "addrbook-contact-deleted",
+  ],
+
+  name: "index_ab",
+  enable() {
+    if (this._log == null) {
+      this._log = console.createInstance({
+        prefix: "gloda.index_ab",
+        maxLogLevel: "Warn",
+        maxLogLevelPref: "gloda.loglevel",
+      });
+    }
+
+    for (let topic of this._notifications) {
+      Services.obs.addObserver(this, topic);
+    }
+  },
+
+  disable() {
+    for (let topic of this._notifications) {
+      Services.obs.removeObserver(this, topic);
+    }
+  },
+
+  // it's a getter so we can reference 'this'
+  get workers() {
+    return [
+      [
+        "ab-card",
+        {
+          worker: this._worker_index_card,
+        },
+      ],
+    ];
+  },
+
+  *_worker_index_card(aJob, aCallbackHandle) {
+    let card = aJob.id;
+
+    if (card.primaryEmail) {
+      // load the identity
+      let query = Gloda.newQuery(GlodaConstants.NOUN_IDENTITY);
+      query.kind("email");
+      // we currently normalize all e-mail addresses to be lowercase
+      query.value(card.primaryEmail.toLowerCase());
+      let identityCollection = query.getCollection(aCallbackHandle);
+      yield GlodaConstants.kWorkAsync;
+
+      if (identityCollection.items.length) {
+        let identity = identityCollection.items[0];
+        // force the identity to know it has an associated ab card.
+        identity._hasAddressBookCard = true;
+
+        this._log.debug("Found identity, processing card.");
+        yield aCallbackHandle.pushAndGo(
+          Gloda.grokNounItem(
+            identity.contact,
+            { card },
+            false,
+            false,
+            aCallbackHandle
+          )
+        );
+        this._log.debug("Done processing card.");
+      }
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  initialSweep() {},
+
+  observe(subject, topic, data) {
+    subject.QueryInterface(Ci.nsIAbCard);
+
+    switch (topic) {
+      case "addrbook-contact-created": {
+        // When an address book card is added, update the cached GlodaIdentity
+        // object's cached idea of whether the identity has an ab card.
+        this._log.debug("Received Card Add Notification");
+
+        let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+          GlodaConstants.NOUN_IDENTITY,
+          "email@" + subject.primaryEmail.toLowerCase()
+        );
+        if (identity) {
+          identity._hasAddressBookCard = true;
+        }
+        break;
+      }
+      case "addrbook-contact-updated": {
+        this._log.debug("Received Card Change Notification");
+
+        let job = new IndexingJob("ab-card", subject);
+        GlodaIndexer.indexJob(job);
+        break;
+      }
+      case "addrbook-contact-deleted": {
+        // When an address book card is added, update the cached GlodaIdentity
+        // object's cached idea of whether the identity has an ab card.
+        this._log.debug("Received Card Removal Notification");
+
+        let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+          GlodaConstants.NOUN_IDENTITY,
+          "email@" + subject.primaryEmail.toLowerCase()
+        );
+        if (identity) {
+          identity._hasAddressBookCard = false;
+        }
+        break;
+      }
+    }
+  },
+};
+GlodaIndexer.registerIndexer(GlodaABIndexer);
+
+var GlodaABAttrs = {
+  providerName: "gloda.ab_attr",
+  _log: null,
+
+  init() {
+    this._log = console.createInstance({
+      prefix: "gloda.abattrs",
+      maxLogLevel: "Warn",
+      maxLogLevelPref: "gloda.loglevel",
+    });
+
+    try {
+      this.defineAttributes();
+    } catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  defineAttributes() {
+    /* ***** Contacts ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "identities",
+      singular: false,
+      special: GlodaConstants.kSpecialColumnChildren,
+      // specialColumnName: "contactID",
+      storageAttributeName: "_identities",
+      subjectNouns: [GlodaConstants.NOUN_CONTACT],
+      objectNoun: GlodaConstants.NOUN_IDENTITY,
+    }); // tested-by: test_attributes_fundamental
+    this._attrContactName = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "name",
+      singular: true,
+      special: GlodaConstants.kSpecialString,
+      specialColumnName: "name",
+      subjectNouns: [GlodaConstants.NOUN_CONTACT],
+      objectNoun: GlodaConstants.NOUN_STRING,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+    this._attrContactPopularity = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "popularity",
+      singular: true,
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "popularity",
+      subjectNouns: [GlodaConstants.NOUN_CONTACT],
+      objectNoun: GlodaConstants.NOUN_NUMBER,
+      canQuery: true,
+    }); // not-tested
+    this._attrContactFrecency = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "frecency",
+      singular: true,
+      special: GlodaConstants.kSpecialColumn,
+      specialColumnName: "frecency",
+      subjectNouns: [GlodaConstants.NOUN_CONTACT],
+      objectNoun: GlodaConstants.NOUN_NUMBER,
+      canQuery: true,
+    }); // not-tested
+
+    /* ***** Identities ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrDerived,
+      attributeName: "contact",
+      singular: true,
+      special: GlodaConstants.kSpecialColumnParent,
+      specialColumnName: "contactID", // the column in the db
+      idStorageAttributeName: "_contactID",
+      valueStorageAttributeName: "_contact",
+      subjectNouns: [GlodaConstants.NOUN_IDENTITY],
+      objectNoun: GlodaConstants.NOUN_CONTACT,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+    this._attrIdentityKind = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "kind",
+      singular: true,
+      special: GlodaConstants.kSpecialString,
+      specialColumnName: "kind",
+      subjectNouns: [GlodaConstants.NOUN_IDENTITY],
+      objectNoun: GlodaConstants.NOUN_STRING,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+    this._attrIdentityValue = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrFundamental,
+      attributeName: "value",
+      singular: true,
+      special: GlodaConstants.kSpecialString,
+      specialColumnName: "value",
+      subjectNouns: [GlodaConstants.NOUN_IDENTITY],
+      objectNoun: GlodaConstants.NOUN_STRING,
+      canQuery: true,
+    }); // tested-by: test_attributes_fundamental
+
+    /* ***** Contact Meta ***** */
+    // Freeform tags; not explicit like thunderbird's fundamental tags.
+    //  we differentiate for now because of fundamental implementation
+    //  differences.
+    this._attrFreeTag = Gloda.defineAttribute({
+      provider: this,
+      extensionName: GlodaConstants.BUILT_IN,
+      attributeType: GlodaConstants.kAttrExplicit,
+      attributeName: "freetag",
+      bind: true,
+      bindName: "freeTags",
+      singular: false,
+      subjectNouns: [GlodaConstants.NOUN_CONTACT],
+      objectNoun: Gloda.lookupNoun("freetag"),
+      parameterNoun: null,
+      canQuery: true,
+    }); // not-tested
+    // we need to find any existing bound freetag attributes, and use them to
+    //  populate to FreeTagNoun's understanding
+    if ("parameterBindings" in this._attrFreeTag) {
+      for (let freeTagName in this._attrFreeTag.parameterBindings) {
+        this._log.debug("Telling FreeTagNoun about: " + freeTagName);
+        FreeTagNoun.getFreeTag(freeTagName);
+      }
+    }
+  },
+
+  *process(aContact, aRawReps, aIsNew, aCallbackHandle) {
+    let card = aRawReps.card;
+    if (aContact.NOUN_ID != GlodaConstants.NOUN_CONTACT) {
+      this._log.warn("Somehow got a non-contact: " + aContact);
+      return; // this will produce an exception; we like.
+    }
+
+    // update the name
+    if (card.displayName && card.displayName != aContact.name) {
+      aContact.name = card.displayName;
+    }
+
+    aContact.freeTags = [];
+
+    let tags = null;
+    try {
+      tags = card.getProperty("Categories", null);
+    } catch (ex) {
+      this._log.error("Problem accessing property: " + ex);
+    }
+    if (tags) {
+      for (let tagName of tags.split(",")) {
+        tagName = tagName.trim();
+        if (tagName) {
+          aContact.freeTags.push(FreeTagNoun.getFreeTag(tagName));
+        }
+      }
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaMsgSearcher.jsm b/comm/mailnews/db/gloda/modules/GlodaMsgSearcher.jsm
new file mode 100644
index 0000000000..f81def2560
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaMsgSearcher.jsm
@@ -0,0 +1,361 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaMsgSearcher"];
+
+const { Gloda } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaPublic.jsm"
+);
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+/**
+ * How much time boost should a 'score point' amount to?  The authoritative,
+ *  incontrivertible answer, across all time and space, is a week.
+ *  Note that gloda stores timestamps as PRTimes for no exceedingly good
+ *  reason.
+ */
+var FUZZSCORE_TIMESTAMP_FACTOR = 1000 * 1000 * 60 * 60 * 24 * 7;
+
+var RANK_USAGE = "glodaRank(matchinfo(messagesText), 1.0, 2.0, 2.0, 1.5, 1.5)";
+
+var DASCORE =
+  "(((" +
+  RANK_USAGE +
+  " + messages.notability) * " +
+  FUZZSCORE_TIMESTAMP_FACTOR +
+  ") + messages.date)";
+
+/**
+ * A new optimization decision we are making is that we do not want to carry
+ *  around any data in our ephemeral tables that is not used for whittling the
+ *  result set.  The idea is that the btree page cache or OS cache is going to
+ *  save us from the disk seeks and carrying around the extra data is just going
+ *  to be CPU/memory churn that slows us down.
+ *
+ * Additionally, we try and avoid row lookups that would have their results
+ *  discarded by the LIMIT.  Because of limitations in FTS3 (which might
+ *  be addressed in FTS4 by a feature request), we can't avoid the 'messages'
+ *  lookup since that has the message's date and static notability but we can
+ *  defer the 'messagesText' lookup.
+ *
+ * This is the access pattern we are after here:
+ * 1) Order the matches with minimized lookup and result storage costs.
+ * - The innermost MATCH does the doclist magic and provides us with
+ *    matchinfo() support which does not require content row retrieval
+ *    from messagesText.  Unfortunately, this is not enough to whittle anything
+ *    because we still need static interestingness, so...
+ * - Based on the match we retrieve the date and notability for that row from
+ *    'messages' using this in conjunction with matchinfo() to provide a score
+ *    that we can then use to LIMIT our results.
+ * 2) We reissue the MATCH query so that we will be able to use offsets(), but
+ *    we intersect the results of this MATCH against our LIMITed results from
+ *    step 1.
+ * - We use 'docid IN (phase 1 query)' to accomplish this because it results in
+ *    efficient lookup.  If we just use a join, we get O(mn) performance because
+ *    a cartesian join ends up being performed where either we end up performing
+ *    the fulltext query M times and table scan intersect with the results from
+ *    phase 1 or we do the fulltext once but traverse the entire result set from
+ *    phase 1 N times.
+ * - We believe that the re-execution of the MATCH query should have no disk
+ *    costs because it should still be cached by SQLite or the OS.  In the case
+ *    where memory is so constrained this is not true our behavior is still
+ *    probably preferable than the old way because that would have caused lots
+ *    of swapping.
+ * - This part of the query otherwise resembles the basic gloda query but with
+ *    the inclusion of the offsets() invocation.  The messages table lookup
+ *    should not involve any disk traffic because the pages should still be
+ *    cached (SQLite or OS) from phase 1.  The messagesText lookup is new, and
+ *    this is the major disk-seek reduction optimization we are making.  (Since
+ *    we avoid this lookup for all of the documents that were excluded by the
+ *    LIMIT.)  Since offsets() also needs to retrieve the row from messagesText
+ *    there is a nice synergy there.
+ */
+var NUEVO_FULLTEXT_SQL =
+  "SELECT messages.*, messagesText.*, offsets(messagesText) AS osets " +
+  "FROM messagesText, messages " +
+  "WHERE" +
+  " messagesText MATCH ?1 " +
+  " AND messagesText.docid IN (" +
+  "SELECT docid " +
+  "FROM messagesText JOIN messages ON messagesText.docid = messages.id " +
+  "WHERE messagesText MATCH ?1 " +
+  "ORDER BY " +
+  DASCORE +
+  " DESC " +
+  "LIMIT ?2" +
+  " )" +
+  " AND messages.id = messagesText.docid " +
+  " AND +messages.deleted = 0" +
+  " AND +messages.folderID IS NOT NULL" +
+  " AND +messages.messageKey IS NOT NULL";
+
+function identityFunc(x) {
+  return x;
+}
+
+function oneLessMaxZero(x) {
+  if (x <= 1) {
+    return 0;
+  }
+  return x - 1;
+}
+
+function reduceSum(accum, curValue) {
+  return accum + curValue;
+}
+
+/*
+ * Columns are: body, subject, attachment names, author, recipients
+ */
+
+/**
+ * Scores if all search terms match in a column.  We bias against author
+ *  slightly and recipient a bit more in this case because a search that
+ *  entirely matches just on a person should give a mention of that person
+ *  in the subject or attachment a fighting chance.
+ * Keep in mind that because of our indexing in the face of address book
+ *  contacts (namely, we index the name used in the e-mail as well as the
+ *  display name on the address book card associated with the e-mail address)
+ *  a contact is going to bias towards matching multiple times.
+ */
+var COLUMN_ALL_MATCH_SCORES = [4, 20, 20, 16, 12];
+/**
+ * Score for each distinct term that matches in the column.  This is capped
+ *  by COLUMN_ALL_SCORES.
+ */
+var COLUMN_PARTIAL_PER_MATCH_SCORES = [1, 4, 4, 4, 3];
+/**
+ * If a term matches multiple times, what is the marginal score for each
+ *  additional match.  We count the total number of matches beyond the
+ *  first match for each term.  In other words, if we have 3 terms which
+ *  matched 5, 3, and 0 times, then the total from our perspective is
+ *  (5 - 1) + (3 - 1) + 0 = 4 + 2 + 0 = 6.  We take the minimum of that value
+ *  and the value in COLUMN_MULTIPLE_MATCH_LIMIT and multiply by the value in
+ *  COLUMN_MULTIPLE_MATCH_SCORES.
+ */
+var COLUMN_MULTIPLE_MATCH_SCORES = [1, 0, 0, 0, 0];
+var COLUMN_MULTIPLE_MATCH_LIMIT = [10, 0, 0, 0, 0];
+
+/**
+ * Score the message on its offsets (from stashedColumns).
+ */
+function scoreOffsets(aMessage, aContext) {
+  let score = 0;
+
+  let termTemplate = aContext.terms.map(_ => 0);
+  // for each column, a list of the incidence of each term
+  let columnTermIncidence = [
+    termTemplate.concat(),
+    termTemplate.concat(),
+    termTemplate.concat(),
+    termTemplate.concat(),
+    termTemplate.concat(),
+  ];
+
+  // we need a friendlyParseInt because otherwise the radix stuff happens
+  //  because of the extra arguments map parses.  curse you, map!
+  let offsetNums = aContext.stashedColumns[aMessage.id][0]
+    .split(" ")
+    .map(x => parseInt(x));
+  for (let i = 0; i < offsetNums.length; i += 4) {
+    let columnIndex = offsetNums[i];
+    let termIndex = offsetNums[i + 1];
+    columnTermIncidence[columnIndex][termIndex]++;
+  }
+
+  for (let iColumn = 0; iColumn < COLUMN_ALL_MATCH_SCORES.length; iColumn++) {
+    let termIncidence = columnTermIncidence[iColumn];
+    if (termIncidence.every(identityFunc)) {
+      // Bestow all match credit.
+      score += COLUMN_ALL_MATCH_SCORES[iColumn];
+    } else if (termIncidence.some(identityFunc)) {
+      // Bestow partial match credit.
+      score += Math.min(
+        COLUMN_ALL_MATCH_SCORES[iColumn],
+        COLUMN_PARTIAL_PER_MATCH_SCORES[iColumn] *
+          termIncidence.filter(identityFunc).length
+      );
+    }
+    // Bestow multiple match credit.
+    score +=
+      Math.min(
+        termIncidence.map(oneLessMaxZero).reduce(reduceSum, 0),
+        COLUMN_MULTIPLE_MATCH_LIMIT[iColumn]
+      ) * COLUMN_MULTIPLE_MATCH_SCORES[iColumn];
+  }
+
+  return score;
+}
+
+/**
+ * The searcher basically looks like a query, but is specialized for fulltext
+ *  search against messages.  Most of the explicit specialization involves
+ *  crafting a SQL query that attempts to order the matches by likelihood that
+ *  the user was looking for it.  This is based on full-text matches combined
+ *  with an explicit (generic) interest score value placed on the message at
+ *  indexing time.  This is followed by using the more generic gloda scoring
+ *  mechanism to explicitly score the messages given the search context in
+ *  addition to the more generic score adjusting rules.
+ */
+function GlodaMsgSearcher(aListener, aSearchString, aAndTerms) {
+  this.listener = aListener;
+
+  this.searchString = aSearchString;
+  this.fulltextTerms = this.parseSearchString(aSearchString);
+  this.andTerms = aAndTerms != null ? aAndTerms : true;
+
+  this.query = null;
+  this.collection = null;
+
+  this.scores = null;
+}
+GlodaMsgSearcher.prototype = {
+  /**
+   * Number of messages to retrieve initially.
+   */
+  get retrievalLimit() {
+    return Services.prefs.getIntPref(
+      "mailnews.database.global.search.msg.limit"
+    );
+  },
+
+  /**
+   * Parse the string into terms/phrases by finding matching double-quotes.
+   */
+  parseSearchString(aSearchString) {
+    aSearchString = aSearchString.trim();
+    let terms = [];
+
+    /*
+     * Add the term as long as the trim on the way in didn't obliterate it.
+     *
+     * In the future this might have other helper logic; it did once before.
+     */
+    function addTerm(aTerm) {
+      if (aTerm) {
+        terms.push(aTerm);
+      }
+    }
+
+    while (aSearchString) {
+      if (aSearchString.startsWith('"')) {
+        let endIndex = aSearchString.indexOf(aSearchString[0], 1);
+        // eat the quote if it has no friend
+        if (endIndex == -1) {
+          aSearchString = aSearchString.substring(1);
+          continue;
+        }
+
+        addTerm(aSearchString.substring(1, endIndex).trim());
+        aSearchString = aSearchString.substring(endIndex + 1);
+        continue;
+      }
+
+      let spaceIndex = aSearchString.indexOf(" ");
+      if (spaceIndex == -1) {
+        addTerm(aSearchString);
+        break;
+      }
+
+      addTerm(aSearchString.substring(0, spaceIndex));
+      aSearchString = aSearchString.substring(spaceIndex + 1);
+    }
+
+    return terms;
+  },
+
+  buildFulltextQuery() {
+    let query = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE, {
+      noMagic: true,
+      explicitSQL: NUEVO_FULLTEXT_SQL,
+      limitClauseAlreadyIncluded: true,
+      // osets is 0-based column number 14 (volatile to column changes)
+      // save the offset column for extra analysis
+      stashColumns: [14],
+    });
+
+    let fulltextQueryString = "";
+
+    for (let [iTerm, term] of this.fulltextTerms.entries()) {
+      if (iTerm) {
+        fulltextQueryString += this.andTerms ? " " : " OR ";
+      }
+
+      // Put our term in quotes.  This is needed for the tokenizer to be able
+      //  to do useful things.  The exception is people clever enough to use
+      //  NEAR.
+      if (/^NEAR(\/\d+)?$/.test(term)) {
+        fulltextQueryString += term;
+      } else if (term.length == 1 && term.charCodeAt(0) >= 0x2000) {
+        // This is a single-character CJK search query, so add a wildcard.
+        // Our tokenizer treats anything at/above 0x2000 as CJK for now.
+        fulltextQueryString += term + "*";
+      } else if (
+        (term.length == 2 &&
+          term.charCodeAt(0) >= 0x2000 &&
+          term.charCodeAt(1) >= 0x2000) ||
+        term.length >= 3
+      ) {
+        fulltextQueryString += '"' + term + '"';
+      }
+    }
+
+    query.fulltextMatches(fulltextQueryString);
+    query.limit(this.retrievalLimit);
+
+    return query;
+  },
+
+  getCollection(aListenerOverride, aData) {
+    if (aListenerOverride) {
+      this.listener = aListenerOverride;
+    }
+
+    this.query = this.buildFulltextQuery();
+    this.collection = this.query.getCollection(this, aData);
+    this.completed = false;
+
+    return this.collection;
+  },
+
+  sortBy: "-dascore",
+
+  onItemsAdded(aItems, aCollection) {
+    let newScores = Gloda.scoreNounItems(
+      aItems,
+      {
+        terms: this.fulltextTerms,
+        stashedColumns: aCollection.stashedColumns,
+      },
+      [scoreOffsets]
+    );
+    if (this.scores) {
+      this.scores = this.scores.concat(newScores);
+    } else {
+      this.scores = newScores;
+    }
+
+    if (this.listener) {
+      this.listener.onItemsAdded(aItems, aCollection);
+    }
+  },
+  onItemsModified(aItems, aCollection) {
+    if (this.listener) {
+      this.listener.onItemsModified(aItems, aCollection);
+    }
+  },
+  onItemsRemoved(aItems, aCollection) {
+    if (this.listener) {
+      this.listener.onItemsRemoved(aItems, aCollection);
+    }
+  },
+  onQueryCompleted(aCollection) {
+    this.completed = true;
+    if (this.listener) {
+      this.listener.onQueryCompleted(aCollection);
+    }
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaPublic.jsm b/comm/mailnews/db/gloda/modules/GlodaPublic.jsm
new file mode 100644
index 0000000000..555a6d8921
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaPublic.jsm
@@ -0,0 +1,45 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["Gloda"];
+
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+/* nothing to import, just run some code */ ChromeUtils.import(
+  "resource:///modules/gloda/Everybody.jsm"
+);
+const { GlodaIndexer } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaIndexer.jsm"
+);
+// initialize the indexer! (who was actually imported as a nested dep by the
+//  things Everybody.jsm imported.)  We waited until now so it could know about
+//  its indexers.
+GlodaIndexer._init();
+const { GlodaMsgIndexer } = ChromeUtils.import(
+  "resource:///modules/gloda/IndexMsg.jsm"
+);
+
+/**
+ * Expose some junk
+ */
+function proxy(aSourceObj, aSourceAttr, aDestObj, aDestAttr) {
+  aDestObj[aDestAttr] = function (...aArgs) {
+    return aSourceObj[aSourceAttr](...aArgs);
+  };
+}
+
+proxy(GlodaIndexer, "addListener", Gloda, "addIndexerListener");
+proxy(GlodaIndexer, "removeListener", Gloda, "removeIndexerListener");
+proxy(GlodaMsgIndexer, "isMessageIndexed", Gloda, "isMessageIndexed");
+proxy(
+  GlodaMsgIndexer,
+  "setFolderIndexingPriority",
+  Gloda,
+  "setFolderIndexingPriority"
+);
+proxy(
+  GlodaMsgIndexer,
+  "resetFolderIndexingPriority",
+  Gloda,
+  "resetFolderIndexingPriority"
+);
diff --git a/comm/mailnews/db/gloda/modules/GlodaQueryClassFactory.jsm b/comm/mailnews/db/gloda/modules/GlodaQueryClassFactory.jsm
new file mode 100644
index 0000000000..2e53cf5925
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaQueryClassFactory.jsm
@@ -0,0 +1,642 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaQueryClassFactory"];
+
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+/**
+ * @class Query class core; each noun gets its own sub-class where attributes
+ *  have helper methods bound.
+ *
+ * @param aOptions A dictionary of options.  Current legal options are:
+ *     - noMagic: Indicates that the noun's dbQueryJoinMagic should be ignored.
+ *                Currently, this means that messages will not have their
+ *                full-text indexed values re-attached.  This is planned to be
+ *                offset by having queries/cache lookups that do not request
+ *                noMagic to ensure that their data does get loaded.
+ *     - explicitSQL: A hand-rolled alternate representation for the core
+ *           SELECT portion of the SQL query.  The queryFromQuery logic still
+ *           generates its normal query, we just ignore its result in favor of
+ *           your provided value.  This means that the positional parameter
+ *           list is still built and you should/must rely on those bound
+ *           parameters (using '?').  The replacement occurs prior to the
+ *           outerWrapColumns, ORDER BY, and LIMIT contributions to the query.
+ *     - outerWrapColumns: If provided, wraps the query in a "SELECT *,blah
+ *           FROM (actual query)" where blah is your list of outerWrapColumns
+ *           made comma-delimited.  The idea is that this allows you to
+ *           reference the result of expressions inside the query using their
+ *           names rather than having to duplicate the logic.  In practice,
+ *           this makes things more readable but is unlikely to improve
+ *           performance.  (Namely, my use of 'offsets' for full-text stuff
+ *           ends up in the EXPLAIN plan twice despite this.)
+ *     - noDbQueryValidityConstraints: Indicates that any validity constraints
+ *           should be ignored. This should be used when you need to get every
+ *           match regardless of whether it's valid.
+ *
+ * @property _owner The query instance that holds the list of unions...
+ * @property _constraints A list of (lists of OR constraints) that are ANDed
+ *     together.  For example [[FROM bob, FROM jim], [DATE last week]] would
+ *     be requesting us to find all the messages from either bob or jim, and
+ *     sent in the last week.
+ * @property _unions A list of other queries whose results are unioned with our
+ *     own.  There is no concept of nesting or sub-queries apart from this
+ *     mechanism.
+ */
+function GlodaQueryClass(aOptions) {
+  this.options = aOptions != null ? aOptions : {};
+
+  // if we are an 'or' clause, who is our parent whom other 'or' clauses should
+  //  spawn from...
+  this._owner = null;
+  // our personal chain of and-ing.
+  this._constraints = [];
+  // the other instances we union with
+  this._unions = [];
+
+  this._order = [];
+  this._limit = 0;
+}
+
+GlodaQueryClass.prototype = {
+  WILDCARD: {},
+
+  get constraintCount() {
+    return this._constraints.length;
+  },
+
+  or() {
+    let owner = this._owner || this;
+    let orQuery = new this._queryClass();
+    orQuery._owner = owner;
+    owner._unions.push(orQuery);
+    return orQuery;
+  },
+
+  orderBy(...aArgs) {
+    this._order.push(...aArgs);
+    return this;
+  },
+
+  limit(aLimit) {
+    this._limit = aLimit;
+    return this;
+  },
+
+  /**
+   * Return a collection asynchronously populated by this collection.  You must
+   *  provide a listener to receive notifications from the collection as it
+   *  receives updates.  The listener object should implement onItemsAdded,
+   *  onItemsModified, and onItemsRemoved methods, all of which take a single
+   *  argument which is the list of items which have been added, modified, or
+   *  removed respectively.
+   *
+   * @param aListener The collection listener.
+   * @param [aData] The data attribute to set on the collection.
+   * @param [aArgs.becomeExplicit] Make the collection explicit so that the
+   *     collection will only ever contain results found from the database
+   *     query and the query will not be updated as new items are indexed that
+   *     also match the query.
+   * @param [aArgs.becomeNull] Change the collection's query to a null query so
+   *     that it will never receive any additional added/modified/removed events
+   *     apart from the underlying database query.  This is really only intended
+   *     for gloda internal use but may be acceptable for non-gloda use.  Please
+   *     ask on mozilla.dev.apps.thunderbird first to make sure there isn't a
+   *     better solution for your use-case.  (Note: removals will still happen
+   *     when things get fully deleted.)
+   */
+  getCollection(aListener, aData, aArgs) {
+    this.completed = false;
+    return this._nounDef.datastore.queryFromQuery(
+      this,
+      aListener,
+      aData,
+      /* aExistingCollection */ null,
+      /* aMasterCollection */ null,
+      aArgs
+    );
+  },
+
+  /* eslint-disable complexity */
+  /**
+   * Test whether the given first-class noun instance satisfies this query.
+   *
+   * @testpoint gloda.query.test
+   */
+  test(aObj) {
+    // when changing this method, be sure that GlodaDatastore's queryFromQuery
+    //  method likewise has any required changes made.
+    let unionQueries = [this].concat(this._unions);
+
+    for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
+      let curQuery = unionQueries[iUnion];
+
+      // assume success until a specific (or) constraint proves us wrong
+      let querySatisfied = true;
+      for (
+        let iConstraint = 0;
+        iConstraint < curQuery._constraints.length;
+        iConstraint++
+      ) {
+        let constraint = curQuery._constraints[iConstraint];
+        let [constraintType, attrDef] = constraint;
+        let boundName = attrDef ? attrDef.boundName : "id";
+        if (
+          boundName in aObj &&
+          aObj[boundName] === GlodaConstants.IGNORE_FACET
+        ) {
+          querySatisfied = false;
+          break;
+        }
+
+        let constraintValues = constraint.slice(2);
+
+        if (constraintType === GlodaConstants.kConstraintIdIn) {
+          if (!constraintValues.includes(aObj.id)) {
+            querySatisfied = false;
+            break;
+          }
+        } else if (
+          constraintType === GlodaConstants.kConstraintIn ||
+          constraintType === GlodaConstants.kConstraintEquals
+        ) {
+          // @testpoint gloda.query.test.kConstraintIn
+          let objectNounDef = attrDef.objectNounDef;
+
+          // if they provide an equals comparator, use that.
+          // (note: the next case has better optimization possibilities than
+          //  this mechanism, but of course has higher initialization costs or
+          //  code complexity costs...)
+          if (objectNounDef.equals) {
+            let testValues;
+            if (!(boundName in aObj)) {
+              testValues = [];
+            } else if (attrDef.singular) {
+              testValues = [aObj[boundName]];
+            } else {
+              testValues = aObj[boundName];
+            }
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (
+              constraintValues.length == 0 &&
+              testValues.length &&
+              testValues[0] != null
+            ) {
+              continue;
+            }
+
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier) {
+                continue;
+              }
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              for (let value of constraintValues) {
+                if (objectNounDef.equals(testValue, value)) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch) {
+                break;
+              }
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          } else {
+            // otherwise, we need to convert everyone to their param/value form
+            //  in order to test for equality
+            // let's just do the simple, obvious thing for now.  which is
+            //  what we did in the prior case but exploding values using
+            //  toParamAndValue, and then comparing.
+            let testValues;
+            if (!(boundName in aObj)) {
+              testValues = [];
+            } else if (attrDef.singular) {
+              testValues = [aObj[boundName]];
+            } else {
+              testValues = aObj[boundName];
+            }
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (
+              constraintValues.length == 0 &&
+              testValues.length &&
+              testValues[0] != null
+            ) {
+              continue;
+            }
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier) {
+                continue;
+              }
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              let [aParam, aValue] = objectNounDef.toParamAndValue(testValue);
+              for (let value of constraintValues) {
+                // skip empty set check sentinel values
+                if (value == null && attrDef.emptySetIsSignificant) {
+                  continue;
+                }
+                let [bParam, bValue] = objectNounDef.toParamAndValue(value);
+                if (aParam == bParam && aValue == bValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch) {
+                break;
+              }
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          }
+        } else if (constraintType === GlodaConstants.kConstraintRanges) {
+          // @testpoint gloda.query.test.kConstraintRanges
+          let objectNounDef = attrDef.objectNounDef;
+
+          let testValues;
+          if (!(boundName in aObj)) {
+            testValues = [];
+          } else if (attrDef.singular) {
+            testValues = [aObj[boundName]];
+          } else {
+            testValues = aObj[boundName];
+          }
+
+          let foundMatch = false;
+          for (let testValue of testValues) {
+            let [tParam, tValue] = objectNounDef.toParamAndValue(testValue);
+            for (let rangeTuple of constraintValues) {
+              let [lowerRValue, upperRValue] = rangeTuple;
+              if (lowerRValue == null) {
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                if (tParam == upperParam && tValue <= upperValue) {
+                  foundMatch = true;
+                  break;
+                }
+              } else if (upperRValue == null) {
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if (tParam == lowerParam && tValue >= lowerValue) {
+                  foundMatch = true;
+                  break;
+                }
+              } else {
+                // no one is null
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if (
+                  tParam == lowerParam &&
+                  tValue >= lowerValue &&
+                  tParam == upperParam &&
+                  tValue <= upperValue
+                ) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+            }
+            if (foundMatch) {
+              break;
+            }
+          }
+          if (!foundMatch) {
+            querySatisfied = false;
+            break;
+          }
+        } else if (constraintType === GlodaConstants.kConstraintStringLike) {
+          // @testpoint gloda.query.test.kConstraintStringLike
+          let curIndex = 0;
+          let value = boundName in aObj ? aObj[boundName] : "";
+          // the attribute must be singular, we don't support arrays of strings.
+          for (let valuePart of constraintValues) {
+            if (typeof valuePart == "string") {
+              let index = value.indexOf(valuePart);
+              // if curIndex is null, we just need any match
+              // if it's not null, it must match the offset of our found match
+              if (curIndex === null) {
+                if (index == -1) {
+                  querySatisfied = false;
+                } else {
+                  curIndex = index + valuePart.length;
+                }
+              } else if (index != curIndex) {
+                querySatisfied = false;
+              } else {
+                curIndex = index + valuePart.length;
+              }
+              if (!querySatisfied) {
+                break;
+              }
+            } else {
+              // wild!
+              curIndex = null;
+            }
+          }
+          // curIndex must be null or equal to the length of the string
+          if (querySatisfied && curIndex !== null && curIndex != value.length) {
+            querySatisfied = false;
+          }
+        } else if (constraintType === GlodaConstants.kConstraintFulltext) {
+          // @testpoint gloda.query.test.kConstraintFulltext
+          // this is beyond our powers. Even if we have the fulltext content in
+          //  memory, which we may not, the tokenization and such to perform
+          //  the testing gets very complicated in the face of i18n, etc.
+          // so, let's fail if the item is not already in the collection, and
+          //  let the testing continue if it is.  (some other constraint may no
+          //  longer apply...)
+          if (!(aObj.id in this.collection._idMap)) {
+            querySatisfied = false;
+          }
+        }
+
+        if (!querySatisfied) {
+          break;
+        }
+      }
+
+      if (querySatisfied) {
+        return true;
+      }
+    }
+    return false;
+  },
+  /* eslint-enable complexity */
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  traditional in/equals constraint.  The goal is to let them build a range
+   *  without having to know how we structure |_constraints|.
+   *
+   * @protected
+   */
+  _inConstraintHelper(aAttrDef, aValues) {
+    let constraint = [GlodaConstants.kConstraintIn, aAttrDef].concat(aValues);
+    this._constraints.push(constraint);
+    return this;
+  },
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  range.  The goal is to let them build a range without having to know how
+   *  we structure |_constraints| or requiring them to mark themselves as
+   *  continuous to get a "Range".
+   *
+   * @protected
+   */
+  _rangedConstraintHelper(aAttrDef, aRanges) {
+    let constraint = [GlodaConstants.kConstraintRanges, aAttrDef].concat(
+      aRanges
+    );
+    this._constraints.push(constraint);
+    return this;
+  },
+};
+
+/**
+ * @class A query that never matches anything.
+ *
+ * Collections corresponding to this query are intentionally frozen in time and
+ *  do not want to be notified of any updates.  We need the collection to be
+ *  registered with the collection manager so that the noun instances in the
+ *  collection are always 'reachable' via the collection for as long as we might
+ *  be handing out references to the instances.  (The other way to avoid updates
+ *  would be to not register the collection, but then items might not be
+ *  reachable.)
+ * This is intended to be used in implementation details behind the gloda
+ *  abstraction barrier.  For example, the message indexer likes to be able
+ *  to represent 'ghost' and deleted messages, but these should never be exposed
+ *  to the user.  For code simplicity, it wants to be able to use the query
+ *  mechanism.  But it doesn't want updates that are effectively
+ *  nonsensical.  For example, a ghost message that is reused by message
+ *  indexing may already be present in a collection; when the collection manager
+ *  receives an itemsAdded event, a GlodaExplicitQueryClass would result in
+ *  an item added notification in that case, which would wildly not be desired.
+ */
+function GlodaNullQueryClass() {}
+
+GlodaNullQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for null queries.
+   */
+  options: {},
+
+  /**
+   * Provide a duck-typing way of indicating to GlodaCollectionManager that our
+   *  associated collection just doesn't want anything to change.  Our test
+   *  function is able to convey most of it, but special-casing has to happen
+   *  somewhere, so it happens here.
+   */
+  frozen: true,
+
+  /**
+   * Since our query never matches anything, it doesn't make sense to let
+   *  someone attempt to construct a boolean OR involving us.
+   *
+   * @returns null
+   */
+  or() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on a null query.  This method is normally used (on a normal query)
+   *  to return a collection populated by the constraints of the query.  We
+   *  match nothing, so we should return nothing.  More importantly, you are
+   *  currently doing something wrong if you try and do this, so null is
+   *  appropriate.  It may turn out that it makes sense for us to return an
+   *  empty collection in the future for sentinel value purposes, but we'll
+   *  cross that bridge when we come to it.
+   *
+   * @returns null
+   */
+  getCollection() {
+    return null;
+  },
+
+  /**
+   * Never matches anything.
+   *
+   * @param aObj The object someone wants us to test for relevance to our
+   *     associated collection.  But we don't care!  Not a fig!
+   * @returns false
+   */
+  test(aObj) {
+    return false;
+  },
+};
+
+/**
+ * @class A query that only 'tests' for already belonging to the collection.
+ *
+ * This type of collection is useful for when you (or rather your listener)
+ *  are interested in hearing about modifications to your collection or removals
+ *  from your collection because of deletion, but do not want to be notified
+ *  about newly indexed items matching your normal query constraints.
+ *
+ * @param aCollection The collection this query belongs to.  This needs to be
+ *     passed-in here or the collection should set the attribute directly when
+ *     the query is passed in to a collection's constructor.
+ */
+function GlodaExplicitQueryClass(aCollection) {
+  this.collection = aCollection;
+}
+
+GlodaExplicitQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for explicit queries.
+   */
+  options: {},
+
+  /**
+   * Since our query is intended to only match the contents of our collection,
+   *  it doesn't make sense to let someone attempt to construct a boolean OR
+   *  involving us.
+   *
+   * @returns null
+   */
+  or() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on an explicit query.  This method is normally used (on a normal
+   *  query) to return a collection populated by the constraints of the query.
+   *  In the case of an explicit query, we expect it will be associated with
+   *  either a hand-created collection or the results of a normal query that is
+   *  immediately converted into an explicit query.  In all likelihood, calling
+   *  this method on an instance of this type is an error, so it is helpful to
+   *  return null because people will error hard.
+   *
+   * @returns null
+   */
+  getCollection() {
+    return null;
+  },
+
+  /**
+   * Matches only items that are already in the collection associated with this
+   *  query (by id).
+   *
+   * @param aObj The object/item to test for already being in the associated
+   *     collection.
+   * @returns true when the object is in the associated collection, otherwise
+   *     false.
+   */
+  test(aObj) {
+    return aObj.id in this.collection._idMap;
+  },
+};
+
+/**
+ * @class A query that 'tests' true for everything.  Intended for debugging purposes
+ *  only.
+ */
+function GlodaWildcardQueryClass() {}
+
+GlodaWildcardQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation.
+   */
+  options: {},
+
+  // don't let people try and mess with us
+  or() {
+    return null;
+  },
+  // don't let people try and query on us (until we have a real use case for
+  //  that...)
+  getCollection() {
+    return null;
+  },
+  /**
+   * Everybody wins!
+   */
+  test(aObj) {
+    return true;
+  },
+};
+
+/**
+ * Factory method to effectively create per-noun subclasses of GlodaQueryClass,
+ *  GlodaNullQueryClass, GlodaExplicitQueryClass, and GlodaWildcardQueryClass.
+ *  For GlodaQueryClass this allows us to add per-noun helpers.  For the others,
+ *  this is merely a means of allowing us to attach the (per-noun) nounDef to
+ *  the 'class'.
+ */
+function GlodaQueryClassFactory(aNounDef) {
+  let newQueryClass = function (aOptions) {
+    GlodaQueryClass.call(this, aOptions);
+  };
+  newQueryClass.prototype = new GlodaQueryClass();
+  newQueryClass.prototype._queryClass = newQueryClass;
+  newQueryClass.prototype._nounDef = aNounDef;
+
+  let newNullClass = function (aCollection) {
+    GlodaNullQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newNullClass.prototype = new GlodaNullQueryClass();
+  newNullClass.prototype._queryClass = newNullClass;
+  newNullClass.prototype._nounDef = aNounDef;
+
+  let newExplicitClass = function (aCollection) {
+    GlodaExplicitQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newExplicitClass.prototype = new GlodaExplicitQueryClass();
+  newExplicitClass.prototype._queryClass = newExplicitClass;
+  newExplicitClass.prototype._nounDef = aNounDef;
+
+  let newWildcardClass = function (aCollection) {
+    GlodaWildcardQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newWildcardClass.prototype = new GlodaWildcardQueryClass();
+  newWildcardClass.prototype._queryClass = newWildcardClass;
+  newWildcardClass.prototype._nounDef = aNounDef;
+
+  return [newQueryClass, newNullClass, newExplicitClass, newWildcardClass];
+}
diff --git a/comm/mailnews/db/gloda/modules/GlodaSyntheticView.jsm b/comm/mailnews/db/gloda/modules/GlodaSyntheticView.jsm
new file mode 100644
index 0000000000..2e0fb7b5be
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaSyntheticView.jsm
@@ -0,0 +1,175 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file is charged with providing you a way to have a pretty gloda-backed
+ *  nsIMsgDBView.
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaSyntheticView"];
+
+/**
+ * Create a synthetic view suitable for passing to |FolderDisplayWidget.show|.
+ * You must pass a query, collection, or conversation in.
+ *
+ * @param {GlodaQuery} [aArgs.query] A gloda query to run.
+ * @param {GlodaCollection} [aArgs.collection] An already-populated collection
+ *     to display.  Do not call getCollection on a query and hand us that.  We
+ *     will not register ourselves as a listener and things will not work.
+ * @param {GlodaConversation} [aArgs.conversation] A conversation whose messages
+ *     you want to display.
+ */
+function GlodaSyntheticView(aArgs) {
+  if ("query" in aArgs) {
+    this.query = aArgs.query;
+    this.collection = this.query.getCollection(this);
+    this.completed = false;
+    this.viewType = "global";
+  } else if ("collection" in aArgs) {
+    this.query = null;
+    this.collection = aArgs.collection;
+    this.completed = true;
+    this.viewType = "global";
+  } else if ("conversation" in aArgs) {
+    this.collection = aArgs.conversation.getMessagesCollection(this);
+    this.query = this.collection.query;
+    this.completed = false;
+    this.viewType = "conversation";
+    this.selectedMessage = aArgs.message.folderMessage;
+  } else {
+    throw new Error("You need to pass a query or collection");
+  }
+
+  this.customColumns = [];
+}
+GlodaSyntheticView.prototype = {
+  defaultSort: [
+    [Ci.nsMsgViewSortType.byDate, Ci.nsMsgViewSortOrder.descending],
+  ],
+
+  /**
+   * Request the search be performed and notification provided to
+   *  aSearchListener.  If results are already available, they should
+   *  be provided to aSearchListener without re-performing the search.
+   */
+  search(aSearchListener, aCompletionCallback) {
+    this.searchListener = aSearchListener;
+    this.completionCallback = aCompletionCallback;
+
+    this.searchListener.onNewSearch();
+    if (this.completed) {
+      this.reportResults(this.collection.items);
+      // we're not really aborting, but it closes things out nicely
+      this.abortSearch();
+    }
+  },
+
+  abortSearch() {
+    if (this.searchListener) {
+      this.searchListener.onSearchDone(Cr.NS_OK);
+    }
+    if (this.completionCallback) {
+      this.completionCallback();
+    }
+    this.searchListener = null;
+    this.completionCallback = null;
+  },
+
+  reportResults(aItems) {
+    for (let item of aItems) {
+      let hdr = item.folderMessage;
+      if (hdr) {
+        this.searchListener.onSearchHit(hdr, hdr.folder);
+      }
+    }
+  },
+
+  /**
+   * Helper function used by |DBViewWrapper.getMsgHdrForMessageID| since there
+   *  are no actual backing folders for it to check.
+   */
+  getMsgHdrForMessageID(aMessageId) {
+    for (let item of this.collection.items) {
+      if (item.headerMessageID == aMessageId) {
+        let hdr = item.folderMessage;
+        if (hdr) {
+          return hdr;
+        }
+      }
+    }
+    return null;
+  },
+
+  /**
+   * The default set of columns to show.
+   */
+  DEFAULT_COLUMN_STATES: {
+    threadCol: {
+      visible: true,
+    },
+    flaggedCol: {
+      visible: true,
+    },
+    subjectCol: {
+      visible: true,
+    },
+    correspondentCol: {
+      visible: Services.prefs.getBoolPref("mail.threadpane.use_correspondents"),
+    },
+    senderCol: {
+      visible: !Services.prefs.getBoolPref(
+        "mail.threadpane.use_correspondents"
+      ),
+    },
+    dateCol: {
+      visible: true,
+    },
+    locationCol: {
+      visible: true,
+    },
+  },
+
+  // --- settings persistence
+  getPersistedSetting(aSetting) {
+    try {
+      return JSON.parse(
+        Services.prefs.getCharPref(
+          "mailnews.database.global.views." + this.viewType + "." + aSetting
+        )
+      );
+    } catch (e) {
+      return this.getDefaultSetting(aSetting);
+    }
+  },
+  setPersistedSetting(aSetting, aValue) {
+    Services.prefs.setCharPref(
+      "mailnews.database.global.views." + this.viewType + "." + aSetting,
+      JSON.stringify(aValue)
+    );
+  },
+  getDefaultSetting(aSetting) {
+    if (aSetting == "columns") {
+      return this.DEFAULT_COLUMN_STATES;
+    }
+    return undefined;
+  },
+
+  // --- collection listener
+  onItemsAdded(aItems, aCollection) {
+    if (this.searchListener) {
+      this.reportResults(aItems);
+    }
+  },
+  onItemsModified(aItems, aCollection) {},
+  onItemsRemoved(aItems, aCollection) {},
+  onQueryCompleted(aCollection) {
+    this.completed = true;
+    if (this.searchListener) {
+      this.searchListener.onSearchDone(Cr.NS_OK);
+    }
+    if (this.completionCallback) {
+      this.completionCallback();
+    }
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/GlodaUtils.jsm b/comm/mailnews/db/gloda/modules/GlodaUtils.jsm
new file mode 100644
index 0000000000..a2b7fe4174
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/GlodaUtils.jsm
@@ -0,0 +1,84 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["GlodaUtils"];
+
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+
+/**
+ * @namespace A holding place for logic that is not gloda-specific and should
+ *  reside elsewhere.
+ */
+var GlodaUtils = {
+  /**
+   * This Regexp is super-complicated and used at least in two different parts of
+   * the code, so let's expose it from one single location.
+   */
+  PART_RE: new RegExp(
+    "^[^?]+\\?(?:/;section=\\d+\\?)?(?:[^&]+&)*part=([^&]+)(?:&[^&]+)*$"
+  ),
+
+  deMime(aString) {
+    return MailServices.mimeConverter.decodeMimeHeader(
+      aString,
+      null,
+      false,
+      true
+    );
+  },
+
+  _headerParser: MailServices.headerParser,
+
+  /**
+   * Parses an RFC 2822 list of e-mail addresses and returns an object with
+   *  4 attributes, as described below.  We will use the example of the user
+   *  passing an argument of '"Bob Smith" <bob@example.com>'.
+   *
+   * This method (by way of nsIMsgHeaderParser) takes care of decoding mime
+   *  headers, but is not aware of folder-level character set overrides.
+   *
+   * count: the number of addresses parsed. (ex: 1)
+   * addresses: a list of e-mail addresses (ex: ["bob@example.com"])
+   * names: a list of names (ex: ["Bob Smith"])
+   * fullAddresses: aka the list of name and e-mail together (ex: ['"Bob Smith"
+   *  <bob@example.com>']).
+   *
+   * This method is a convenience wrapper around nsIMsgHeaderParser.
+   */
+  parseMailAddresses(aMailAddresses) {
+    let addresses = this._headerParser.parseEncodedHeader(aMailAddresses);
+    return {
+      names: addresses.map(a => a.name || null),
+      addresses: addresses.map(a => a.email),
+      fullAddresses: addresses.map(a => a.toString()),
+      count: addresses.length,
+    };
+  },
+
+  /**
+   * MD5 hash a string and return the hex-string result. Impl from nsICryptoHash
+   *  docs.
+   */
+  md5HashString(aString) {
+    let data = [...new TextEncoder().encode(aString)];
+
+    let hasher = Cc["@mozilla.org/security/hash;1"].createInstance(
+      Ci.nsICryptoHash
+    );
+    hasher.init(Ci.nsICryptoHash.MD5);
+    hasher.update(data, data.length);
+    let hash = hasher.finish(false);
+
+    // return the two-digit hexadecimal code for a byte
+    function toHexString(charCode) {
+      return ("0" + charCode.toString(16)).slice(-2);
+    }
+
+    // convert the binary hash data to a hex string.
+    let hex = Object.keys(hash).map(i => toHexString(hash.charCodeAt(i)));
+    return hex.join("");
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/IndexMsg.jsm b/comm/mailnews/db/gloda/modules/IndexMsg.jsm
new file mode 100644
index 0000000000..9a4add589e
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/IndexMsg.jsm
@@ -0,0 +1,3464 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/*
+ * This file currently contains a fairly general implementation of asynchronous
+ *  indexing with a very explicit message indexing implementation.  As gloda
+ *  will eventually want to index more than just messages, the message-specific
+ *  things should ideally lose their special hold on this file.  This will
+ *  benefit readability/size as well.
+ */
+
+const EXPORTED_SYMBOLS = ["GlodaMsgIndexer"];
+
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+const { GlodaDatastore } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaDatastore.jsm"
+);
+const { GlodaContact, GlodaFolder } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaDataModel.jsm"
+);
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaCollectionManager } = ChromeUtils.import(
+  "resource:///modules/gloda/Collection.jsm"
+);
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+const { GlodaIndexer, IndexingJob } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaIndexer.jsm"
+);
+const { MsgHdrToMimeMessage } = ChromeUtils.import(
+  "resource:///modules/gloda/MimeMessage.jsm"
+);
+
+const lazy = {};
+ChromeUtils.defineModuleGetter(
+  lazy,
+  "MailUtils",
+  "resource:///modules/MailUtils.jsm"
+);
+
+// Cr does not have mailnews error codes!
+var NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE = 0x80550005;
+
+var GLODA_MESSAGE_ID_PROPERTY = "gloda-id";
+/**
+ * Message header property to track dirty status; one of
+ *  |GlodaIndexer.kMessageClean|, |GlodaIndexer.kMessageDirty|,
+ *  |GlodaIndexer.kMessageFilthy|.
+ */
+var GLODA_DIRTY_PROPERTY = "gloda-dirty";
+
+/**
+ * The sentinel GLODA_MESSAGE_ID_PROPERTY value indicating that a message fails
+ *  to index and we should not bother trying again, at least not until a new
+ *  release is made.
+ *
+ * This should ideally just flip between 1 and 2, with GLODA_OLD_BAD_MESSAGE_ID
+ *  flipping in the other direction.  If we start having more trailing badness,
+ *  _indexerGetEnumerator and GLODA_OLD_BAD_MESSAGE_ID will need to be altered.
+ *
+ * When flipping this, be sure to update glodaTestHelper.js's copy.
+ */
+var GLODA_BAD_MESSAGE_ID = 2;
+/**
+ * The gloda id we used to use to mark messages as bad, but now should be
+ *  treated as eligible for indexing.  This is only ever used for consideration
+ *  when creating msg header enumerators with `_indexerGetEnumerator` which
+ *  means we only will re-index such messages in an indexing sweep.  Accordingly
+ *  event-driven indexing will still treat such messages as unindexed (and
+ *  unindexable) until an indexing sweep picks them up.
+ */
+var GLODA_OLD_BAD_MESSAGE_ID = 1;
+var GLODA_FIRST_VALID_MESSAGE_ID = 32;
+
+var JUNK_SCORE_PROPERTY = "junkscore";
+var JUNK_SPAM_SCORE_STR = Ci.nsIJunkMailPlugin.IS_SPAM_SCORE.toString();
+
+/**
+ * The processing flags that tell us that a message header has not yet been
+ *  reported to us via msgsClassified.  If it has one of these flags, it is
+ *  still being processed.
+ */
+var NOT_YET_REPORTED_PROCESSING_FLAGS =
+  Ci.nsMsgProcessingFlags.NotReportedClassified |
+  Ci.nsMsgProcessingFlags.ClassifyJunk;
+
+// for list comprehension fun
+function* range(begin, end) {
+  for (let i = begin; i < end; ++i) {
+    yield i;
+  }
+}
+
+/**
+ * We do not set properties on the messages until we perform a DB commit; this
+ *  helper class tracks messages that we have indexed but are not yet marked
+ *  as such on their header.
+ */
+var PendingCommitTracker = {
+  /**
+   * Maps message URIs to their gloda ids.
+   *
+   * I am not entirely sure why I chose the URI for the key rather than
+   *  gloda folder ID + message key.  Most likely it was to simplify debugging
+   *  since the gloda folder ID is opaque while the URI is very informative.  It
+   *  is also possible I was afraid of IMAP folder renaming triggering a UID
+   *  renumbering?
+   */
+  _indexedMessagesPendingCommitByKey: {},
+  /**
+   * Map from the pending commit gloda id to a tuple of [the corresponding
+   *  message header, dirtyState].
+   */
+  _indexedMessagesPendingCommitByGlodaId: {},
+  /**
+   * Do we have a post-commit handler registered with this transaction yet?
+   */
+  _pendingCommit: false,
+
+  /**
+   * The function gets called when the commit actually happens to flush our
+   *  message id's.
+   *
+   * It is very possible that by the time this call happens we have left the
+   *  folder and nulled out msgDatabase on the folder.  Since nulling it out
+   *  is what causes the commit, if we set the headers here without somehow
+   *  forcing a commit, we will lose.  Badly.
+   * Accordingly, we make a list of all the folders that the headers belong to
+   *  as we iterate, make sure to re-attach their msgDatabase before forgetting
+   *  the headers, then make sure to zero the msgDatabase again, triggering a
+   *  commit.  If there were a way to directly get the nsIMsgDatabase from the
+   *  header we could do that and call commit directly.  We don't track
+   *  databases along with the headers since the headers can change because of
+   *  moves and that would increase the number of moving parts.
+   */
+  _commitCallback() {
+    let foldersByURI = {};
+    let lastFolder = null;
+
+    for (let glodaId in PendingCommitTracker._indexedMessagesPendingCommitByGlodaId) {
+      let [msgHdr, dirtyState] =
+        PendingCommitTracker._indexedMessagesPendingCommitByGlodaId[glodaId];
+      // Mark this message as indexed.
+      // It's conceivable the database could have gotten blown away, in which
+      //  case the message headers are going to throw exceptions when we try
+      //  and touch them.  So we wrap this in a try block that complains about
+      //  this unforeseen circumstance.  (noteFolderDatabaseGettingBlownAway
+      //  should have been called and avoided this situation in all known
+      //  situations.)
+      try {
+        let curGlodaId = msgHdr.getUint32Property(GLODA_MESSAGE_ID_PROPERTY);
+        if (curGlodaId != glodaId) {
+          msgHdr.setUint32Property(GLODA_MESSAGE_ID_PROPERTY, glodaId);
+        }
+        let headerDirty = msgHdr.getUint32Property(GLODA_DIRTY_PROPERTY);
+        if (headerDirty != dirtyState) {
+          msgHdr.setUint32Property(GLODA_DIRTY_PROPERTY, dirtyState);
+        }
+
+        // Make sure this folder is in our foldersByURI map.
+        if (lastFolder == msgHdr.folder) {
+          continue;
+        }
+        lastFolder = msgHdr.folder;
+        let folderURI = lastFolder.URI;
+        if (!(folderURI in foldersByURI)) {
+          foldersByURI[folderURI] = lastFolder;
+        }
+      } catch (ex) {
+        GlodaMsgIndexer._log.error(
+          "Exception while attempting to mark message with gloda state after" +
+            "db commit",
+          ex
+        );
+      }
+    }
+
+    // it is vitally important to do this before we forget about the headers!
+    for (let uri in foldersByURI) {
+      let folder = foldersByURI[uri];
+      // This will not cause a parse.  The database is in-memory since we have
+      //  a header that belongs to it.  This just causes the folder to
+      //  re-acquire a reference from the database manager.
+      folder.msgDatabase;
+      // And this will cause a commit.  (And must be done since we don't want
+      //  to cause a leak.)
+      folder.msgDatabase = null;
+    }
+
+    PendingCommitTracker._indexedMessagesPendingCommitByGlodaId = {};
+    PendingCommitTracker._indexedMessagesPendingCommitByKey = {};
+
+    PendingCommitTracker._pendingCommit = false;
+  },
+
+  /**
+   * Track a message header that should be marked with the given gloda id when
+   *  the database commits.
+   */
+  track(aMsgHdr, aGlodaId) {
+    let pendingKey = aMsgHdr.folder.URI + "#" + aMsgHdr.messageKey;
+    this._indexedMessagesPendingCommitByKey[pendingKey] = aGlodaId;
+    this._indexedMessagesPendingCommitByGlodaId[aGlodaId] = [
+      aMsgHdr,
+      GlodaMsgIndexer.kMessageClean,
+    ];
+
+    if (!this._pendingCommit) {
+      GlodaDatastore.runPostCommit(this._commitCallback);
+      this._pendingCommit = true;
+    }
+  },
+
+  /**
+   * Get the current state of a message header given that we cannot rely on just
+   *  looking at the header's properties because we defer setting those
+   *  until the SQLite commit happens.
+   *
+   * @returns Tuple of [gloda id, dirty status].
+   */
+  getGlodaState(aMsgHdr) {
+    // If it's in the pending commit table, then the message is basically
+    //  clean.  Return that info.
+    let pendingKey = aMsgHdr.folder.URI + "#" + aMsgHdr.messageKey;
+    if (pendingKey in this._indexedMessagesPendingCommitByKey) {
+      let glodaId =
+        PendingCommitTracker._indexedMessagesPendingCommitByKey[pendingKey];
+      return [glodaId, this._indexedMessagesPendingCommitByGlodaId[glodaId][1]];
+    }
+
+    // Otherwise the header's concept of state is correct.
+    let glodaId = aMsgHdr.getUint32Property(GLODA_MESSAGE_ID_PROPERTY);
+    let glodaDirty = aMsgHdr.getUint32Property(GLODA_DIRTY_PROPERTY);
+    return [glodaId, glodaDirty];
+  },
+
+  /**
+   * Update our structure to reflect moved headers.  Moves are currently
+   *  treated as weakly interesting and do not require a reindexing
+   *  although collections will get notified.  So our job is to to fix-up
+   *  the pending commit information if the message has a pending commit.
+   */
+  noteMove(aOldHdr, aNewHdr) {
+    let oldKey = aOldHdr.folder.URI + "#" + aOldHdr.messageKey;
+    if (!(oldKey in this._indexedMessagesPendingCommitByKey)) {
+      return;
+    }
+
+    let glodaId = this._indexedMessagesPendingCommitByKey[oldKey];
+    delete this._indexedMessagesPendingCommitByKey[oldKey];
+
+    let newKey = aNewHdr.folder.URI + "#" + aNewHdr.messageKey;
+    this._indexedMessagesPendingCommitByKey[newKey] = glodaId;
+
+    // only clobber the header, not the dirty state
+    this._indexedMessagesPendingCommitByGlodaId[glodaId][0] = aNewHdr;
+  },
+
+  /**
+   * A blind move is one where we have the source header but not the destination
+   *  header.  This happens for IMAP messages that do not involve offline fake
+   *  headers.
+   * XXX Since IMAP moves will propagate the gloda-id/gloda-dirty bits for us,
+   *  we could detect the other side of the move when it shows up as a
+   *  msgsClassified event and restore the mapping information.  Since the
+   *  offline fake header case should now cover the bulk of IMAP move
+   *  operations, we probably do not need to pursue this.
+   *
+   * We just re-dispatch to noteDirtyHeader because we can't do anything more
+   *  clever.
+   */
+  noteBlindMove(aOldHdr) {
+    this.noteDirtyHeader(aOldHdr);
+  },
+
+  /**
+   * If a message is dirty we should stop tracking it for post-commit
+   *  purposes.  This is not because we don't want to write to its header
+   *  when we commit as much as that we want to avoid |getHeaderGlodaState|
+   *  reporting that the message is clean.  We could complicate our state
+   *  by storing that information, but this is easier and ends up the same
+   *  in the end.
+   */
+  noteDirtyHeader(aMsgHdr) {
+    let pendingKey = aMsgHdr.folder.URI + "#" + aMsgHdr.messageKey;
+    if (!(pendingKey in this._indexedMessagesPendingCommitByKey)) {
+      return;
+    }
+
+    // (It is important that we get the gloda id from our own structure!)
+    let glodaId = this._indexedMessagesPendingCommitByKey[pendingKey];
+    this._indexedMessagesPendingCommitByGlodaId[glodaId][1] =
+      GlodaMsgIndexer.kMessageDirty;
+  },
+
+  /**
+   * Sometimes a folder database gets blown away.  This happens for one of two
+   *  expected reasons right now:
+   * - Folder compaction.
+   * - Explicit reindexing of a folder via the folder properties "rebuild index"
+   *    button.
+   *
+   * When this happens, we are basically out of luck and need to discard
+   *  everything about the folder.  The good news is that the folder compaction
+   *  pass is clever enough to re-establish the linkages that are being lost
+   *  when we drop these things on the floor.  Reindexing of a folder is not
+   *  clever enough to deal with this but is an exceptional case of last resort
+   *  (the user should not normally be performing a reindex as part of daily
+   *  operation), so we accept that messages may be redundantly indexed.
+   */
+  noteFolderDatabaseGettingBlownAway(aMsgFolder) {
+    let uri = aMsgFolder.URI + "#";
+    for (let key of Object.keys(this._indexedMessagesPendingCommitByKey)) {
+      // this is not as efficient as it could be, but compaction is relatively
+      //  rare and the number of pending headers is generally going to be
+      //  small.
+      if (key.indexOf(uri) == 0) {
+        delete this._indexedMessagesPendingCommitByKey[key];
+      }
+    }
+  },
+};
+
+/**
+ * This callback handles processing the asynchronous query results of
+ *  |GlodaMsgIndexer.getMessagesByMessageID|.
+ */
+function MessagesByMessageIdCallback(
+  aMsgIDToIndex,
+  aResults,
+  aCallback,
+  aCallbackThis
+) {
+  this.msgIDToIndex = aMsgIDToIndex;
+  this.results = aResults;
+  this.callback = aCallback;
+  this.callbackThis = aCallbackThis;
+}
+
+MessagesByMessageIdCallback.prototype = {
+  _log: console.createInstance({
+    prefix: "gloda.index_msg.mbm",
+    maxLogLevel: "Warn",
+    maxLogLevelPref: "gloda.loglevel",
+  }),
+
+  onItemsAdded(aItems, aCollection) {
+    // just outright bail if we are shutdown
+    if (GlodaDatastore.datastoreIsShutdown) {
+      return;
+    }
+
+    this._log.debug("getting results...");
+    for (let message of aItems) {
+      this.results[this.msgIDToIndex[message.headerMessageID]].push(message);
+    }
+  },
+  onItemsModified() {},
+  onItemsRemoved() {},
+  onQueryCompleted(aCollection) {
+    // just outright bail if we are shutdown
+    if (GlodaDatastore.datastoreIsShutdown) {
+      return;
+    }
+
+    this._log.debug("query completed, notifying... " + this.results);
+
+    this.callback.call(this.callbackThis, this.results);
+  },
+};
+
+/**
+ * The message indexer!
+ *
+ * === Message Indexing Strategy
+ * To these ends, we implement things like so:
+ *
+ * Message State Tracking
+ * - We store a property on all indexed headers indicating their gloda message
+ *   id.  This allows us to tell whether a message is indexed from the header,
+ *   without having to consult the SQL database.
+ * - When we receive an event that indicates that a message's meta-data has
+ *   changed and gloda needs to re-index the message, we set a property on the
+ *   header that indicates the message is dirty.  This property can indicate
+ *   that the message needs to be re-indexed but the gloda-id is valid (dirty)
+ *   or that the message's gloda-id is invalid (filthy) because the gloda
+ *   database has been blown away.
+ * - We track whether a folder is up-to-date on our GlodaFolder representation
+ *   using a concept of dirtiness, just like messages.  Like messages, a folder
+ *   can be dirty or filthy.  A dirty folder has at least one dirty message in
+ *   it which means we should scan the folder.  A filthy folder means that
+ *   every message in the folder should be considered filthy.  Folders start
+ *   out filthy when Gloda is first told about them indicating we cannot
+ *   trust any of the gloda-id's in the folders.  Filthy folders are downgraded
+ *   to dirty folders after we mark all of the headers with gloda-id's filthy.
+ *
+ * Indexing Message Control
+ * - We index the headers of all IMAP messages. We index the bodies of all IMAP
+ *   messages that are offline.  We index all local messages.  We plan to avoid
+ *   indexing news messages.
+ * - We would like a way to express desires about indexing that either don't
+ *   confound offline storage with indexing, or actually allow some choice.
+ *
+ * Indexing Messages
+ * - We have two major modes of indexing: sweep and event-driven.  When we
+ *   start up we kick off an indexing sweep.  We use event-driven indexing
+ *   as we receive events for eligible messages, but if we get too many
+ *   events we start dropping them on the floor and just flag that an indexing
+ *   sweep is required.
+ * - The sweep initiates folder indexing jobs based on the priorities assigned
+ *   to folders.  Folder indexing uses a filtered message enumerator to find
+ *   messages that need to be indexed, minimizing wasteful exposure of message
+ *   headers to XPConnect that we would not end up indexing.
+ * - For local folders, we use GetDatabaseWithReparse to ensure that the .msf
+ *   file exists.  For IMAP folders, we simply use GetDatabase because we know
+ *   the auto-sync logic will make sure that the folder is up-to-date and we
+ *   want to avoid creating problems through use of updateFolder.
+ *
+ * Junk Mail
+ * - We do not index junk.  We do not index messages until the junk/non-junk
+ *   determination has been made.  If a message gets marked as junk, we act like
+ *   it was deleted.
+ * - We know when a message is actively queued for junk processing thanks to
+ *   folder processing flags.  nsMsgDBFolder::CallFilterPlugins does this
+ *   prior to initiating spam processing.  Unfortunately, this method does not
+ *   get called until after we receive the notification about the existence of
+ *   the header.  How long after can vary on different factors.  The longest
+ *   delay is in the IMAP case where there is a filter that requires the
+ *   message body to be present; the method does not get called until all the
+ *   bodies are downloaded.
+ *
+ */
+var GlodaMsgIndexer = {
+  /**
+   * A partial attempt to generalize to support multiple databases.  Each
+   *  database would have its own datastore would have its own indexer.  But
+   *  we rather inter-mingle our use of this field with the singleton global
+   *  GlodaDatastore.
+   */
+  _datastore: GlodaDatastore,
+  _log: console.createInstance({
+    prefix: "gloda.index_msg",
+    maxLogLevel: "Warn",
+    maxLogLevelPref: "gloda.loglevel",
+  }),
+
+  _junkService: MailServices.junk,
+
+  name: "index_msg",
+  /**
+   * Are we enabled, read: are we processing change events?
+   */
+  _enabled: false,
+  get enabled() {
+    return this._enabled;
+  },
+
+  enable() {
+    // initialize our listeners' this pointers
+    this._databaseAnnouncerListener.indexer = this;
+    this._msgFolderListener.indexer = this;
+
+    // register for:
+    // - folder loaded events, so we know when getDatabaseWithReparse has
+    //   finished updating the index/what not (if it wasn't immediately
+    //   available)
+    // - property changes (so we know when a message's read/starred state have
+    //   changed.)
+    this._folderListener._init(this);
+    MailServices.mailSession.AddFolderListener(
+      this._folderListener,
+      Ci.nsIFolderListener.intPropertyChanged |
+        Ci.nsIFolderListener.propertyFlagChanged |
+        Ci.nsIFolderListener.event
+    );
+
+    MailServices.mfn.addListener(
+      this._msgFolderListener,
+      // note: intentionally no msgAdded or msgUnincorporatedMoved.
+      Ci.nsIMsgFolderNotificationService.msgsClassified |
+        Ci.nsIMsgFolderNotificationService.msgsJunkStatusChanged |
+        Ci.nsIMsgFolderNotificationService.msgsDeleted |
+        Ci.nsIMsgFolderNotificationService.msgsMoveCopyCompleted |
+        Ci.nsIMsgFolderNotificationService.msgKeyChanged |
+        Ci.nsIMsgFolderNotificationService.folderAdded |
+        Ci.nsIMsgFolderNotificationService.folderDeleted |
+        Ci.nsIMsgFolderNotificationService.folderMoveCopyCompleted |
+        Ci.nsIMsgFolderNotificationService.folderRenamed |
+        Ci.nsIMsgFolderNotificationService.folderCompactStart |
+        Ci.nsIMsgFolderNotificationService.folderCompactFinish |
+        Ci.nsIMsgFolderNotificationService.folderReindexTriggered
+    );
+
+    this._enabled = true;
+
+    this._considerSchemaMigration();
+
+    this._log.info("Event-Driven Indexing is now " + this._enabled);
+  },
+  disable() {
+    // remove FolderLoaded notification listener
+    MailServices.mailSession.RemoveFolderListener(this._folderListener);
+
+    MailServices.mfn.removeListener(this._msgFolderListener);
+
+    this._indexerLeaveFolder(); // nop if we aren't "in" a folder
+
+    this._enabled = false;
+
+    this._log.info("Event-Driven Indexing is now " + this._enabled);
+  },
+
+  /**
+   * Indicates that we have pending deletions to process, meaning that there
+   *  are gloda message rows flagged for deletion.  If this value is a boolean,
+   *  it means the value is known reliably.  If this value is null, it means
+   *  that we don't know, likely because we have started up and have not checked
+   *  the database.
+   */
+  pendingDeletions: null,
+
+  /**
+   * The message (or folder state) is believed up-to-date.
+   */
+  kMessageClean: 0,
+  /**
+   * The message (or folder) is known to not be up-to-date. In the case of
+   *  folders, this means that some of the messages in the folder may be dirty.
+   *  However, because of the way our indexing works, it is possible there may
+   *  actually be no dirty messages in a folder.  (We attempt to process
+   *  messages in an event-driven fashion for a finite number of messages, but
+   *  because we can quit without completing processing of the queue, we need to
+   *  mark the folder dirty, just-in-case.)  (We could do some extra leg-work
+   *  and do a better job of marking the folder clean again.)
+   */
+  kMessageDirty: 1,
+  /**
+   * We have not indexed the folder at all, but messages in the folder think
+   *  they are indexed.  We downgrade the folder to just kMessageDirty after
+   *  marking all the messages in the folder as dirty.  We do this so that if we
+   *  have to stop indexing the folder we can still build on our progress next
+   *  time we enter the folder.
+   * We mark all folders filthy when (re-)creating the database because there
+   *  may be previous state left over from an earlier database.
+   */
+  kMessageFilthy: 2,
+
+  /**
+   * A message addition job yet to be (completely) processed.  Since message
+   *  addition events come to us one-by-one, in order to aggregate them into a
+   *  job, we need something like this.  It's up to the indexing loop to
+   *  decide when to null this out; it can either do it when it first starts
+   *  processing it, or when it has processed the last thing.  It's really a
+   *  question of whether we want retrograde motion in the folder progress bar
+   *  or the message progress bar.
+   */
+  _pendingAddJob: null,
+
+  /**
+   * The number of messages that we should queue for processing before letting
+   *  them fall on the floor and relying on our folder-walking logic to ensure
+   *  that the messages are indexed.
+   * The reason we allow for queueing messages in an event-driven fashion is
+   *  that once we have reached a steady-state, it is preferable to be able to
+   *  deal with new messages and modified meta-data in a prompt fashion rather
+   *  than having to (potentially) walk every folder in the system just to find
+   *  the message that the user changed the tag on.
+   */
+  _indexMaxEventQueueMessages: 20,
+
+  /**
+   * Unit testing hook to get us to emit additional logging that verges on
+   *  inane for general usage but is helpful in unit test output to get a lay
+   *  of the land and for paranoia reasons.
+   */
+  _unitTestSuperVerbose: false,
+
+  /** The GlodaFolder corresponding to the folder we are indexing. */
+  _indexingGlodaFolder: null,
+  /** The nsIMsgFolder we are currently indexing. */
+  _indexingFolder: null,
+  /** The nsIMsgDatabase we are currently indexing. */
+  _indexingDatabase: null,
+  /**
+   * The iterator we are using to iterate over the headers in
+   *  this._indexingDatabase.
+   */
+  _indexingIterator: null,
+
+  /** folder whose entry we are pending on */
+  _pendingFolderEntry: null,
+
+  /**
+   * Async common logic that we want to deal with the given folder ID.  Besides
+   *  cutting down on duplicate code, this ensures that we are listening on
+   *  the folder in case it tries to go away when we are using it.
+   *
+   * @returns true when the folder was successfully entered, false when we need
+   *     to pend on notification of updating of the folder (due to re-parsing
+   *     or what have you).  In the event of an actual problem, an exception
+   *     will escape.
+   */
+  _indexerEnterFolder(aFolderID) {
+    // leave the folder if we haven't explicitly left it.
+    if (this._indexingFolder !== null) {
+      this._indexerLeaveFolder();
+    }
+
+    this._indexingGlodaFolder = GlodaDatastore._mapFolderID(aFolderID);
+    this._indexingFolder = this._indexingGlodaFolder.getXPCOMFolder(
+      this._indexingGlodaFolder.kActivityIndexing
+    );
+
+    if (this._indexingFolder) {
+      this._log.debug("Entering folder: " + this._indexingFolder.URI);
+    }
+
+    try {
+      // The msf may need to be created or otherwise updated for local folders.
+      // This may require yielding until such time as the msf has been created.
+      try {
+        if (this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder) {
+          this._indexingDatabase = this._indexingFolder.getDatabaseWithReparse(
+            null,
+            null
+          );
+        }
+        // we need do nothing special for IMAP, news, or other
+      } catch (e) {
+        // getDatabaseWithReparse can return either NS_ERROR_NOT_INITIALIZED or
+        //  NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE if the net result is that it
+        //  is going to send us a notification when the reparse has completed.
+        // (note that although internally NS_MSG_ERROR_FOLDER_SUMMARY_MISSING
+        //  might get flung around, it won't make it out to us, and will instead
+        //  be permuted into an NS_ERROR_NOT_INITIALIZED.)
+        if (
+          e.result == Cr.NS_ERROR_NOT_INITIALIZED ||
+          e.result == NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE
+        ) {
+          // this means that we need to pend on the update; the listener for
+          //  FolderLoaded events will call _indexerCompletePendingFolderEntry.
+          this._log.debug("Pending on folder load...");
+          this._pendingFolderEntry = this._indexingFolder;
+          return GlodaConstants.kWorkAsync;
+        }
+        throw e;
+      }
+      // we get an nsIMsgDatabase out of this (unsurprisingly) which
+      //  explicitly inherits from nsIDBChangeAnnouncer, which has the
+      //  addListener call we want.
+      if (this._indexingDatabase == null) {
+        this._indexingDatabase = this._indexingFolder.msgDatabase;
+      }
+      this._indexingDatabase.addListener(this._databaseAnnouncerListener);
+    } catch (ex) {
+      this._log.error(
+        "Problem entering folder: " +
+          (this._indexingFolder ? this._indexingFolder.prettyName : "unknown") +
+          ", skipping. Error was: " +
+          ex.fileName +
+          ":" +
+          ex.lineNumber +
+          ": " +
+          ex
+      );
+      this._indexingGlodaFolder.indexing = false;
+      this._indexingFolder = null;
+      this._indexingGlodaFolder = null;
+      this._indexingDatabase = null;
+      this._indexingEnumerator = null;
+
+      // re-throw, we just wanted to make sure this junk is cleaned up and
+      //  get localized error logging...
+      throw ex;
+    }
+
+    return GlodaConstants.kWorkSync;
+  },
+
+  /**
+   * If the folder was still parsing/updating when we tried to enter, then this
+   *  handler will get called by the listener who got the FolderLoaded message.
+   * All we need to do is get the database reference, register a listener on
+   *  the db, and retrieve an iterator if desired.
+   */
+  _indexerCompletePendingFolderEntry() {
+    this._indexingDatabase = this._indexingFolder.msgDatabase;
+    this._indexingDatabase.addListener(this._databaseAnnouncerListener);
+    this._log.debug("...Folder Loaded!");
+
+    // the load is no longer pending; we certainly don't want more notifications
+    this._pendingFolderEntry = null;
+    // indexerEnterFolder returned kWorkAsync, which means we need to notify
+    //  the callback driver to get things going again.
+    GlodaIndexer.callbackDriver();
+  },
+
+  /**
+   * Enumerate all messages in the folder.
+   */
+  kEnumAllMsgs: 0,
+  /**
+   * Enumerate messages that look like they need to be indexed.
+   */
+  kEnumMsgsToIndex: 1,
+  /**
+   * Enumerate messages that are already indexed.
+   */
+  kEnumIndexedMsgs: 2,
+
+  /**
+   * Synchronous helper to get an enumerator for the current folder (as found
+   *  in |_indexingFolder|.
+   *
+   * @param aEnumKind One of |kEnumAllMsgs|, |kEnumMsgsToIndex|, or
+   *     |kEnumIndexedMsgs|.
+   * @param [aAllowPreBadIds=false] Only valid for |kEnumIndexedMsgs|, tells us
+   *     that we should treat message with any gloda-id as dirty, not just
+   *     messages that have non-bad message id's.
+   */
+  _indexerGetEnumerator(aEnumKind, aAllowPreBadIds) {
+    if (aEnumKind == this.kEnumMsgsToIndex) {
+      // We need to create search terms for messages to index. Messages should
+      //  be indexed if they're indexable (local or offline and not expunged)
+      //  and either: haven't been indexed, are dirty, or are marked with with
+      //  a former GLODA_BAD_MESSAGE_ID that is no longer our bad marker.  (Our
+      //  bad marker can change on minor schema revs so that we can try and
+      //  reindex those messages exactly once and without needing to go through
+      //  a pass to mark them as needing one more try.)
+      // The basic search expression is:
+      //  ((GLODA_MESSAGE_ID_PROPERTY Is 0) ||
+      //   (GLODA_MESSAGE_ID_PROPERTY Is GLODA_OLD_BAD_MESSAGE_ID) ||
+      //   (GLODA_DIRTY_PROPERTY Isnt 0)) &&
+      //  (JUNK_SCORE_PROPERTY Isnt 100)
+      // If the folder !isLocal we add the terms:
+      //  - if the folder is offline -- && (Status Is nsMsgMessageFlags.Offline)
+      //  - && (Status Isnt nsMsgMessageFlags.Expunged)
+
+      let searchSession = Cc[
+        "@mozilla.org/messenger/searchSession;1"
+      ].createInstance(Ci.nsIMsgSearchSession);
+      let searchTerms = [];
+      let isLocal = this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder;
+
+      searchSession.addScopeTerm(
+        Ci.nsMsgSearchScope.offlineMail,
+        this._indexingFolder
+      );
+      let nsMsgSearchAttrib = Ci.nsMsgSearchAttrib;
+      let nsMsgSearchOp = Ci.nsMsgSearchOp;
+
+      // first term: (GLODA_MESSAGE_ID_PROPERTY Is 0
+      let searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = false; // actually don't care here
+      searchTerm.beginsGrouping = true;
+      searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
+      searchTerm.op = nsMsgSearchOp.Is;
+      let value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.status = 0;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = GLODA_MESSAGE_ID_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      // second term: || GLODA_MESSAGE_ID_PROPERTY Is GLODA_OLD_BAD_MESSAGE_ID
+      searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = false; // OR
+      searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
+      searchTerm.op = nsMsgSearchOp.Is;
+      value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.status = GLODA_OLD_BAD_MESSAGE_ID;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = GLODA_MESSAGE_ID_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      //  third term: || GLODA_DIRTY_PROPERTY Isnt 0 )
+      searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = false;
+      searchTerm.endsGrouping = true;
+      searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
+      searchTerm.op = nsMsgSearchOp.Isnt;
+      value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.status = 0;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = GLODA_DIRTY_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      // JUNK_SCORE_PROPERTY Isnt 100
+      // For symmetry with our event-driven stuff, we just directly deal with
+      //  the header property.
+      searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = true;
+      searchTerm.attrib = nsMsgSearchAttrib.HdrProperty;
+      searchTerm.op = nsMsgSearchOp.Isnt;
+      value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.str = JUNK_SPAM_SCORE_STR;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = JUNK_SCORE_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      if (!isLocal) {
+        // If the folder is offline, then the message should be too
+        if (this._indexingFolder.getFlag(Ci.nsMsgFolderFlags.Offline)) {
+          // third term: && Status Is nsMsgMessageFlags.Offline
+          searchTerm = searchSession.createTerm();
+          searchTerm.booleanAnd = true;
+          searchTerm.attrib = nsMsgSearchAttrib.MsgStatus;
+          searchTerm.op = nsMsgSearchOp.Is;
+          value = searchTerm.value;
+          value.attrib = searchTerm.attrib;
+          value.status = Ci.nsMsgMessageFlags.Offline;
+          searchTerm.value = value;
+          searchTerms.push(searchTerm);
+        }
+
+        // fourth term: && Status Isnt nsMsgMessageFlags.Expunged
+        searchTerm = searchSession.createTerm();
+        searchTerm.booleanAnd = true;
+        searchTerm.attrib = nsMsgSearchAttrib.MsgStatus;
+        searchTerm.op = nsMsgSearchOp.Isnt;
+        value = searchTerm.value;
+        value.attrib = searchTerm.attrib;
+        value.status = Ci.nsMsgMessageFlags.Expunged;
+        searchTerm.value = value;
+        searchTerms.push(searchTerm);
+      }
+
+      this._indexingEnumerator = this._indexingDatabase.getFilterEnumerator(
+        searchTerms,
+        true
+      );
+    } else if (aEnumKind == this.kEnumIndexedMsgs) {
+      // Enumerate only messages that are already indexed.  This comes out to:
+      //  ((GLODA_MESSAGE_ID_PROPERTY > GLODA_FIRST_VALID_MESSAGE_ID-1) &&
+      //   (GLODA_DIRTY_PROPERTY Isnt kMessageFilthy))
+      // In English, a message is indexed if (by clause):
+      // 1) The message has a gloda-id and that gloda-id is in the valid range
+      //    (and not in the bad message marker range).
+      // 2) The message has not been marked filthy (which invalidates the
+      //    gloda-id.)  We also assume that the folder would not have been
+      //    entered at all if it was marked filthy.
+      let searchSession = Cc[
+        "@mozilla.org/messenger/searchSession;1"
+      ].createInstance(Ci.nsIMsgSearchSession);
+      let searchTerms = [];
+
+      searchSession.addScopeTerm(
+        Ci.nsMsgSearchScope.offlineMail,
+        this._indexingFolder
+      );
+      let nsMsgSearchAttrib = Ci.nsMsgSearchAttrib;
+      let nsMsgSearchOp = Ci.nsMsgSearchOp;
+
+      // first term: (GLODA_MESSAGE_ID_PROPERTY > GLODA_FIRST_VALID_MESSAGE_ID-1
+      let searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = false; // actually don't care here
+      searchTerm.beginsGrouping = true;
+      searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
+      // use != 0 if we're allow pre-bad ids.
+      searchTerm.op = aAllowPreBadIds
+        ? nsMsgSearchOp.Isnt
+        : nsMsgSearchOp.IsGreaterThan;
+      let value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.status = aAllowPreBadIds ? 0 : GLODA_FIRST_VALID_MESSAGE_ID - 1;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = GLODA_MESSAGE_ID_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      //  second term: && GLODA_DIRTY_PROPERTY Isnt kMessageFilthy)
+      searchTerm = searchSession.createTerm();
+      searchTerm.booleanAnd = true;
+      searchTerm.endsGrouping = true;
+      searchTerm.attrib = nsMsgSearchAttrib.Uint32HdrProperty;
+      searchTerm.op = nsMsgSearchOp.Isnt;
+      value = searchTerm.value;
+      value.attrib = searchTerm.attrib;
+      value.status = this.kMessageFilthy;
+      searchTerm.value = value;
+      searchTerm.hdrProperty = GLODA_DIRTY_PROPERTY;
+      searchTerms.push(searchTerm);
+
+      // The use-case of already indexed messages does not want them reversed;
+      //  we care about seeing the message keys in order.
+      this._indexingEnumerator = this._indexingDatabase.getFilterEnumerator(
+        searchTerms,
+        false
+      );
+    } else if (aEnumKind == this.kEnumAllMsgs) {
+      this._indexingEnumerator =
+        this._indexingDatabase.reverseEnumerateMessages();
+    } else {
+      throw new Error("Unknown enumerator type requested:" + aEnumKind);
+    }
+  },
+
+  _indexerLeaveFolder() {
+    if (this._indexingFolder !== null) {
+      if (this._indexingDatabase) {
+        this._indexingDatabase.commit(Ci.nsMsgDBCommitType.kLargeCommit);
+        // remove our listener!
+        this._indexingDatabase.removeListener(this._databaseAnnouncerListener);
+      }
+      // let the gloda folder know we are done indexing
+      this._indexingGlodaFolder.indexing = false;
+      // null everyone out
+      this._indexingFolder = null;
+      this._indexingGlodaFolder = null;
+      this._indexingDatabase = null;
+      this._indexingEnumerator = null;
+    }
+  },
+
+  /**
+   * Event fed to us by our nsIFolderListener when a folder is loaded.  We use
+   *  this event to know when a folder we were trying to open to index is
+   *  actually ready to be indexed.  (The summary may have not existed, may have
+   *  been out of date, or otherwise.)
+   *
+   * @param aFolder An nsIMsgFolder, already QI'd.
+   */
+  _onFolderLoaded(aFolder) {
+    if (
+      this._pendingFolderEntry !== null &&
+      aFolder.URI == this._pendingFolderEntry.URI
+    ) {
+      this._indexerCompletePendingFolderEntry();
+    }
+  },
+
+  // it's a getter so we can reference 'this'.  we could memoize.
+  get workers() {
+    return [
+      [
+        "folderSweep",
+        {
+          worker: this._worker_indexingSweep,
+          jobCanceled: this._cleanup_indexingSweep,
+          cleanup: this._cleanup_indexingSweep,
+        },
+      ],
+      [
+        "folder",
+        {
+          worker: this._worker_folderIndex,
+          recover: this._recover_indexMessage,
+          cleanup: this._cleanup_indexing,
+        },
+      ],
+      [
+        "folderCompact",
+        {
+          worker: this._worker_folderCompactionPass,
+          // compaction enters the folder so needs to know how to leave
+          cleanup: this._cleanup_indexing,
+        },
+      ],
+      [
+        "message",
+        {
+          worker: this._worker_messageIndex,
+          onSchedule: this._schedule_messageIndex,
+          jobCanceled: this._canceled_messageIndex,
+          recover: this._recover_indexMessage,
+          cleanup: this._cleanup_indexing,
+        },
+      ],
+      [
+        "delete",
+        {
+          worker: this._worker_processDeletes,
+        },
+      ],
+
+      [
+        "fixMissingContacts",
+        {
+          worker: this._worker_fixMissingContacts,
+        },
+      ],
+    ];
+  },
+
+  _schemaMigrationInitiated: false,
+  _considerSchemaMigration() {
+    if (
+      !this._schemaMigrationInitiated &&
+      GlodaDatastore._actualSchemaVersion === 26
+    ) {
+      let job = new IndexingJob("fixMissingContacts", null);
+      GlodaIndexer.indexJob(job);
+      this._schemaMigrationInitiated = true;
+    }
+  },
+
+  initialSweep() {
+    this.indexingSweepNeeded = true;
+  },
+
+  _indexingSweepActive: false,
+  /**
+   * Indicate that an indexing sweep is desired.  We kick-off an indexing
+   *  sweep at start-up and whenever we receive an event-based notification
+   *  that we either can't process as an event or that we normally handle
+   *  during the sweep pass anyways.
+   */
+  set indexingSweepNeeded(aNeeded) {
+    if (!this._indexingSweepActive && aNeeded) {
+      let job = new IndexingJob("folderSweep", null);
+      job.mappedFolders = false;
+      GlodaIndexer.indexJob(job);
+      this._indexingSweepActive = true;
+    }
+  },
+
+  /**
+   * Performs the folder sweep, locating folders that should be indexed, and
+   *  creating a folder indexing job for them, and rescheduling itself for
+   *  execution after that job is completed.  Once it indexes all the folders,
+   *  if we believe we have deletions to process (or just don't know), it kicks
+   *  off a deletion processing job.
+   *
+   * Folder traversal logic is based off the spotlight/vista indexer code; we
+   *  retrieve the list of servers and folders each time want to find a new
+   *  folder to index.  This avoids needing to maintain a perfect model of the
+   *  folder hierarchy at all times.  (We may eventually want to do that, but
+   *  this is sufficient and safe for now.)  Although our use of dirty flags on
+   *  the folders allows us to avoid tracking the 'last folder' we processed,
+   *  we do so to avoid getting 'trapped' in a folder with a high rate of
+   *  changes.
+   */
+  *_worker_indexingSweep(aJob) {
+    if (!aJob.mappedFolders) {
+      // Walk the folders and make sure all the folders we would want to index
+      //  are mapped.  Build up a list of GlodaFolders as we go, so that we can
+      //  sort them by their indexing priority.
+      let foldersToProcess = (aJob.foldersToProcess = []);
+
+      for (let folder of MailServices.accounts.allFolders) {
+        if (this.shouldIndexFolder(folder)) {
+          foldersToProcess.push(Gloda.getFolderForFolder(folder));
+        }
+      }
+
+      // sort the folders by priority (descending)
+      foldersToProcess.sort(function (a, b) {
+        return b.indexingPriority - a.indexingPriority;
+      });
+
+      aJob.mappedFolders = true;
+    }
+
+    // -- process the folders (in sorted order)
+    while (aJob.foldersToProcess.length) {
+      let glodaFolder = aJob.foldersToProcess.shift();
+      // ignore folders that:
+      // - have been deleted out of existence!
+      // - are not dirty/have not been compacted
+      // - are actively being compacted
+      if (
+        glodaFolder._deleted ||
+        (!glodaFolder.dirtyStatus && !glodaFolder.compacted) ||
+        glodaFolder.compacting
+      ) {
+        continue;
+      }
+
+      // If the folder is marked as compacted, give it a compaction job.
+      if (glodaFolder.compacted) {
+        GlodaIndexer.indexJob(new IndexingJob("folderCompact", glodaFolder.id));
+      }
+
+      // add a job for the folder indexing if it was dirty
+      if (glodaFolder.dirtyStatus) {
+        GlodaIndexer.indexJob(new IndexingJob("folder", glodaFolder.id));
+      }
+
+      // re-schedule this job (although this worker will die)
+      GlodaIndexer.indexJob(aJob);
+      yield GlodaConstants.kWorkDone;
+    }
+
+    // consider deletion
+    if (this.pendingDeletions || this.pendingDeletions === null) {
+      GlodaIndexer.indexJob(new IndexingJob("delete", null));
+    }
+
+    // we don't have any more work to do...
+    this._indexingSweepActive = false;
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * The only state we need to cleanup is that there is no longer an active
+   *  indexing sweep.
+   */
+  _cleanup_indexingSweep(aJob) {
+    this._indexingSweepActive = false;
+  },
+
+  /**
+   * The number of headers to look at before yielding with kWorkSync.  This
+   *  is for time-slicing purposes so we still yield to the UI periodically.
+   */
+  HEADER_CHECK_SYNC_BLOCK_SIZE: 25,
+
+  FOLDER_COMPACTION_PASS_BATCH_SIZE: 512,
+  /**
+   * Special indexing pass for (local) folders than have been compacted.  The
+   *  compaction can cause message keys to change because message keys in local
+   *  folders are simply offsets into the mbox file.  Accordingly, we need to
+   *  update the gloda records/objects to point them at the new message key.
+   *
+   * Our general algorithm is to perform two traversals in parallel.  The first
+   *  is a straightforward enumeration of the message headers in the folder that
+   *  apparently have been already indexed.  These provide us with the message
+   *  key and the "gloda-id" property.
+   * The second is a list of tuples containing a gloda message id, its current
+   *  message key per the gloda database, and the message-id header.  We re-fill
+   *  the list with batches on-demand.  This allows us to both avoid dispatching
+   *  needless UPDATEs as well as deal with messages that were tracked by the
+   *  PendingCommitTracker but were discarded by the compaction notification.
+   *
+   * We end up processing two streams of gloda-id's and some extra info.  In
+   *  the normal case we expect these two streams to line up exactly and all
+   *  we need to do is update the message key if it has changed.
+   *
+   * There are a few exceptional cases where things do not line up:
+   * 1) The gloda database knows about a message that the enumerator does not
+   *    know about...
+   *   a) This message exists in the folder (identified using its message-id
+   *      header).  This means the message got indexed but PendingCommitTracker
+   *      had to forget about the info when the compaction happened.  We
+   *      re-establish the link and track the message in PendingCommitTracker
+   *      again.
+   *   b) The message does not exist in the folder.  This means the message got
+   *      indexed, PendingCommitTracker had to forget about the info, and
+   *      then the message either got moved or deleted before now.  We mark
+   *      the message as deleted; this allows the gloda message to be reused
+   *      if the move target has not yet been indexed or purged if it already
+   *      has been and the gloda message is a duplicate.  And obviously, if the
+   *      event that happened was actually a delete, then the delete is the
+   *      right thing to do.
+   * 2) The enumerator knows about a message that the gloda database does not
+   *    know about.  This is unexpected and should not happen.  We log a
+   *    warning.  We are able to differentiate this case from case #1a by
+   *    retrieving the message header associated with the next gloda message
+   *    (using the message-id header per 1a again).  If the gloda message's
+   *    message key is after the enumerator's message key then we know this is
+   *    case #2.  (It implies an insertion in the enumerator stream which is how
+   *    we define the unexpected case.)
+   *
+   * Besides updating the database rows, we also need to make sure that
+   *  in-memory representations are updated.  Immediately after dispatching
+   *  UPDATE changes to the database we use the same set of data to walk the
+   *  live collections and update any affected messages.  We are then able to
+   *  discard the information.  Although this means that we will have to
+   *  potentially walk the live collections multiple times, unless something
+   *  has gone horribly wrong, the number of collections should be reasonable
+   *  and the lookups are cheap.  We bias batch sizes accordingly.
+   *
+   * Because we operate based on chunks we need to make sure that when we
+   *  actually deal with multiple chunks that we don't step on our own feet with
+   *  our database updates.  Since compaction of message key K results in a new
+   *  message key K' such that K' <= K, we can reliably issue database
+   *  updates for all values <= K.  Which means our feet are safe no matter
+   *  when we issue the update command.  For maximum cache benefit, we issue
+   *  our updates prior to our new query since they should still be maximally
+   *  hot at that point.
+   */
+  *_worker_folderCompactionPass(aJob, aCallbackHandle) {
+    yield this._indexerEnterFolder(aJob.id);
+
+    // It's conceivable that with a folder sweep we might end up trying to
+    //  compact a folder twice.  Bail early in this case.
+    if (!this._indexingGlodaFolder.compacted) {
+      yield GlodaConstants.kWorkDone;
+    }
+
+    // this is a forward enumeration (sometimes we reverse enumerate; not here)
+    this._indexerGetEnumerator(this.kEnumIndexedMsgs);
+
+    const HEADER_CHECK_SYNC_BLOCK_SIZE = this.HEADER_CHECK_SYNC_BLOCK_SIZE;
+    const FOLDER_COMPACTION_PASS_BATCH_SIZE =
+      this.FOLDER_COMPACTION_PASS_BATCH_SIZE;
+
+    // Tuples of [gloda id, message key, message-id header] from
+    //  folderCompactionPassBlockFetch
+    let glodaIdsMsgKeysHeaderIds = [];
+    // Unpack each tuple from glodaIdsMsgKeysHeaderIds into these guys.
+    // (Initialize oldMessageKey because we use it to kickstart our query.)
+    let oldGlodaId,
+      oldMessageKey = -1,
+      oldHeaderMessageId;
+    // parallel lists of gloda ids and message keys to pass to
+    //  GlodaDatastore.updateMessageLocations
+    let updateGlodaIds = [];
+    let updateMessageKeys = [];
+    // list of gloda id's to mark deleted
+    let deleteGlodaIds = [];
+
+    // for GC reasons we need to track the number of headers seen
+    let numHeadersSeen = 0;
+
+    // We are consuming two lists; our loop structure has to reflect that.
+    let headerIter = this._indexingEnumerator[Symbol.iterator]();
+    let mayHaveMoreGlodaMessages = true;
+    let keepIterHeader = false;
+    let keepGlodaTuple = false;
+    let msgHdr = null;
+    while (headerIter || mayHaveMoreGlodaMessages) {
+      let glodaId;
+      if (headerIter) {
+        if (!keepIterHeader) {
+          let result = headerIter.next();
+          if (result.done) {
+            headerIter = null;
+            msgHdr = null;
+            // do the loop check again
+            continue;
+          }
+          msgHdr = result.value;
+        } else {
+          keepIterHeader = false;
+        }
+      }
+
+      if (msgHdr) {
+        numHeadersSeen++;
+        if (numHeadersSeen % HEADER_CHECK_SYNC_BLOCK_SIZE == 0) {
+          yield GlodaConstants.kWorkSync;
+        }
+
+        // There is no need to check with PendingCommitTracker.  If a message
+        //  somehow got indexed between the time the compaction killed
+        //  everything and the time we run, that is a bug.
+        glodaId = msgHdr.getUint32Property(GLODA_MESSAGE_ID_PROPERTY);
+        // (there is also no need to check for gloda dirty since the enumerator
+        //  filtered that for us.)
+      }
+
+      // get more [gloda id, message key, message-id header] tuples if out
+      if (!glodaIdsMsgKeysHeaderIds.length && mayHaveMoreGlodaMessages) {
+        // Since we operate on blocks, getting a new block implies we should
+        //  flush the last block if applicable.
+        if (updateGlodaIds.length) {
+          GlodaDatastore.updateMessageLocations(
+            updateGlodaIds,
+            updateMessageKeys,
+            aJob.id,
+            true
+          );
+          updateGlodaIds = [];
+          updateMessageKeys = [];
+        }
+
+        if (deleteGlodaIds.length) {
+          GlodaDatastore.markMessagesDeletedByIDs(deleteGlodaIds);
+          deleteGlodaIds = [];
+        }
+
+        GlodaDatastore.folderCompactionPassBlockFetch(
+          aJob.id,
+          oldMessageKey + 1,
+          FOLDER_COMPACTION_PASS_BATCH_SIZE,
+          aCallbackHandle.wrappedCallback
+        );
+        glodaIdsMsgKeysHeaderIds = yield GlodaConstants.kWorkAsync;
+        // Reverse so we can use pop instead of shift and I don't need to be
+        //  paranoid about performance.
+        glodaIdsMsgKeysHeaderIds.reverse();
+
+        if (!glodaIdsMsgKeysHeaderIds.length) {
+          mayHaveMoreGlodaMessages = false;
+
+          // We shouldn't be in the loop anymore if headerIter is dead now.
+          if (!headerIter) {
+            break;
+          }
+        }
+      }
+
+      if (!keepGlodaTuple) {
+        if (mayHaveMoreGlodaMessages) {
+          [oldGlodaId, oldMessageKey, oldHeaderMessageId] =
+            glodaIdsMsgKeysHeaderIds.pop();
+        } else {
+          oldGlodaId = oldMessageKey = oldHeaderMessageId = null;
+        }
+      } else {
+        keepGlodaTuple = false;
+      }
+
+      // -- normal expected case
+      if (glodaId == oldGlodaId) {
+        // only need to do something if the key is not right
+        if (msgHdr.messageKey != oldMessageKey) {
+          updateGlodaIds.push(glodaId);
+          updateMessageKeys.push(msgHdr.messageKey);
+        }
+      } else {
+        // -- exceptional cases
+        // This should always return a value unless something is very wrong.
+        //  We do not want to catch the exception if one happens.
+        let idBasedHeader = oldHeaderMessageId
+          ? this._indexingDatabase.getMsgHdrForMessageID(oldHeaderMessageId)
+          : false;
+        // - Case 1b.
+        // We want to mark the message as deleted.
+        if (idBasedHeader == null) {
+          deleteGlodaIds.push(oldGlodaId);
+        } else if (
+          idBasedHeader &&
+          ((msgHdr && idBasedHeader.messageKey < msgHdr.messageKey) || !msgHdr)
+        ) {
+          // - Case 1a
+          // The expected case is that the message referenced by the gloda
+          //  database precedes the header the enumerator told us about.  This
+          //  is expected because if PendingCommitTracker did not mark the
+          //  message as indexed/clean then the enumerator would not tell us
+          //  about it.
+          // Also, if we ran out of headers from the enumerator, this is a dead
+          //  giveaway that this is the expected case.
+          // tell the pending commit tracker about the gloda database one
+          PendingCommitTracker.track(idBasedHeader, oldGlodaId);
+          // and we might need to update the message key too
+          if (idBasedHeader.messageKey != oldMessageKey) {
+            updateGlodaIds.push(oldGlodaId);
+            updateMessageKeys.push(idBasedHeader.messageKey);
+          }
+          // Take another pass through the loop so that we check the
+          //  enumerator header against the next message in the gloda
+          //  database.
+          keepIterHeader = true;
+        } else if (msgHdr) {
+          // - Case 2
+          // Whereas if the message referenced by gloda has a message key
+          //  greater than the one returned by the enumerator, then we have a
+          //  header claiming to be indexed by gloda that gloda does not
+          //  actually know about.  This is exceptional and gets a warning.
+          this._log.warn(
+            "Observed header that claims to be gloda indexed " +
+              "but that gloda has never heard of during " +
+              "compaction." +
+              " In folder: " +
+              msgHdr.folder.URI +
+              " sketchy key: " +
+              msgHdr.messageKey +
+              " subject: " +
+              msgHdr.mime2DecodedSubject
+          );
+          // Keep this tuple around for the next enumerator provided header
+          keepGlodaTuple = true;
+        }
+      }
+    }
+    // If we don't flush the update, no one will!
+    if (updateGlodaIds.length) {
+      GlodaDatastore.updateMessageLocations(
+        updateGlodaIds,
+        updateMessageKeys,
+        aJob.id,
+        true
+      );
+    }
+    if (deleteGlodaIds.length) {
+      GlodaDatastore.markMessagesDeletedByIDs(deleteGlodaIds);
+    }
+
+    this._indexingGlodaFolder._setCompactedState(false);
+
+    this._indexerLeaveFolder();
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Index the contents of a folder.
+   */
+  *_worker_folderIndex(aJob, aCallbackHandle) {
+    yield this._indexerEnterFolder(aJob.id);
+
+    if (!this.shouldIndexFolder(this._indexingFolder)) {
+      aJob.safelyInvokeCallback(true);
+      yield GlodaConstants.kWorkDone;
+    }
+
+    // Make sure listeners get notified about this job.
+    GlodaIndexer._notifyListeners();
+
+    // there is of course a cost to all this header investigation even if we
+    //  don't do something.  so we will yield with kWorkSync for every block.
+    const HEADER_CHECK_SYNC_BLOCK_SIZE = this.HEADER_CHECK_SYNC_BLOCK_SIZE;
+
+    // we can safely presume if we are here that this folder has been selected
+    //  for offline processing...
+
+    // -- Filthy Folder
+    // A filthy folder may have misleading properties on the message that claim
+    //  the message is indexed.  They are misleading because the database, for
+    //  whatever reason, does not have the messages (accurately) indexed.
+    // We need to walk all the messages and mark them filthy if they have a
+    //  dirty property.  Once we have done this, we can downgrade the folder's
+    //  dirty status to plain dirty.  We do this rather than trying to process
+    //  everyone in one go in a filthy context because if we have to terminate
+    //  indexing before we quit, we don't want to have to re-index messages next
+    //  time.  (This could even lead to never completing indexing in a
+    //  pathological situation.)
+    let glodaFolder = GlodaDatastore._mapFolder(this._indexingFolder);
+    if (glodaFolder.dirtyStatus == glodaFolder.kFolderFilthy) {
+      this._indexerGetEnumerator(this.kEnumIndexedMsgs, true);
+      let count = 0;
+      for (let msgHdr of this._indexingEnumerator) {
+        // we still need to avoid locking up the UI, pause periodically...
+        if (++count % HEADER_CHECK_SYNC_BLOCK_SIZE == 0) {
+          yield GlodaConstants.kWorkSync;
+        }
+
+        let glodaMessageId = msgHdr.getUint32Property(
+          GLODA_MESSAGE_ID_PROPERTY
+        );
+        // if it has a gloda message id, we need to mark it filthy
+        if (glodaMessageId != 0) {
+          msgHdr.setUint32Property(GLODA_DIRTY_PROPERTY, this.kMessageFilthy);
+        }
+        // if it doesn't have a gloda message id, we will definitely index it,
+        //  so no action is required.
+      }
+      // Commit the filthy status changes to the message database.
+      this._indexingDatabase.commit(Ci.nsMsgDBCommitType.kLargeCommit);
+
+      // this will automatically persist to the database
+      glodaFolder._downgradeDirtyStatus(glodaFolder.kFolderDirty);
+    }
+
+    // Figure out whether we're supposed to index _everything_ or just what
+    //  has not yet been indexed.
+    let force = "force" in aJob && aJob.force;
+    let enumeratorType = force ? this.kEnumAllMsgs : this.kEnumMsgsToIndex;
+
+    // Pass 1: count the number of messages to index.
+    //  We do this in order to be able to report to the user what we're doing.
+    // TODO: give up after reaching a certain number of messages in folders
+    //  with ridiculous numbers of messages and make the interface just say
+    //  something like "over N messages to go."
+
+    this._indexerGetEnumerator(enumeratorType);
+
+    let numMessagesToIndex = 0;
+    // eslint-disable-next-line no-unused-vars
+    for (let ignore of this._indexingEnumerator) {
+      // We're only counting, so do bigger chunks on this pass.
+      ++numMessagesToIndex;
+      if (numMessagesToIndex % (HEADER_CHECK_SYNC_BLOCK_SIZE * 8) == 0) {
+        yield GlodaConstants.kWorkSync;
+      }
+    }
+
+    aJob.goal = numMessagesToIndex;
+
+    if (numMessagesToIndex > 0) {
+      // We used up the iterator, get a new one.
+      this._indexerGetEnumerator(enumeratorType);
+
+      // Pass 2: index the messages.
+      let count = 0;
+      for (let msgHdr of this._indexingEnumerator) {
+        // per above, we want to periodically release control while doing all
+        // this header traversal/investigation.
+        if (++count % HEADER_CHECK_SYNC_BLOCK_SIZE == 0) {
+          yield GlodaConstants.kWorkSync;
+        }
+
+        // To keep our counts more accurate, increment the offset before
+        //  potentially skipping any messages.
+        ++aJob.offset;
+
+        // Skip messages that have not yet been reported to us as existing via
+        //  msgsClassified.
+        if (
+          this._indexingFolder.getProcessingFlags(msgHdr.messageKey) &
+          NOT_YET_REPORTED_PROCESSING_FLAGS
+        ) {
+          continue;
+        }
+
+        // Because the gloda id could be in-flight, we need to double-check the
+        //  enumerator here since it can't know about our in-memory stuff.
+        let [glodaId, glodaDirty] = PendingCommitTracker.getGlodaState(msgHdr);
+        // if the message seems valid and we are not forcing indexing, skip it.
+        //  (that means good gloda id and not dirty)
+        if (
+          !force &&
+          glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+          glodaDirty == this.kMessageClean
+        ) {
+          continue;
+        }
+
+        this._log.debug(">>>  calling _indexMessage");
+        yield aCallbackHandle.pushAndGo(
+          this._indexMessage(msgHdr, aCallbackHandle),
+          { what: "indexMessage", msgHdr }
+        );
+        GlodaIndexer._indexedMessageCount++;
+        this._log.debug("<<<  back from _indexMessage");
+      }
+    }
+
+    // This will trigger an (async) db update which cannot hit the disk prior to
+    //  the actual database records that constitute the clean state.
+    // XXX There is the slight possibility that, in the event of a crash, this
+    //  will hit the disk but the gloda-id properties on the headers will not
+    //  get set.  This should ideally be resolved by detecting a non-clean
+    //  shutdown and marking all folders as dirty.
+    glodaFolder._downgradeDirtyStatus(glodaFolder.kFolderClean);
+
+    // by definition, it's not likely we'll visit this folder again anytime soon
+    this._indexerLeaveFolder();
+
+    aJob.safelyInvokeCallback(true);
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Invoked when a "message" job is scheduled so that we can clear
+   *  _pendingAddJob if that is the job.  We do this so that work items are not
+   *  added to _pendingAddJob while it is being processed.
+   */
+  _schedule_messageIndex(aJob, aCallbackHandle) {
+    // we do not want new work items to be added as we are processing, so
+    //  clear _pendingAddJob.  A new job will be created as needed.
+    if (aJob === this._pendingAddJob) {
+      this._pendingAddJob = null;
+    }
+    // update our goal from the items length
+    aJob.goal = aJob.items.length;
+  },
+  /**
+   * If the job gets canceled, we need to make sure that we clear out pending
+   *  add job or our state will get wonky.
+   */
+  _canceled_messageIndex(aJob) {
+    if (aJob === this._pendingAddJob) {
+      this._pendingAddJob = null;
+    }
+  },
+
+  /**
+   * Index a specific list of messages that we know to index from
+   *  event-notification hints.
+   */
+  *_worker_messageIndex(aJob, aCallbackHandle) {
+    // if we are already in the correct folder, our "get in the folder" clause
+    //  will not execute, so we need to make sure this value is accurate in
+    //  that case.  (and we want to avoid multiple checks...)
+    for (; aJob.offset < aJob.items.length; aJob.offset++) {
+      let item = aJob.items[aJob.offset];
+      // item is either [folder ID, message key] or
+      //                [folder ID, message ID]
+
+      let glodaFolderId = item[0];
+      // If the folder has been deleted since we queued, skip this message
+      if (!GlodaDatastore._folderIdKnown(glodaFolderId)) {
+        continue;
+      }
+      let glodaFolder = GlodaDatastore._mapFolderID(glodaFolderId);
+
+      // Stay out of folders that:
+      // - are compacting / compacted and not yet processed
+      // - got deleted (this would be redundant if we had a stance on id nukage)
+      // (these things could have changed since we queued the event)
+      if (
+        glodaFolder.compacting ||
+        glodaFolder.compacted ||
+        glodaFolder._deleted
+      ) {
+        continue;
+      }
+
+      // get in the folder
+      if (this._indexingGlodaFolder != glodaFolder) {
+        yield this._indexerEnterFolder(glodaFolderId);
+
+        // Now that we have the real nsIMsgFolder, sanity-check that we should
+        //  be indexing it.  (There are some checks that require the
+        //  nsIMsgFolder.)
+        if (!this.shouldIndexFolder(this._indexingFolder)) {
+          continue;
+        }
+      }
+
+      let msgHdr;
+      // GetMessageHeader can be affected by the use cache, so we need to check
+      //  ContainsKey first to see if the header is really actually there.
+      if (typeof item[1] == "number") {
+        msgHdr =
+          this._indexingDatabase.containsKey(item[1]) &&
+          this._indexingFolder.GetMessageHeader(item[1]);
+      } else {
+        // Same deal as in move processing.
+        // TODO fixme to not assume singular message-id's.
+        msgHdr = this._indexingDatabase.getMsgHdrForMessageID(item[1]);
+      }
+
+      if (msgHdr) {
+        yield aCallbackHandle.pushAndGo(
+          this._indexMessage(msgHdr, aCallbackHandle),
+          { what: "indexMessage", msgHdr }
+        );
+      } else {
+        yield GlodaConstants.kWorkSync;
+      }
+    }
+
+    // There is no real reason to stay 'in' the folder.  If we are going to get
+    //  more events from the folder, its database would have to be open for us
+    //  to get the events, so it's not like we're creating an efficiency
+    //  problem where we unload a folder just to load it again in 2 seconds.
+    // (Well, at least assuming the views are good about holding onto the
+    //  database references even though they go out of their way to avoid
+    //  holding onto message header references.)
+    this._indexerLeaveFolder();
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Recover from a "folder" or "message" job failing inside a call to
+   *  |_indexMessage|, marking the message bad.  If we were not in an
+   *  |_indexMessage| call, then fail to recover.
+   *
+   * @param aJob The job that was being worked.  We ignore this for now.
+   * @param aContextStack The callbackHandle mechanism's context stack.  When we
+   *     invoke pushAndGo for _indexMessage we put something in so we can
+   *     detect when it is on the async stack.
+   * @param aException The exception that is necessitating we attempt to
+   *     recover.
+   *
+   * @returns 1 if we were able to recover (because we want the call stack
+   *     popped down to our worker), false if we can't.
+   */
+  _recover_indexMessage(aJob, aContextStack, aException) {
+    // See if indexMessage is on the stack...
+    if (
+      aContextStack.length >= 2 &&
+      aContextStack[1] &&
+      "what" in aContextStack[1] &&
+      aContextStack[1].what == "indexMessage"
+    ) {
+      // it is, so this is probably recoverable.
+
+      this._log.debug(
+        "Exception while indexing message, marking it bad (gloda id of 1)."
+      );
+
+      // -- Mark the message as bad
+      let msgHdr = aContextStack[1].msgHdr;
+      // (In the worst case, the header is no longer valid, which will result in
+      //  exceptions.  We need to be prepared for that.)
+      try {
+        msgHdr.setUint32Property(
+          GLODA_MESSAGE_ID_PROPERTY,
+          GLODA_BAD_MESSAGE_ID
+        );
+        // clear the dirty bit if it has one
+        if (msgHdr.getUint32Property(GLODA_DIRTY_PROPERTY)) {
+          msgHdr.setUint32Property(GLODA_DIRTY_PROPERTY, 0);
+        }
+      } catch (ex) {
+        // If we are indexing a folder and the message header is no longer
+        //  valid, then it's quite likely the whole folder is no longer valid.
+        //  But since in the event-driven message indexing case we could have
+        //  other valid things to look at, let's try and recover.  The folder
+        //  indexing case will come back to us shortly and we will indicate
+        //  recovery is not possible at that point.
+        // So do nothing here since by popping the indexing of the specific
+        //  message out of existence we are recovering.
+      }
+      return 1;
+    }
+    return false;
+  },
+
+  /**
+   * Cleanup after an aborted "folder" or "message" job.
+   */
+  _cleanup_indexing(aJob) {
+    this._indexerLeaveFolder();
+    aJob.safelyInvokeCallback(false);
+  },
+
+  /**
+   * Maximum number of deleted messages to process at a time.  Arbitrary; there
+   *  are no real known performance constraints at this point.
+   */
+  DELETED_MESSAGE_BLOCK_SIZE: 32,
+
+  /**
+   * Process pending deletes...
+   */
+  *_worker_processDeletes(aJob, aCallbackHandle) {
+    // Count the number of messages we will eventually process.  People freak
+    //  out when the number is constantly increasing because they think gloda
+    //  has gone rogue.  (Note: new deletions can still accumulate during
+    //  our execution, so we may 'expand' our count a little still.)
+    this._datastore.countDeletedMessages(aCallbackHandle.wrappedCallback);
+    aJob.goal = yield GlodaConstants.kWorkAsync;
+    this._log.debug(
+      "There are currently " +
+        aJob.goal +
+        " messages awaiting" +
+        " deletion processing."
+    );
+
+    // get a block of messages to delete.
+    let query = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE, {
+      noDbQueryValidityConstraints: true,
+    });
+    query._deleted(1);
+    query.limit(this.DELETED_MESSAGE_BLOCK_SIZE);
+    let deletedCollection = query.getCollection(aCallbackHandle);
+    yield GlodaConstants.kWorkAsync;
+
+    while (deletedCollection.items.length) {
+      for (let message of deletedCollection.items) {
+        // If it turns out our count is wrong (because some new deletions
+        //  happened since we entered this worker), let's issue a new count
+        //  and use that to accurately update our goal.
+        if (aJob.offset >= aJob.goal) {
+          this._datastore.countDeletedMessages(aCallbackHandle.wrappedCallback);
+          aJob.goal += yield GlodaConstants.kWorkAsync;
+        }
+
+        yield aCallbackHandle.pushAndGo(
+          this._deleteMessage(message, aCallbackHandle)
+        );
+        aJob.offset++;
+        yield GlodaConstants.kWorkSync;
+      }
+
+      deletedCollection = query.getCollection(aCallbackHandle);
+      yield GlodaConstants.kWorkAsync;
+    }
+    this.pendingDeletions = false;
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  *_worker_fixMissingContacts(aJob, aCallbackHandle) {
+    let identityContactInfos = [];
+
+    // -- asynchronously get a list of all identities without contacts
+    // The upper bound on the number of messed up contacts is the number of
+    //  contacts in the user's address book.  This should be small enough
+    //  (and the data size small enough) that this won't explode thunderbird.
+    let queryStmt = GlodaDatastore._createAsyncStatement(
+      "SELECT identities.id, identities.contactID, identities.value " +
+        "FROM identities " +
+        "LEFT JOIN contacts ON identities.contactID = contacts.id " +
+        "WHERE identities.kind = 'email' AND contacts.id IS NULL",
+      true
+    );
+    queryStmt.executeAsync({
+      handleResult(aResultSet) {
+        let row;
+        while ((row = aResultSet.getNextRow())) {
+          identityContactInfos.push({
+            identityId: row.getInt64(0),
+            contactId: row.getInt64(1),
+            email: row.getString(2),
+          });
+        }
+      },
+      handleError(aError) {},
+      handleCompletion(aReason) {
+        GlodaDatastore._asyncCompleted();
+        aCallbackHandle.wrappedCallback();
+      },
+    });
+    queryStmt.finalize();
+    GlodaDatastore._pendingAsyncStatements++;
+    yield GlodaConstants.kWorkAsync;
+
+    // -- perform fixes only if there were missing contacts
+    if (identityContactInfos.length) {
+      const yieldEvery = 64;
+      // - create the missing contacts
+      for (let i = 0; i < identityContactInfos.length; i++) {
+        if (i % yieldEvery === 0) {
+          yield GlodaConstants.kWorkSync;
+        }
+
+        let info = identityContactInfos[i],
+          card = MailServices.ab.cardForEmailAddress(info.email),
+          contact = new GlodaContact(
+            GlodaDatastore,
+            info.contactId,
+            null,
+            null,
+            card ? card.displayName || info.email : info.email,
+            0,
+            0
+          );
+        GlodaDatastore.insertContact(contact);
+
+        // update the in-memory rep of the identity to know about the contact
+        //  if there is one.
+        let identity = GlodaCollectionManager.cacheLookupOne(
+          GlodaConstants.NOUN_IDENTITY,
+          info.identityId,
+          false
+        );
+        if (identity) {
+          // Unfortunately, although this fixes the (reachable) Identity and
+          //  exposes the Contact, it does not make the Contact reachable from
+          //  the collection manager.  This will make explicit queries that look
+          //  up the contact potentially see the case where
+          //  contact.identities[0].contact !== contact.  Alternately, that
+          //  may not happen and instead the "contact" object we created above
+          //  may become unlinked.  (I'd have to trace some logic I don't feel
+          //  like tracing.)  Either way, The potential fallout is minimal
+          //  since the object identity invariant will just lapse and popularity
+          //  on the contact may become stale, and neither of those meaningfully
+          //  affect the operation of anything in Thunderbird.
+          // If we really cared, we could find all the dominant collections
+          //  that reference the identity and update their corresponding
+          //  contact collection to make it reachable.  That use-case does not
+          //  exist outside of here, which is why we're punting.
+          identity._contact = contact;
+          contact._identities = [identity];
+        }
+
+        // NOTE: If the addressbook indexer did anything useful other than
+        //  adapting to name changes, we could schedule indexing of the cards at
+        //  this time.  However, as of this writing, it doesn't, and this task
+        //  is a one-off relevant only to the time of this writing.
+      }
+
+      // - mark all folders as dirty, initiate indexing sweep
+      this.dirtyAllKnownFolders();
+      this.indexingSweepNeeded = true;
+    }
+
+    // -- mark the schema upgrade, be done
+    GlodaDatastore._updateSchemaVersion(GlodaDatastore._schemaVersion);
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Determine whether a folder is suitable for indexing.
+   *
+   * @param aMsgFolder An nsIMsgFolder you want to see if we should index.
+   *
+   * @returns true if we want to index messages in this type of folder, false if
+   *     we do not.
+   */
+  shouldIndexFolder(aMsgFolder) {
+    let folderFlags = aMsgFolder.flags;
+    // Completely ignore non-mail and virtual folders.  They should never even
+    //  get to be GlodaFolder instances.
+    if (
+      !(folderFlags & Ci.nsMsgFolderFlags.Mail) ||
+      folderFlags & Ci.nsMsgFolderFlags.Virtual
+    ) {
+      return false;
+    }
+
+    // Some folders do not really exist; we can detect this by getStringProperty
+    //  exploding when we call it.  This is primarily a concern because
+    //  _mapFolder calls said exploding method, but we also don't want to
+    //  even think about indexing folders that don't exist.  (Such folders are
+    //  likely the result of a messed up profile.)
+    try {
+      // flags is used because it should always be in the cache avoiding a miss
+      //  which would compel an msf open.
+      aMsgFolder.getStringProperty("flags");
+    } catch (ex) {
+      return false;
+    }
+
+    // Now see what our gloda folder information has to say about the folder.
+    let glodaFolder = GlodaDatastore._mapFolder(aMsgFolder);
+    return glodaFolder.indexingPriority != glodaFolder.kIndexingNeverPriority;
+  },
+
+  /**
+   * Sets the indexing priority for this folder and persists it both to Gloda,
+   * and, for backup purposes, to the nsIMsgFolder via string property as well.
+   *
+   * Setting this priority may cause the indexer to either reindex this folder,
+   * or remove this folder from the existing index.
+   *
+   * @param {nsIMsgFolder} aFolder
+   * @param {number} aPriority (one of the priority constants from GlodaFolder)
+   */
+  setFolderIndexingPriority(aFolder, aPriority) {
+    let glodaFolder = GlodaDatastore._mapFolder(aFolder);
+
+    // if there's been no change, we're done
+    if (aPriority == glodaFolder.indexingPriority) {
+      return;
+    }
+
+    // save off the old priority, and set the new one
+    let previousPrio = glodaFolder.indexingPriority;
+    glodaFolder._indexingPriority = aPriority;
+
+    // persist the new priority
+    GlodaDatastore.updateFolderIndexingPriority(glodaFolder);
+    aFolder.setStringProperty("indexingPriority", Number(aPriority).toString());
+
+    // if we've been told never to index this folder...
+    if (aPriority == glodaFolder.kIndexingNeverPriority) {
+      // stop doing so
+      if (this._indexingFolder == aFolder) {
+        GlodaIndexer.killActiveJob();
+      }
+
+      // mark all existing messages as deleted
+      GlodaDatastore.markMessagesDeletedByFolderID(glodaFolder.id);
+
+      // re-index
+      GlodaMsgIndexer.indexingSweepNeeded = true;
+    } else if (previousPrio == glodaFolder.kIndexingNeverPriority) {
+      // there's no existing index, but the user now wants one
+      glodaFolder._dirtyStatus = glodaFolder.kFolderFilthy;
+      GlodaDatastore.updateFolderDirtyStatus(glodaFolder);
+      GlodaMsgIndexer.indexingSweepNeeded = true;
+    }
+  },
+
+  /**
+   * Resets the indexing priority on the given folder to whatever the default
+   * is for folders of that type.
+   *
+   * @note Calls setFolderIndexingPriority under the hood, so has identical
+   *       potential reindexing side-effects
+   *
+   * @param {nsIMsgFolder} aFolder
+   * @param {boolean} aAllowSpecialFolderIndexing
+   */
+  resetFolderIndexingPriority(aFolder, aAllowSpecialFolderIndexing) {
+    this.setFolderIndexingPriority(
+      aFolder,
+      GlodaDatastore.getDefaultIndexingPriority(
+        aFolder,
+        aAllowSpecialFolderIndexing
+      )
+    );
+  },
+
+  /**
+   * Queue all of the folders of all of the accounts of the current profile
+   *  for indexing.  We traverse all folders and queue them immediately to try
+   *  and have an accurate estimate of the number of folders that need to be
+   *  indexed.  (We previously queued accounts rather than immediately
+   *  walking their list of folders.)
+   */
+  indexEverything() {
+    this._log.info("Queueing all accounts for indexing.");
+
+    GlodaDatastore._beginTransaction();
+    for (let account of MailServices.accounts.accounts) {
+      this.indexAccount(account);
+    }
+    GlodaDatastore._commitTransaction();
+  },
+
+  /**
+   * Queue all of the folders belonging to an account for indexing.
+   */
+  indexAccount(aAccount) {
+    let rootFolder = aAccount.incomingServer.rootFolder;
+    if (rootFolder instanceof Ci.nsIMsgFolder) {
+      this._log.info("Queueing account folders for indexing: " + aAccount.key);
+
+      for (let folder of rootFolder.descendants) {
+        if (this.shouldIndexFolder(folder)) {
+          GlodaIndexer.indexJob(
+            new IndexingJob("folder", GlodaDatastore._mapFolder(folder).id)
+          );
+        }
+      }
+    } else {
+      this._log.info("Skipping Account, root folder not nsIMsgFolder");
+    }
+  },
+
+  /**
+   * Queue a single folder for indexing given an nsIMsgFolder.
+   *
+   * @param [aOptions.callback] A callback to invoke when the folder finishes
+   *     indexing.  First argument is true if the task ran to completion
+   *     successfully, false if we had to abort for some reason.
+   * @param [aOptions.force=false] Should we force the indexing of all messages
+   *     in the folder (true) or just index what hasn't been indexed (false).
+   * @returns true if we are going to index the folder, false if not.
+   */
+  indexFolder(aMsgFolder, aOptions) {
+    if (!this.shouldIndexFolder(aMsgFolder)) {
+      return false;
+    }
+    let glodaFolder = GlodaDatastore._mapFolder(aMsgFolder);
+    // stay out of compacting/compacted folders
+    if (glodaFolder.compacting || glodaFolder.compacted) {
+      return false;
+    }
+
+    this._log.info("Queue-ing folder for indexing: " + aMsgFolder.prettyName);
+    let job = new IndexingJob("folder", glodaFolder.id);
+    if (aOptions) {
+      if ("callback" in aOptions) {
+        job.callback = aOptions.callback;
+      }
+      if ("force" in aOptions) {
+        job.force = true;
+      }
+    }
+    GlodaIndexer.indexJob(job);
+    return true;
+  },
+
+  /**
+   * Queue a list of messages for indexing.
+   *
+   * @param aFoldersAndMessages List of [nsIMsgFolder, message key] tuples.
+   */
+  indexMessages(aFoldersAndMessages) {
+    let job = new IndexingJob("message", null);
+    job.items = aFoldersAndMessages.map(fm => [
+      GlodaDatastore._mapFolder(fm[0]).id,
+      fm[1],
+    ]);
+    GlodaIndexer.indexJob(job);
+  },
+
+  /**
+   * Mark all known folders as dirty so that the next indexing sweep goes
+   *  into all folders and checks their contents to see if they need to be
+   *  indexed.
+   *
+   * This is being added for the migration case where we want to try and reindex
+   *  all of the messages that had been marked with GLODA_BAD_MESSAGE_ID but
+   *  which is now GLODA_OLD_BAD_MESSAGE_ID and so we should attempt to reindex
+   *  them.
+   */
+  dirtyAllKnownFolders() {
+    // Just iterate over the datastore's folder map and tell each folder to
+    //  be dirty if its priority is not disabled.
+    for (let folderID in GlodaDatastore._folderByID) {
+      let glodaFolder = GlodaDatastore._folderByID[folderID];
+      if (glodaFolder.indexingPriority !== glodaFolder.kIndexingNeverPriority) {
+        glodaFolder._ensureFolderDirty();
+      }
+    }
+  },
+
+  /**
+   * Given a message header, return whether this message is likely to have
+   * been indexed or not.
+   *
+   * This means the message must:
+   * - Be in a folder eligible for gloda indexing. (Not News, etc.)
+   * - Be in a non-filthy folder.
+   * - Be gloda-indexed and non-filthy.
+   *
+   * @param aMsgHdr A message header.
+   * @returns true if the message is likely to have been indexed.
+   */
+  isMessageIndexed(aMsgHdr) {
+    // If it's in a folder that we flat out do not index, say no.
+    if (!this.shouldIndexFolder(aMsgHdr.folder)) {
+      return false;
+    }
+    let glodaFolder = GlodaDatastore._mapFolder(aMsgHdr.folder);
+    let [glodaId, glodaDirty] = PendingCommitTracker.getGlodaState(aMsgHdr);
+    return (
+      glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+      glodaDirty != GlodaMsgIndexer.kMessageFilthy &&
+      glodaFolder &&
+      glodaFolder.dirtyStatus != glodaFolder.kFolderFilthy
+    );
+  },
+
+  /* *********** Event Processing *********** */
+
+  /**
+   * Tracks messages we have received msgKeyChanged notifications for in order
+   *  to provide batching and to suppress needless reindexing when we receive
+   *  the expected follow-up msgsClassified notification.
+   *
+   * The entries in this dictionary should be extremely short-lived as we
+   *  receive the msgKeyChanged notification as the offline fake header is
+   *  converted into a real header (which is accompanied by a msgAdded
+   *  notification we don't pay attention to).  Once the headers finish
+   *  updating, the message classifier will get its at-bat and should likely
+   *  find that the messages have already been classified and so fast-path
+   *  them.
+   *
+   * The keys in this dictionary are chosen to be consistent with those of
+   *  PendingCommitTracker: the folder.URI + "#" + the (new) message key.
+   * The values in the dictionary are either an object with "id" (the gloda
+   *  id), "key" (the new message key), and "dirty" (is it dirty and so
+   *  should still be queued for indexing) attributes, or null indicating that
+   *  no change in message key occurred and so no database changes are required.
+   */
+  _keyChangedBatchInfo: {},
+
+  /**
+   * Common logic for things that want to feed event-driven indexing.  This gets
+   *  called by both |_msgFolderListener.msgsClassified| when we are first
+   *  seeing a message as well as by |_folderListener| when things happen to
+   *  existing messages.  Although we could slightly specialize for the
+   *  new-to-us case, it works out to be cleaner to just treat them the same
+   *  and take a very small performance hit.
+   *
+   * @param aMsgHdrs array of messages to treat as potentially changed.
+   * @param aDirtyingEvent Is this event inherently dirtying?  Receiving a
+   *     msgsClassified notification is not inherently dirtying because it is
+   *     just telling us that a message exists.  We use this knowledge to
+   *     ignore the msgsClassified notifications for messages we have received
+   *     msgKeyChanged notifications for and fast-pathed.  Since it is possible
+   *     for user action to do something that dirties the message between the
+   *     time we get the msgKeyChanged notification and when we receive the
+   *     msgsClassified notification, we want to make sure we don't get
+   *     confused.  (Although since we remove the message from our ignore-set
+   *     after the first notification, we would likely just mistakenly treat
+   *     the msgsClassified notification as something dirtying, so it would
+   *     still work out...)
+   */
+  _reindexChangedMessages(aMsgHdrs, aDirtyingEvent) {
+    let glodaIdsNeedingDeletion = null;
+    let messageKeyChangedIds = null,
+      messageKeyChangedNewKeys = null;
+    for (let msgHdr of aMsgHdrs) {
+      // -- Index this folder?
+      let msgFolder = msgHdr.folder;
+      if (!this.shouldIndexFolder(msgFolder)) {
+        continue;
+      }
+      // -- Ignore messages in filthy folders!
+      // A filthy folder can only be processed by an indexing sweep, and at
+      //  that point the message will get indexed.
+      let glodaFolder = GlodaDatastore._mapFolder(msgHdr.folder);
+      if (glodaFolder.dirtyStatus == glodaFolder.kFolderFilthy) {
+        continue;
+      }
+
+      // -- msgKeyChanged event follow-up
+      if (!aDirtyingEvent) {
+        let keyChangedKey = msgHdr.folder.URI + "#" + msgHdr.messageKey;
+        if (keyChangedKey in this._keyChangedBatchInfo) {
+          var keyChangedInfo = this._keyChangedBatchInfo[keyChangedKey];
+          delete this._keyChangedBatchInfo[keyChangedKey];
+
+          // Null means to ignore this message because the key did not change
+          //  (and the message was not dirty so it is safe to ignore.)
+          if (keyChangedInfo == null) {
+            continue;
+          }
+          // (the key may be null if we only generated the entry because the
+          //  message was dirty)
+          if (keyChangedInfo.key !== null) {
+            if (messageKeyChangedIds == null) {
+              messageKeyChangedIds = [];
+              messageKeyChangedNewKeys = [];
+            }
+            messageKeyChangedIds.push(keyChangedInfo.id);
+            messageKeyChangedNewKeys.push(keyChangedInfo.key);
+          }
+          // ignore the message because it was not dirty
+          if (!keyChangedInfo.isDirty) {
+            continue;
+          }
+        }
+      }
+
+      // -- Index this message?
+      // We index local messages, IMAP messages that are offline, and IMAP
+      // messages that aren't offline but whose folders aren't offline either
+      let isFolderLocal = msgFolder instanceof Ci.nsIMsgLocalMailFolder;
+      if (!isFolderLocal) {
+        if (
+          !(msgHdr.flags & Ci.nsMsgMessageFlags.Offline) &&
+          msgFolder.getFlag(Ci.nsMsgFolderFlags.Offline)
+        ) {
+          continue;
+        }
+      }
+      // Ignore messages whose processing flags indicate it has not yet been
+      //  classified.  In the IMAP case if the Offline flag is going to get set
+      //  we are going to see it before the msgsClassified event so this is
+      //  very important.
+      if (
+        msgFolder.getProcessingFlags(msgHdr.messageKey) &
+        NOT_YET_REPORTED_PROCESSING_FLAGS
+      ) {
+        continue;
+      }
+
+      let [glodaId, glodaDirty] = PendingCommitTracker.getGlodaState(msgHdr);
+
+      let isSpam =
+        msgHdr.getStringProperty(JUNK_SCORE_PROPERTY) == JUNK_SPAM_SCORE_STR;
+
+      // -- Is the message currently gloda indexed?
+      if (
+        glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+        glodaDirty != this.kMessageFilthy
+      ) {
+        // - Is the message spam?
+        if (isSpam) {
+          // Treat this as a deletion...
+          if (!glodaIdsNeedingDeletion) {
+            glodaIdsNeedingDeletion = [];
+          }
+          glodaIdsNeedingDeletion.push(glodaId);
+          // and skip to the next message
+          continue;
+        }
+
+        // - Mark the message dirty if it is clean.
+        // (This is the only case in which we need to mark dirty so that the
+        //  indexing sweep takes care of things if we don't process this in
+        //  an event-driven fashion.  If the message has no gloda-id or does
+        //  and it's already dirty or filthy, it is already marked for
+        //  indexing.)
+        if (glodaDirty == this.kMessageClean) {
+          msgHdr.setUint32Property(GLODA_DIRTY_PROPERTY, this.kMessageDirty);
+        }
+        // if the message is pending clean, this change invalidates that.
+        PendingCommitTracker.noteDirtyHeader(msgHdr);
+      } else if (isSpam) {
+        // If it's not indexed but is spam, ignore it.
+        continue;
+      }
+      // (we want to index the message if we are here)
+
+      // mark the folder dirty too, so we know to look inside
+      glodaFolder._ensureFolderDirty();
+
+      if (this._pendingAddJob == null) {
+        this._pendingAddJob = new IndexingJob("message", null);
+        GlodaIndexer.indexJob(this._pendingAddJob);
+      }
+      // only queue the message if we haven't overflowed our event-driven budget
+      if (this._pendingAddJob.items.length < this._indexMaxEventQueueMessages) {
+        this._pendingAddJob.items.push([
+          GlodaDatastore._mapFolder(msgFolder).id,
+          msgHdr.messageKey,
+        ]);
+      } else {
+        this.indexingSweepNeeded = true;
+      }
+    }
+
+    // Process any message key changes (from earlier msgKeyChanged events)
+    if (messageKeyChangedIds != null) {
+      GlodaDatastore.updateMessageKeys(
+        messageKeyChangedIds,
+        messageKeyChangedNewKeys
+      );
+    }
+
+    // If we accumulated any deletions in there, batch them off now.
+    if (glodaIdsNeedingDeletion) {
+      GlodaDatastore.markMessagesDeletedByIDs(glodaIdsNeedingDeletion);
+      this.pendingDeletions = true;
+    }
+  },
+
+  /* ***** Folder Changes ***** */
+  /**
+   * All additions and removals are queued for processing.  Indexing messages
+   *  is potentially phenomenally expensive, and deletion can still be
+   *  relatively expensive due to our need to delete the message, its
+   *  attributes, and all attributes that reference it.  Additionally,
+   *  attribute deletion costs are higher than attribute look-up because
+   *  there is the actual row plus its 3 indices, and our covering indices are
+   *  no help there.
+   *
+   */
+  _msgFolderListener: {
+    indexer: null,
+
+    /**
+     * We no longer use the msgAdded notification, instead opting to wait until
+     *  junk/trait classification has run (or decided not to run) and all
+     *  filters have run.  The msgsClassified notification provides that for us.
+     */
+    msgAdded(aMsgHdr) {
+      // we are never called! we do not enable this bit!
+    },
+
+    /**
+     * Process (apparently newly added) messages that have been looked at by
+     *  the message classifier.  This ensures that if the message was going
+     *  to get marked as spam, this will have already happened.
+     *
+     * Besides truly new (to us) messages, We will also receive this event for
+     *  messages that are the result of IMAP message move/copy operations,
+     *  including both moves that generated offline fake headers and those that
+     *  did not.  In the offline fake header case, however, we are able to
+     *  ignore their msgsClassified events because we will have received a
+     *  msgKeyChanged notification sometime in the recent past.
+     */
+    msgsClassified(aMsgHdrs, aJunkClassified, aTraitClassified) {
+      this.indexer._log.debug("msgsClassified notification");
+      try {
+        GlodaMsgIndexer._reindexChangedMessages(aMsgHdrs, false);
+      } catch (ex) {
+        this.indexer._log.error("Explosion in msgsClassified handling:", ex);
+      }
+    },
+
+    /**
+     * Any messages which have had their junk state changed are marked for
+     * reindexing.
+     */
+    msgsJunkStatusChanged(messages) {
+      this.indexer._log.debug("JunkStatusChanged notification");
+      GlodaMsgIndexer._reindexChangedMessages(messages, true);
+    },
+
+    /**
+     * Handle real, actual deletion (move to trash and IMAP deletion model
+     *  don't count); we only see the deletion here when it becomes forever,
+     *  or rather _just before_ it becomes forever.  Because the header is
+     *  going away, we need to either process things immediately or extract the
+     *  information required to purge it later without the header.
+     * To this end, we mark all messages that were indexed in the gloda message
+     *  database as deleted.  We set our pending deletions flag to let our
+     *  indexing logic know that after its next wave of folder traversal, it
+     *  should perform a deletion pass.  If it turns out the messages are coming
+     *  back, the fact that deletion is thus deferred can be handy, as we can
+     *  reuse the existing gloda message.
+     */
+    msgsDeleted(aMsgHdrs) {
+      this.indexer._log.debug("msgsDeleted notification");
+      let glodaMessageIds = [];
+
+      for (let msgHdr of aMsgHdrs) {
+        let [glodaId, glodaDirty] = PendingCommitTracker.getGlodaState(msgHdr);
+        if (
+          glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+          glodaDirty != GlodaMsgIndexer.kMessageFilthy
+        ) {
+          glodaMessageIds.push(glodaId);
+        }
+      }
+
+      if (glodaMessageIds.length) {
+        GlodaMsgIndexer._datastore.markMessagesDeletedByIDs(glodaMessageIds);
+        GlodaMsgIndexer.pendingDeletions = true;
+      }
+    },
+
+    /**
+     * Process a move or copy.
+     *
+     * Moves to a local folder or an IMAP folder where we are generating offline
+     *  fake headers are dealt with efficiently because we get both the source
+     *  and destination headers.  The main ingredient to having offline fake
+     *  headers is that allowUndo was true when the operation was performance.
+     *  The only non-obvious thing is that we need to make sure that we deal
+     *  with the impact of filthy folders and messages on gloda-id's (they
+     *  invalidate the gloda-id).
+     *
+     * Moves to an IMAP folder that do not generate offline fake headers do not
+     *  provide us with the target header, but the IMAP SetPendingAttributes
+     *  logic will still attempt to propagate the properties on the message
+     *  header so when we eventually see it in the msgsClassified notification,
+     *  it should have the properties of the source message copied over.
+     * We make sure that gloda-id's do not get propagated when messages are
+     *  moved from IMAP folders that are marked filthy or are marked as not
+     *  supposed to be indexed by clearing the pending attributes for the header
+     *  being tracked by the destination IMAP folder.
+     * We could fast-path the IMAP move case in msgsClassified by noticing that
+     *  a message is showing up with a gloda-id header already and just
+     *  performing an async location update.
+     *
+     * Moves that occur involving 'compacted' folders are fine and do not
+     *  require special handling here.  The one tricky super-edge-case that
+     *  can happen (and gets handled by the compaction pass) is the move of a
+     *  message that got gloda indexed that did not already have a gloda-id and
+     *  PendingCommitTracker did not get to flush the gloda-id before the
+     *  compaction happened.  In that case our move logic cannot know to do
+     *  anything and the gloda database still thinks the message lives in our
+     *  folder.  The compaction pass will deal with this by marking the message
+     *  as deleted.  The rationale being that marking it deleted allows the
+     *  message to be re-used if it gets indexed in the target location, or if
+     *  the target location has already been indexed, we no longer need the
+     *  duplicate and it should be deleted.  (Also, it is unable to distinguish
+     *  between a case where the message got deleted versus moved.)
+     *
+     * Because copied messages are, by their nature, duplicate messages, we
+     *  do not particularly care about them.  As such, we defer their processing
+     *  to the automatic sync logic that will happen much later on.  This is
+     *  potentially desirable in case the user deletes some of the original
+     *  messages, allowing us to reuse the gloda message representations when
+     *  we finally get around to indexing the messages.  We do need to mark the
+     *  folder as dirty, though, to clue in the sync logic.
+     */
+    msgsMoveCopyCompleted(aMove, aSrcMsgHdrs, aDestFolder, aDestMsgHdrs) {
+      this.indexer._log.debug("MoveCopy notification.  Move: " + aMove);
+      try {
+        // ---- Move
+        if (aMove) {
+          // -- Effectively a deletion?
+          // If the destination folder is not indexed, it's like these messages
+          //  are being deleted.
+          if (!GlodaMsgIndexer.shouldIndexFolder(aDestFolder)) {
+            this.msgsDeleted(aSrcMsgHdrs);
+            return;
+          }
+
+          // -- Avoid propagation of filthy gloda-id's.
+          // If the source folder is filthy or should not be indexed (and so
+          //  any gloda-id's found in there are gibberish), our only job is to
+          //  strip the gloda-id's off of all the destination headers because
+          //  none of the gloda-id's are valid (and so we certainly don't want
+          //  to try and use them as a basis for updating message keys.)
+          let srcMsgFolder = aSrcMsgHdrs[0].folder;
+          if (
+            !this.indexer.shouldIndexFolder(srcMsgFolder) ||
+            GlodaDatastore._mapFolder(srcMsgFolder).dirtyStatus ==
+              GlodaFolder.prototype.kFolderFilthy
+          ) {
+            // Local case, just modify the destination headers directly.
+            if (aDestMsgHdrs.length > 0) {
+              for (let destMsgHdr of aDestMsgHdrs) {
+                // zero it out if it exists
+                // (no need to deal with pending commit issues here; a filthy
+                //  folder by definition has nothing indexed in it.)
+                let glodaId = destMsgHdr.getUint32Property(
+                  GLODA_MESSAGE_ID_PROPERTY
+                );
+                if (glodaId) {
+                  destMsgHdr.setUint32Property(GLODA_MESSAGE_ID_PROPERTY, 0);
+                }
+              }
+
+              // Since we are moving messages from a folder where they were
+              //  effectively not indexed, it is up to us to make sure the
+              //  messages now get indexed.
+              this.indexer._reindexChangedMessages(aDestMsgHdrs);
+              return;
+            }
+
+            // IMAP move case, we need to operate on the pending headers using
+            //  the source header to get the pending header and as the
+            //  indication of what has been already set on the pending header.
+            let destDb;
+            // so, this can fail, and there's not much we can do about it.
+            try {
+              destDb = aDestFolder.msgDatabase;
+            } catch (ex) {
+              this.indexer._log.warn(
+                "Destination database for " +
+                  aDestFolder.prettyName +
+                  " not ready on IMAP move." +
+                  " Gloda corruption possible."
+              );
+              return;
+            }
+            for (let srcMsgHdr of aSrcMsgHdrs) {
+              // zero it out if it exists
+              // (no need to deal with pending commit issues here; a filthy
+              //  folder by definition has nothing indexed in it.)
+              let glodaId = srcMsgHdr.getUint32Property(
+                GLODA_MESSAGE_ID_PROPERTY
+              );
+              if (glodaId) {
+                destDb.setUint32AttributeOnPendingHdr(
+                  srcMsgHdr,
+                  GLODA_MESSAGE_ID_PROPERTY,
+                  0
+                );
+              }
+            }
+
+            // Nothing remains to be done.  The msgClassified event will take
+            //  care of making sure the message gets indexed.
+            return;
+          }
+
+          // --- Have destination headers (local case):
+          if (aDestMsgHdrs.length > 0) {
+            // -- Update message keys for valid gloda-id's.
+            // (Which means ignore filthy gloda-id's.)
+            let glodaIds = [];
+            let newMessageKeys = [];
+            // Track whether we see any messages that are not gloda indexed so
+            //  we know if we have to mark the destination folder dirty.
+            let sawNonGlodaMessage = false;
+            for (let iMsg = 0; iMsg < aSrcMsgHdrs.length; iMsg++) {
+              let srcMsgHdr = aSrcMsgHdrs[iMsg];
+              let destMsgHdr = aDestMsgHdrs[iMsg];
+
+              let [glodaId, dirtyStatus] =
+                PendingCommitTracker.getGlodaState(srcMsgHdr);
+              if (
+                glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+                dirtyStatus != GlodaMsgIndexer.kMessageFilthy
+              ) {
+                // we may need to update the pending commit map (it checks)
+                PendingCommitTracker.noteMove(srcMsgHdr, destMsgHdr);
+                // but we always need to update our database
+                glodaIds.push(glodaId);
+                newMessageKeys.push(destMsgHdr.messageKey);
+              } else {
+                sawNonGlodaMessage = true;
+              }
+            }
+
+            // this method takes care to update the in-memory representations
+            //  too; we don't need to do anything
+            if (glodaIds.length) {
+              GlodaDatastore.updateMessageLocations(
+                glodaIds,
+                newMessageKeys,
+                aDestFolder
+              );
+            }
+
+            // Mark the destination folder dirty if we saw any messages that
+            //  were not already gloda indexed.
+            if (sawNonGlodaMessage) {
+              let destGlodaFolder = GlodaDatastore._mapFolder(aDestFolder);
+              destGlodaFolder._ensureFolderDirty();
+              this.indexer.indexingSweepNeeded = true;
+            }
+          } else {
+            // --- No dest headers (IMAP case):
+            // Update any valid gloda indexed messages into their new folder to
+            //  make the indexer's life easier when it sees the messages in their
+            //  new folder.
+            let glodaIds = [];
+
+            let srcFolderIsLocal =
+              srcMsgFolder instanceof Ci.nsIMsgLocalMailFolder;
+            for (let msgHdr of aSrcMsgHdrs) {
+              let [glodaId, dirtyStatus] =
+                PendingCommitTracker.getGlodaState(msgHdr);
+              if (
+                glodaId >= GLODA_FIRST_VALID_MESSAGE_ID &&
+                dirtyStatus != GlodaMsgIndexer.kMessageFilthy
+              ) {
+                // we may need to update the pending commit map (it checks)
+                PendingCommitTracker.noteBlindMove(msgHdr);
+                // but we always need to update our database
+                glodaIds.push(glodaId);
+
+                // XXX UNDO WORKAROUND
+                // This constitutes a move from a local folder to an IMAP
+                //  folder.  Undo does not currently do the right thing for us,
+                //  but we have a chance of not orphaning the message if we
+                //  mark the source header as dirty so that when the message
+                //  gets re-added we see it.  (This does require that we enter
+                //  the folder; we set the folder dirty after the loop to
+                //  increase the probability of this but it's not foolproof
+                //  depending on when the next indexing sweep happens and when
+                //  the user performs an undo.)
+                msgHdr.setUint32Property(
+                  GLODA_DIRTY_PROPERTY,
+                  GlodaMsgIndexer.kMessageDirty
+                );
+              }
+            }
+            // XXX ALSO UNDO WORKAROUND
+            if (srcFolderIsLocal) {
+              let srcGlodaFolder = GlodaDatastore._mapFolder(srcMsgFolder);
+              srcGlodaFolder._ensureFolderDirty();
+            }
+
+            // quickly move them to the right folder, zeroing their message keys
+            GlodaDatastore.updateMessageFoldersByKeyPurging(
+              glodaIds,
+              aDestFolder
+            );
+            // we _do not_ need to mark the folder as dirty, because the
+            //  message added events will cause that to happen.
+          }
+        } else {
+          // ---- Copy case
+          // -- Do not propagate gloda-id's for copies
+          // (Only applies if we have the destination header, which means local)
+          for (let destMsgHdr of aDestMsgHdrs) {
+            let glodaId = destMsgHdr.getUint32Property(
+              GLODA_MESSAGE_ID_PROPERTY
+            );
+            if (glodaId) {
+              destMsgHdr.setUint32Property(GLODA_MESSAGE_ID_PROPERTY, 0);
+            }
+          }
+
+          // mark the folder as dirty; we'll get to it later.
+          let destGlodaFolder = GlodaDatastore._mapFolder(aDestFolder);
+          destGlodaFolder._ensureFolderDirty();
+          this.indexer.indexingSweepNeeded = true;
+        }
+      } catch (ex) {
+        this.indexer._log.error(
+          "Problem encountered during message move/copy:",
+          ex.stack
+        );
+      }
+    },
+
+    /**
+     * Queue up message key changes that are a result of offline fake headers
+     *  being made real for the actual update during the msgsClassified
+     *  notification that is expected after this.  We defer the
+     *  actual work (if there is any to be done; the fake header might have
+     *  guessed the right UID correctly) so that we can batch our work.
+     *
+     * The expectation is that there will be no meaningful time window between
+     *  this notification and the msgsClassified notification since the message
+     *  classifier should not actually need to classify the messages (they
+     *  should already have been classified) and so can fast-path them.
+     */
+    msgKeyChanged(aOldMsgKey, aNewMsgHdr) {
+      try {
+        let val = null,
+          newKey = aNewMsgHdr.messageKey;
+        let [glodaId, glodaDirty] =
+          PendingCommitTracker.getGlodaState(aNewMsgHdr);
+        // If we haven't indexed this message yet, take no action, and leave it
+        // up to msgsClassified to take proper action.
+        if (glodaId < GLODA_FIRST_VALID_MESSAGE_ID) {
+          return;
+        }
+        // take no action on filthy messages,
+        // generate an entry if dirty or the keys don't match.
+        if (
+          glodaDirty !== GlodaMsgIndexer.kMessageFilthy &&
+          (glodaDirty === GlodaMsgIndexer.kMessageDirty ||
+            aOldMsgKey !== newKey)
+        ) {
+          val = {
+            id: glodaId,
+            key: aOldMsgKey !== newKey ? newKey : null,
+            isDirty: glodaDirty === GlodaMsgIndexer.kMessageDirty,
+          };
+        }
+
+        let key = aNewMsgHdr.folder.URI + "#" + aNewMsgHdr.messageKey;
+        this.indexer._keyChangedBatchInfo[key] = val;
+      } catch (ex) {
+        // this is more for the unit test to fail rather than user error reporting
+        this.indexer._log.error(
+          "Problem encountered during msgKeyChanged" +
+            " notification handling: " +
+            ex +
+            "\n\n" +
+            ex.stack +
+            " \n\n"
+        );
+      }
+    },
+
+    /**
+     * Detect newly added folders before they get messages so we map them before
+     * they get any messages added to them.  If we only hear about them after
+     * they get their 1st message, then we will mark them filthy, but if we mark
+     * them before that, they get marked clean.
+     */
+    folderAdded(aMsgFolder) {
+      // This is invoked for its side-effect of invoking _mapFolder and doing so
+      // only after filtering out folders we don't care about.
+      GlodaMsgIndexer.shouldIndexFolder(aMsgFolder);
+    },
+
+    /**
+     * Handles folder no-longer-exists-ence.  We mark all messages as deleted
+     *  and remove the folder from our URI table.  Currently, if a folder that
+     *  contains other folders is deleted, we may either receive one
+     *  notification for the folder that is deleted, or a notification for the
+     *  folder and one for each of its descendents.  This depends upon the
+     *  underlying account implementation, so we explicitly handle each case.
+     *  Namely, we treat it as if we're only planning on getting one, but we
+     *  handle if the children are already gone for some reason.
+     */
+    folderDeleted(aFolder) {
+      this.indexer._log.debug("folderDeleted notification");
+      try {
+        let delFunc = function (aFolder, indexer) {
+          if (indexer._datastore._folderKnown(aFolder)) {
+            indexer._log.info(
+              "Processing deletion of folder " + aFolder.prettyName + "."
+            );
+            let glodaFolder = GlodaDatastore._mapFolder(aFolder);
+            indexer._datastore.markMessagesDeletedByFolderID(glodaFolder.id);
+            indexer._datastore.deleteFolderByID(glodaFolder.id);
+            GlodaDatastore._killGlodaFolderIntoTombstone(glodaFolder);
+          } else {
+            indexer._log.info(
+              "Ignoring deletion of folder " +
+                aFolder.prettyName +
+                " because it is unknown to gloda."
+            );
+          }
+        };
+
+        let descendentFolders = aFolder.descendants;
+        // (the order of operations does not matter; child, non-child, whatever.)
+        // delete the parent
+        delFunc(aFolder, this.indexer);
+        // delete all its descendents
+        for (let folder of descendentFolders) {
+          delFunc(folder, this.indexer);
+        }
+
+        this.indexer.pendingDeletions = true;
+      } catch (ex) {
+        this.indexer._log.error(
+          "Problem encountered during folder deletion" +
+            ": " +
+            ex +
+            "\n\n" +
+            ex.stack +
+            "\n\n"
+        );
+      }
+    },
+
+    /**
+     * Handle a folder being copied or moved.
+     * Moves are handled by a helper function shared with _folderRenameHelper
+     *  (which takes care of any nesting involved).
+     * Copies are actually ignored, because our periodic indexing traversal
+     *  should discover these automatically.  We could hint ourselves into
+     *  action, but arguably a set of completely duplicate messages is not
+     *  a high priority for indexing.
+     */
+    folderMoveCopyCompleted(aMove, aSrcFolder, aDestFolder) {
+      this.indexer._log.debug(
+        "folderMoveCopy notification (Move: " + aMove + ")"
+      );
+      if (aMove) {
+        let srcURI = aSrcFolder.URI;
+        let targetURI =
+          aDestFolder.URI + srcURI.substring(srcURI.lastIndexOf("/"));
+        this._folderRenameHelper(aSrcFolder, targetURI);
+      } else {
+        this.indexer.indexingSweepNeeded = true;
+      }
+    },
+
+    /**
+     * We just need to update the URI <-> ID maps and the row in the database,
+     *  all of which is actually done by the datastore for us.
+     * This method needs to deal with the complexity where local folders will
+     *  generate a rename notification for each sub-folder, but IMAP folders
+     *  will generate only a single notification.  Our logic primarily handles
+     *  this by not exploding if the original folder no longer exists.
+     */
+    _folderRenameHelper(aOrigFolder, aNewURI) {
+      let newFolder = lazy.MailUtils.getOrCreateFolder(aNewURI);
+      let specialFolderFlags =
+        Ci.nsMsgFolderFlags.Trash | Ci.nsMsgFolderFlags.Junk;
+      if (newFolder.isSpecialFolder(specialFolderFlags, true)) {
+        let descendentFolders = newFolder.descendants;
+
+        // First thing to do: make sure we don't index the resulting folder and
+        //  its descendents.
+        GlodaMsgIndexer.resetFolderIndexingPriority(newFolder);
+        for (let folder of descendentFolders) {
+          GlodaMsgIndexer.resetFolderIndexingPriority(folder);
+        }
+
+        // Remove from the index messages from the original folder
+        this.folderDeleted(aOrigFolder);
+      } else {
+        let descendentFolders = aOrigFolder.descendants;
+
+        let origURI = aOrigFolder.URI;
+        // this rename is straightforward.
+        GlodaDatastore.renameFolder(aOrigFolder, aNewURI);
+
+        for (let folder of descendentFolders) {
+          let oldSubURI = folder.URI;
+          // mangle a new URI from the old URI.  we could also try and do a
+          //  parallel traversal of the new folder hierarchy, but that seems like
+          //  more work.
+          let newSubURI = aNewURI + oldSubURI.substring(origURI.length);
+          this.indexer._datastore.renameFolder(oldSubURI, newSubURI);
+        }
+
+        this.indexer._log.debug(
+          "folder renamed: " + origURI + " to " + aNewURI
+        );
+      }
+    },
+
+    /**
+     * Handle folder renames, dispatching to our rename helper (which also
+     *  takes care of any nested folder issues.)
+     */
+    folderRenamed(aOrigFolder, aNewFolder) {
+      this._folderRenameHelper(aOrigFolder, aNewFolder.URI);
+    },
+
+    /**
+     * Helper used by folderCompactStart/folderReindexTriggered.
+     */
+    _reindexFolderHelper(folder, isCompacting) {
+      // ignore folders we ignore...
+      if (!GlodaMsgIndexer.shouldIndexFolder(folder)) {
+        return;
+      }
+
+      let glodaFolder = GlodaDatastore._mapFolder(folder);
+      if (isCompacting) {
+        glodaFolder.compacting = true;
+      }
+
+      // Purge any explicit indexing of said folder.
+      GlodaIndexer.purgeJobsUsingFilter(function (aJob) {
+        return aJob.jobType == "folder" && aJob.id == folder.id;
+      });
+
+      // Abort the active job if it's in the folder (this covers both
+      //  event-driven indexing that happens to be in the folder as well
+      //  explicit folder indexing of the folder).
+      if (GlodaMsgIndexer._indexingFolder == folder) {
+        GlodaIndexer.killActiveJob();
+      }
+
+      // Tell the PendingCommitTracker to throw away anything it is tracking
+      //  about the folder.  We will pick up the pieces in the compaction
+      //  pass.
+      PendingCommitTracker.noteFolderDatabaseGettingBlownAway(folder);
+
+      // (We do not need to mark the folder dirty because if we were indexing
+      //  it, it already must have been marked dirty.)
+    },
+
+    /**
+     * folderCompactStart: Mark the folder as compacting in our in-memory
+     * representation.  This should keep any new indexing out of the folder
+     * until it is done compacting.  Also, kill any active or existing jobs
+     * to index the folder.
+     */
+    folderCompactStart(folder) {
+      this._reindexFolderHelper(folder, true);
+    },
+
+    /**
+     * folderReindexTriggered: We do the same thing as folderCompactStart
+     * but don't mark the folder as compacting.
+     */
+    folderReindexTriggered(folder) {
+      this._reindexFolderHelper(folder, false);
+    },
+
+    /**
+     * folderCompactFinish: Mark the folder as done compacting in our
+     * in-memory representation.  Assuming the folder was known to us and
+     * not marked filthy, queue a compaction job.
+     */
+    folderCompactFinish(folder) {
+      // ignore folders we ignore...
+      if (!GlodaMsgIndexer.shouldIndexFolder(folder)) {
+        return;
+      }
+
+      let glodaFolder = GlodaDatastore._mapFolder(folder);
+      glodaFolder.compacting = false;
+      glodaFolder._setCompactedState(true);
+
+      // Queue compaction unless the folder was filthy (in which case there
+      //  are no valid gloda-id's to update.)
+      if (glodaFolder.dirtyStatus != glodaFolder.kFolderFilthy) {
+        GlodaIndexer.indexJob(new IndexingJob("folderCompact", glodaFolder.id));
+      }
+
+      // Queue indexing of the folder if it is dirty.  We are doing this
+      //  mainly in case we were indexing it before the compaction started.
+      //  It should be reasonably harmless if we weren't.
+      // (It would probably be better to just make sure that there is an
+      //  indexing sweep queued or active, and if it's already active that
+      //  this folder is in the queue to be processed.)
+      if (glodaFolder.dirtyStatus == glodaFolder.kFolderDirty) {
+        GlodaIndexer.indexJob(new IndexingJob("folder", glodaFolder.id));
+      }
+    },
+  },
+
+  /**
+   * A nsIFolderListener (listening on nsIMsgMailSession so we get all of
+   *  these events) PRIMARILY to get folder loaded notifications.  Because of
+   *  deficiencies in the nsIMsgFolderListener's events at this time, we also
+   *  get our folder-added and newsgroup notifications from here for now.  (This
+   *  will be rectified.)
+   */
+  _folderListener: {
+    indexer: null,
+
+    _init(aIndexer) {
+      this.indexer = aIndexer;
+    },
+
+    onFolderAdded(parentFolder, child) {},
+    onMessageAdded(parentFolder, msg) {},
+    onFolderRemoved(parentFolder, child) {},
+    onMessageRemoved(parentFolder, msg) {},
+    onFolderPropertyChanged(aItem, aProperty, aOldValue, aNewValue) {},
+    /**
+     * Detect changes to folder flags and reset our indexing priority.  This
+     * is important because (all?) folders start out without any flags and
+     * then get their flags added to them.
+     */
+    onFolderIntPropertyChanged(aFolderItem, aProperty, aOldValue, aNewValue) {
+      if (aProperty !== "FolderFlag") {
+        return;
+      }
+      if (!GlodaMsgIndexer.shouldIndexFolder(aFolderItem)) {
+        return;
+      }
+      // Only reset priority if folder Special Use changes.
+      if (
+        (aOldValue & Ci.nsMsgFolderFlags.SpecialUse) ==
+        (aNewValue & Ci.nsMsgFolderFlags.SpecialUse)
+      ) {
+        return;
+      }
+      GlodaMsgIndexer.resetFolderIndexingPriority(aFolderItem);
+    },
+    onFolderBoolPropertyChanged(aItem, aProperty, aOldValue, aNewValue) {},
+    onFolderUnicharPropertyChanged(aItem, aProperty, aOldValue, aNewValue) {},
+    /**
+     * Notice when user activity adds/removes tags or changes a message's
+     *  status.
+     */
+    onFolderPropertyFlagChanged(aMsgHdr, aProperty, aOldValue, aNewValue) {
+      if (
+        aProperty == "Keywords" ||
+        // We could care less about the new flag changing.
+        (aProperty == "Status" &&
+          (aOldValue ^ aNewValue) != Ci.nsMsgMessageFlags.New &&
+          // We do care about IMAP deletion, but msgsDeleted tells us that, so
+          //  ignore IMAPDeleted too...
+          (aOldValue ^ aNewValue) != Ci.nsMsgMessageFlags.IMAPDeleted) ||
+        aProperty == "Flagged"
+      ) {
+        GlodaMsgIndexer._reindexChangedMessages([aMsgHdr], true);
+      }
+    },
+
+    /**
+     * Get folder loaded notifications for folders that had to do some
+     *  (asynchronous) processing before they could be opened.
+     */
+    onFolderEvent(aFolder, aEvent) {
+      if (aEvent == "FolderLoaded") {
+        this.indexer._onFolderLoaded(aFolder);
+      }
+    },
+  },
+
+  /* ***** Rebuilding / Reindexing ***** */
+  /**
+   * Allow us to invalidate an outstanding folder traversal because the
+   *  underlying database is going away.  We use other means for detecting
+   *  modifications of the message (labeling, marked (un)read, starred, etc.)
+   *
+   * This is an nsIDBChangeListener listening to an nsIDBChangeAnnouncer.  To
+   *  add ourselves, we get us a nice nsMsgDatabase, query it to the announcer,
+   *  then call addListener.
+   */
+  _databaseAnnouncerListener: {
+    indexer: null,
+    /**
+     * XXX We really should define the operations under which we expect this to
+     *  occur.  While we know this must be happening as the result of a
+     *  ForceClosed call, we don't have a comprehensive list of when this is
+     *  expected to occur.  Some reasons:
+     * - Compaction (although we should already have killed the job thanks to
+     *    our compaction notification)
+     * - UID validity rolls.
+     * - Folder Rename
+     * - Folder Delete
+     * The fact that we already have the database open when getting this means
+     *  that it had to be valid before we opened it, which hopefully rules out
+     *  modification of the mbox file by an external process (since that is
+     *  forbidden when we are running) and many other exotic things.
+     *
+     * So this really ends up just being a correctness / safety protection
+     *  mechanism.  At least now that we have better compaction support.
+     */
+    onAnnouncerGoingAway(aDBChangeAnnouncer) {
+      // The fact that we are getting called means we have an active folder and
+      //  that we therefore are the active job.  As such, we must kill the
+      //  active job.
+      // XXX In the future, when we support interleaved event-driven indexing
+      //  that bumps long-running indexing tasks, the semantics of this will
+      //  have to change a bit since we will want to maintain being active in a
+      //  folder even when bumped.  However, we will probably have a more
+      //  complex notion of indexing contexts on a per-job basis.
+      GlodaIndexer.killActiveJob();
+    },
+
+    onHdrFlagsChanged(aHdrChanged, aOldFlags, aNewFlags, aInstigator) {},
+    onHdrDeleted(aHdrChanged, aParentKey, aFlags, aInstigator) {},
+    onHdrAdded(aHdrChanged, aParentKey, aFlags, aInstigator) {},
+    onParentChanged(aKeyChanged, aOldParent, aNewParent, aInstigator) {},
+    onReadChanged(aInstigator) {},
+    onJunkScoreChanged(aInstigator) {},
+    onHdrPropertyChanged(aHdrToChange, aPreChange, aStatus, aInstigator) {},
+    onEvent(aDB, aEvent) {},
+  },
+
+  /**
+   * Given a list of Message-ID's, return a matching list of lists of messages
+   *  matching those Message-ID's.  So if you pass an array with three
+   *  Message-ID's ["a", "b", "c"], you would get back an array containing
+   *  3 lists, where the first list contains all the messages with a message-id
+   *  of "a", and so forth.  The reason a list is returned rather than null/a
+   *  message is that we accept the reality that we have multiple copies of
+   *  messages with the same ID.
+   * This call is asynchronous because it depends on previously created messages
+   *  to be reflected in our results, which requires us to execute on the async
+   *  thread where all our writes happen.  This also turns out to be a
+   *  reasonable thing because we could imagine pathological cases where there
+   *  could be a lot of message-id's and/or a lot of messages with those
+   *  message-id's.
+   *
+   * The returned collection will include both 'ghost' messages (messages
+   *  that exist for conversation-threading purposes only) as well as deleted
+   *  messages in addition to the normal 'live' messages that non-privileged
+   *  queries might return.
+   */
+  getMessagesByMessageID(aMessageIDs, aCallback, aCallbackThis) {
+    let msgIDToIndex = {};
+    let results = [];
+    for (let iID = 0; iID < aMessageIDs.length; ++iID) {
+      let msgID = aMessageIDs[iID];
+      results.push([]);
+      msgIDToIndex[msgID] = iID;
+    }
+
+    // (Note: although we are performing a lookup with no validity constraints
+    //  and using the same object-relational-mapper-ish layer used by things
+    //  that do have constraints, we are not at risk of exposing deleted
+    //  messages to other code and getting it confused.  The only way code
+    //  can find a message is if it shows up in their queries or gets announced
+    //  via GlodaCollectionManager.itemsAdded, neither of which will happen.)
+    let query = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE, {
+      noDbQueryValidityConstraints: true,
+    });
+    query.headerMessageID.apply(query, aMessageIDs);
+    query.frozen = true;
+
+    let listener = new MessagesByMessageIdCallback(
+      msgIDToIndex,
+      results,
+      aCallback,
+      aCallbackThis
+    );
+    return query.getCollection(listener, null, { becomeNull: true });
+  },
+
+  /**
+   * A reference to MsgHdrToMimeMessage that unit testing can clobber when it
+   *  wants to cause us to hang or inject a fault.  If you are not
+   *  glodaTestHelper.js then _do not touch this_.
+   */
+  _MsgHdrToMimeMessageFunc: MsgHdrToMimeMessage,
+  /**
+   * Primary message indexing logic.  This method is mainly concerned with
+   *  getting all the information about the message required for threading /
+   *  conversation building and subsequent processing.  It is responsible for
+   *  determining whether to reuse existing gloda messages or whether a new one
+   *  should be created.  Most attribute stuff happens in fund_attr.js or
+   *  expl_attr.js.
+   *
+   * Prior to calling this method, the caller must have invoked
+   *  |_indexerEnterFolder|, leaving us with the following true invariants
+   *  below.
+   *
+   * @pre aMsgHdr.folder == this._indexingFolder
+   * @pre aMsgHdr.folder.msgDatabase == this._indexingDatabase
+   */
+  *_indexMessage(aMsgHdr, aCallbackHandle) {
+    this._log.debug(
+      "*** Indexing message: " + aMsgHdr.messageKey + " : " + aMsgHdr.subject
+    );
+
+    // If the message is offline, then get the message body as well
+    let aMimeMsg;
+    if (
+      aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline ||
+      aMsgHdr.folder instanceof Ci.nsIMsgLocalMailFolder
+    ) {
+      this._MsgHdrToMimeMessageFunc(
+        aMsgHdr,
+        aCallbackHandle.callbackThis,
+        aCallbackHandle.callback,
+        false,
+        {
+          saneBodySize: true,
+        }
+      );
+      aMimeMsg = (yield GlodaConstants.kWorkAsync)[1];
+    } else {
+      this._log.debug("  * Message is not offline -- only headers indexed");
+    }
+
+    this._log.debug("  * Got message, subject " + aMsgHdr.subject);
+
+    if (this._unitTestSuperVerbose) {
+      if (aMimeMsg) {
+        this._log.debug("  * Got Mime " + aMimeMsg.prettyString());
+      } else {
+        this._log.debug("  * NO MIME MESSAGE!!!\n");
+      }
+    }
+
+    // -- Find/create the conversation the message belongs to.
+    // Our invariant is that all messages that exist in the database belong to
+    //  a conversation.
+
+    // - See if any of the ancestors exist and have a conversationID...
+    // (references are ordered from old [0] to new [n-1])
+    let references = Array.from(range(0, aMsgHdr.numReferences)).map(i =>
+      aMsgHdr.getStringReference(i)
+    );
+    // also see if we already know about the message...
+    references.push(aMsgHdr.messageId);
+
+    this.getMessagesByMessageID(
+      references,
+      aCallbackHandle.callback,
+      aCallbackHandle.callbackThis
+    );
+    // (ancestorLists has a direct correspondence to the message ids)
+    let ancestorLists = yield GlodaConstants.kWorkAsync;
+
+    this._log.debug("ancestors raw: " + ancestorLists);
+    this._log.debug(
+      "ref len: " + references.length + " anc len: " + ancestorLists.length
+    );
+    this._log.debug("references: " + references);
+    this._log.debug("ancestors: " + ancestorLists);
+
+    // pull our current message lookup results off
+    references.pop();
+    let candidateCurMsgs = ancestorLists.pop();
+
+    let conversationID = null;
+    let conversation = null;
+    // -- figure out the conversation ID
+    // if we have a clone/already exist, just use his conversation ID
+    if (candidateCurMsgs.length > 0) {
+      conversationID = candidateCurMsgs[0].conversationID;
+      conversation = candidateCurMsgs[0].conversation;
+    } else {
+      // otherwise check out our ancestors
+      // (walk from closest to furthest ancestor)
+      for (
+        let iAncestor = ancestorLists.length - 1;
+        iAncestor >= 0;
+        --iAncestor
+      ) {
+        let ancestorList = ancestorLists[iAncestor];
+
+        if (ancestorList.length > 0) {
+          // we only care about the first instance of the message because we are
+          //  able to guarantee the invariant that all messages with the same
+          //  message id belong to the same conversation.
+          let ancestor = ancestorList[0];
+          if (conversationID === null) {
+            conversationID = ancestor.conversationID;
+            conversation = ancestor.conversation;
+          } else if (conversationID != ancestor.conversationID) {
+            // XXX this inconsistency is known and understood and tracked by
+            //  bug 478162 https://bugzilla.mozilla.org/show_bug.cgi?id=478162
+            // this._log.error("Inconsistency in conversations invariant on " +
+            //                ancestor.headerMessageID + ".  It has conv id " +
+            //                ancestor.conversationID + " but expected " +
+            //                conversationID + ". ID: " + ancestor.id);
+          }
+        }
+      }
+    }
+
+    // nobody had one?  create a new conversation
+    if (conversationID === null) {
+      // (the create method could issue the id, making the call return
+      //  without waiting for the database...)
+      conversation = this._datastore.createConversation(
+        aMsgHdr.mime2DecodedSubject,
+        null,
+        null
+      );
+      conversationID = conversation.id;
+    }
+
+    // Walk from furthest to closest ancestor, creating the ancestors that don't
+    //  exist. (This is possible if previous messages that were consumed in this
+    //  thread only had an in-reply-to or for some reason did not otherwise
+    //  provide the full references chain.)
+    for (let iAncestor = 0; iAncestor < ancestorLists.length; ++iAncestor) {
+      let ancestorList = ancestorLists[iAncestor];
+
+      if (ancestorList.length == 0) {
+        this._log.debug(
+          "creating message with: null, " +
+            conversationID +
+            ", " +
+            references[iAncestor] +
+            ", null."
+        );
+        let ancestor = this._datastore.createMessage(
+          null,
+          null, // ghost
+          conversationID,
+          null,
+          references[iAncestor],
+          null, // no subject
+          null, // no body
+          null
+        ); // no attachments
+        this._datastore.insertMessage(ancestor);
+        ancestorLists[iAncestor].push(ancestor);
+      }
+    }
+    // now all our ancestors exist, though they may be ghost-like...
+
+    // find if there's a ghost version of our message or we already have indexed
+    //  this message.
+    let curMsg = null;
+    this._log.debug(candidateCurMsgs.length + " candidate messages");
+    for (let iCurCand = 0; iCurCand < candidateCurMsgs.length; iCurCand++) {
+      let candMsg = candidateCurMsgs[iCurCand];
+
+      this._log.debug(
+        "candidate folderID: " +
+          candMsg.folderID +
+          " messageKey: " +
+          candMsg.messageKey
+      );
+
+      if (candMsg.folderURI == this._indexingFolder.URI) {
+        // if we are in the same folder and we have the same message key, we
+        //  are definitely the same, stop looking.
+        if (candMsg.messageKey == aMsgHdr.messageKey) {
+          curMsg = candMsg;
+          break;
+        }
+        // if (we are in the same folder and) the candidate message has a null
+        //  message key, we treat it as our best option unless we find an exact
+        //  key match. (this would happen because the 'move' notification case
+        //  has to deal with not knowing the target message key.  this case
+        //  will hopefully be somewhat improved in the future to not go through
+        //  this path which mandates re-indexing of the message in its entirety)
+        if (candMsg.messageKey === null) {
+          curMsg = candMsg;
+        } else if (
+          curMsg === null &&
+          !this._indexingDatabase.containsKey(candMsg.messageKey)
+        ) {
+          // (We are in the same folder and) the candidate message's underlying
+          // message no longer exists/matches. Assume we are the same but
+          // were betrayed by a re-indexing or something, but we have to make
+          // sure a perfect match doesn't turn up.
+          curMsg = candMsg;
+        }
+      } else if (curMsg === null && candMsg.folderID === null) {
+        // a ghost/deleted message is fine
+        curMsg = candMsg;
+      }
+    }
+
+    let attachmentNames = aMimeMsg?.allAttachments.map(att => att.name) || null;
+
+    let isConceptuallyNew, isRecordNew, insertFulltext;
+    if (curMsg === null) {
+      curMsg = this._datastore.createMessage(
+        aMsgHdr.folder,
+        aMsgHdr.messageKey,
+        conversationID,
+        aMsgHdr.date,
+        aMsgHdr.messageId
+      );
+      curMsg._conversation = conversation;
+      isConceptuallyNew = isRecordNew = insertFulltext = true;
+    } else {
+      isRecordNew = false;
+      // the message is conceptually new if it was a ghost or dead.
+      isConceptuallyNew = curMsg._isGhost || curMsg._isDeleted;
+      // insert fulltext if it was a ghost
+      insertFulltext = curMsg._isGhost;
+      curMsg._folderID = this._datastore._mapFolder(aMsgHdr.folder).id;
+      curMsg._messageKey = aMsgHdr.messageKey;
+      curMsg.date = new Date(aMsgHdr.date / 1000);
+      // the message may have been deleted; tell it to make sure it's not.
+      curMsg._ensureNotDeleted();
+      // note: we are assuming that our matching logic is flawless in that
+      //  if this message was not a ghost, we are assuming the 'body'
+      //  associated with the id is still exactly the same.  It is conceivable
+      //  that there are cases where this is not true.
+    }
+
+    if (aMimeMsg) {
+      let bodyPlain = aMimeMsg.coerceBodyToPlaintext(aMsgHdr.folder);
+      if (bodyPlain) {
+        curMsg._bodyLines = bodyPlain.split(/\r?\n/);
+        // curMsg._content gets set by GlodaFundAttr.jsm
+      }
+    }
+
+    // Mark the message as new (for the purposes of fulltext insertion)
+    if (insertFulltext) {
+      curMsg._isNew = true;
+    }
+
+    curMsg._subject = aMsgHdr.mime2DecodedSubject;
+    curMsg._attachmentNames = attachmentNames;
+
+    // curMsg._indexAuthor gets set by GlodaFundAttr.jsm
+    // curMsg._indexRecipients gets set by GlodaFundAttr.jsm
+
+    // zero the notability so everything in grokNounItem can just increment
+    curMsg.notability = 0;
+
+    yield aCallbackHandle.pushAndGo(
+      Gloda.grokNounItem(
+        curMsg,
+        { header: aMsgHdr, mime: aMimeMsg, bodyLines: curMsg._bodyLines },
+        isConceptuallyNew,
+        isRecordNew,
+        aCallbackHandle
+      )
+    );
+
+    delete curMsg._bodyLines;
+    delete curMsg._content;
+    delete curMsg._isNew;
+    delete curMsg._indexAuthor;
+    delete curMsg._indexRecipients;
+
+    // we want to update the header for messages only after the transaction
+    //  irrevocably hits the disk.  otherwise we could get confused if the
+    //  transaction rolls back or what not.
+    PendingCommitTracker.track(aMsgHdr, curMsg.id);
+
+    yield GlodaConstants.kWorkDone;
+  },
+
+  /**
+   * Wipe a message out of existence from our index.  This is slightly more
+   *  tricky than one would first expect because there are potentially
+   *  attributes not immediately associated with this message that reference
+   *  the message.  Not only that, but deletion of messages may leave a
+   *  conversation possessing only ghost messages, which we don't want, so we
+   *  need to nuke the moot conversation and its moot ghost messages.
+   * For now, we are actually punting on that trickiness, and the exact
+   *  nuances aren't defined yet because we have not decided whether to store
+   *  such attributes redundantly.  For example, if we have subject-pred-object,
+   *  we could actually store this as attributes (subject, id, object) and
+   *  (object, id, subject).  In such a case, we could query on (subject, *)
+   *  and use the results to delete the (object, id, subject) case.  If we
+   *  don't redundantly store attributes, we can deal with the problem by
+   *  collecting up all the attributes that accept a message as their object
+   *  type and issuing a delete against that.  For example, delete (*, [1,2,3],
+   *  message id).
+   * (We are punting because we haven't implemented support for generating
+   *  attributes like that yet.)
+   *
+   * @TODO: implement deletion of attributes that reference (deleted) messages
+   */
+  *_deleteMessage(aMessage, aCallbackHandle) {
+    this._log.debug("*** Deleting message: " + aMessage);
+
+    // -- delete our attributes
+    // delete the message's attributes (if we implement the cascade delete, that
+    //  could do the honors for us... right now we define the trigger in our
+    //  schema but the back-end ignores it)
+    GlodaDatastore.clearMessageAttributes(aMessage);
+
+    // -- delete our message or ghost us, and maybe nuke the whole conversation
+    // Look at the other messages in the conversation.
+    // (Note: although we are performing a lookup with no validity constraints
+    //  and using the same object-relational-mapper-ish layer used by things
+    //  that do have constraints, we are not at risk of exposing deleted
+    //  messages to other code and getting it confused.  The only way code
+    //  can find a message is if it shows up in their queries or gets announced
+    //  via GlodaCollectionManager.itemsAdded, neither of which will happen.)
+    let convPrivQuery = Gloda.newQuery(GlodaConstants.NOUN_MESSAGE, {
+      noDbQueryValidityConstraints: true,
+    });
+    convPrivQuery.conversation(aMessage.conversation);
+    let conversationCollection = convPrivQuery.getCollection(aCallbackHandle);
+    yield GlodaConstants.kWorkAsync;
+
+    let conversationMsgs = conversationCollection.items;
+
+    // Count the number of ghosts messages we see to determine if we are
+    //  the last message alive.
+    let ghostCount = 0;
+    let twinMessageExists = false;
+    for (let convMsg of conversationMsgs) {
+      // ignore our own message
+      if (convMsg.id == aMessage.id) {
+        continue;
+      }
+
+      if (convMsg._isGhost) {
+        ghostCount++;
+      } else if (
+        // This message is our (living) twin if it is not a ghost, not deleted,
+        // and has the same message-id header.
+        !convMsg._isDeleted &&
+        convMsg.headerMessageID == aMessage.headerMessageID
+      ) {
+        twinMessageExists = true;
+      }
+    }
+
+    // -- If everyone else is a ghost, blow away the conversation.
+    // If there are messages still alive or deleted but we have not yet gotten
+    //  to them yet _deleteMessage, then do not do this.  (We will eventually
+    //  hit this case if they are all deleted.)
+    if (conversationMsgs.length - 1 == ghostCount) {
+      // - Obliterate each message
+      for (let msg of conversationMsgs) {
+        GlodaDatastore.deleteMessageByID(msg.id);
+      }
+      // - Obliterate the conversation
+      GlodaDatastore.deleteConversationByID(aMessage.conversationID);
+      // *no one* should hold a reference or use aMessage after this point,
+      //  trash it so such ne'er do'wells are made plain.
+      aMessage._objectPurgedMakeYourselfUnpleasant();
+    } else if (twinMessageExists) {
+      // -- Ghost or purge us as appropriate
+      // Purge us if we have a (living) twin; no ghost required.
+      GlodaDatastore.deleteMessageByID(aMessage.id);
+      // *no one* should hold a reference or use aMessage after this point,
+      //  trash it so such ne'er do'wells are made plain.
+      aMessage._objectPurgedMakeYourselfUnpleasant();
+    } else {
+      // No twin, a ghost is required, we become the ghost.
+      aMessage._ghost();
+      GlodaDatastore.updateMessage(aMessage);
+      // ghosts don't have fulltext. purge it.
+      GlodaDatastore.deleteMessageTextByID(aMessage.id);
+    }
+
+    yield GlodaConstants.kWorkDone;
+  },
+};
+GlodaIndexer.registerIndexer(GlodaMsgIndexer);
diff --git a/comm/mailnews/db/gloda/modules/MimeMessage.jsm b/comm/mailnews/db/gloda/modules/MimeMessage.jsm
new file mode 100644
index 0000000000..8859f10877
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/MimeMessage.jsm
@@ -0,0 +1,821 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = [
+  "MsgHdrToMimeMessage",
+  "MimeMessage",
+  "MimeContainer",
+  "MimeBody",
+  "MimeUnknown",
+  "MimeMessageAttachment",
+];
+
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+
+/**
+ * The URL listener is surplus because the CallbackStreamListener ends up
+ *  getting the same set of events, effectively.
+ */
+var dumbUrlListener = {
+  OnStartRunningUrl(aUrl) {},
+  OnStopRunningUrl(aUrl, aExitCode) {},
+};
+
+/**
+ * Maintain a list of all active stream listeners so that we can cancel them all
+ *  during shutdown.  If we don't cancel them, we risk calls into javascript
+ *  from C++ after the various XPConnect contexts have already begun their
+ *  teardown process.
+ */
+var activeStreamListeners = {};
+
+var shutdownCleanupObserver = {
+  _initialized: false,
+  ensureInitialized() {
+    if (this._initialized) {
+      return;
+    }
+
+    Services.obs.addObserver(this, "quit-application");
+
+    this._initialized = true;
+  },
+
+  observe(aSubject, aTopic, aData) {
+    if (aTopic == "quit-application") {
+      Services.obs.removeObserver(this, "quit-application");
+
+      for (let uri in activeStreamListeners) {
+        let streamListener = activeStreamListeners[uri];
+        if (streamListener._request) {
+          streamListener._request.cancel(Cr.NS_BINDING_ABORTED);
+        }
+      }
+    }
+  },
+};
+
+function CallbackStreamListener(aMsgHdr, aCallbackThis, aCallback) {
+  this._msgHdr = aMsgHdr;
+  // Messages opened from file or attachments do not have a folder property, but
+  // have their url stored as a string property.
+  let hdrURI = aMsgHdr.folder
+    ? aMsgHdr.folder.getUriForMsg(aMsgHdr)
+    : aMsgHdr.getStringProperty("dummyMsgUrl");
+
+  this._request = null;
+  this._stream = null;
+  if (aCallback === undefined) {
+    this._callbacksThis = [null];
+    this._callbacks = [aCallbackThis];
+  } else {
+    this._callbacksThis = [aCallbackThis];
+    this._callbacks = [aCallback];
+  }
+  activeStreamListeners[hdrURI] = this;
+}
+
+/**
+ * @implements {nsIRequestObserver}
+ * @implements {nsIStreamListener}
+ */
+CallbackStreamListener.prototype = {
+  QueryInterface: ChromeUtils.generateQI(["nsIStreamListener"]),
+
+  // nsIRequestObserver part
+  onStartRequest(aRequest) {
+    this._request = aRequest;
+  },
+  onStopRequest(aRequest, aStatusCode) {
+    // Messages opened from file or attachments do not have a folder property,
+    // but have their url stored as a string property.
+    let msgURI = this._msgHdr.folder
+      ? this._msgHdr.folder.getUriForMsg(this._msgHdr)
+      : this._msgHdr.getStringProperty("dummyMsgUrl");
+    delete activeStreamListeners[msgURI];
+
+    aRequest.QueryInterface(Ci.nsIChannel);
+    let message = MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aRequest.URI.spec];
+    if (message === undefined) {
+      message = null;
+    }
+
+    delete MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aRequest.URI.spec];
+
+    for (let i = 0; i < this._callbacksThis.length; i++) {
+      try {
+        this._callbacks[i].call(this._callbacksThis[i], this._msgHdr, message);
+      } catch (e) {
+        // Most of the time, exceptions will silently disappear into the endless
+        // deeps of XPConnect, and never reach the surface ever again. At least
+        // warn the user if he has dump enabled.
+        dump(
+          "The MsgHdrToMimeMessage callback threw an exception: " + e + "\n"
+        );
+        // That one will probably never make it to the original caller.
+        throw e;
+      }
+    }
+
+    this._msgHdr = null;
+    this._request = null;
+    this._stream = null;
+    this._callbacksThis = null;
+    this._callbacks = null;
+  },
+
+  // nsIStreamListener part
+
+  /**
+   * Our onDataAvailable should actually never be called. The stream converter
+   * is actually eating everything except the start and stop notification.
+   */
+  onDataAvailable(aRequest, aInputStream, aOffset, aCount) {
+    throw new Error(
+      `The stream converter should have grabbed the data for ${aRequest?.URI.spec}`
+    );
+  },
+};
+
+function stripEncryptedParts(aPart) {
+  if (aPart.parts && aPart.isEncrypted) {
+    aPart.parts = []; // Show an empty container.
+  } else if (aPart.parts) {
+    aPart.parts = aPart.parts.map(stripEncryptedParts);
+  }
+  return aPart;
+}
+
+/**
+ * Starts retrieval of a MimeMessage instance for the given message header.
+ *  Your callback will be called with the message header you provide and the
+ *
+ * @param aMsgHdr The message header to retrieve the body for and build a MIME
+ *     representation of the message.
+ * @param aCallbackThis The (optional) 'this' to use for your callback function.
+ * @param aCallback The callback function to invoke on completion of message
+ *     parsing or failure.  The first argument passed will be the nsIMsgDBHdr
+ *     you passed to this function.  The second argument will be the MimeMessage
+ *     instance resulting from the processing on success, and null on failure.
+ * @param [aAllowDownload=false] Should we allow the message to be downloaded
+ *     for this streaming request?  The default is false, which means that we
+ *     require that the message be available offline.  If false is passed and
+ *     the message is not available offline, we will propagate an exception
+ *     thrown by the underlying code.
+ * @param [aOptions] Optional options.
+ * @param [aOptions.saneBodySize] Limit body sizes to a 'reasonable' size in
+ *     order to combat corrupt offline/message stores creating pathological
+ *     situations where we have erroneously multi-megabyte messages.  This
+ *     also likely reduces the impact of legitimately ridiculously large
+ *     messages.
+ * @param [aOptions.examineEncryptedParts] By default, we won't reveal the
+ *     contents of multipart/encrypted parts to the consumers, unless explicitly
+ *     requested. In the case of MIME/PGP messages, for instance, the message
+ *     will appear as an empty multipart/encrypted container, unless this option
+ *     is used.
+ */
+function MsgHdrToMimeMessage(
+  aMsgHdr,
+  aCallbackThis,
+  aCallback,
+  aAllowDownload,
+  aOptions
+) {
+  shutdownCleanupObserver.ensureInitialized();
+
+  let requireOffline = !aAllowDownload;
+  // Messages opened from file or attachments do not have a folder property, but
+  // have their url stored as a string property.
+  let msgURI = aMsgHdr.folder
+    ? aMsgHdr.folder.getUriForMsg(aMsgHdr)
+    : aMsgHdr.getStringProperty("dummyMsgUrl");
+
+  let msgService = MailServices.messageServiceFromURI(msgURI);
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = aOptions;
+  // By default, Enigmail only decrypts a message streamed via libmime if it's
+  // the one currently on display in the message reader. With this option, we're
+  // letting Enigmail know that it should decrypt the message since the client
+  // explicitly asked for it.
+  let encryptedStr =
+    aOptions && aOptions.examineEncryptedParts
+      ? "&examineEncryptedParts=true"
+      : "";
+
+  // S/MIME, our other encryption backend, is not that smart, and always
+  // decrypts data. In order to protect sensitive data (e.g. not index it in
+  // Gloda), unless the client asked for encrypted data, we pass to the client
+  // callback a stripped-down version of the MIME structure where encrypted
+  // parts have been removed.
+  let wrapCallback = function (aCallback, aCallbackThis) {
+    if (aOptions && aOptions.examineEncryptedParts) {
+      return aCallback;
+    }
+    return (aMsgHdr, aMimeMsg) =>
+      aCallback.call(aCallbackThis, aMsgHdr, stripEncryptedParts(aMimeMsg));
+  };
+
+  // Apparently there used to be an old syntax where the callback was the second
+  // argument...
+  let callback = aCallback ? aCallback : aCallbackThis;
+  let callbackThis = aCallback ? aCallbackThis : null;
+
+  // if we're already streaming this msg, just add the callback
+  // to the listener.
+  let listenerForURI = activeStreamListeners[msgURI];
+  if (listenerForURI != undefined) {
+    listenerForURI._callbacks.push(wrapCallback(callback, callbackThis));
+    listenerForURI._callbacksThis.push(callbackThis);
+    return;
+  }
+  let streamListener = new CallbackStreamListener(
+    aMsgHdr,
+    callbackThis,
+    wrapCallback(callback, callbackThis)
+  );
+
+  try {
+    msgService.streamMessage(
+      msgURI,
+      streamListener, // consumer
+      null, // nsIMsgWindow
+      dumbUrlListener, // nsIUrlListener
+      true, // have them create the converter
+      // additional uri payload, note that "header=" is prepended automatically
+      "filter&emitter=js" + encryptedStr,
+      requireOffline
+    );
+  } catch (ex) {
+    // If streamMessage throws an exception, we should make sure to clear the
+    // activeStreamListener, or any subsequent attempt at sreaming this URI
+    // will silently fail
+    if (activeStreamListeners[msgURI]) {
+      delete activeStreamListeners[msgURI];
+    }
+    MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+    throw ex;
+  }
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+}
+
+/**
+ * Let the jsmimeemitter provide us with results.  The poor emitter (if I am
+ *  understanding things correctly) is evaluated outside of the C.u.import
+ *  world, so if we were to import him, we would not see him, but rather a new
+ *  copy of him.  This goes for his globals, etc.  (and is why we live in this
+ *  file right here).  Also, it appears that the XPCOM JS wrappers aren't
+ *  magically unified so that we can try and pass data as expando properties
+ *  on things like the nsIUri instances either.  So we have the jsmimeemitter
+ *  import us and poke things into RESULT_RENDEVOUZ.  We put it here on this
+ *  function to try and be stealthy and avoid polluting the namespaces (or
+ *  encouraging bad behaviour) of our importers.
+ *
+ * If you can come up with a prettier way to shuttle this data, please do.
+ */
+MsgHdrToMimeMessage.RESULT_RENDEVOUZ = {};
+/**
+ * Cram rich options here for the MimeMessageEmitter to grab from.  We
+ *  leverage the known control-flow to avoid needing a whole dictionary here.
+ *  We set this immediately before constructing the emitter and clear it
+ *  afterwards.  Control flow is never yielded during the process and reentrancy
+ *  cannot happen via any other means.
+ */
+MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+
+var HeaderHandlerBase = {
+  /**
+   * Look-up a header that should be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @param aDefaultValue The value to return if the header was not found, null
+   *     if left unspecified.
+   * @returns the value of the header if present, and the default value if not
+   *  (defaults to null).  If the header was present multiple times, the first
+   *  instance of the header is returned.  Use getAll if you want all of the
+   *  values for the multiply-defined header.
+   */
+  get(aHeaderName, aDefaultValue) {
+    if (aDefaultValue === undefined) {
+      aDefaultValue = null;
+    }
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers) {
+      // we require that the list cannot be empty if present
+      return this.headers[lowerHeader][0];
+    }
+    return aDefaultValue;
+  },
+  /**
+   * Look-up a header that can be present multiple times.  Use get for headers
+   *  that you only expect to be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @returns An array containing the values observed, which may mean a zero
+   *     length array.
+   */
+  getAll(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers) {
+      return this.headers[lowerHeader];
+    }
+    return [];
+  },
+  /**
+   * @param aHeaderName Header name to test for its presence.
+   * @returns true if the message has (at least one value for) the given header
+   *     name.
+   */
+  has(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    return lowerHeader in this.headers;
+  },
+  _prettyHeaderString(aIndent) {
+    if (aIndent === undefined) {
+      aIndent = "";
+    }
+    let s = "";
+    for (let header in this.headers) {
+      let values = this.headers[header];
+      s += "\n        " + aIndent + header + ": " + values;
+    }
+    return s;
+  },
+};
+
+/**
+ * @ivar partName The MIME part, ex "1.2.2.1".  The partName of a (top-level)
+ *     message is "1", its first child is "1.1", its second child is "1.2",
+ *     its first child's first child is "1.1.1", etc.
+ * @ivar headers Maps lower-cased header field names to a list of the values
+ *     seen for the given header.  Use get or getAll as convenience helpers.
+ * @ivar parts The list of the MIME part children of this message.  Children
+ *     will be either MimeMessage instances, MimeMessageAttachment instances,
+ *     MimeContainer instances, or MimeUnknown instances.  The latter two are
+ *     the result of limitations in the Javascript representation generation
+ *     at this time, combined with the need to most accurately represent the
+ *     MIME structure.
+ */
+function MimeMessage() {
+  this.partName = null;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeMessage.prototype = {
+  __proto__: HeaderHandlerBase,
+  contentType: "message/rfc822",
+
+  /**
+   * @returns a list of all attachments contained in this message and all its
+   *     sub-messages.  Only MimeMessageAttachment instances will be present in
+   *     the list (no sub-messages).
+   */
+  get allAttachments() {
+    let results = []; // messages are not attachments, don't include self
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+
+  /**
+   * @returns a list of all attachments contained in this message and all its
+   *     sub-messages, including the sub-messages.
+   */
+  get allInlineAttachments() {
+    // Do not include the top message, but only sub-messages.
+    let results = this.partName ? [this] : [];
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allInlineAttachments);
+    }
+    return results;
+  },
+
+  /**
+   * @returns a list of all attachments contained in this message, with
+   *    included/forwarded messages treated as real attachments. Attachments
+   *    contained in inner messages won't be shown.
+   */
+  get allUserAttachments() {
+    if (this.url) {
+      // The jsmimeemitter camouflaged us as a MimeAttachment
+      return [this];
+    }
+    return this.parts
+      .map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+
+  /**
+   * @returns the total size of this message, that is, the size of all subparts
+   */
+  get size() {
+    return this.parts
+      .map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+
+  /**
+   * In the case of attached messages, libmime considers them as attachments,
+   * and if the body is, say, quoted-printable encoded, then libmime will start
+   * counting bytes and notify the js mime emitter about it. The JS mime emitter
+   * being a nice guy, it will try to set a size on us. While this is the
+   * expected behavior for MimeMsgAttachments, we must make sure we can handle
+   * that (failing to write a setter results in exceptions being thrown).
+   */
+  set size(whatever) {
+    // nop
+  },
+
+  /**
+   * @param aMsgFolder A message folder, any message folder.  Because this is
+   *    a hack.
+   * @returns The concatenation of all of the body parts where parts
+   *    available as text/plain are pulled as-is, and parts only available
+   *    as text/html are converted to plaintext form first.  In other words,
+   *    if we see a multipart/alternative with a text/plain, we take the
+   *    text/plain.  If we see a text/html without an alternative, we convert
+   *    that to text.
+   */
+  coerceBodyToPlaintext(aMsgFolder) {
+    let bodies = [];
+    for (let part of this.parts) {
+      // an undefined value for something not having the method is fine
+      let body =
+        part.coerceBodyToPlaintext && part.coerceBodyToPlaintext(aMsgFolder);
+      if (body) {
+        bodies.push(body);
+      }
+    }
+    if (bodies) {
+      return bodies.join("");
+    }
+    return "";
+  },
+
+  /**
+   * Convert the message and its hierarchy into a "pretty string".  The message
+   *  and each MIME part get their own line.  The string never ends with a
+   *  newline.  For a non-multi-part message, only a single line will be
+   *  returned.
+   * Messages have their subject displayed, attachments have their filename and
+   *  content-type (ex: image/jpeg) displayed.  "Filler" classes simply have
+   *  their class displayed.
+   */
+  prettyString(aVerbose, aIndent, aDumpBody) {
+    if (aIndent === undefined) {
+      aIndent = "";
+    }
+    let nextIndent = aIndent + "  ";
+
+    let s =
+      "Message " +
+        (this.isEncrypted ? "[encrypted] " : "") +
+        "(" +
+        this.size +
+        " bytes): " +
+        "subject" in
+      this.headers
+        ? this.headers.subject
+        : "";
+    if (aVerbose) {
+      s += this._prettyHeaderString(nextIndent);
+    }
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s +=
+        "\n" +
+        nextIndent +
+        (iPart + 1) +
+        " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+};
+
+/**
+ * @ivar contentType The content-type of this container.
+ * @ivar parts The parts held by this container.  These can be instances of any
+ *      of the classes found in this file.
+ */
+function MimeContainer(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeContainer.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    let results = [];
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+  get allInlineAttachments() {
+    let results = [];
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allInlineAttachments);
+    }
+    return results;
+  },
+  get allUserAttachments() {
+    return this.parts
+      .map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size() {
+    return this.parts
+      .map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+  set size(whatever) {
+    // nop
+  },
+  coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "multipart/alternative") {
+      let htmlPart;
+      // pick the text/plain if we can find one, otherwise remember the HTML one
+      for (let part of this.parts) {
+        if (part.contentType == "text/plain") {
+          return part.body;
+        }
+        if (part.contentType == "text/html") {
+          htmlPart = part;
+        } else if (!htmlPart && part.contentType == "text/enriched") {
+          // text/enriched gets transformed into HTML, so use it if we don't
+          // already have an HTML part.
+          htmlPart = part;
+        }
+      }
+      // convert the HTML part if we have one
+      if (htmlPart) {
+        return aMsgFolder.convertMsgSnippetToPlainText(htmlPart.body);
+      }
+    }
+    // if it's not alternative, recurse/aggregate using MimeMessage logic
+    return MimeMessage.prototype.coerceBodyToPlaintext.call(this, aMsgFolder);
+  },
+  prettyString(aVerbose, aIndent, aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s =
+      "Container " +
+      (this.isEncrypted ? "[encrypted] " : "") +
+      "(" +
+      this.size +
+      " bytes): " +
+      this.contentType;
+    if (aVerbose) {
+      s += this._prettyHeaderString(nextIndent);
+    }
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s +=
+        "\n" +
+        nextIndent +
+        (iPart + 1) +
+        " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+  toString() {
+    return "Container: " + this.contentType;
+  },
+};
+
+/**
+ * @class Represents a body portion that we understand and do not believe to be
+ *  a proper attachment.  This means text/plain or text/html and it has no
+ *  filename.  (A filename suggests an attachment.)
+ *
+ * @ivar contentType The content type of this body materal; text/plain or
+ *     text/html.
+ * @ivar body The actual body content.
+ */
+function MimeBody(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.body = "";
+  this.isEncrypted = false;
+}
+
+MimeBody.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return []; // we are a leaf
+  },
+  get allInlineAttachments() {
+    return []; // we are a leaf
+  },
+  get allUserAttachments() {
+    return []; // we are a leaf
+  },
+  get size() {
+    return this.body.length;
+  },
+  set size(whatever) {
+    // nop
+  },
+  appendBody(aBuf) {
+    this.body += aBuf;
+  },
+  coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "text/plain") {
+      return this.body;
+    }
+    // text/enriched gets transformed into HTML by libmime
+    if (
+      this.contentType == "text/html" ||
+      this.contentType == "text/enriched"
+    ) {
+      return aMsgFolder.convertMsgSnippetToPlainText(this.body);
+    }
+    return "";
+  },
+  prettyString(aVerbose, aIndent, aDumpBody) {
+    let s =
+      "Body: " +
+      (this.isEncrypted ? "[encrypted] " : "") +
+      "" +
+      this.contentType +
+      " (" +
+      this.body.length +
+      " bytes" +
+      (aDumpBody ? ": '" + this.body + "'" : "") +
+      ")";
+    if (aVerbose) {
+      s += this._prettyHeaderString(aIndent + "  ");
+    }
+    return s;
+  },
+  toString() {
+    return "Body: " + this.contentType + " (" + this.body.length + " bytes)";
+  },
+};
+
+/**
+ * @class A MIME Leaf node that doesn't have a filename so we assume it's not
+ *  intended to be an attachment proper.  This is probably meant for inline
+ *  display or is the result of someone amusing themselves by composing messages
+ *  by hand or a bad client.  This class should probably be renamed or we should
+ *  introduce a better named class that we try and use in preference to this
+ *  class.
+ *
+ * @ivar contentType The content type of this part.
+ */
+function MimeUnknown(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  // Looks like libmime does not always interpret us as an attachment, which
+  //  means we'll have to have a default size. Returning undefined would cause
+  //  the recursive size computations to fail.
+  this._size = 0;
+  this.isEncrypted = false;
+  // We want to make sure MimeUnknown has a part property: S/MIME encrypted
+  // messages have a topmost MimeUnknown part, with the encrypted bit set to 1,
+  // and we need to ensure all other encrypted parts are children of this
+  // topmost part.
+  this.parts = [];
+}
+
+MimeUnknown.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return this.parts
+      .map(child => child.allAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get allInlineAttachments() {
+    return this.parts
+      .map(child => child.allInlineAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get allUserAttachments() {
+    return this.parts
+      .map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size() {
+    return (
+      this._size +
+      this.parts
+        .map(child => child.size)
+        .reduce((a, b) => a + Math.max(b, 0), 0)
+    );
+  },
+  set size(aSize) {
+    this._size = aSize;
+  },
+  prettyString(aVerbose, aIndent, aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s =
+      "Unknown: " +
+      (this.isEncrypted ? "[encrypted] " : "") +
+      "" +
+      this.contentType +
+      " (" +
+      this.size +
+      " bytes)";
+    if (aVerbose) {
+      s += this._prettyHeaderString(aIndent + "  ");
+    }
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s +=
+        "\n" +
+        nextIndent +
+        (iPart + 1) +
+        " " +
+        (part ? part.prettyString(aVerbose, nextIndent, aDumpBody) : "NULL");
+    }
+    return s;
+  },
+  toString() {
+    return "Unknown: " + this.contentType;
+  },
+};
+
+/**
+ * @class An attachment proper.  We think it's an attachment because it has a
+ *  filename that libmime was able to figure out.
+ *
+ * @ivar partName @see{MimeMessage.partName}
+ * @ivar name The filename of this attachment.
+ * @ivar contentType The MIME content type of this part.
+ * @ivar url The URL to stream if you want the contents of this part.
+ * @ivar isExternal Is the attachment stored someplace else than in the message?
+ * @ivar size The size of the attachment if available, -1 otherwise (size is set
+ *  after initialization by jsmimeemitter.js)
+ */
+function MimeMessageAttachment(
+  aPartName,
+  aName,
+  aContentType,
+  aUrl,
+  aIsExternal
+) {
+  this.partName = aPartName;
+  this.name = aName;
+  this.contentType = aContentType;
+  this.url = aUrl;
+  this.isExternal = aIsExternal;
+  this.headers = {};
+  this.isEncrypted = false;
+  // parts is copied over from the part instance that preceded us
+  // headers is copied over from the part instance that preceded us
+  // isEncrypted is copied over from the part instance that preceded us
+}
+
+MimeMessageAttachment.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return [this]; // we are a leaf, so just us.
+  },
+  get allInlineAttachments() {
+    return [this]; // we are a leaf, so just us.
+  },
+  get allUserAttachments() {
+    return [this];
+  },
+  prettyString(aVerbose, aIndent, aDumpBody) {
+    let s =
+      "Attachment " +
+      (this.isEncrypted ? "[encrypted] " : "") +
+      "(" +
+      this.size +
+      " bytes): " +
+      this.name +
+      ", " +
+      this.contentType;
+    if (aVerbose) {
+      s += this._prettyHeaderString(aIndent + "  ");
+    }
+    return s;
+  },
+  toString() {
+    return this.prettyString(false, "");
+  },
+};
diff --git a/comm/mailnews/db/gloda/modules/NounFreetag.jsm b/comm/mailnews/db/gloda/modules/NounFreetag.jsm
new file mode 100644
index 0000000000..cb169645f1
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/NounFreetag.jsm
@@ -0,0 +1,91 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["FreeTag", "FreeTagNoun"];
+
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+
+function FreeTag(aTagName) {
+  this.name = aTagName;
+}
+
+FreeTag.prototype = {
+  toString() {
+    return this.name;
+  },
+};
+
+/**
+ * @namespace Tag noun provider.  Since the tag unique value is stored as a
+ *  parameter, we are an odd case and semantically confused.
+ */
+var FreeTagNoun = {
+  _log: console.createInstance({
+    prefix: "gloda.noun.freetag",
+    maxLogLevel: "Warn",
+    maxLogLevelPref: "gloda.loglevel",
+  }),
+
+  name: "freetag",
+  clazz: FreeTag,
+  allowsArbitraryAttrs: false,
+  usesParameter: true,
+
+  _listeners: [],
+  addListener(aListener) {
+    this._listeners.push(aListener);
+  },
+  removeListener(aListener) {
+    let index = this._listeners.indexOf(aListener);
+    if (index >= 0) {
+      this._listeners.splice(index, 1);
+    }
+  },
+
+  populateKnownFreeTags() {
+    for (let attr of this.objectNounOfAttributes) {
+      let attrDB = attr.dbDef;
+      for (let param in attrDB.parameterBindings) {
+        this.getFreeTag(param);
+      }
+    }
+  },
+
+  knownFreeTags: {},
+  getFreeTag(aTagName) {
+    let tag = this.knownFreeTags[aTagName];
+    if (!tag) {
+      tag = this.knownFreeTags[aTagName] = new FreeTag(aTagName);
+      for (let listener of this._listeners) {
+        listener.onFreeTagAdded(tag);
+      }
+    }
+    return tag;
+  },
+
+  comparator(a, b) {
+    if (a == null) {
+      if (b == null) {
+        return 0;
+      }
+      return 1;
+    } else if (b == null) {
+      return -1;
+    }
+    return a.name.localeCompare(b.name);
+  },
+
+  toParamAndValue(aTag) {
+    return [aTag.name, null];
+  },
+
+  toJSON(aTag) {
+    return aTag.name;
+  },
+  fromJSON(aTagName) {
+    return this.getFreeTag(aTagName);
+  },
+};
+
+Gloda.defineNoun(FreeTagNoun);
diff --git a/comm/mailnews/db/gloda/modules/NounMimetype.jsm b/comm/mailnews/db/gloda/modules/NounMimetype.jsm
new file mode 100644
index 0000000000..fef1a33bc7
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/NounMimetype.jsm
@@ -0,0 +1,582 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["MimeType", "MimeTypeNoun"];
+
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+var LOG = console.createInstance({
+  prefix: "gloda.noun.mimetype",
+  maxLogLevel: "Warn",
+  maxLogLevelPref: "gloda.loglevel",
+});
+
+var CategoryStringMap = {};
+
+/**
+ * Input data structure to allow us to build a fast mapping from mime type to
+ *  category name.  The keys in MimeCategoryMapping are the top-level
+ *  categories.  Each value can either be a list of MIME types or a nested
+ *  object which recursively defines sub-categories.  We currently do not use
+ *  the sub-categories.  They are just there to try and organize the MIME types
+ *  a little and open the door to future enhancements.
+ *
+ * Do _not_ add additional top-level categories unless you have added
+ *  corresponding entries to gloda.properties under the
+ *  "gloda.mimetype.category" branch and are making sure localizers are aware
+ *  of the change and have time to localize it.
+ *
+ * Entries with wildcards in them are part of a fallback strategy by the
+ *  |mimeTypeNoun| and do not actually use regular expressions or anything like
+ *  that.  Everything is a straight string lookup.  Given "foo/bar" we look for
+ *  "foo/bar", then "foo/*", and finally "*".
+ */
+var MimeCategoryMapping = {
+  archives: [
+    "application/java-archive",
+    "application/x-java-archive",
+    "application/x-jar",
+    "application/x-java-jnlp-file",
+
+    "application/mac-binhex40",
+    "application/vnd.ms-cab-compressed",
+
+    "application/x-arc",
+    "application/x-arj",
+    "application/x-compress",
+    "application/x-compressed-tar",
+    "application/x-cpio",
+    "application/x-cpio-compressed",
+    "application/x-deb",
+
+    "application/x-bittorrent",
+
+    "application/x-rar",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/zip",
+    "application/x-zip-compressed",
+    "application/x-zip",
+
+    "application/x-bzip",
+    "application/x-bzip-compressed-tar",
+    "application/x-bzip2",
+    "application/x-gzip",
+    "application/x-tar",
+    "application/x-tar-gz",
+    "application/x-tarz",
+  ],
+  documents: {
+    database: [
+      "application/vnd.ms-access",
+      "application/x-msaccess",
+      "application/msaccess",
+      "application/vnd.msaccess",
+      "application/x-msaccess",
+      "application/mdb",
+      "application/x-mdb",
+
+      "application/vnd.oasis.opendocument.database",
+    ],
+    graphics: [
+      "application/postscript",
+      "application/x-bzpostscript",
+      "application/x-dvi",
+      "application/x-gzdvi",
+
+      "application/illustrator",
+
+      "application/vnd.corel-draw",
+      "application/cdr",
+      "application/coreldraw",
+      "application/x-cdr",
+      "application/x-coreldraw",
+      "image/cdr",
+      "image/x-cdr",
+      "zz-application/zz-winassoc-cdr",
+
+      "application/vnd.oasis.opendocument.graphics",
+      "application/vnd.oasis.opendocument.graphics-template",
+      "application/vnd.oasis.opendocument.image",
+
+      "application/x-dia-diagram",
+    ],
+    presentation: [
+      "application/vnd.ms-powerpoint.presentation.macroenabled.12",
+      "application/vnd.ms-powerpoint.template.macroenabled.12",
+      "application/vnd.ms-powerpoint",
+      "application/powerpoint",
+      "application/mspowerpoint",
+      "application/x-mspowerpoint",
+      "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      "application/vnd.openxmlformats-officedocument.presentationml.template",
+
+      "application/vnd.oasis.opendocument.presentation",
+      "application/vnd.oasis.opendocument.presentation-template",
+    ],
+    spreadsheet: [
+      "application/vnd.lotus-1-2-3",
+      "application/x-lotus123",
+      "application/x-123",
+      "application/lotus123",
+      "application/wk1",
+
+      "application/x-quattropro",
+
+      "application/vnd.ms-excel.sheet.binary.macroenabled.12",
+      "application/vnd.ms-excel.sheet.macroenabled.12",
+      "application/vnd.ms-excel.template.macroenabled.12",
+      "application/vnd.ms-excel",
+      "application/msexcel",
+      "application/x-msexcel",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+
+      "application/vnd.oasis.opendocument.formula",
+      "application/vnd.oasis.opendocument.formula-template",
+      "application/vnd.oasis.opendocument.chart",
+      "application/vnd.oasis.opendocument.chart-template",
+      "application/vnd.oasis.opendocument.spreadsheet",
+      "application/vnd.oasis.opendocument.spreadsheet-template",
+
+      "application/x-gnumeric",
+    ],
+    wordProcessor: [
+      "application/msword",
+      "application/vnd.ms-word",
+      "application/x-msword",
+      "application/msword-template",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+      "application/vnd.ms-word.document.macroenabled.12",
+      "application/vnd.ms-word.template.macroenabled.12",
+      "application/x-mswrite",
+      "application/x-pocket-word",
+
+      "application/rtf",
+      "text/rtf",
+
+      "application/vnd.oasis.opendocument.text",
+      "application/vnd.oasis.opendocument.text-master",
+      "application/vnd.oasis.opendocument.text-template",
+      "application/vnd.oasis.opendocument.text-web",
+
+      "application/vnd.wordperfect",
+
+      "application/x-abiword",
+      "application/x-amipro",
+    ],
+    suite: ["application/vnd.ms-works"],
+  },
+  images: ["image/*"],
+  media: {
+    audio: ["audio/*"],
+    video: ["video/*"],
+    container: [
+      "application/ogg",
+
+      "application/smil",
+      "application/vnd.ms-asf",
+      "application/vnd.rn-realmedia",
+      "application/x-matroska",
+      "application/x-quicktime-media-link",
+      "application/x-quicktimeplayer",
+    ],
+  },
+  other: ["*"],
+  pdf: [
+    "application/pdf",
+    "application/x-pdf",
+    "image/pdf",
+    "file/pdf",
+    "application/x-bzpdf",
+    "application/x-gzpdf",
+  ],
+};
+
+/**
+ * Mime type abstraction that exists primarily so we can map mime types to
+ *  integer id's.
+ *
+ * Instances of this class should only be retrieved via |MimeTypeNoun|; no one
+ *  should ever create an instance directly.
+ */
+function MimeType(aID, aType, aSubType, aFullType, aCategory) {
+  this._id = aID;
+  this._type = aType;
+  this._subType = aSubType;
+  this._fullType = aFullType;
+  this._category = aCategory;
+}
+
+MimeType.prototype = {
+  /**
+   * The integer id we have associated with the mime type.  This is stable for
+   *  the lifetime of the database, which means that anything in the Gloda
+   *  database can use this without fear.  Things not persisted in the database
+   *  should use the actual string mime type, retrieval via |fullType|.
+   */
+  get id() {
+    return this._id;
+  },
+  /**
+   * The first part of the MIME type; "text/plain" gets you "text".
+   */
+  get type() {
+    return this._type;
+  },
+  set fullType(aFullType) {
+    if (!this._fullType) {
+      this._fullType = aFullType;
+      [this._type, this._subType] = this._fullType.split("/");
+      this._category = MimeTypeNoun._getCategoryForMimeType(
+        aFullType,
+        this._type
+      );
+    }
+  },
+  /**
+   * If the |fullType| is "text/plain", subType is "plain".
+   */
+  get subType() {
+    return this._subType;
+  },
+  /**
+   * The full MIME type; "text/plain" returns "text/plain".
+   */
+  get fullType() {
+    return this._fullType;
+  },
+  toString() {
+    return this.fullType;
+  },
+
+  /**
+   * @returns the category we believe this mime type belongs to.  This category
+   *     name should never be shown directly to the user.  Instead, use
+   *     |categoryLabel| to get the localized name for the category.  The
+   *     category mapping comes from mimeTypesCategories.js.
+   */
+  get category() {
+    return this._category;
+  },
+  /**
+   * @returns The localized label for the category from gloda.properties in the
+   *     "gloda.mimetype.category.CATEGORY.label" definition using the value
+   *     from |category|.
+   */
+  get categoryLabel() {
+    return CategoryStringMap[this._category];
+  },
+};
+
+/**
+ * Mime type noun provider.
+ *
+ * The set of MIME Types is sufficiently limited that we can keep them all in
+ *  memory.  In theory it is also sufficiently limited that we could use the
+ *  parameter mechanism in the database.  However, it is more efficient, for
+ *  both space and performance reasons, to store the specific mime type as a
+ *  value.  For future-proofing reasons, we opt to use a database table to
+ *  persist the mapping rather than a hard-coded list.  A preferences file or
+ *  other text file would arguably suffice, but for consistency reasons, the
+ *  database is not a bad thing.
+ */
+var MimeTypeNoun = {
+  name: "mime-type",
+  clazz: MimeType, // gloda supports clazz as well as class
+  allowsArbitraryAttrs: false,
+
+  _strings: Services.strings.createBundle(
+    "chrome://messenger/locale/gloda.properties"
+  ),
+
+  // note! update test_noun_mimetype if you change our internals!
+  _mimeTypes: {},
+  _mimeTypesByID: {},
+  TYPE_BLOCK_SIZE: 16384,
+  _mimeTypeHighID: {},
+  _mimeTypeRangeDummyObjects: {},
+  _highID: 0,
+
+  // we now use the exciting 'schema' mechanism of defineNoun to get our table
+  //  created for us, plus some helper methods that we simply don't use.
+  schema: {
+    name: "mimeTypes",
+    columns: [
+      ["id", "INTEGER PRIMARY KEY", "_id"],
+      ["mimeType", "TEXT", "fullType"],
+    ],
+  },
+
+  _init() {
+    LOG.debug("loading MIME types");
+    this._loadCategoryMapping();
+    this._loadMimeTypes();
+  },
+
+  /**
+   * A map from MIME type to category name.
+   */
+  _mimeTypeToCategory: {},
+  /**
+   * Load the contents of MimeTypeCategories and populate
+   */
+  _loadCategoryMapping() {
+    let mimeTypeToCategory = this._mimeTypeToCategory;
+
+    function procMapObj(aSubTree, aCategories) {
+      for (let key in aSubTree) {
+        let value = aSubTree[key];
+        // Add this category to our nested categories list.  Use concat since
+        //  the list will be long-lived and each list needs to be distinct.
+        let categories = aCategories.concat();
+        categories.push(key);
+
+        if (categories.length == 1) {
+          CategoryStringMap[key] = MimeTypeNoun._strings.GetStringFromName(
+            "gloda.mimetype.category." + key + ".label"
+          );
+        }
+
+        // Is it an array? If so, just process this depth
+        if (Array.isArray(value)) {
+          for (let mimeTypeStr of value) {
+            mimeTypeToCategory[mimeTypeStr] = categories;
+          }
+        } else {
+          // it's yet another sub-tree branch
+          procMapObj(value, categories);
+        }
+      }
+    }
+    procMapObj(MimeCategoryMapping, []);
+  },
+
+  /**
+   * Lookup the category associated with a MIME type given its full type and
+   *  type.  (So, "foo/bar" and "foo" for "foo/bar".)
+   */
+  _getCategoryForMimeType(aFullType, aType) {
+    if (aFullType in this._mimeTypeToCategory) {
+      return this._mimeTypeToCategory[aFullType][0];
+    }
+    let wildType = aType + "/*";
+    if (wildType in this._mimeTypeToCategory) {
+      return this._mimeTypeToCategory[wildType][0];
+    }
+    return this._mimeTypeToCategory["*"][0];
+  },
+
+  /**
+   * In order to allow the gloda query mechanism to avoid hitting the database,
+   *  we need to either define the noun type as cacheable and have a super-large
+   *  cache or simply have a collection with every MIME type in it that stays
+   *  alive forever.
+   * This is that collection.  It is initialized by |_loadMimeTypes|.  As new
+   *  MIME types are created, we add them to the collection.
+   */
+  _universalCollection: null,
+
+  /**
+   * Kick off a query of all the mime types in our database, leaving
+   *  |_processMimeTypes| to actually do the legwork.
+   */
+  _loadMimeTypes() {
+    // get all the existing mime types!
+    let query = Gloda.newQuery(this.id);
+    let nullFunc = function () {};
+    this._universalCollection = query.getCollection(
+      {
+        onItemsAdded: nullFunc,
+        onItemsModified: nullFunc,
+        onItemsRemoved: nullFunc,
+        onQueryCompleted(aCollection) {
+          MimeTypeNoun._processMimeTypes(aCollection.items);
+        },
+      },
+      null
+    );
+  },
+
+  /**
+   * For the benefit of our Category queryHelper, we need dummy ranged objects
+   *  that cover the numerical address space allocated to the category.  We
+   *  can't use a real object for the upper-bound because the upper-bound is
+   *  constantly growing and there is the chance the query might get persisted,
+   *  which means these values need to be long-lived.  Unfortunately, our
+   *  solution to this problem (dummy objects) complicates the second case,
+   *  should it ever occur.  (Because the dummy objects cannot be persisted
+   *  on their own... but there are other issues that will come up that we will
+   *  just have to deal with then.)
+   */
+  _createCategoryDummies(aId, aCategory) {
+    let blockBottom = aId - (aId % this.TYPE_BLOCK_SIZE);
+    let blockTop = blockBottom + this.TYPE_BLOCK_SIZE - 1;
+    this._mimeTypeRangeDummyObjects[aCategory] = [
+      new MimeType(
+        blockBottom,
+        "!category-dummy!",
+        aCategory,
+        "!category-dummy!/" + aCategory,
+        aCategory
+      ),
+      new MimeType(
+        blockTop,
+        "!category-dummy!",
+        aCategory,
+        "!category-dummy!/" + aCategory,
+        aCategory
+      ),
+    ];
+  },
+
+  _processMimeTypes(aMimeTypes) {
+    for (let mimeType of aMimeTypes) {
+      if (mimeType.id > this._highID) {
+        this._highID = mimeType.id;
+      }
+      this._mimeTypes[mimeType] = mimeType;
+      this._mimeTypesByID[mimeType.id] = mimeType;
+
+      let blockHighID =
+        mimeType.category in this._mimeTypeHighID
+          ? this._mimeTypeHighID[mimeType.category]
+          : undefined;
+      // create the dummy range objects
+      if (blockHighID === undefined) {
+        this._createCategoryDummies(mimeType.id, mimeType.category);
+      }
+      if (blockHighID === undefined || mimeType.id > blockHighID) {
+        this._mimeTypeHighID[mimeType.category] = mimeType.id;
+      }
+    }
+  },
+
+  _addNewMimeType(aMimeTypeName) {
+    let [typeName, subTypeName] = aMimeTypeName.split("/");
+    let category = this._getCategoryForMimeType(aMimeTypeName, typeName);
+
+    if (!(category in this._mimeTypeHighID)) {
+      let nextID =
+        this._highID -
+        (this._highID % this.TYPE_BLOCK_SIZE) +
+        this.TYPE_BLOCK_SIZE;
+      this._mimeTypeHighID[category] = nextID;
+      this._createCategoryDummies(nextID, category);
+    }
+
+    let nextID = ++this._mimeTypeHighID[category];
+
+    let mimeType = new MimeType(
+      nextID,
+      typeName,
+      subTypeName,
+      aMimeTypeName,
+      category
+    );
+    if (mimeType.id > this._highID) {
+      this._highID = mimeType.id;
+    }
+
+    this._mimeTypes[aMimeTypeName] = mimeType;
+    this._mimeTypesByID[nextID] = mimeType;
+
+    // As great as the gloda extension mechanisms are, we don't think it makes
+    //  a lot of sense to use them in this case.  So we directly trigger object
+    //  insertion without any of the grokNounItem stuff.
+    this.objInsert.call(this.datastore, mimeType);
+    // Since we bypass grokNounItem and its fun, we need to explicitly add the
+    //  new MIME-type to _universalCollection ourselves.  Don't try this at
+    //  home, kids.
+    this._universalCollection._onItemsAdded([mimeType]);
+
+    return mimeType;
+  },
+
+  /**
+   * Map a mime type to a |MimeType| instance, creating it if necessary.
+   *
+   * @param aMimeTypeName The mime type.  It may optionally include parameters
+   *     (which will be ignored).  A mime type is of the form "type/subtype".
+   *     A type with parameters would look like 'type/subtype; param="value"'.
+   */
+  getMimeType(aMimeTypeName) {
+    // first, lose any parameters
+    let semiIndex = aMimeTypeName.indexOf(";");
+    if (semiIndex >= 0) {
+      aMimeTypeName = aMimeTypeName.substring(0, semiIndex);
+    }
+    aMimeTypeName = aMimeTypeName.trim().toLowerCase();
+
+    if (aMimeTypeName in this._mimeTypes) {
+      return this._mimeTypes[aMimeTypeName];
+    }
+    return this._addNewMimeType(aMimeTypeName);
+  },
+
+  /**
+   * Query helpers contribute additional functions to the query object for the
+   *  attributes that use the noun type.  For example, we define Category, so
+   *  for the "attachmentTypes" attribute, "attachmentTypesCategory" would be
+   *  exposed.
+   */
+  queryHelpers: {
+    /**
+     * Query for MIME type categories based on one or more MIME type objects
+     *  passed in.  We want the range to span the entire block allocated to the
+     *  category.
+     *
+     * @param aAttrDef The attribute that is using us.
+     * @param aArguments The actual arguments object that
+     */
+    Category(aAttrDef, aArguments) {
+      let rangePairs = [];
+      // If there are no arguments then we want to fall back to the 'in'
+      //  constraint which matches on any attachment.
+      if (!aArguments || aArguments.length == 0) {
+        return this._inConstraintHelper(aAttrDef, []);
+      }
+
+      for (let iArg = 0; iArg < aArguments.length; iArg++) {
+        let arg = aArguments[iArg];
+        rangePairs.push(MimeTypeNoun._mimeTypeRangeDummyObjects[arg.category]);
+      }
+      return this._rangedConstraintHelper(aAttrDef, rangePairs);
+    },
+  },
+
+  comparator(a, b) {
+    if (a == null) {
+      if (b == null) {
+        return 0;
+      }
+      return 1;
+    } else if (b == null) {
+      return -1;
+    }
+    return a.fullType.localeCompare(b.fullType);
+  },
+
+  toParamAndValue(aMimeType) {
+    return [null, aMimeType.id];
+  },
+  toJSON(aMimeType) {
+    return aMimeType.id;
+  },
+  fromJSON(aMimeTypeID) {
+    return this._mimeTypesByID[aMimeTypeID];
+  },
+};
+Gloda.defineNoun(MimeTypeNoun, GlodaConstants.NOUN_MIME_TYPE);
+try {
+  MimeTypeNoun._init();
+} catch (ex) {
+  LOG.error(
+    "problem init-ing: " + ex.fileName + ":" + ex.lineNumber + ": " + ex
+  );
+}
diff --git a/comm/mailnews/db/gloda/modules/NounTag.jsm b/comm/mailnews/db/gloda/modules/NounTag.jsm
new file mode 100644
index 0000000000..1e5db85a42
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/NounTag.jsm
@@ -0,0 +1,97 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["TagNoun"];
+
+const { MailServices } = ChromeUtils.import(
+  "resource:///modules/MailServices.jsm"
+);
+
+const { Gloda } = ChromeUtils.import("resource:///modules/gloda/Gloda.jsm");
+const { GlodaConstants } = ChromeUtils.import(
+  "resource:///modules/gloda/GlodaConstants.jsm"
+);
+
+/**
+ * @namespace Tag noun provider.
+ */
+var TagNoun = {
+  name: "tag",
+  clazz: Ci.nsIMsgTag,
+  usesParameter: true,
+  allowsArbitraryAttrs: false,
+  idAttr: "key",
+  _msgTagService: null,
+  _tagMap: null,
+  _tagList: null,
+
+  _init() {
+    // This reference can be substituted for testing purposes.
+    this._msgTagService = MailServices.tags;
+    this._updateTagMap();
+  },
+
+  getAllTags() {
+    if (this._tagList == null) {
+      this._updateTagMap();
+    }
+    return this._tagList;
+  },
+
+  _updateTagMap() {
+    this._tagMap = {};
+    let tagArray = (this._tagList = this._msgTagService.getAllTags());
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      this._tagMap[tag.key] = tag;
+    }
+  },
+
+  comparator(a, b) {
+    if (a == null) {
+      if (b == null) {
+        return 0;
+      }
+      return 1;
+    } else if (b == null) {
+      return -1;
+    }
+    return a.tag.localeCompare(b.tag);
+  },
+  userVisibleString(aTag) {
+    return aTag.tag;
+  },
+
+  // we cannot be an attribute value
+
+  toParamAndValue(aTag) {
+    return [aTag.key, null];
+  },
+  toJSON(aTag) {
+    return aTag.key;
+  },
+  fromJSON(aTagKey, aIgnored) {
+    let tag = this._tagMap.hasOwnProperty(aTagKey)
+      ? this._tagMap[aTagKey]
+      : undefined;
+    // you will note that if a tag is removed, we are unable to aggressively
+    //  deal with this.  we are okay with this, but it would be nice to be able
+    //  to listen to the message tag service to know when we should rebuild.
+    if (tag === undefined && this._msgTagService.isValidKey(aTagKey)) {
+      this._updateTagMap();
+      tag = this._tagMap[aTagKey];
+    }
+    // we intentionally are returning undefined if the tag doesn't exist
+    return tag;
+  },
+  /**
+   * Convenience helper to turn a tag key into a tag name.
+   */
+  getTag(aTagKey) {
+    return this.fromJSON(aTagKey);
+  },
+};
+
+TagNoun._init();
+Gloda.defineNoun(TagNoun, GlodaConstants.NOUN_TAG);
diff --git a/comm/mailnews/db/gloda/modules/SuffixTree.jsm b/comm/mailnews/db/gloda/modules/SuffixTree.jsm
new file mode 100644
index 0000000000..239993e180
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/SuffixTree.jsm
@@ -0,0 +1,381 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["SuffixTree", "MultiSuffixTree"];
+
+/**
+ * Given a list of strings and a corresponding map of items that those strings
+ *  correspond to, build a suffix tree.
+ */
+function MultiSuffixTree(aStrings, aItems) {
+  if (aStrings.length != aItems.length) {
+    throw new Error("Array lengths need to be the same.");
+  }
+
+  let s = "";
+  let offsetsToItems = [];
+  let lastLength = 0;
+  for (let i = 0; i < aStrings.length; i++) {
+    s += aStrings[i];
+    offsetsToItems.push(lastLength, s.length, aItems[i]);
+    lastLength = s.length;
+  }
+
+  this._construct(s);
+  this._offsetsToItems = offsetsToItems;
+  this._numItems = aItems.length;
+}
+
+/**
+ * @class
+ */
+function State(aStartIndex, aEndIndex, aSuffix) {
+  this.start = aStartIndex;
+  this.end = aEndIndex;
+  this.suffix = aSuffix;
+}
+
+/**
+ * Since objects are basically hash-tables anyways, we simply create an
+ *  attribute whose name is the first letter of the edge string.  (So, the
+ *  edge string can conceptually be a multi-letter string, but since we would
+ *  split it were there any ambiguity, it's okay to just use the single letter.)
+ *  This avoids having to update the attribute name or worry about tripping our
+ *  implementation up.
+ */
+State.prototype = {
+  get isExplicit() {
+    // our end is not inclusive...
+    return this.end <= this.start;
+  },
+  get isImplicit() {
+    // our end is not inclusive...
+    return this.end > this.start;
+  },
+
+  get length() {
+    return this.end - this.start;
+  },
+
+  toString() {
+    return (
+      "[Start: " +
+      this.start +
+      " End: " +
+      this.end +
+      (this.suffix ? " non-null suffix]" : " null suffix]")
+    );
+  },
+};
+
+/**
+ * Suffix tree implemented using Ukkonen's algorithm.
+ *
+ * @class
+ */
+function SuffixTree(aStr) {
+  this._construct(aStr);
+}
+
+/**
+ * States are
+ */
+SuffixTree.prototype = {
+  /**
+   * Find all items matching the provided substring.
+   */
+  findMatches(aSubstring) {
+    let results = [];
+    let state = this._root;
+    let index = 0;
+    let end = aSubstring.length;
+    while (index < end) {
+      state = state[aSubstring[index]];
+      // bail if there was no edge
+      if (state === undefined) {
+        return results;
+      }
+      // bail if the portion of the edge we traversed is not equal to that
+      //  portion of our pattern
+      let actualTraverseLength = Math.min(state.length, end - index);
+      if (
+        this._str.substring(state.start, state.start + actualTraverseLength) !=
+        aSubstring.substring(index, index + actualTraverseLength)
+      ) {
+        return results;
+      }
+      index += state.length;
+    }
+
+    // state should now be the node which itself and all its children match...
+    // The delta is to adjust us to the offset of the last letter of our match;
+    //  the edge we traversed to get here may have found us traversing more
+    //  than we wanted.
+    // index - end captures the over-shoot of the edge traversal,
+    // index - end + 1 captures the fact that we want to find the last letter
+    //  that matched, not just the first letter beyond it
+    // However, if this state is a leaf node (end == 'infinity'), then 'end'
+    //  isn't describing an edge at all and we want to avoid accounting for it.
+    let delta;
+    /*
+    if (state.end != this._infinity)
+      //delta = index - end + 1;
+      delta = end - (index - state.length);
+    else */
+    delta = index - state.length - end + 1;
+
+    this._resultGather(state, results, {}, end, delta, true);
+    return results;
+  },
+
+  _resultGather(
+    aState,
+    aResults,
+    aPresence,
+    aPatLength,
+    aDelta,
+    alreadyAdjusted
+  ) {
+    // find the item that this state originated from based on the state's
+    //  start character.  offsetToItem holds [string start index, string end
+    //  index (exclusive), item reference].  So we want to binary search to
+    //  find the string whose start/end index contains the state's start index.
+    let low = 0;
+    let high = this._numItems - 1;
+    let mid, stringStart, stringEnd;
+
+    let patternLast = aState.start - aDelta;
+    while (low <= high) {
+      mid = low + Math.floor((high - low) / 2); // excessive, especially with js nums
+      stringStart = this._offsetsToItems[mid * 3];
+      let startDelta = stringStart - patternLast;
+      stringEnd = this._offsetsToItems[mid * 3 + 1];
+      let endDelta = stringEnd - patternLast;
+      if (startDelta > 0) {
+        high = mid - 1;
+      } else if (endDelta <= 0) {
+        low = mid + 1;
+      } else {
+        break;
+      }
+    }
+
+    // - The match occurred completely inside a source string.  Success.
+    // - The match spans more than one source strings, and is therefore not
+    //   a match.
+
+    // at this point, we have located the origin string that corresponds to the
+    //  start index of this state.
+    // - The match terminated with the end of the preceding string, and does
+    //   not match us at all.  We, and potentially our children, are merely
+    //   serving as a unique terminal.
+    // - The
+
+    let patternFirst = patternLast - (aPatLength - 1);
+
+    if (patternFirst >= stringStart) {
+      if (!(stringStart in aPresence)) {
+        aPresence[stringStart] = true;
+        aResults.push(this._offsetsToItems[mid * 3 + 2]);
+      }
+    }
+
+    // bail if we had it coming OR
+    // if the result terminates at/part-way through this state, meaning any
+    //  of its children are not going to be actual results, just hangers
+    //  on.
+    /*
+    if (bail || (end <= aState.end)) {
+dump("  bailing! (bail was: " + bail + ")\n");
+      return;
+    }
+*/
+    // process our children...
+    for (let key in aState) {
+      // edges have attributes of length 1...
+      if (key.length == 1) {
+        let statePrime = aState[key];
+        this._resultGather(
+          statePrime,
+          aResults,
+          aPresence,
+          aPatLength,
+          aDelta + aState.length, // (alreadyAdjusted ? 0 : aState.length),
+          false
+        );
+      }
+    }
+  },
+
+  /**
+   * Given a reference 'pair' of a state and a string (may be 'empty'=explicit,
+   *  which means no work to do and we return immediately) follow that state
+   *  (and then the successive states)'s transitions until we run out of
+   *  transitions.  This happens either when we find an explicit state, or
+   *  find ourselves partially along an edge (conceptually speaking).  In
+   *  the partial case, we return the state prior to the edge traversal.
+   * (The information about the 'edge' is contained on its target State;
+   *  we can do this because a state is only referenced by one other state.)
+   */
+  _canonize(aState, aStart, aEnd) {
+    if (aEnd <= aStart) {
+      return [aState, aStart];
+    }
+
+    let statePrime;
+    // we treat an aState of null as 'bottom', which has transitions for every
+    //  letter in the alphabet to 'root'.  rather than create all those
+    //  transitions, we special-case here.
+    if (aState === null) {
+      statePrime = this._root;
+    } else {
+      statePrime = aState[this._str[aStart]];
+    }
+    while (statePrime.length <= aEnd - aStart) {
+      // (no 1 adjustment required)
+      aStart += statePrime.length;
+      aState = statePrime;
+      if (aStart < aEnd) {
+        statePrime = aState[this._str[aStart]];
+      }
+    }
+    return [aState, aStart];
+  },
+
+  /**
+   * Given a reference 'pair' whose state may or may not be explicit (and for
+   *  which we will perform the required splitting to make it explicit), test
+   *  whether it already possesses a transition corresponding to the provided
+   *  character.
+   *
+   * @returns A list of: whether we had to make it explicit, the (potentially)
+   *    new explicit state.
+   */
+  _testAndSplit(aState, aStart, aEnd, aChar) {
+    if (aStart < aEnd) {
+      // it's not explicit
+      let statePrime = aState[this._str[aStart]];
+      let length = aEnd - aStart;
+      if (aChar == this._str[statePrime.start + length]) {
+        return [true, aState];
+      }
+
+      // do splitting... aState -> rState -> statePrime
+      let rState = new State(statePrime.start, statePrime.start + length);
+      aState[this._str[statePrime.start]] = rState;
+      statePrime.start += length;
+      rState[this._str[statePrime.start]] = statePrime;
+      return [false, rState];
+    }
+
+    // it's already explicit
+    if (aState === null) {
+      // bottom case... shouldn't happen, but hey.
+      return [true, aState];
+    }
+    return [aChar in aState, aState];
+  },
+
+  _update(aState, aStart, aIndex) {
+    let oldR = this._root;
+    let textAtIndex = this._str[aIndex]; // T sub i (0-based corrected...)
+    // because of the way we store the 'end' value as a one-past form, we do
+    //  not need to subtract 1 off of aIndex.
+    let [endPoint, rState] = this._testAndSplit(
+      aState,
+      aStart,
+      aIndex, // no -1
+      textAtIndex
+    );
+    while (!endPoint) {
+      let rPrime = new State(aIndex, this._infinity);
+      rState[textAtIndex] = rPrime;
+      if (oldR !== this._root) {
+        oldR.suffix = rState;
+      }
+      oldR = rState;
+      [aState, aStart] = this._canonize(aState.suffix, aStart, aIndex); // no -1
+      [endPoint, rState] = this._testAndSplit(
+        aState,
+        aStart,
+        aIndex, // no -1
+        textAtIndex
+      );
+    }
+    if (oldR !== this._root) {
+      oldR.suffix = aState;
+    }
+
+    return [aState, aStart];
+  },
+
+  _construct(aStr) {
+    this._str = aStr;
+    // just needs to be longer than the string.
+    this._infinity = aStr.length + 1;
+
+    // this._bottom = new State(0, -1, null);
+    this._root = new State(-1, 0, null); // null === bottom
+    let state = this._root;
+    let start = 0;
+
+    for (let i = 0; i < aStr.length; i++) {
+      [state, start] = this._update(state, start, i); // treat as flowing -1...
+      [state, start] = this._canonize(state, start, i + 1); // 1-length string
+    }
+  },
+
+  dump(aState, aIndent, aKey) {
+    if (aState === undefined) {
+      aState = this._root;
+    }
+    if (aIndent === undefined) {
+      aIndent = "";
+      aKey = ".";
+    }
+
+    if (aState.isImplicit) {
+      let snip;
+      if (aState.length > 10) {
+        snip =
+          this._str.slice(
+            aState.start,
+            Math.min(aState.start + 10, this._str.length)
+          ) + "...";
+      } else {
+        snip = this._str.slice(
+          aState.start,
+          Math.min(aState.end, this._str.length)
+        );
+      }
+      dump(
+        aIndent +
+          aKey +
+          ":" +
+          snip +
+          "(" +
+          aState.start +
+          ":" +
+          aState.end +
+          ")\n"
+      );
+    } else {
+      dump(
+        aIndent +
+          aKey +
+          ": (explicit:" +
+          aState.start +
+          ":" +
+          aState.end +
+          ")\n"
+      );
+    }
+    let nextIndent = aIndent + "  ";
+    let keys = Object.keys(aState).filter(c => c.length == 1);
+    for (let key of keys) {
+      this.dump(aState[key], nextIndent, key);
+    }
+  },
+};
+MultiSuffixTree.prototype = SuffixTree.prototype;
diff --git a/comm/mailnews/db/gloda/modules/moz.build b/comm/mailnews/db/gloda/modules/moz.build
new file mode 100644
index 0000000000..54978c24ea
--- /dev/null
+++ b/comm/mailnews/db/gloda/modules/moz.build
@@ -0,0 +1,31 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXTRA_JS_MODULES.gloda += [
+    "Collection.jsm",
+    "Everybody.jsm",
+    "Facet.jsm",
+    "Gloda.jsm",
+    "GlodaConstants.jsm",
+    "GlodaContent.jsm",
+    "GlodaDatabind.jsm",
+    "GlodaDataModel.jsm",
+    "GlodaDatastore.jsm",
+    "GlodaExplicitAttr.jsm",
+    "GlodaFundAttr.jsm",
+    "GlodaIndexer.jsm",
+    "GlodaMsgIndexer.jsm",
+    "GlodaMsgSearcher.jsm",
+    "GlodaPublic.jsm",
+    "GlodaQueryClassFactory.jsm",
+    "GlodaSyntheticView.jsm",
+    "GlodaUtils.jsm",
+    "IndexMsg.jsm",
+    "MimeMessage.jsm",
+    "NounFreetag.jsm",
+    "NounMimetype.jsm",
+    "NounTag.jsm",
+    "SuffixTree.jsm",
+]