summaryrefslogtreecommitdiffstats
path: root/mailnews/db/gloda/modules/gloda.js
blob: 3a32e6041c8dc80523f0519deadc1179501695a0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

this.EXPORTED_SYMBOLS = ['Gloda'];

var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;

Cu.import("resource:///modules/gloda/log4moz.js");

Cu.import("resource:///modules/gloda/datastore.js");
Cu.import("resource:///modules/gloda/datamodel.js");
Cu.import("resource:///modules/gloda/databind.js");
Cu.import("resource:///modules/gloda/collection.js");
Cu.import("resource:///modules/gloda/connotent.js");
Cu.import("resource:///modules/gloda/query.js");
Cu.import("resource:///modules/gloda/utils.js");

Cu.import("resource:///modules/iteratorUtils.jsm");
Cu.import("resource:///modules/IOUtils.js");
Cu.import("resource:///modules/mailServices.js");
Cu.import("resource://gre/modules/Services.jsm");

/**
 * @see |Gloda.BadItemContentsError|
 */
function BadItemContentsError(aMessage) {
  this.message = aMessage;
}
BadItemContentsError.prototype = {
  toString: function BadItemContentsError_toString() {
    return this.message;
  }
};


/**
 * Provides the user-visible (and extension visible) global database
 *  functionality.  There is currently a dependency/ordering
 *  problem in that the concept of 'gloda' also includes some logic that is
 *  contributed by built-in extensions, if you will.  Those built-in extensions
 *  (fundattr.js, explattr.js) also import this file.  To avoid a circular
 *  dependency, those built-in extensions are loaded by everybody.js.  The
 *  simplest/best solution is probably to move everybody.js to be gloda.js and
 *  have it re-export only 'Gloda'.  gloda.js (this file) can then move to be
 *  gloda_int.js (or whatever our eventual naming scheme is), which built-in
 *  extensions can explicitly rely upon.
 *
 * === Concepts
 *
 * == Nouns
 *
 * Inspired by reasonable uses of triple-stores, I have tried to leverage
 *  existing model and terminology rather than rolling out own for everything.
 *  The idea with triple-stores is that you have a subject, a predicate, and an
 *  object.  For example, if we are talking about a message, that is the
 *  subject, the predicate could roughly be sent-by, and the object a person.
 *  We can generalize this idea to say that the subject and objects are nouns.
 * Since we want to be more flexible than only dealing with messages, we
 *  therefore introduce the concept of nouns as an organizing principle.
 *
 * == Attributes
 *
 * Our attributes definitions are basically our predicates.  When we define
 *  an attribute, it's a label with a bunch of meta-data.  Our attribute
 *  instances are basically a 'triple' in a triple-store.  The attributes
 *  are stored in database rows that imply a specific noun-type (ex: the
 *  messageAttributes table), with an ID identifying the message which is our
 *  subject, an attribute ID which identifies the attribute definition in use
 *  (and therefore the predicate), plus an object ID (given context aka the
 *  noun type by the attribute's meta-data) which identifies the 'object'.
 *
 * == But...
 *
 * Things aren't entirely as clear as they could be right now, terminology/
 *  concept/implementation-wise.  Some work is probably still in order.
 *
 * === Implementation
 *
 * == Nouns
 *
 * So, we go and define the nouns that are roughly the classes in our data
 *  model.  Every 'class' we define in datamodel.js is a noun that gets defined
 *  here in the Gloda core.  We provide sufficient meta-data about the noun to
 *  serialize/deserialize its representation from our database representation.
 *  Nouns do not have to be defined in this class, but can also be contributed
 *  by external code.
 * We have a concept of 'first class' nouns versus non-first class nouns.  The
 *  distinction is meant to be whether we can store meta-information about those
 *  nouns using attributes.  Right now, only message are real first-class nouns,
 *  but we want to expand that to include contacts and eventually events and
 *  tasks as lightning-integration occurs.  In practice, we are stretching the
 *  definition of first-class nouns slightly to include things we can't store
 *  meta-data about, but want to be able to query about.  We do want to resolve
 *  this.
 *
 * == Attributes
 *
 * Attributes are defined by "attribute providers" who are responsible for
 *  taking an instance of a first-class noun (for which they are registered)
 *  plus perhaps some other meta-data, and returning a list of attributes
 *  extracted from that noun.  For now, this means messages.  Attribute
 *  providers may create new data records as a side-effect of the indexing
 *  process, although we have not yet fully dealt with the problem of deleting
 *  these records should they become orphaned in the database due to the
 *  purging of a message and its attributes.
 * All of the 'core' gloda attributes are provided by the fundattr.js and
 *  explattr.js providers.
 *
 * === (Notable) Future Work
 *
 * == Attributes
 *
 * Attribute mechanisms currently lack any support for 'overriding' attributes
 *  provided by other attribute providers.  For example, the fundattr provider
 *  tells us who a message is 'from' based on the e-mail address present.
 *  However, other plugins may actually know better.  For example, the bugzilla
 *  daemon e-mails based on bug activity although the daemon gets the credit
 *  as the official sender.  A bugzilla plugin can easily extract the actual
 *  person/e-mail addressed who did something on the bug to cause the
 *  notification to be sent.  In practice, we would like that person to be
 *  the 'sender' of the bugmail.  But we can't really do that right, yet.
 *
 * @namespace
 */
var Gloda = {
  /**
   * Initialize logging, the datastore (SQLite database), the core nouns and
   *  attributes, and the contact and identities that belong to the presumed
   *  current user (based on accounts).
   *
   * Additional nouns and the core attribute providers are initialized by the
   *  everybody.js module which ensures all of those dependencies are loaded
   *  (and initialized).
   */
  _init: function gloda_ns_init() {
    this._initLogging();
    GlodaDatastore._init(this._nounIDToDef);
    this._initAttributes();
    this._initMyIdentities();
  },

  _log: null,
  /**
   * Initialize logging; the error console window gets Warning/Error, and stdout
   *  (via dump) gets everything.
   */
  _initLogging: function gloda_ns_initLogging() {
    let formatter = new Log4Moz.BasicFormatter();
    Log4Moz.repository.rootLogger.level = Log4Moz.Level.Debug;

    let enableConsoleLogging = false;
    let enableDumpLogging = false;
    // should we assume there is someone else consuming our log4moz stream?
    let enableUpstreamLogging = false;
    let considerNetLogging = false;

    let glodaLog = Log4Moz.repository.getLogger("gloda");
    glodaLog.level = Log4Moz.Level.Warn;

    try {
      // figure out if event-driven indexing should be enabled...
      let branch = Services.prefs.getBranch("mailnews.database.global.logging.");
      enableConsoleLogging = branch.getBoolPref("console");
      enableDumpLogging = branch.getBoolPref("dump");
      enableUpstreamLogging = branch.getBoolPref("upstream");
      considerNetLogging = branch.getBoolPref("net");
    } catch (ex) {}

    if (enableConsoleLogging) {
      let capp = new Log4Moz.ConsoleAppender(formatter);
      capp.level = Log4Moz.Level.Warn;
      glodaLog.addAppender(capp);
    }

    if (enableDumpLogging) {
      let dapp = new Log4Moz.DumpAppender(formatter);
      dapp.level = Log4Moz.Level.All;
      glodaLog.level = Log4Moz.Level.All;
      glodaLog.addAppender(dapp);
    }

    if (enableUpstreamLogging) {
      glodaLog.level = Log4Moz.Level.All;
    }

    if (considerNetLogging) {
      let file = Services.dirsvc.get("TmpD", Ci.nsIFile);
      file.append("chainsaw.ptr");
      if (file.exists()) {
        let data = IOUtils.loadFileToString(file);
        data = data.trim();
        let [host, port] = data.split(":");
        let xf = new Log4Moz.XMLFormatter();
        let sapp = new Log4Moz.SocketAppender(host, Number(port), xf);
        sapp.level = Log4Moz.Level.All;
        glodaLog.level = Log4Moz.Level.All;
        glodaLog.addAppender(sapp);
      }
    }

    this._log = Log4Moz.repository.getLogger("gloda.NS");
    this._log.info("Logging Initialized");
  },

  /**
   * The indexer is idle.
   */
  kIndexerIdle: 0,
  /**
   * The indexer is doing something.  We used to have other specific states, but
   *  they have been rendered irrelevant and wiped from existence.
   */
  kIndexerIndexing: 1,

  /**
   * Synchronous activities performed that can be thought of as one processing
   *  token.  Potentially yield the event-loop and re-schedule for later based
   *  on how long we've actually taken/etc.  The goal here is that code that
   *  is doing stuff synchronously yields with kWorkSync periodically to make
   *  sure that it doesn't dominate the event-loop.  Unless the processing
   *  in question is particularly intensive, it should be reasonable to apply
   *  some decimation factor (ex: 32 or 64) with the general goal of yielding
   *  every 3-10 milliseconds.
   */
  kWorkSync: 0,
  /**
   * Asynchronous activity performed, you need to relinquish flow control and
   *  trust us to call callbackDriver later.
   */
  kWorkAsync: 1,
  /**
   * We are all done with our task, close us and figure out something else to do.
   */
  kWorkDone: 2,
  /**
   * We are not done with our task, but we think it's a good idea to take a
   *  breather because we believe we have tied up the event loop for a
   *  non-trivial amount of time.  So please re-schedule us in the future.
   *
   * This is currently only used internally by the indexer's batching logic;
   *  minor changes may be required if used by actual indexers.
   */
  kWorkPause: 3,
  /**
   * We are done with our task, and have a result that we are returning.  This
   *  should only be used by your callback handler's doneWithResult method.
   *  Ex: you are passed aCallbackHandle, and you do
   *  "yield aCallbackHandle.doneWithResult(myResult);".
   */
  kWorkDoneWithResult: 4,

  /**
   * Callers should access the unique ID for the GlodaDatastore
   * with this getter. If the GlodaDatastore has not been
   * initialized, this value is null.
   *
   * @return a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
   */
  get datastoreID() {
    return GlodaDatastore._datastoreID;
  },

  /**
   * Lookup a gloda message from an nsIMsgDBHdr, with the result returned as a
   *  collection.  Keep in mind that the message may not be indexed, so you
   *  may end up with an empty collection.  (Also keep in mind that this query
   *  is asynchronous, so you will want your action-taking logic to be found
   *  in your listener's onQueryCompleted method; the result will not be in
   *  the collection when this method returns.)
   *
   * @param aMsgHdr The header of the message you want the gloda message for.
   * @param aListener The listener that should be registered with the collection
   * @param aData The (optional) value to set as the data attribute on the
   *     collection.
   *
   * @return The collection that will receive the results.
   *
   * @testpoint gloda.ns.getMessageCollectionForHeader()
   */
  getMessageCollectionForHeader: function gloda_ns_getMessageForHeader(aMsgHdr,
      aListener, aData) {
    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
    query.folder(aMsgHdr.folder).messageKey(aMsgHdr.messageKey);
    return query.getCollection(aListener, aData);
  },

  /**
   * Given a list of message headers, return a collection containing the gloda
   *  messages that correspond to those headers.  Keep in mind that gloda may
   *  not have indexed all the messages, so the returned collection may not have
   *  a message for each header you provide. (Also keep in mind that this query
   *  is asynchronous, so you will want your action-taking logic to be found
   *  in your listener's onQueryCompleted method; no results will be present in
   *  the collection when this method returns.)
   *
   * @param aHeaders A javascript Array or and XPCOM list that fixIterator can
   *     can handle.
   * @param aListener The listener that should be registered with the collection
   * @param aData The (optional) value to set as the data attribute on the
   *     collection.
   *
   * @return The collection that will receive the results.
   *
   * @testpoint gloda.ns.getMessageCollectionForHeaders()
   */
  getMessageCollectionForHeaders: function gloda_ns_getMessagesForHeaders(
      aHeaders, aListener, aData) {
    // group the headers by the folder they are found in
    let headersByFolder = {};
    let iter;
    for (let header in fixIterator(aHeaders)) {
      let folderURI = header.folder.URI;
      let headersForFolder = headersByFolder[folderURI];
      if (headersForFolder === undefined)
        headersByFolder[folderURI] = [header];
      else
        headersForFolder.push(header);
    }

    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
    let clause;
    // build a query, using a separate union clause for each folder.
    for (let folderURI in headersByFolder) {
      let headersForFolder = headersByFolder[folderURI];
      let folder = this.getFolderForFolder(headersForFolder[0].folder);
      // if this is the first or clause, just use the query itself
      if (!clause)
        clause = query;
      else // create a new query clause via the 'or' command
        clause = query.or();

      clause.folder(folder);
      let messageKeys = headersForFolder.map(hdr => hdr.messageKey);
      clause.messageKey.apply(clause, messageKeys);
    }

    return query.getCollection(aListener, aData);
  },

  /**
   * @testpoint gloda.ns.getMessageContent
   */
  getMessageContent: function gloda_ns_getMessageContent(aGlodaMessage, aMimeMsg) {
    return mimeMsgToContentAndMeta(aMimeMsg, aGlodaMessage.folderMessage.folder)[0];
  },

  getFolderForFolder: function gloda_ns_getFolderForFolder(aMsgFolder) {
    return GlodaDatastore._mapFolder(aMsgFolder);
  },

  /**
   * Takes one or more strings containing lists of comma-delimited e-mail
   *  addresses with optional display names, and returns a list of sub-lists of
   *  identities, where each sub-list corresponds to each of the strings passed
   *  as arguments.  These identities are loaded from the database if they
   *  already exist, or created if they do not yet exist.
   * If the identities need to be created, they will also result in the
   *  creation of a gloda contact.  If a display name was provided with the
   *  e-mail address, it will become the name of the gloda contact.  If a
   *  display name was not provided, the e-mail address will also serve as the
   *  contact name.
   * This method uses the indexer's callback handle mechanism, and does not
   *  obey traditional return semantics.
   *
   * We normalize all e-mail addresses to be lowercase as a normative measure.
   *
   * @param aCallbackHandle The GlodaIndexer callback handle (or equivalent)
   *   that you are operating under.
   * @param ... One or more strings.  Each string can contain zero or more
   *   e-mail addresses with display name.  If more than one address is given,
   *   they should be comma-delimited.  For example
   *   '"Bob Smith" <bob@example.com>' is an address with display name.  Mime
   *   header decoding is performed, but is ignorant of any folder-level
   *   character set overrides.
   * @returns via the callback handle mechanism, a list containing one sub-list
   *   for each string argument passed.  Each sub-list containts zero or more
   *   GlodaIdentity instances corresponding to the addresses provided.
   */
  getOrCreateMailIdentities:
      function* gloda_ns_getOrCreateMailIdentities(aCallbackHandle) {
    let addresses = {};
    let resultLists = [];

    // parse the strings
    for (let iArg = 1; iArg < arguments.length; iArg++) {
      let aMailAddresses = arguments[iArg];
      let parsed = GlodaUtils.parseMailAddresses(aMailAddresses);

      let resultList = [];
      resultLists.push(resultList);

      let identities = [];
      for (let iAddress = 0; iAddress < parsed.count; iAddress++) {
        let address = parsed.addresses[iAddress].toLowerCase();
        if (address in addresses)
          addresses[address].push(resultList);
        else
          addresses[address] = [parsed.names[iAddress], resultList];
      }
    }

    let addressList = Object.keys(addresses);
    if (addressList.length == 0) {
      yield aCallbackHandle.doneWithResult(resultLists);
      // we should be stopped before we reach this point, but safety first.
      return;
    }

    let query = this.newQuery(this.NOUN_IDENTITY);
    query.kind("email");
    query.value.apply(query, addressList);
    let collection = query.getCollection(aCallbackHandle);
    yield this.kWorkAsync;

    // put the identities in the appropriate result lists
    for (let identity of collection.items) {
      let nameAndResultLists = addresses[identity.value];
      this._log.debug(" found identity for '" + nameAndResultLists[0] + "' (" +
                      identity.value + ")");
      // index 0 is the name, skip it
      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
        nameAndResultLists[iResList].push(identity);
      }
      delete addresses[identity.value];
    }

    // create the identities that did not exist yet
    for (let address in addresses) {
      let nameAndResultLists = addresses[address];
      let name = nameAndResultLists[0];

      this._log.debug(" creating contact for '" + name + "' (" + address + ")");

      // try and find an existing address book contact.
      let card = GlodaUtils.getCardForEmail(address);
      // XXX when we have the address book GUID stuff, we need to use that to
      //  find existing contacts... (this will introduce a new query phase
      //  where we batch all the GUIDs for an async query)
      // XXX when the address book supports multiple e-mail addresses, we
      //  should also just create identities for any that don't yet exist

      // if there is no name, just use the e-mail (the ab indexer actually
      //  processes the card's displayName for synchronization, so we don't
      //  need to do that.)
      if (!name)
        name = address;

      let contact = GlodaDatastore.createContact(null, null, name, 0, 0);

      // we must create the identity.  use a blank description because there's
      //  nothing to differentiate it from other identities, as this contact
      //  only has one initially (us).
      // XXX when we have multiple e-mails and there is a meaning associated
      //  with each e-mail, try and use that to populate the description.
      // XXX we are creating the identity here before we insert the contact.
      //  conceptually it is good for us to be creating the identity before
      //  exposing it to the address-book indexer, but we could get our id's
      //  in a bad way from not deferring the identity insertion until after
      //  the contact insertion.
      let identity = GlodaDatastore.createIdentity(contact.id, contact,
        "email", address, /* description */ "", /* relay? */ false);
      contact._identities = [identity];

      // give the address book indexer a chance if we have a card.
      // (it will fix-up the name based on the card as appropriate)
      if (card)
        yield aCallbackHandle.pushAndGo(
          Gloda.grokNounItem(contact, {card: card}, true, true,
                             aCallbackHandle));
      else // grokNounItem will issue the insert for us...
        GlodaDatastore.insertContact(contact);

      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++) {
        nameAndResultLists[iResList].push(identity);
      }
    }

    yield aCallbackHandle.doneWithResult(resultLists);
  },

  /**
   * Dictionary of the user's known identities; key is the identity id, value
   *  is the actual identity.  This is populated by _initMyIdentities based on
   *  the accounts defined.
   */
  myIdentities: {},
  /**
   * The contact corresponding to the current user.  We are assuming that only
   *  a single user/human being uses the current profile.  This is known to be
   *  a flawed assumption, but is the best first approximation available.
   *
   * @TODO attempt to deal with multile people using the same profile
   */
  myContact: null,
  /**
   * Populate myIdentities with all of our identities.  Currently we do this
   *  by assuming that there is one human/user per profile, and that all of the
   *  accounts defined in the profile belong to them.  The single contact is
   *  stored on myContact.
   *
   * @TODO deal with account addition/modification/removal
   * @TODO attempt to deal with multiple people using the same profile
   */
  _initMyIdentities: function gloda_ns_initMyIdentities() {
    let myContact = null;
    let myIdentities = {};
    let myEmailAddresses = {}; // process each email at most once; stored here

    let fullName, fallbackName;
    let existingIdentities = [];
    let identitiesToCreate = [];

    let numIdentities = MailServices.accounts.allIdentities.length;

    // nothing to do if there are no accounts/identities.
    if (!numIdentities)
      return;

    for (let iIdentity = 0; iIdentity < numIdentities; iIdentity++) {
      let msgIdentity =
        MailServices.accounts.allIdentities.queryElementAt(iIdentity,
                                                           Ci.nsIMsgIdentity);

      if (!fullName)
        fullName = msgIdentity.fullName;
      if (!fallbackName)
        fallbackName = msgIdentity.email;

      let emailAddress = msgIdentity.email;
      let replyTo = msgIdentity.replyTo;

      // find the identities if they exist, flag to create them if they don't
      if (emailAddress) {
        let parsed = GlodaUtils.parseMailAddresses(emailAddress);
        if (!(parsed.addresses[0] in myEmailAddresses)) {
          let identity = GlodaDatastore.getIdentity("email",
                                                    parsed.addresses[0]);
          if (identity)
            existingIdentities.push(identity);
          else
            identitiesToCreate.push(parsed.addresses[0]);
          myEmailAddresses[parsed.addresses[0]] = true;
        }
      }
      if (replyTo) {
        let parsed = GlodaUtils.parseMailAddresses(replyTo);
        if (!(parsed.addresses[0] in myEmailAddresses)) {
          let identity = GlodaDatastore.getIdentity("email",
                                                    parsed.addresses[0]);
          if (identity)
            existingIdentities.push(identity);
          else
            identitiesToCreate.push(parsed.addresses[0]);
          myEmailAddresses[parsed.addresses[0]] = true;
        }
      }
    }

    // we need to establish the identity.contact portions of the relationship
    for (let identity of existingIdentities) {
      identity._contact = GlodaDatastore.getContactByID(identity.contactID);
    }

    if (existingIdentities.length) {
      // just use the first guy's contact
      myContact = existingIdentities[0].contact;
    }
    else {
      // create a new contact
      myContact = GlodaDatastore.createContact(null, null,
                                               fullName || fallbackName,
                                               0, 0);
      GlodaDatastore.insertContact(myContact);
    }

    if (identitiesToCreate.length) {
      for (let iIdentity = 0; iIdentity < identitiesToCreate.length;
          iIdentity++) {
        let emailAddress = identitiesToCreate[iIdentity];
        // XXX this won't always be of type "email" as we add new account types
        // XXX the blank string could be trying to differentiate; we do have
        //  enough info to do it.
        let identity = GlodaDatastore.createIdentity(myContact.id, myContact,
                                                     "email",
                                                     emailAddress,
                                                     "", false);
        existingIdentities.push(identity);
      }
    }

    for (let iIdentity = 0; iIdentity < existingIdentities.length;
        iIdentity++) {
      let identity = existingIdentities[iIdentity];
      myIdentities[identity.id] = identity;
    }

    this.myContact = myContact;
    this.myIdentities = myIdentities;
    myContact._identities = Object.keys(myIdentities).
      map(id => myIdentities[id]);

    // we need contacts to make these objects reachable via the collection
    //  manager.
    this._myContactCollection = this.explicitCollection(this.NOUN_CONTACT,
                                                        [this.myContact]);
    this._myIdentitiesCollection =
      this.explicitCollection(this.NOUN_IDENTITY, this.myContact._identities);
  },

  /**
   * An attribute that is a defining characteristic of the subject.
   */
  kAttrFundamental: 0,
  /**
   * An attribute that is an optimization derived from two or more fundamental
   *  attributes and exists solely to improve database query performance.
   */
  kAttrOptimization: 1,
  /**
   * An attribute that is derived from the content of the subject.  For example,
   *  a message that references a bugzilla bug could have a "derived" attribute
   *  that captures the bugzilla reference.  This is not
   */
  kAttrDerived: 2,
  /**
   * An attribute that is the result of an explicit and intentional user action
   *  upon the subject.  For example, a tag placed on a message by a user (or
   *  at the user's request by a filter) is explicit.
   */
  kAttrExplicit: 3,
  /**
   * An attribute that is indirectly the result of a user's behaviour.  For
   *  example, if a user consults a message multiple times, we may conclude that
   *  the user finds the message interesting.  It is "implied", if you will,
   *  that the message is interesting.
   */
  kAttrImplicit: 4,

  /**
   * This attribute is not 'special'; it is stored as a (thing id, attribute id,
   *  attribute id) tuple in the database rather than on thing's row or on
   *  thing's fulltext row.  (Where "thing" could be a message or any other
   *  first class noun.)
   */
  kSpecialNotAtAll: GlodaDatastore.kSpecialNotAtAll,
  /**
   * This attribute is stored as a numeric column on the row for the noun.  The
   *  attribute definition should include this value as 'special' and the
   *  column name that stores the attribute as 'specialColumnName'.
   */
  kSpecialColumn: GlodaDatastore.kSpecialColumn,
  kSpecialColumnChildren: GlodaDatastore.kSpecialColumnChildren,
  kSpecialColumnParent: GlodaDatastore.kSpecialColumnParent,
  /**
   * This attribute is stored as a string column on the row for the noun.  It
   *  differs from kSpecialColumn in that it is a string, which once had
   *  query ramifications and one day may have them again.
   */
  kSpecialString: GlodaDatastore.kSpecialString,
  /**
   * This attribute is stored as a fulltext column on the fulltext table for
   *  the noun.  The attribute defintion should include this value as 'special'
   *  and the column name that stores the table as 'specialColumnName'.
   */
  kSpecialFulltext: GlodaDatastore.kSpecialFulltext,

  /**
   * The extensionName used for the attributes defined by core gloda plugins
   *  such as fundattr.js and explattr.js.
   */
  BUILT_IN: "built-in",

  /**
   * Special sentinel value that will cause facets to skip a noun instance
   * when an attribute has this value.
   */
  IGNORE_FACET: GlodaDatastore.IGNORE_FACET,

  /*
   * The following are explicit noun IDs.  While most extension-provided nouns
   *  will have dynamically allocated id's that are looked up by name, these
   *  id's can be relied upon to exist and be accessible via these
   *  pseudo-constants.  It's not really clear that we need these, although it
   *  does potentially simplify code to not have to look up all of their nouns
   *  at initialization time.
   */
  /**
   * Boolean values, expressed as 0/1 in the database and non-continuous for
   *  constraint purposes.  Like numbers, such nouns require their attributes
   *  to provide them with context, lacking any of their own.
   * Having this as a noun type may be a bad idea; a change of nomenclature
   *  (so that we are not claiming a boolean value is a noun, but still using
   *  it in the same way) or implementation to require each boolean noun
   *  actually be its own noun may be in order.
   */
  NOUN_BOOLEAN: 1,
  /**
   * A number, which could mean an integer or floating point values.  We treat
   *  these as continuous, meaning that queries on them can have ranged
   *  constraints expressed on them.  Lacking any inherent context, numbers
   *  depend on their attributes to parameterize them as required.
   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
   */
  NOUN_NUMBER: 2,
  /**
   * A (non-fulltext) string.
   * Same deal as with NOUN_BOOLEAN, we may need to change this up conceptually.
   */
  NOUN_STRING: 3,
  /** A date, encoded as a PRTime, represented as a js Date object. */
  NOUN_DATE: 10,
  /**
   * Fulltext search support, somewhat magical.  This is only intended to be
   *  used for kSpecialFulltext attributes, and exclusively as a constraint
   *  mechanism.  The values are always represented as strings.  It is presumed
   *  that the user of this functionality knows how to generate SQLite FTS3
   *  style MATCH queries, or is okay with us just gluing them together with
   *  " OR " when used in an or-constraint case.  Gloda's query mechanism
   *  currently lacks the ability to to compile Gloda-style and-constraints
   *  into a single MATCH query, but it will turn out okay, just less
   *  efficiently than it could.
   */
  NOUN_FULLTEXT: 20,
  /**
   * Represents a MIME Type.  We currently lack any human-intelligible
   *  descriptions of mime types.
   */
  NOUN_MIME_TYPE: 40,
  /**
   * Captures a message tag as well as when the tag's presence was observed,
   *  hoping to approximate when the tag was applied.  It's a somewhat dubious
   *  attempt to not waste our opporunity to store a value along with the tag.
   *  (The tag is actually stored as an attribute parameter on the attribute
   *  definition, rather than a value in the attribute 'instance' for the
   *  message.)
   */
  NOUN_TAG: 50,
  /**
   * Doesn't actually work owing to a lack of an object to represent a folder.
   *  We do expose the folderURI and folderID of a message, but need to map that
   *  to a good abstraction.  Probably something thin around a SteelFolder or
   *  the like; we would contribute the functionality to easily move from a
   *  folder to the list of gloda messages in that folder, as well as the
   *  indexing preferences for that folder.
   * @TODO folder noun and related abstraction
   */
  NOUN_FOLDER: GlodaFolder.prototype.NOUN_ID, // 100
  /**
   * All messages belong to a conversation.  See datamodel.js for the
   *  definition of the GlodaConversation class.
   */
  NOUN_CONVERSATION: GlodaConversation.prototype.NOUN_ID, // 101
  /**
   * A one-to-one correspondence with underlying (indexed) nsIMsgDBHdr
   *  instances.  See datamodel.js for the definition of the GlodaMessage class.
   */
  NOUN_MESSAGE: GlodaMessage.prototype.NOUN_ID, // 102
  /**
   * Corresponds to a human being, who may have multiple electronic identities
   *  (a la NOUN_IDENTITY).  There is no requirement for association with an
   *  address book contact, although when the address book contact exists,
   *  we want to be associated with it.  See datamodel.js for the definition
   *  of the GlodaContact class.
   */
  NOUN_CONTACT: GlodaContact.prototype.NOUN_ID, // 103
  /**
   * A single identity of a contact, who may have one or more.  E-mail accounts,
   *  instant messaging accounts, social network site accounts, etc. are each
   *  identities.  See datamodel.js for the definition of the GlodaIdentity
   *  class.
   */
  NOUN_IDENTITY: GlodaIdentity.prototype.NOUN_ID, // 104
  /**
   * An attachment to a message. A message may have many different attachments.
   */
  NOUN_ATTACHMENT: GlodaAttachment.prototype.NOUN_ID, // 105
  /**
   * An account related to a message. A message can have only one account.
   */
  NOUN_ACCOUNT: GlodaAccount.prototype.NOUN_ID, // 106

  /**
   * Parameterized identities, for use in the from-me, to-me, cc-me optimization
   *  cases.  Not for reuse without some thought.  These nouns use the parameter
   *  to store the 'me' identity that we are talking about, and the value to
   *  store the identity of the other party.  So in both the from-me and to-me
   *  cases involving 'me' and 'foo@bar', the 'me' identity is always stored via
   *  the attribute parameter, and the 'foo@bar' identity is always stored as
   *  the attribute value.  See fundattr.js for more information on this, but
   *  you probably shouldn't be touching this unless you are fundattr.
   */
  NOUN_PARAM_IDENTITY: 200,

  /** Next Noun ID to hand out, these don't need to be persisted (for now). */
  _nextNounID: 1000,

  /**
   * Maps noun names to noun IDs.
   */
  _nounNameToNounID: {},
  /**
   * Maps noun IDs to noun definition dictionaries.  (Noun definition
   *  dictionaries provided to us at the time a noun was defined, plus some
   *  additional stuff we put in there.)
   */
  _nounIDToDef: {},

  _managedToJSON: function gloda_ns_managedToJSON(aItem) {
    return aItem.id;
  },

  /**
   * Define a noun.  Takes a dictionary with the following keys/values:
   *
   * @param aNounDef.name The name of the noun.  This is not a display name
   *     (anything being displayed needs to be localized, after all), but simply
   *     the canonical name for debugging purposes and for people to pass to
   *     lookupNoun.  The suggested convention is lower-case-dash-delimited,
   *     with names being singular (since it's a single noun we are referring
   *     to.)
   * @param aNounDef.class The 'class' to which an instance of the noun will
   *     belong (aka will pass an instanceof test).  You may also provide this
   *     as 'clazz' if the keyword makes your IDE angry.
   * @param aNounDef.allowsArbitraryAttrs Is this a 'first class noun'/can it be
   *     a subject, AKA can this noun have attributes stored on it that relate
   *     it to other things?  For example, a message is first-class; we store
   *     attributes of messages.  A date is not first-class now, nor is it
   *     likely to be; we will not store attributes about a date, although dates
   *     will be the objects of other subjects.  (For example: we might
   *     associate a date with a calendar event, but the date is an attribute of
   *     the calendar event and not vice versa.)
   * @param aNounDef.usesParameter A boolean indicating whether this noun
   *     requires use of the 'parameter' BLOB storage field on the attribute
   *     bindings in the database to persist itself.  Use of parameters should
   *     be limited to a reasonable number of values (16-32 is okay, more than
   *     that is pushing it and 256 should be considered an absolute upper
   *     bound) because of the database organization.  When false, your
   *     toParamAndValue function is expected to return null for the parameter
   *     and likewise your fromParamAndValue should expect ignore and generally
   *     ignore the argument.
   * @param aNounDef.toParamAndValue A function that takes an instantiated noun
   *     instance and returns a 2-element list of [parameter, value] where
   *     parameter may only be non-null if you passed a usesParameter of true.
   *     Parameter may be of any type (BLOB), and value must be numeric (pass
   *     0 if you don't need the value).
   *
   * @param aNounDef.isPrimitive True when the noun instance is a raw numeric
   *     value/string/boolean.  False when the instance is an object.  When
   *     false, it is assumed the attribute that serves as a unique identifier
   *     for the value is "id" unless 'idAttr' is provided.
   * @param [aNounDef.idAttr="id"] For non-primitive nouns, this is the
   *     attribute on the object that uniquely identifies it.
   *
   * @param aNounDef.schema Unsupported mechanism by which you can define a
   *     table that corresponds to this noun.  The table will be created if it
   *     does not exist.
   *     - name The table name; don't conflict with other things!
   *     - columns A list of [column name, sqlite type] tuples.  You should
   *       always include a definition like ["id", "INTEGER PRIMARY KEY"] for
   *       now (and it should be the first column name too.)  If you care about
   *       how the attributes are poked into your object (for example, you want
   *       underscores used for some of them because the attributes should be
   *       immutable), then you can include a third string that is the name of
   *       the attribute to use.
   *     - indices A dictionary of lists of column names, where the key name
   *       becomes the index name.  Ex: {foo: ["bar"]} results in an index on
   *       the column "bar" where the index is named "foo".
  */
  defineNoun: function gloda_ns_defineNoun(aNounDef, aNounID) {
    this._log.info("Defining noun: " + aNounDef.name);
    if (aNounID === undefined)
      aNounID = this._nextNounID++;
    aNounDef.id = aNounID;

    // Let people whose editors get angry about illegal attribute names use
    //  clazz instead of class.
    if (aNounDef.clazz)
      aNounDef.class = aNounDef.clazz;

    if (!("idAttr" in aNounDef))
      aNounDef.idAttr = "id";
    if (!("comparator" in aNounDef)) {
      aNounDef.comparator = function() {
        throw new Error("Noun type '" + aNounDef.name +
                        "' lacks a real comparator.");
      };
    }

    // We allow nouns to have data tables associated with them where we do all
    //  the legwork.  The schema attribute is the gateway to this magical world
    //  of functionality.  Said door is officially unsupported.
    if (aNounDef.schema) {
      if (!aNounDef.tableName) {
        if (aNounDef.schema.name)
          aNounDef.tableName = "ext_" + aNounDef.schema.name;
        else
          aNounDef.tableName = "ext_" + aNounDef.name;
      }
      // this creates the data table and binder and hooks everything up
      GlodaDatastore.createNounTable(aNounDef);

      if (!aNounDef.toParamAndValue)
        aNounDef.toParamAndValue = function (aThing) {
          if (aThing instanceof aNounDef.class)
            return [null, aThing.id];
          else // assume they're just passing the id directly
            return [null, aThing];
        };
    }

    // if it has a table, you can query on it.  seems straight-forward.
    if (aNounDef.tableName) {
      [aNounDef.queryClass, aNounDef.nullQueryClass,
       aNounDef.explicitQueryClass, aNounDef.wildcardQueryClass] =
          GlodaQueryClassFactory(aNounDef);
      aNounDef._dbMeta = {};
      aNounDef.class.prototype.NOUN_ID = aNounDef.id;
      aNounDef.class.prototype.NOUN_DEF = aNounDef;
      aNounDef.toJSON = this._managedToJSON;

      aNounDef.specialLoadAttribs = [];

      // - define the 'id' constrainer
      let idConstrainer = function() {
        let constraint = [GlodaDatastore.kConstraintIdIn, null];
        for (let iArg = 0; iArg < arguments.length; iArg++) {
          constraint.push(arguments[iArg]);
        }
        this._constraints.push(constraint);
        return this;
      };
      aNounDef.queryClass.prototype.id = idConstrainer;
    }
    if (aNounDef.cache) {
      let cacheCost = aNounDef.cacheCost || 1024;
      let cacheBudget = aNounDef.cacheBudget || 128 * 1024;
      let cacheSize = Math.floor(cacheBudget / cacheCost);
      if (cacheSize)
        GlodaCollectionManager.defineCache(aNounDef, cacheSize);
    }
    aNounDef.attribsByBoundName = {};
    aNounDef.domExposeAttribsByBoundName = {};

    aNounDef.objectNounOfAttributes = [];

    this._nounNameToNounID[aNounDef.name] = aNounID;
    this._nounIDToDef[aNounID] = aNounDef;
    aNounDef.actions = [];

    this._attrProviderOrderByNoun[aNounDef.id] = [];
    this._attrOptimizerOrderByNoun[aNounDef.id] = [];
    this._attrProvidersByNoun[aNounDef.id] = {};

    return aNounDef;
  },

  /**
   * Lookup a noun (ID) suitable for passing to defineAttribute's various
   *  noun arguments.  Throws an exception if the noun with the given name
   *  cannot be found; the assumption is that you can't live without the noun.
   */
  lookupNoun: function gloda_ns_lookupNoun(aNounName) {
    if (aNounName in this._nounNameToNounID)
      return this._nounNameToNounID[aNounName];

    throw Error("Unable to locate noun with name '" + aNounName + "', but I " +
                "do know about: " +
                Object.keys(this._nounNameToNounID).join(", "));
  },

  /**
   * Lookup a noun def given a name.
   */
  lookupNounDef: function gloda_ns_lookupNoun(aNounName) {
    return this._nounIDToDef[this.lookupNoun(aNounName)];
  },


  /**
   * Define an action on a noun.  During the prototype stage, this was conceived
   *  of as a way to expose all the constraints possible given a noun.  For
   *  example, if you have an identity or a contact, you could use this to
   *  see all the messages sent from/to a given contact.  It was likewise
   *  thought potentially usable for future expansion.  For example, you could
   *  also decide to send an e-mail to a contact when you have the contact
   *  instance available.
   * Outside of the 'expmess' checkbox-happy prototype, this functionality is
   *  not used.  As such, this functionality should be considered in flux and
   *  subject to changes.  Also, very open to specific suggestsions motivated
   *  by use cases.
   * One conceptual issue raised by this mechanism is the interaction of actions
   *  with facts like "this message is read".  We currently implement the 'fact'
   *  by defining an attribute with a 'boolean' noun type.  To deal with this,
   *  in various places we pass-in the attribute as well as the noun value.
   *  Since the relationships for booleans and integers in these cases is
   *  standard and well-defined, this works out pretty well, but suggests we
   *  need to think things through.
   *
   * @param aNounID The ID of the noun you want to define an action on.
   * @param aActionMeta The dictionary describing the noun.  The dictionary
   *     should have the following fields:
   * - actionType: a string indicating the type of action.  Currently, only
   *   "filter" is a legal value.
   * - actionTarget: the noun ID of the noun type on which this action is
   *   applicable.  For example,
   *
   * The following should be present for actionType=="filter";
   * - shortName: The name that should be used to display this constraint.  For
   *   example, a checkbox-heavy UI might display a checkbox for each constraint
   *   using shortName as the label.
   * - makeConstraint: A function that takes the attribute that is the source
   *   of the noun and the noun instance as arguments, and returns APV-style
   *   constraints.  Since the APV-style query mechanism is now deprecated,
   *   this signature is deprecated.  Probably the way to update this would be
   *   to pass in the query instance that constraints should be contributed to.
   */
  defineNounAction: function gloda_ns_defineNounAction(aNounID, aActionMeta) {
    let nounDef = this._nounIDToDef[aNounID];
    nounDef.actions.push(aActionMeta);
  },

  /**
   * Retrieve all of the actions (as defined using defineNounAction) for the
   *  given noun type (via noun ID) with the given action type (ex: filter).
   */
  getNounActions: function gloda_ns_getNounActions(aNounID, aActionType) {
    let nounDef = this._nounIDToDef[aNounID];
    if (!nounDef)
      return [];
    return nounDef.actions.
      filter(action => !aActionType || (action.actionType == aActionType));
  },

  /** Attribute providers in the sequence to process them. */
  _attrProviderOrderByNoun: {},
  /** Attribute providers that provide optimizers, in the sequence to proc. */
  _attrOptimizerOrderByNoun: {},
  /** Maps attribute providers to the list of attributes they provide */
  _attrProviders: {},
  /**
   * Maps nouns to their attribute providers to a list of the attributes they
   *  provide for the noun.
   */
  _attrProvidersByNoun: {},

  /**
   * Define the core nouns (that are not defined elsewhere) and a few noun
   *  actions.  Core nouns could be defined in other files, assuming dependency
   *  issues are resolved via the everybody.js mechanism or something else.
   *  Right now, noun_tag defines the tag noun.  If we broke more of these out,
   *  we would probably want to move the 'class' code from datamodel.js, the
   *  SQL table def and helper code from datastore.js (and this code) to their
   *  own noun_*.js files.  There are some trade-offs to be made, and I think
   *  we can deal with those once we start to integrate lightning/calendar and
   *  our noun space gets large and more heterogeneous.
   */
  _initAttributes: function gloda_ns_initAttributes() {
    this.defineNoun({
      name: "bool",
      clazz: Boolean, allowsArbitraryAttrs: false,
      isPrimitive: true,
      // favor true before false
      comparator: function gloda_bool_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return b - a;
      },
      toParamAndValue: function(aBool) {
        return [null, aBool ? 1 : 0];
      }}, this.NOUN_BOOLEAN);
    this.defineNoun({
      name: "number",
      clazz: Number, allowsArbitraryAttrs: false, continuous: true,
      isPrimitive: true,
      comparator: function gloda_number_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a - b;
      },
      toParamAndValue: function(aNum) {
        return [null, aNum];
      }}, this.NOUN_NUMBER);
    this.defineNoun({
      name: "string",
      clazz: String, allowsArbitraryAttrs: false,
      isPrimitive: true,
      comparator: function gloda_string_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.localeCompare(b);
      },
      toParamAndValue: function(aString) {
        return [null, aString];
      }}, this.NOUN_STRING);
    this.defineNoun({
      name: "date",
      clazz: Date, allowsArbitraryAttrs: false, continuous: true,
      isPrimitive: true,
      comparator: function gloda_data_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a - b;
      },
      toParamAndValue: function(aDate) {
        return [null, aDate.valueOf() * 1000];
      }}, this.NOUN_DATE);
    this.defineNoun({
      name: "fulltext",
      clazz: String, allowsArbitraryAttrs: false, continuous: false,
      isPrimitive: true,
      comparator: function gloda_fulltext_comparator(a, b) {
        throw new Error("Fulltext nouns are not comparable!");
      },
      // as noted on NOUN_FULLTEXT, we just pass the string around.  it never
      //  hits the database, so it's okay.
      toParamAndValue: function(aString) {
        return [null, aString];
      }}, this.NOUN_FULLTEXT);

    this.defineNoun({
      name: "folder",
      clazz: GlodaFolder,
      allowsArbitraryAttrs: false,
      isPrimitive: false,
      queryHelpers: {
        /**
         * Query for accounts based on the account associated with folders.  We
         *  walk all of the folders associated with an account and put them in
         *  the list of folders that match if gloda would index them.  This is
         *  unsuitable for producing a persistable constraint since it does not
         *  adapt for added/deleted folders.  However, it is sufficient for
         *  faceting.  Also, we don't persist constraints yet.
         *
         * @TODO The long-term solution is to move towards using arithmetic
         *     encoding on folder-id's like we use for MIME types and friends.
         */
        Account: function(aAttrDef, aArguments) {
          let folderValues = [];
          let seenRootFolders = {};
          for (let iArg = 0; iArg < aArguments.length; iArg++) {
            let givenFolder = aArguments[iArg];
            let givenMsgFolder = givenFolder.getXPCOMFolder(
                                   givenFolder.kActivityFolderOnlyNoData);
            let rootFolder = givenMsgFolder.rootFolder;

            // skip processing this folder if we have already processed its
            //  root folder.
            if (rootFolder.URI in seenRootFolders)
              continue;
            seenRootFolders[rootFolder.URI] = true;

            let allFolders = rootFolder.descendants;
            for (let folder in fixIterator(allFolders, Ci.nsIMsgFolder)) {
              let folderFlags = folder.flags;

              // Ignore virtual folders, non-mail folders.
              // XXX this is derived from GlodaIndexer's shouldIndexFolder.
              //  This should probably just use centralized code or the like.
              if (!(folderFlags & Ci.nsMsgFolderFlags.Mail) ||
                  (folderFlags & Ci.nsMsgFolderFlags.Virtual))
                continue;
              // we only index local or IMAP folders
              if (!(folder instanceof Ci.nsIMsgLocalMailFolder) &&
                !(folder instanceof Ci.nsIMsgImapMailFolder))
                continue;

              let glodaFolder = Gloda.getFolderForFolder(folder);
              folderValues.push(glodaFolder);
            }
          }
          return this._inConstraintHelper(aAttrDef, folderValues);
        }
      },
      comparator: function gloda_folder_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.name.localeCompare(b.name);
      },
      toParamAndValue: function(aFolderOrGlodaFolder) {
        if (aFolderOrGlodaFolder instanceof GlodaFolder)
          return [null, aFolderOrGlodaFolder.id];
        else
          return [null, GlodaDatastore._mapFolder(aFolderOrGlodaFolder).id];
      }}, this.NOUN_FOLDER);
    this.defineNoun({
      name: "account",
      clazz: GlodaAccount,
      allowsArbitraryAttrs: false,
      isPrimitive: false,
      equals: function(a, b) {
        if (a && !b || !a && b)
          return false;
        if (!a && !b)
          return true;
        return a.id == b.id;
      },
      comparator: function gloda_account_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.name.localeCompare(b.name);
      }}, this.NOUN_ACCOUNT);
    this.defineNoun({
      name: "conversation",
      clazz: GlodaConversation,
      allowsArbitraryAttrs: false,
      isPrimitive: false,
      cache: true, cacheCost: 512,
      tableName: "conversations",
      attrTableName: "messageAttributes", attrIDColumnName: "conversationID",
      datastore: GlodaDatastore,
      objFromRow: GlodaDatastore._conversationFromRow,
      comparator: function gloda_conversation_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.subject.localeCompare(b.subject);
      },
      toParamAndValue: function(aConversation) {
        if (aConversation instanceof GlodaConversation)
          return [null, aConversation.id];
        else // assume they're just passing the id directly
          return [null, aConversation];
      }}, this.NOUN_CONVERSATION);
    this.defineNoun({
      name: "message",
      clazz: GlodaMessage,
      allowsArbitraryAttrs: true,
      isPrimitive: false,
      cache: true, cacheCost: 2048,
      tableName: "messages",
      // we will always have a fulltext row, even for messages where we don't
      //  have the body available.  this is because we want the subject indexed.
      dbQueryJoinMagic:
        " INNER JOIN messagesText ON messages.id = messagesText.rowid",
      attrTableName: "messageAttributes", attrIDColumnName: "messageID",
      datastore: GlodaDatastore, objFromRow: GlodaDatastore._messageFromRow,
      dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
      dbQueryValidityConstraintSuffix:
        " AND +deleted = 0 AND +folderID IS NOT NULL AND +messageKey IS NOT NULL",
      // This is what's used when we have no validity constraints, i.e. we allow
      // for ghost messages, which do not have a row in the messagesText table.
      dbQueryJoinMagicWithNoValidityConstraints:
        " LEFT JOIN messagesText ON messages.id = messagesText.rowid",
      objInsert: GlodaDatastore.insertMessage,
      objUpdate: GlodaDatastore.updateMessage,
      toParamAndValue: function(aMessage) {
        if (aMessage instanceof GlodaMessage)
          return [null, aMessage.id];
        else // assume they're just passing the id directly
          return [null, aMessage];
      }}, this.NOUN_MESSAGE);
    this.defineNoun({
      name: "contact",
      clazz: GlodaContact,
      allowsArbitraryAttrs: true,
      isPrimitive: false,
      cache: true, cacheCost: 128,
      tableName: "contacts",
      attrTableName: "contactAttributes", attrIDColumnName: "contactID",
      datastore: GlodaDatastore, objFromRow: GlodaDatastore._contactFromRow,
      dbAttribAdjuster: GlodaDatastore.adjustAttributes,
      objInsert: GlodaDatastore.insertContact,
      objUpdate: GlodaDatastore.updateContact,
      comparator: function gloda_contact_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.name.localeCompare(b.name);
      },
      toParamAndValue: function(aContact) {
        if (aContact instanceof GlodaContact)
          return [null, aContact.id];
        else // assume they're just passing the id directly
          return [null, aContact];
      }}, this.NOUN_CONTACT);
    this.defineNoun({
      name: "identity",
      clazz: GlodaIdentity,
      allowsArbitraryAttrs: false,
      isPrimitive: false,
      cache: true, cacheCost: 128,
      usesUniqueValue: true,
      tableName: "identities",
      datastore: GlodaDatastore, objFromRow: GlodaDatastore._identityFromRow,
      /**
       * Short string is the contact name, long string includes the identity
       *  value too, delimited by a colon.  Not tremendously localizable.
       */
      userVisibleString: function(aIdentity, aLong) {
        if (!aLong)
          return aIdentity.contact.name;
        if (aIdentity.contact.name == aIdentity.value)
          return aIdentity.value;
        return aIdentity.contact.name + " (" + aIdentity.value + ")";
      },
      comparator: function gloda_identity_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        return a.contact.name.localeCompare(b.contact.name);
      },
      toParamAndValue: function(aIdentity) {
        if (aIdentity instanceof GlodaIdentity)
          return [null, aIdentity.id];
        else // assume they're just passing the id directly
          return [null, aIdentity];
      }}, this.NOUN_IDENTITY);
    this.defineNoun({
      name: "attachment-infos",
      clazz: GlodaAttachment,
      allowsArbitraryAttrs: false,
      isPrimitive: false,
      toJSON: function (x) {
          return [
            x._name,
            x._contentType,
            x._size,
            x._part,
            x._externalUrl,
            x._isExternal
          ]
        },
      fromJSON: function (x, aGlodaMessage) {
          let [name, contentType, size, _part, _externalUrl, isExternal] = x;
          return new GlodaAttachment(aGlodaMessage, name, contentType, size, _part, _externalUrl, isExternal);
        },
      }, this.NOUN_ATTACHMENT);

    // parameterized identity is just two identities; we store the first one
    //  (whose value set must be very constrainted, like the 'me' identities)
    //  as the parameter, the second (which does not need to be constrained)
    //  as the value.
    this.defineNoun({
      name: "parameterized-identity",
      clazz: null,
      allowsArbitraryAttrs: false,
      comparator: function gloda_fulltext_comparator(a, b) {
        if (a == null) {
          if (b == null)
            return 0;
          else
            return 1;
        }
        else if (b == null) {
          return -1;
        }
        // First sort by the first identity in the tuple
        // Since our general use-case is for the first guy to be "me", we only
        //  compare the identity value, not the name.
        let fic = a[0].value.localeCompare(b[0].value);
        if (fic)
          return fic;
        // Next compare the second identity in the tuple, but use the contact
        //  this time to be consistent with our identity comparator.
        return a[1].contact.name.localeCompare(b[1].contact.name);
      },
      computeDelta: function(aCurValues, aOldValues) {
        let oldMap = {};
        for (let tupe of aOldValues) {
          let [originIdentity, targetIdentity] = tupe;
          let targets = oldMap[originIdentity];
          if (targets === undefined)
            targets = oldMap[originIdentity] = {};
          targets[targetIdentity] = true;
        }

        let added = [], removed = [];
        for (let tupe of aCurValues) {
          let [originIdentity, targetIdentity] = tupe;
          let targets = oldMap[originIdentity];
          if ((targets === undefined) || !(targetIdentity in targets))
            added.push(tupe);
          else
            delete targets[targetIdentity];
        }

        for (let originIdentity in oldMap) {
          let targets = oldMap[originIdentity];
          for (let targetIdentity in targets) {
            removed.push([originIdentity, targetIdentity]);
          }
        }

        return [added, removed];
      },
      contributeObjDependencies: function(aJsonValues, aReferencesByNounID,
          aInverseReferencesByNounID) {
        // nothing to do with a zero-length list
        if (aJsonValues.length == 0)
          return false;

        let nounIdentityDef = Gloda._nounIDToDef[Gloda.NOUN_IDENTITY];
        let references = aReferencesByNounID[nounIdentityDef.id];
        if (references === undefined)
          references = aReferencesByNounID[nounIdentityDef.id] = {};

        for (let tupe of aJsonValues) {
          let [originIdentityID, targetIdentityID] = tupe;
          if (!(originIdentityID in references))
            references[originIdentityID] = null;
          if (!(targetIdentityID in references))
            references[targetIdentityID] = null;
        }

        return true;
      },
      resolveObjDependencies: function(aJsonValues, aReferencesByNounID,
          aInverseReferencesByNounID) {
        let references =
          aReferencesByNounID[Gloda.NOUN_IDENTITY];

        let results = [];
        for (let tupe of aJsonValues) {
          let [originIdentityID, targetIdentityID] = tupe;
          results.push([references[originIdentityID],
                        references[targetIdentityID]]);
        }

        return results;
      },
      toJSON: function (aIdentityTuple) {
        return [aIdentityTuple[0].id, aIdentityTuple[1].id];
      },
      toParamAndValue: function(aIdentityTuple) {
        return [aIdentityTuple[0].id, aIdentityTuple[1].id];
      }}, this.NOUN_PARAM_IDENTITY);

    GlodaDatastore.getAllAttributes();
  },

  /**
   * Create accessor functions to 'bind' an attribute to underlying normalized
   *  attribute storage, as well as creating the appropriate query object
   *  constraint helper functions.  This name is somewhat of a misnomer because
   *  special attributes are not 'bound' (because specific/non-generic per-class
   *  code provides the properties) but still depend on this method to
   *  establish their constraint helper methods.
   *
   * @XXX potentially rename to not suggest binding is required.
   */
  _bindAttribute: function gloda_ns_bindAttr(aAttrDef, aSubjectNounDef) {
    let objectNounDef = aAttrDef.objectNounDef;

    // -- the query constraint helpers
    if (aSubjectNounDef.queryClass !== undefined) {
      let constrainer;
      let canQuery = true;
      if (aAttrDef.special == this.kSpecialFulltext) {
        constrainer = function() {
          let constraint = [GlodaDatastore.kConstraintFulltext, aAttrDef];
          for (let iArg = 0; iArg < arguments.length; iArg++) {
            constraint.push(arguments[iArg]);
          }
          this._constraints.push(constraint);
          return this;
        };
      }
      else if (aAttrDef.canQuery || aAttrDef.attributeName.startsWith("_")) {
        constrainer = function() {
          let constraint = [GlodaDatastore.kConstraintIn, aAttrDef];
          for (let iArg = 0; iArg < arguments.length; iArg++) {
            constraint.push(arguments[iArg]);
          }
          this._constraints.push(constraint);
          return this;
        };
      } else {
        constrainer = function() {
          throw new Error(
              "Cannot query on attribute "+aAttrDef.attributeName
            + " because its canQuery parameter hasn't been set to true."
            + " Reading the comments about Gloda.defineAttribute may be a"
            + " sensible thing to do now.");
        }
        canQuery = false;
      }

      aSubjectNounDef.queryClass.prototype[aAttrDef.boundName] = constrainer;

      // Don't bind extra query-able attributes if we're unable to perform a
      // search on the attribute.
      if (!canQuery)
        return;

      // - ranged value helper: fooRange
      if (objectNounDef.continuous) {
        // takes one or more tuples of [lower bound, upper bound]
        let rangedConstrainer = function() {
          let constraint = [GlodaDatastore.kConstraintRanges, aAttrDef];
          for (let iArg = 0; iArg < arguments.length; iArg++ ) {
            constraint.push(arguments[iArg]);
          }
          this._constraints.push(constraint);
          return this;
        };

        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Range"] =
          rangedConstrainer;
      }

      // - string LIKE helper for special on-row attributes: fooLike
      // (it is impossible to store a string as an indexed attribute, which is
      //  why we do this for on-row only.)
      if (aAttrDef.special == this.kSpecialString) {
        let likeConstrainer = function() {
          let constraint = [GlodaDatastore.kConstraintStringLike, aAttrDef];
          for (let iArg = 0; iArg < arguments.length; iArg++) {
            constraint.push(arguments[iArg]);
          }
          this._constraints.push(constraint);
          return this;
        };

        aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Like"] =
          likeConstrainer;
      }

      // - Custom helpers provided by the noun type...
      if ("queryHelpers" in objectNounDef) {
        for (let name in objectNounDef.queryHelpers) {
          let helper = objectNounDef.queryHelpers[name];
          // we need a new closure...
          let helperFunc = helper;
          aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + name] =
            function() {
              return helperFunc.call(this, aAttrDef, arguments);
            };
        }
      }
    }
  },

  /**
   * Names of attribute-specific localized strings and the JS attribute they are
   *  exposed as in the attribute's "strings" attribute (if the provider has a
   *  string bundle exposed on its "strings" attribute).  They are rooted at
   *  "gloda.SUBJECT-NOUN-NAME.attr.ATTR-NAME.*".
   *
   * Please consult the localization notes in gloda.properties to understand
   *  what these are used for.
   */
  _ATTR_LOCALIZED_STRINGS: {
    /* - Faceting */
    facetNameLabel: "facetNameLabel",
    includeLabel: "includeLabel",
    excludeLabel: "excludeLabel",
    remainderLabel: "remainderLabel",
    mustMatchLabel: "mustMatchLabel",
    cantMatchLabel: "cantMatchLabel",
    mayMatchLabel: "mayMatchLabel",
    mustMatchNoneLabel: "mustMatchNoneLabel",
    mustMatchSomeLabel: "mustMatchSomeLabel",
    mayMatchAnyLabel: "mayMatchAnyLabel",
  },
  /**
   * Define an attribute and all its meta-data.  Takes a single dictionary as
   *  its argument, with the following required properties:
   *
   * @param aAttrDef.provider The object instance providing a 'process' method.
   * @param aAttrDef.extensionName The name of the extension providing these
   *     attributes.
   * @param aAttrDef.attributeType The type of attribute, one of the values from
   *     the kAttr* enumeration.
   * @param aAttrDef.attributeName The name of the attribute, which also doubles
   *     as the bound property name if you pass 'bind' a value of true.  You are
   *     responsible for avoiding collisions, which presumably will mean
   *     checking/updating a wiki page in the future, or just prefixing your
   *     attribute name with your extension name or something like that.
   * @param aAttrDef.bind Should this attribute be 'bound' as a convenience
   *     attribute on the subject's object (true/false)?  For example, with an
   *     attributeName of "foo" and passing true for 'bind' with a subject noun
   *     of NOUN_MESSAGE, GlodaMessage instances will expose a "foo" getter that
   *     returns the value of the attribute.  If 'singular' is true, this means
   *     an instance of the object class corresponding to the noun type or null
   *     if the attribute does not exist.  If 'singular' is false, this means a
   *     list of instances of the object class corresponding to the noun type,
   *     where the list may be empty if no instances of the attribute are
   *     present.
   * @param aAttrDef.bindName Optional override of attributeName for purposes of
   *     the binding property's name.
   * @param aAttrDef.singular Is the attribute going to happen at most once
   *     (true), or potentially multiple times (false).  This affects whether
   *     the binding returns a list or just a single item (which is null when
   *     the attribute is not present).
   * @param [aAttrDef.emptySetIsSignificant=false] Should we
   * @param aAttrDef.subjectNouns A list of object types (NOUNs) that this
   *     attribute can be set on.  Each element in the list should be one of the
   *     NOUN_* constants or a dynamically registered noun type.
   * @param aAttrDef.objectNoun The object type (one of the NOUN_* constants or
   *     a dynamically registered noun types) that is the 'object' in the
   *     traditional RDF triple.  More pragmatically, in the database row used
   *     to represent an attribute, we store the subject (ex: message ID),
   *     attribute ID, and an integer which is the integer representation of the
   *     'object' whose type you are defining right here.
   */
  defineAttribute: function gloda_ns_defineAttribute(aAttrDef) {
    // ensure required properties exist on aAttrDef
    if (!("provider" in aAttrDef) ||
        !("extensionName" in aAttrDef) ||
        !("attributeType" in aAttrDef) ||
        !("attributeName" in aAttrDef) ||
        !("singular" in aAttrDef) ||
        !("subjectNouns" in aAttrDef) ||
        !("objectNoun" in aAttrDef))
      // perhaps we should have a list of required attributes, perchance with
      //  and explanation of what it holds, and use that to be friendlier?
      throw Error("You omitted a required attribute defining property, please" +
                  " consult the documentation as penance.");

    // -- Fill in defaults
    if (!("emptySetIsSignificant" in aAttrDef))
      aAttrDef.emptySetIsSignificant = false;

    if (!("canQuery" in aAttrDef))
      aAttrDef.canQuery = aAttrDef.facet ? true : false;

    // return if the attribute has already been defined
    if (aAttrDef.dbDef)
      return aAttrDef;

    // - first time we've seen a provider init logic
    if (!(aAttrDef.provider.providerName in this._attrProviders)) {
      this._attrProviders[aAttrDef.provider.providerName] = [];
      if (aAttrDef.provider.contentWhittle)
        whittlerRegistry.registerWhittler(aAttrDef.provider);
    }

    let compoundName = aAttrDef.extensionName + ":" + aAttrDef.attributeName;
    // -- Database Definition
    let attrDBDef;
    if (compoundName in GlodaDatastore._attributeDBDefs) {
      // the existence of the GlodaAttributeDBDef means that either it has
      //  already been fully defined, or has been loaded from the database but
      //  not yet 'bound' to a provider (and had important meta-info that
      //  doesn't go in the db copied over)
      attrDBDef = GlodaDatastore._attributeDBDefs[compoundName];
    }
    // we need to create the attribute definition in the database
    else {
      let attrID = null;
      attrID = GlodaDatastore._createAttributeDef(aAttrDef.attributeType,
                                                  aAttrDef.extensionName,
                                                  aAttrDef.attributeName,
                                                  null);

      attrDBDef = new GlodaAttributeDBDef(GlodaDatastore, attrID, compoundName,
        aAttrDef.attributeType, aAttrDef.extensionName, aAttrDef.attributeName);
      GlodaDatastore._attributeDBDefs[compoundName] = attrDBDef;
      GlodaDatastore._attributeIDToDBDefAndParam[attrID] = [attrDBDef, null];
    }

    aAttrDef.dbDef = attrDBDef;
    attrDBDef.attrDef = aAttrDef;

    aAttrDef.id = aAttrDef.dbDef.id;

    if ("bindName" in aAttrDef)
      aAttrDef.boundName = aAttrDef.bindName;
    else
      aAttrDef.boundName = aAttrDef.attributeName;

    aAttrDef.objectNounDef = this._nounIDToDef[aAttrDef.objectNoun];
    aAttrDef.objectNounDef.objectNounOfAttributes.push(aAttrDef);

    // -- Facets
    function normalizeFacetDef(aFacetDef) {
      if (!("groupIdAttr" in aFacetDef))
        aFacetDef.groupIdAttr = aAttrDef.objectNounDef.idAttr;
      if (!("groupComparator" in aFacetDef))
        aFacetDef.groupComparator = aAttrDef.objectNounDef.comparator;
      if (!("filter" in aFacetDef))
        aFacetDef.filter = null;
    }
    // No facet attribute means no facet desired; set an explicit null so that
    //  code can check without doing an "in" check.
    if (!("facet" in aAttrDef))
      aAttrDef.facet = null;
    // Promote "true" facet values to the defaults.  Where attributes have
    //  specified values, make sure we fill in any missing defaults.
    else {
      if (aAttrDef.facet == true) {
        aAttrDef.facet = {
          type: "default",
          groupIdAttr: aAttrDef.objectNounDef.idAttr,
          groupComparator: aAttrDef.objectNounDef.comparator,
          filter: null,
        };
      }
      else {
        normalizeFacetDef(aAttrDef.facet);
      }
    }
    if ("extraFacets" in aAttrDef) {
      for (let facetDef of aAttrDef.extraFacets) {
        normalizeFacetDef(facetDef);
      }
    }

    function gatherLocalizedStrings(aBundle, aPropRoot, aStickIn) {
      for (let propName in Gloda._ATTR_LOCALIZED_STRINGS) {
        let attrName = Gloda._ATTR_LOCALIZED_STRINGS[propName];
        try {
          aStickIn[attrName] = aBundle.get(aPropRoot + propName);
        }
        catch (ex) {
          // do nothing.  nsIStringBundle throws exceptions because it is a
          //  standard nsresult type of API and our helper buddy does nothing
          //  to help us.  (StringBundle.js, that is.)
        }
      }
    }

    // -- L10n.
    // If the provider has a string bundle, populate a "strings" attribute with
    //  our standard attribute strings that can be UI exposed.
    if (("strings" in aAttrDef.provider) && (aAttrDef.facet)) {
      let bundle = aAttrDef.provider.strings;

      // -- attribute strings
      let attrStrings = aAttrDef.facet.strings = {};
      // we use the first subject the attribute applies to as the basis of
      //  where to get the string from.  Mainly because we currently don't have
      //  any attributes with multiple subjects nor a use-case where we expose
      //  multiple noun types via the UI.  (Just messages right now.)
      let canonicalSubject = this._nounIDToDef[aAttrDef.subjectNouns[0]];
      let propRoot = "gloda." + canonicalSubject.name + ".attr." +
                       aAttrDef.attributeName + ".";
      gatherLocalizedStrings(bundle, propRoot, attrStrings);

      // -- alias strings for synthetic facets
      if ("extraFacets" in aAttrDef) {
        for (let facetDef of aAttrDef.extraFacets) {
          facetDef.strings = {};
          let aliasPropRoot = "gloda." + canonicalSubject.name + ".attr." +
                                facetDef.alias + ".";
          gatherLocalizedStrings(bundle, aliasPropRoot, facetDef.strings);
        }
      }
    }

    // -- Subject Noun Binding
    for (let iSubject = 0; iSubject < aAttrDef.subjectNouns.length;
           iSubject++) {
      let subjectType = aAttrDef.subjectNouns[iSubject];
      let subjectNounDef = this._nounIDToDef[subjectType];
      this._bindAttribute(aAttrDef, subjectNounDef);

      // update the provider maps...
      if (this._attrProviderOrderByNoun[subjectType]
              .indexOf(aAttrDef.provider) == -1) {
        this._attrProviderOrderByNoun[subjectType].push(aAttrDef.provider);
        if (aAttrDef.provider.optimize)
          this._attrOptimizerOrderByNoun[subjectType].push(aAttrDef.provider);
        this._attrProvidersByNoun[subjectType][aAttrDef.provider] = [];
      }
      this._attrProvidersByNoun[subjectType][aAttrDef.provider].push(aAttrDef);

      subjectNounDef.attribsByBoundName[aAttrDef.boundName] = aAttrDef;
      if (aAttrDef.domExpose)
        subjectNounDef.domExposeAttribsByBoundName[aAttrDef.boundName] =
          aAttrDef;

      if (aAttrDef.special & this.kSpecialColumn)
        subjectNounDef.specialLoadAttribs.push(aAttrDef);

      // if this is a parent column attribute, make note of it so that if we
      //  need to do an inverse references lookup, we know what column we are
      //  issuing against.
      if (aAttrDef.special === this.kSpecialColumnParent) {
        subjectNounDef.parentColumnAttr = aAttrDef;
      }

      if (aAttrDef.objectNounDef.tableName ||
          aAttrDef.objectNounDef.contributeObjDependencies) {
        subjectNounDef.hasObjDependencies = true;
      }
    }

    this._attrProviders[aAttrDef.provider.providerName].push(aAttrDef);
    return aAttrDef;
  },

  /**
   * Retrieve the attribute provided by the given extension with the given
   *  attribute name.  The original idea was that plugins would effectively
   *  name-space attributes, helping avoid collisions.  Since we are leaning
   *  towards using binding heavily, this doesn't really help, as the collisions
   *  will just occur on the attribute name instead.  Also, this can turn
   *  extensions into liars as name changes/moves to core/etc. happen.
   * @TODO consider removing the extension name argument parameter requirement
   */
  getAttrDef: function gloda_ns_getAttrDef(aPluginName, aAttrName) {
    let compoundName = aPluginName + ":" + aAttrName;
    return GlodaDatastore._attributeDBDefs[compoundName];
  },

  /**
   * Create a new query instance for the given noun-type.  This provides
   *  a generic way to provide constraint-based queries of any first-class
   *  nouns supported by the system.
   *
   * The idea is that every attribute on an object can be used to express
   *  a constraint on the query object.  Constraints implicitly 'AND' together,
   *  but providing multiple arguments to a constraint function results in an
   *  'OR'ing of those values.  Additionally, you can call or() on the returned
   *  query to create an alternate query that is effectively a giant OR against
   *  all the constraints you create on the main query object (or any other
   *  alternate queries returned by or()).  (Note: there is no nesting of these
   *  alternate queries. query.or().or() is equivalent to query.or())
   * For each attribute, there is a constraint with the same name that takes
   *  one or more arguments.  The arguments represent a set of OR values that
   *  objects matching the query can have.  (If you want the constraint
   *  effectively ANDed together, just invoke the constraint function
   *  multiple times.)  For example, newQuery(NOUN_PERSON).age(25) would
   *  constraint to all the people aged 25, while age(25, 26) would constrain
   *  to all the people age 25 or 26.
   * For each attribute with a 'continuous' noun, there is a constraint with the
   *  attribute name with "Range" appended.  It takes two arguments which are an
   *  inclusive lower bound and an inclusive lower bound for values in the
   *  range.  If you would like an open-ended range on either side, pass null
   *  for that argument.  If you would like to specify multiple ranges that
   *  should be ORed together, simply pass additional (pairs of) arguments.
   *  For example, newQuery(NOUN_PERSON).age(25,100) would constraint to all
   *  the people who are >= 25 and <= 100.  Likewise age(25, null) would just
   *  return all the people who are 25 or older.  And age(25,30,35,40) would
   *  return people who are either 25-30 or 35-30.
   * There are also full-text constraint columns.  In a nutshell, their
   *  arguments are the strings that should be passed to the SQLite FTS3
   *  MATCH clause.
   *
   * @param aNounID The (integer) noun-id of the noun you want to query on.
   * @param aOptions an optional dictionary of query options, see the GlodaQuery
   *     class documentation.
   */
  newQuery: function gloda_ns_newQuery(aNounID, aOptions) {
    let nounDef = this._nounIDToDef[aNounID];
    return new nounDef.queryClass(aOptions);
  },

  /**
   * Create a collection/query for the given noun-type that only matches the
   *  provided items.  This is to be used when you have an explicit set of items
   *  that you would still like to receive updates for.
   */
  explicitCollection: function gloda_ns_explicitCollection(aNounID, aItems) {
    let nounDef = this._nounIDToDef[aNounID];
    let collection = new GlodaCollection(nounDef, aItems, null, null);
    let query = new nounDef.explicitQueryClass(collection);
    collection.query = query;
    GlodaCollectionManager.registerCollection(collection);
    return collection;
  },

  /**
   * Debugging 'wildcard' collection creation support.  A wildcard collection
   *  will 'accept' any new item instances presented to the collection manager
   *  as new.  The result is that it allows you to be notified as new items
   *  as they are indexed, existing items as they are loaded from the database,
   *  etc.
   * Because the items are added to the collection without limit, this will
   *  result in a leak if you don't do something to clean up after the
   *  collection.  (Forgetting about the collection will suffice, as it is still
   *  weakly held.)
   */
  _wildcardCollection: function gloda_ns_wildcardCollection(aNounID, aItems) {
    let nounDef = this._nounIDToDef[aNounID];
    let collection = new GlodaCollection(nounDef, aItems, null, null);
    let query = new nounDef.wildcardQueryClass(collection);
    collection.query = query;
    GlodaCollectionManager.registerCollection(collection);
    return collection;
  },

  /**
   * Attribute providers attempting to index something that experience a fatal
   *  problem should throw one of these.  For example:
   *  "throw new Gloda.BadItemContentsError('Message lacks an author.');".
   *
   * We're not really taking advantage of this yet, but it's a good idea.
   */
  BadItemContentsError: BadItemContentsError,

  /**
   * Populate a gloda representation of an item given the thus-far built
   *  representation, the previous representation, and one or more raw
   *  representations.  The attribute providers/optimizers for the given noun
   *  type are invoked, allowing them to contribute/alter things.  Following
   *  that, we build and persist our attribute representations.
   *
   * The result of the processing ends up with attributes in 3 different forms:
   * - Database attribute rows (to be added and removed).
   * - In-memory representation.
   * - JSON-able representation.
   *
   * @param aItem The noun instance you want processed.
   * @param aRawReps A dictionary that we pass to the attribute providers.
   *     There is a(n implied) contract between the caller of grokNounItem for a
   *     given noun type and the attribute providers for that noun type, and we
   *     have nothing to do with it OTHER THAN inserting a 'trueGlodaRep'
   *     value into it.  In the event of reindexing an existing object, the
   *     gloda representation we pass to the indexers is actually a clone that
   *     allows the asynchronous indexers to mutate the object without
   *     causing visible changes in the existing representation of the gloda
   *     object.  We patch the changes back onto the original item atomically
   *     once indexing completes.  The 'trueGlodaRep' is then useful for
   *     objects that hang off of the gloda instance that need a reference
   *     back to their containing object for API convenience purposes.
   * @param aIsConceptuallyNew Is the item "new" in the sense that it would
   *     never have been visible from within user code?  This translates into
   *     whether this should trigger an itemAdded notification or an
   *     itemModified notification.
   * @param aIsRecordNew Is the item "new" in the sense that we should INSERT
   *     a record rather than UPDATE-ing a record.  For example, when dealing
   *     with messages where we may have a ghost, the ghost message is not a
   *     new record, but is conceptually new.
   * @param aCallbackHandle The GlodaIndexer-style callback handle that is being
   *     used to drive this processing in an async fashion.  (See
   *     GlodaIndexer._callbackHandle).
   * @param aDoCache Should we allow this item to be contributed to its noun
   *     cache?
   */
  grokNounItem: function* gloda_ns_grokNounItem(aItem, aRawReps,
      aIsConceptuallyNew, aIsRecordNew, aCallbackHandle, aDoCache) {
    let itemNounDef = aItem.NOUN_DEF;
    let attribsByBoundName = itemNounDef.attribsByBoundName;

    this._log.info(" ** grokNounItem: " + itemNounDef.name);

    let addDBAttribs = [];
    let removeDBAttribs = [];

    let jsonDict = {};

    let aOldItem;
    aRawReps.trueGlodaRep = aItem;
    if (aIsConceptuallyNew) // there is no old item if we are new.
      aOldItem = {};
    else {
      aOldItem = aItem;
      // we want to create a clone of the existing item so that we can know the
      //  deltas that happened for indexing purposes
      aItem = aItem._clone();
    }

    // Have the attribute providers directly set properties on the aItem
    let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
    for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
      this._log.info("  * provider: " + attrProviders[iProvider].providerName);
      yield aCallbackHandle.pushAndGo(
        attrProviders[iProvider].process(aItem, aRawReps, aIsConceptuallyNew,
                                         aCallbackHandle));
    }

    let attrOptimizers = this._attrOptimizerOrderByNoun[itemNounDef.id];
    for (let iProvider = 0; iProvider < attrOptimizers.length; iProvider++) {
      this._log.info("  * optimizer: " + attrOptimizers[iProvider].providerName);
      yield aCallbackHandle.pushAndGo(
        attrOptimizers[iProvider].optimize(aItem, aRawReps, aIsConceptuallyNew,
                                           aCallbackHandle));
    }
    this._log.info(" ** done with providers.");

    // Iterate over the attributes on the item
    for (let key of Object.keys(aItem)) {
      let value = aItem[key];
      // ignore keys that start with underscores, they are private and not
      //  persisted by our attribute mechanism.  (they are directly handled by
      //  the object implementation.)
      if (key.startsWith("_"))
        continue;
      // find the attribute definition that corresponds to this key
      let attrib = attribsByBoundName[key];
      // if there's no attribute, that's not good, but not horrible.
      if (attrib === undefined) {
        this._log.warn("new proc ignoring attrib: " + key);
        continue;
      }

      let attribDB = attrib.dbDef;
      let objectNounDef = attrib.objectNounDef;

      // - translate for our JSON rep
      if (attrib.singular) {
        if (objectNounDef.toJSON)
          jsonDict[attrib.id] = objectNounDef.toJSON(value);
        else
          jsonDict[attrib.id] = value;
      }
      else {
        if (objectNounDef.toJSON) {
          let toJSON = objectNounDef.toJSON;
          jsonDict[attrib.id] = [];
          for (let [, subValue] in Iterator(value)) {
            jsonDict[attrib.id].push(toJSON(subValue));
          }
        }
        else
          jsonDict[attrib.id] = value;
      }

      let oldValue = aOldItem[key];

      // the 'old' item is still the canonical one; update it
      // do the update now, because we may skip operations on addDBAttribs and
      //  removeDBattribs, if the attribute is not to generate entries in
      //  messageAttributes
      if (oldValue !== undefined || !aIsConceptuallyNew)
        aOldItem[key] = value;

      // the new canQuery property has to be set to true to generate entries
      // in the messageAttributes table. Any other truthy value (like a non
      // empty string), will still make the message query-able but without
      // using the database.
      if (attrib.canQuery !== true) {
        continue;
      }

      // - database index attributes

      // perform a delta analysis against the old value, if we have one
      if (oldValue !== undefined) {
        // in the singular case if they don't match, it's one add and one remove
        if (attrib.singular) {
          // test for identicality, failing that, see if they have explicit
          //  equals support.
          if ((value !== oldValue) &&
              (!value.equals || !value.equals(oldValue))) {
            addDBAttribs.push(attribDB.convertValuesToDBAttributes([value])[0]);
            removeDBAttribs.push(
              attribDB.convertValuesToDBAttributes([oldValue])[0]);
          }
        }
        // in the plural case, we have to figure the deltas accounting for
        //  possible changes in ordering (which is insignificant from an
        //  indexing perspective)
        // some nouns may not meet === equivalence needs, so must provide a
        //  custom computeDelta method to help us out
        else if (objectNounDef.computeDelta) {
          let [valuesAdded, valuesRemoved] =
            objectNounDef.computeDelta(value, oldValue);
          // convert the values to database-style attribute rows
          addDBAttribs.push.apply(addDBAttribs,
            attribDB.convertValuesToDBAttributes(valuesAdded));
          removeDBAttribs.push.apply(removeDBAttribs,
            attribDB.convertValuesToDBAttributes(valuesRemoved));
        }
        else {
          // build a map of the previous values; we will delete the values as
          //  we see them so that we will know what old values are no longer
          //  present in the current set of values.
          let oldValueMap = {};
          for (let anOldValue of oldValue) {
            // remember, the key is just the toString'ed value, so we need to
            //  store and use the actual value as the value!
            oldValueMap[anOldValue] = anOldValue;
          }
          // traverse the current values...
          let valuesAdded = [];
          for (let curValue of value) {
            if (curValue in oldValueMap)
              delete oldValueMap[curValue];
            else
              valuesAdded.push(curValue);
          }
          // anything still on oldValueMap was removed.
          let valuesRemoved = Object.keys(oldValueMap).
            map(key => oldValueMap[key]);
          // convert the values to database-style attribute rows
          addDBAttribs.push.apply(addDBAttribs,
            attribDB.convertValuesToDBAttributes(valuesAdded));
          removeDBAttribs.push.apply(removeDBAttribs,
            attribDB.convertValuesToDBAttributes(valuesRemoved));
        }

        // Add/remove the empty set indicator as appropriate.
        if (attrib.emptySetIsSignificant) {
          // if we are now non-zero but previously were zero, remove.
          if (value.length && !oldValue.length)
            removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
          // if we are now zero length but previously were not, add
          else if (!value.length && oldValue.length)
            addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
        }
      }
      // no old value, all values are new
      else {
        // add the db reps on the new values
        if (attrib.singular)
          value = [value];
        addDBAttribs.push.apply(addDBAttribs,
                                attribDB.convertValuesToDBAttributes(value));
        // Add the empty set indicator for the attribute id if appropriate.
        if (!value.length && attrib.emptySetIsSignificant)
          addDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
      }
    }

    // Iterate over any remaining values in old items for purge purposes.
    for (let key of Object.keys(aOldItem)) {
      let value = aOldItem[key];
      // ignore keys that start with underscores, they are private and not
      //  persisted by our attribute mechanism.  (they are directly handled by
      //  the object implementation.)
      if (key.startsWith("_"))
        continue;
      // ignore things we saw in the new guy
      if (key in aItem)
        continue;

      // find the attribute definition that corresponds to this key
      let attrib = attribsByBoundName[key];
      // if there's no attribute, that's not good, but not horrible.
      if (attrib === undefined) {
        continue;
      }

      // delete these from the old item, as the old item is canonical, and
      //  should no longer have these values
      delete aOldItem[key];

      if (attrib.canQuery !== true) {
        this._log.debug("Not inserting attribute "+attrib.attributeName
            +" into the db, since we don't plan on querying on it");
        continue;
      }

      if (attrib.singular)
        value = [value];
      let attribDB = attrib.dbDef;
      removeDBAttribs.push.apply(removeDBAttribs,
                                 attribDB.convertValuesToDBAttributes(value));
      // remove the empty set marker if there should have been one
      if (!value.length && attrib.emptySetIsSignificant)
        removeDBAttribs.push([GlodaDatastore.kEmptySetAttrId, attribDB.id]);
    }

    aItem._jsonText = JSON.stringify(jsonDict);
    this._log.debug("  json text: " + aItem._jsonText);

    if (aIsRecordNew) {
      this._log.debug(" inserting item");
      itemNounDef.objInsert.call(itemNounDef.datastore, aItem);
    }
    else {
      this._log.debug(" updating item");
      itemNounDef.objUpdate.call(itemNounDef.datastore, aItem);
    }

    this._log.debug(" adjusting attributes, add: " + addDBAttribs + " rem: " +
        removeDBAttribs);
    itemNounDef.dbAttribAdjuster.call(itemNounDef.datastore, aItem,
      addDBAttribs, removeDBAttribs);

    if (!aIsConceptuallyNew && ("_declone" in aOldItem))
      aOldItem._declone(aItem);

    // Cache ramifications...
    if (aDoCache === undefined || aDoCache) {
      if (aIsConceptuallyNew)
        GlodaCollectionManager.itemsAdded(aItem.NOUN_ID, [aItem]);
      else
        GlodaCollectionManager.itemsModified(aOldItem.NOUN_ID, [aOldItem]);
    }

    this._log.debug(" done grokking.");

    yield this.kWorkDone;
  },

  /**
   * Processes a list of noun instances for their score within a given context.
   *  This is primarily intended for use by search ranking mechanisms, but could
   *  be used elsewhere too.  (It does, however, depend on the complicity of the
   *  score method implementations to not get confused.)
   *
   * @param aItems The non-empty list of items to score.
   * @param aContext A noun-specific dictionary that we just pass to the funcs.
   * @param aExtraScoreFuncs A list of extra scoring functions to apply.
   * @returns A list of integer scores equal in length to aItems.
   */
  scoreNounItems: function gloda_ns_grokNounItem(aItems, aContext,
                                                 aExtraScoreFuncs) {
    let scores = [];
    // bail if there is nothing to score
    if (!aItems.length)
      return scores;

    let itemNounDef = aItems[0].NOUN_DEF;
    if (aExtraScoreFuncs == null)
      aExtraScoreFuncs = [];

    for (let item of aItems) {
      let score = 0;
      let attrProviders = this._attrProviderOrderByNoun[itemNounDef.id];
      for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
        let provider = attrProviders[iProvider];
        if (provider.score)
          score += provider.score(item);
      }
      for (let [, extraScoreFunc] in Iterator(aExtraScoreFuncs))
        score += extraScoreFunc(item, aContext);
      scores.push(score);
    }

    return scores;
  }
};

/* and initialize the Gloda object/NS before we return... */
try {
  Gloda._init();
}
catch (ex) {
  Gloda._log.debug("Exception during Gloda init (" + ex.fileName + ":" +
                   ex.lineNumber + "): " + ex);
};
/* but don't forget that we effectively depend on everybody.js too, and
   currently on our importer to be importing that if they need us fully armed
   and operational. */