summaryrefslogtreecommitdiffstats
path: root/toolkit/mozapps/extensions/internal/AddonRepository.sys.mjs
blob: e53e4af7a4834a932d73d81297cb28de8ff06d92 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";

const lazy = {};

ChromeUtils.defineESModuleGetters(lazy, {
  AddonManager: "resource://gre/modules/AddonManager.sys.mjs",
  AddonManagerPrivate: "resource://gre/modules/AddonManager.sys.mjs",
  AsyncShutdown: "resource://gre/modules/AsyncShutdown.sys.mjs",
  DeferredTask: "resource://gre/modules/DeferredTask.sys.mjs",
  NetUtil: "resource://gre/modules/NetUtil.sys.mjs",
  ServiceRequest: "resource://gre/modules/ServiceRequest.sys.mjs",
});

// The current platform as specified in the AMO API:
// http://addons-server.readthedocs.io/en/latest/topics/api/addons.html#addon-detail-platform
ChromeUtils.defineLazyGetter(lazy, "PLATFORM", () => {
  let platform = Services.appinfo.OS;
  switch (platform) {
    case "Darwin":
      return "mac";

    case "Linux":
      return "linux";

    case "Android":
      return "android";

    case "WINNT":
      return "windows";
  }
  return platform;
});

const PREF_GETADDONS_CACHE_ENABLED = "extensions.getAddons.cache.enabled";

XPCOMUtils.defineLazyPreferenceGetter(
  lazy,
  "getAddonsCacheEnabled",
  PREF_GETADDONS_CACHE_ENABLED
);

const PREF_GETADDONS_CACHE_TYPES = "extensions.getAddons.cache.types";
const PREF_GETADDONS_CACHE_ID_ENABLED =
  "extensions.%ID%.getAddons.cache.enabled";
const PREF_GETADDONS_BROWSEADDONS = "extensions.getAddons.browseAddons";
const PREF_GETADDONS_BYIDS = "extensions.getAddons.get.url";
const PREF_GETADDONS_BROWSESEARCHRESULTS =
  "extensions.getAddons.search.browseURL";
const PREF_GETADDONS_DB_SCHEMA = "extensions.getAddons.databaseSchema";
const PREF_GET_LANGPACKS = "extensions.getAddons.langpacks.url";
const PREF_GET_BROWSER_MAPPINGS = "extensions.getAddons.browserMappings.url";

const PREF_METADATA_LASTUPDATE = "extensions.getAddons.cache.lastUpdate";
const PREF_METADATA_UPDATETHRESHOLD_SEC =
  "extensions.getAddons.cache.updateThreshold";
const DEFAULT_METADATA_UPDATETHRESHOLD_SEC = 172800; // two days

const DEFAULT_CACHE_TYPES = "extension,theme,locale,dictionary";

const FILE_DATABASE = "addons.json";
const DB_SCHEMA = 6;
const DB_MIN_JSON_SCHEMA = 5;
const DB_BATCH_TIMEOUT_MS = 50;

const BLANK_DB = function () {
  return {
    addons: new Map(),
    schema: DB_SCHEMA,
  };
};

import { Log } from "resource://gre/modules/Log.sys.mjs";

const LOGGER_ID = "addons.repository";

// Create a new logger for use by the Addons Repository
// (Requires AddonManager.sys.mjs)
var logger = Log.repository.getLogger(LOGGER_ID);

function convertHTMLToPlainText(html) {
  if (!html) {
    return html;
  }
  var converter = Cc[
    "@mozilla.org/widget/htmlformatconverter;1"
  ].createInstance(Ci.nsIFormatConverter);

  var input = Cc["@mozilla.org/supports-string;1"].createInstance(
    Ci.nsISupportsString
  );
  input.data = html.replace(/\n/g, "<br>");

  var output = {};
  converter.convert("text/html", input, "text/plain", output);

  if (output.value instanceof Ci.nsISupportsString) {
    return output.value.data.replace(/\r\n/g, "\n");
  }
  return html;
}

async function getAddonsToCache(aIds) {
  let types = Services.prefs.getStringPref(
    PREF_GETADDONS_CACHE_TYPES,
    DEFAULT_CACHE_TYPES
  );

  types = types.split(",");

  let addons = await lazy.AddonManager.getAddonsByIDs(aIds);
  let enabledIds = [];

  for (let [i, addon] of addons.entries()) {
    var preference = PREF_GETADDONS_CACHE_ID_ENABLED.replace("%ID%", aIds[i]);
    // If the preference doesn't exist caching is enabled by default
    if (!Services.prefs.getBoolPref(preference, true)) {
      continue;
    }

    // The add-ons manager may not know about this ID yet if it is a pending
    // install. In that case we'll just cache it regardless

    // Don't cache add-ons of the wrong types
    if (addon && !types.includes(addon.type)) {
      continue;
    }

    // Don't cache system add-ons
    if (addon && addon.isSystem) {
      continue;
    }

    enabledIds.push(aIds[i]);
  }

  return enabledIds;
}

function AddonSearchResult(aId) {
  this.id = aId;
  this.icons = {};
  this._unsupportedProperties = {};
}

AddonSearchResult.prototype = {
  /**
   * The ID of the add-on
   */
  id: null,

  /**
   * The add-on type (e.g. "extension" or "theme")
   */
  type: null,

  /**
   * The name of the add-on
   */
  name: null,

  /**
   * The version of the add-on
   */
  version: null,

  /**
   * The creator of the add-on
   */
  creator: null,

  /**
   * The developers of the add-on
   */
  developers: null,

  /**
   * A short description of the add-on
   */
  description: null,

  /**
   * The full description of the add-on
   */
  fullDescription: null,

  /**
   * The end-user licensing agreement (EULA) of the add-on
   */
  eula: null,

  /**
   * The url of the add-on's icon
   */
  get iconURL() {
    return this.icons && this.icons[32];
  },

  /**
   * The URLs of the add-on's icons, as an object with icon size as key
   */
  icons: null,

  /**
   * An array of screenshot urls for the add-on
   */
  screenshots: null,

  /**
   * The homepage for the add-on
   */
  homepageURL: null,

  /**
   * The support URL for the add-on
   */
  supportURL: null,

  /**
   * The contribution url of the add-on
   */
  contributionURL: null,

  /**
   * The rating of the add-on, 0-5
   */
  averageRating: null,

  /**
   * The number of reviews for this add-on
   */
  reviewCount: null,

  /**
   * The URL to the list of reviews for this add-on
   */
  reviewURL: null,

  /**
   * The number of times the add-on was downloaded the current week
   */
  weeklyDownloads: null,

  /**
   * The URL to the AMO detail page of this (listed) add-on
   */
  amoListingURL: null,

  /**
   * AddonInstall object generated from the add-on XPI url
   */
  install: null,

  /**
   * nsIURI storing where this add-on was installed from
   */
  sourceURI: null,

  /**
   * The Date that the add-on was most recently updated
   */
  updateDate: null,

  toJSON() {
    let json = {};

    for (let property of Object.keys(this)) {
      let value = this[property];
      if (property.startsWith("_") || typeof value === "function") {
        continue;
      }

      try {
        switch (property) {
          case "sourceURI":
            json.sourceURI = value ? value.spec : "";
            break;

          case "updateDate":
            json.updateDate = value ? value.getTime() : "";
            break;

          default:
            json[property] = value;
        }
      } catch (ex) {
        logger.warn("Error writing property value for " + property);
      }
    }

    for (let property of Object.keys(this._unsupportedProperties)) {
      let value = this._unsupportedProperties[property];
      if (!property.startsWith("_")) {
        json[property] = value;
      }
    }

    return json;
  },
};

/**
 * The add-on repository is a source of add-ons that can be installed. It can
 * be searched in three ways. The first takes a list of IDs and returns a
 * list of the corresponding add-ons. The second returns a list of add-ons that
 * come highly recommended. This list should change frequently. The third is to
 * search for specific search terms entered by the user. Searches are
 * asynchronous and results should be passed to the provided callback object
 * when complete. The results passed to the callback should only include add-ons
 * that are compatible with the current application and are not already
 * installed.
 */
export var AddonRepository = {
  /**
   * The homepage for visiting this repository. If the corresponding preference
   * is not defined, defaults to about:blank.
   */
  get homepageURL() {
    let url = this._formatURLPref(PREF_GETADDONS_BROWSEADDONS, {});
    return url != null ? url : "about:blank";
  },

  get appIsShuttingDown() {
    return Services.startup.shuttingDown;
  },

  /**
   * Retrieves the url that can be visited to see search results for the given
   * terms. If the corresponding preference is not defined, defaults to
   * about:blank.
   *
   * @param  aSearchTerms
   *         Search terms used to search the repository
   */
  getSearchURL(aSearchTerms) {
    let url = this._formatURLPref(PREF_GETADDONS_BROWSESEARCHRESULTS, {
      TERMS: aSearchTerms,
    });
    return url != null ? url : "about:blank";
  },

  /**
   * Whether caching is currently enabled
   */
  get cacheEnabled() {
    return lazy.getAddonsCacheEnabled;
  },

  /**
   * Shut down AddonRepository
   * return: promise{integer} resolves with the result of flushing
   *         the AddonRepository database
   */
  shutdown() {
    return AddonDatabase.shutdown(false);
  },

  metadataAge() {
    let now = Math.round(Date.now() / 1000);
    let lastUpdate = Services.prefs.getIntPref(PREF_METADATA_LASTUPDATE, 0);
    return Math.max(0, now - lastUpdate);
  },

  isMetadataStale() {
    let threshold = Services.prefs.getIntPref(
      PREF_METADATA_UPDATETHRESHOLD_SEC,
      DEFAULT_METADATA_UPDATETHRESHOLD_SEC
    );
    return this.metadataAge() > threshold;
  },

  /**
   * Asynchronously get a cached add-on by id. The add-on (or null if the
   * add-on is not found) is passed to the specified callback. If caching is
   * disabled, null is passed to the specified callback.
   *
   * The callback variant exists only for existing code in XPIProvider.sys.mjs
   * and XPIDatabase.sys.mjs that requires a synchronous callback, yuck.
   *
   * @param  aId
   *         The id of the add-on to get
   */
  async getCachedAddonByID(aId, aCallback) {
    if (!aId || !this.cacheEnabled) {
      if (aCallback) {
        aCallback(null);
      }
      return null;
    }

    if (aCallback && AddonDatabase._loaded) {
      let addon = AddonDatabase.getAddon(aId);
      aCallback(addon);
      return addon;
    }

    await AddonDatabase.openConnection();

    let addon = AddonDatabase.getAddon(aId);
    if (aCallback) {
      aCallback(addon);
    }
    return addon;
  },

  /*
   * Clear and delete the AddonRepository database
   * @return Promise{null} resolves when the database is deleted
   */
  _clearCache() {
    return AddonDatabase.delete().then(() =>
      lazy.AddonManagerPrivate.updateAddonRepositoryData()
    );
  },

  /*
   * Create a ServiceRequest instance.
   * @return ServiceRequest returns a ServiceRequest instance.
   */
  _createServiceRequest() {
    return new lazy.ServiceRequest({ mozAnon: true });
  },

  /**
   * Fetch data from an API where the results may span multiple "pages".
   * This function will take care of issuing multiple requests until all
   * the results have been fetched, and will coalesce them all into a
   * single return value.  The handling here is specific to the way AMO
   * implements paging (ie a JSON result with a "next" property).
   *
   * @param {string} pref
   *                 The pref name that contains the API URL to call.
   * @param {object} params
   *                 A key-value object that contains the parameters to replace
   *                 in the API URL.
   * @param {function} handler
   *                   This function will be called once per page of results,
   *                   it should return an array of objects (the type depends
   *                   on the particular API being called of course).
   *
   * @returns Promise{array} An array of all the individual results from
   *                         the API call(s).
   */
  _fetchPaged(pref, params, handler) {
    const startURL = this._formatURLPref(pref, params);

    let results = [];
    const fetchNextPage = url => {
      return new Promise((resolve, reject) => {
        if (this.appIsShuttingDown) {
          logger.debug(
            "Rejecting AddonRepository._fetchPaged call, shutdown already in progress"
          );
          reject(
            new Error(
              `Reject ServiceRequest for "${url}", shutdown already in progress`
            )
          );
          return;
        }
        let request = this._createServiceRequest();
        request.mozBackgroundRequest = true;
        request.open("GET", url, true);
        request.responseType = "json";

        request.addEventListener("error", aEvent => {
          reject(new Error(`GET ${url} failed`));
        });
        request.addEventListener("timeout", aEvent => {
          reject(new Error(`GET ${url} timed out`));
        });
        request.addEventListener("load", aEvent => {
          let response = request.response;
          if (!response || (request.status != 200 && request.status != 0)) {
            reject(new Error(`GET ${url} failed (status ${request.status})`));
            return;
          }

          try {
            results.push(...handler(response.results));
          } catch (err) {
            reject(err);
          }

          if (response.next) {
            resolve(fetchNextPage(response.next));
          }

          resolve(results);
        });

        request.send(null);
      });
    };

    return fetchNextPage(startURL);
  },

  /**
   * Fetch metadata for a given set of addons from AMO.
   *
   * @param  aIDs
   *         The array of ids to retrieve metadata for.
   * @returns {array<AddonSearchResult>}
   */
  async getAddonsByIDs(aIDs) {
    const idCheck = aIDs.map(id => {
      if (id.startsWith("rta:")) {
        return atob(id.split(":")[1]);
      }
      return id;
    });

    const addons = await this._fetchPaged(
      PREF_GETADDONS_BYIDS,
      { IDS: aIDs.join(",") },
      results =>
        results
          .map(entry => this._parseAddon(entry))
          // Only return the add-ons corresponding the IDs passed to this method.
          .filter(addon => idCheck.includes(addon.id))
    );

    return addons;
  },

  /**
   * Fetch the Firefox add-ons mapped to the list of extension IDs for the
   * browser ID passed to this method.
   *
   * See: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html#browser-mappings
   *
   * @param browserID
   *        The browser ID used to retrieve the mapping of IDs.
   * @param extensionIDs
   *        The array of browser (non-Firefox) extension IDs to retrieve
   *        metadata for.
   * @returns {object} result
   *        The result of the mapping.
   * @returns {array<AddonSearchResult>} result.addons
   *        The AddonSearchResults for the addons that were successfully mapped.
   * @returns {array<string>} result.matchedIDs
   *        The IDs of the extensions that were successfully matched to
   *        equivalents that can be installed in this browser. These are
   *        the IDs before matching to equivalents.
   * @returns {array<string>} result.unmatchedIDs
   *        The IDs of the extensions that were not matched to equivalents.
   */
  async getMappedAddons(browserID, extensionIDs) {
    let matchedExtensionIDs = new Set();
    let unmatchedExtensionIDs = new Set(extensionIDs);

    const addonIds = await this._fetchPaged(
      PREF_GET_BROWSER_MAPPINGS,
      { BROWSER: browserID },
      results =>
        results
          // Filter out all the entries with an extension ID not in the list
          // passed to the method.
          .filter(entry => {
            if (unmatchedExtensionIDs.has(entry.extension_id)) {
              unmatchedExtensionIDs.delete(entry.extension_id);
              matchedExtensionIDs.add(entry.extension_id);
              return true;
            }
            return false;
          })
          // Return the add-on ID (stored as `guid` on AMO).
          .map(entry => entry.addon_guid)
    );

    if (!addonIds.length) {
      return {
        addons: [],
        matchedIDs: [],
        unmatchedIDs: [...unmatchedExtensionIDs],
      };
    }

    return {
      addons: await this.getAddonsByIDs(addonIds),
      matchedIDs: [...matchedExtensionIDs],
      unmatchedIDs: [...unmatchedExtensionIDs],
    };
  },

  /**
   * Asynchronously add add-ons to the cache corresponding to the specified
   * ids. If caching is disabled, the cache is unchanged.
   *
   * @param  aIds
   *         The array of add-on ids to add to the cache
   * @returns {array<AddonSearchResult>} Add-ons to add to the cache.
   */
  async cacheAddons(aIds) {
    logger.debug(
      "cacheAddons: enabled " + this.cacheEnabled + " IDs " + aIds.toSource()
    );
    if (!this.cacheEnabled) {
      return [];
    }

    let ids = await getAddonsToCache(aIds);

    // If there are no add-ons to cache, act as if caching is disabled
    if (!ids.length) {
      return [];
    }

    let addons = [];
    try {
      addons = await this.getAddonsByIDs(ids);
    } catch (err) {
      logger.error(`Error in addon metadata check: ${err.message}`);
    }
    if (addons.length) {
      await AddonDatabase.update(addons);
    }
    return addons;
  },

  /**
   * Get all installed addons from the AddonManager singleton.
   *
   * @return Promise{array<AddonWrapper>} Resolves to an array of AddonWrapper instances.
   */
  _getAllInstalledAddons() {
    return lazy.AddonManager.getAllAddons();
  },

  /**
   * Performs the periodic background update check.
   *
   * In Firefox Desktop builds, the background update check is triggered on a
   * daily basis as part of the AOM background update check and registered
   * from: `toolkit/mozapps/extensions/extensions.manifest`
   *
   * In GeckoView builds, add-ons are checked for updates individually. The
   * `AddonRepository.backgroundUpdateCheck()` method is called by the
   * `updateWebExtension()` method defined in `GeckoViewWebExtensions.sys.mjs`
   * but only when `AddonRepository.isMetadataStale()` returns true.
   *
   * @return Promise{null} Resolves when the metadata update is complete.
   */
  async backgroundUpdateCheck() {
    let shutter = (async () => {
      if (this.appIsShuttingDown) {
        logger.debug(
          "Returning earlier from backgroundUpdateCheck, shutdown already in progress"
        );
        return;
      }

      let allAddons = await this._getAllInstalledAddons();

      // Completely remove cache if caching is not enabled
      if (!this.cacheEnabled) {
        logger.debug("Clearing cache because it is disabled");
        await this._clearCache();
        return;
      }

      let ids = allAddons.map(a => a.id);
      logger.debug("Repopulate add-on cache with " + ids.toSource());

      let addonsToCache = await getAddonsToCache(ids);

      // Completely remove cache if there are no add-ons to cache
      if (!addonsToCache.length) {
        logger.debug("Clearing cache because 0 add-ons were requested");
        await this._clearCache();
        return;
      }

      let addons;
      try {
        addons = await this.getAddonsByIDs(addonsToCache);
      } catch (err) {
        // This is likely to happen if the server is unreachable, e.g. when
        // there is no network connectivity.
        logger.error(`Error in addon metadata lookup: ${err.message}`);
        // Return now to avoid calling repopulate with an empty array;
        // doing so would clear the cache.
        return;
      }

      AddonDatabase.repopulate(addons);

      // Always call AddonManager updateAddonRepositoryData after we refill the cache
      await lazy.AddonManagerPrivate.updateAddonRepositoryData();
    })();
    lazy.AddonManager.beforeShutdown.addBlocker(
      "AddonRepository Background Updater",
      shutter
    );
    await shutter;
    lazy.AddonManager.beforeShutdown.removeBlocker(shutter);
  },

  /*
   * Creates an AddonSearchResult by parsing an entry from the AMO API.
   *
   * @param  aEntry
   *         An entry from the AMO search API to parse.
   * @return Result object containing the parsed AddonSearchResult
   */
  _parseAddon(aEntry) {
    let addon = new AddonSearchResult(aEntry.guid);

    addon.name = aEntry.name;
    if (typeof aEntry.current_version == "object") {
      addon.version = String(aEntry.current_version.version);
      if (Array.isArray(aEntry.current_version.files)) {
        for (let file of aEntry.current_version.files) {
          if (file.platform == "all" || file.platform == lazy.PLATFORM) {
            if (file.url) {
              addon.sourceURI = lazy.NetUtil.newURI(file.url);
            }
            break;
          }
        }
      }
    }
    addon.homepageURL = aEntry.homepage;
    addon.supportURL = aEntry.support_url;
    addon.amoListingURL = aEntry.url;

    addon.description = convertHTMLToPlainText(aEntry.summary);
    addon.fullDescription = convertHTMLToPlainText(aEntry.description);

    addon.weeklyDownloads = aEntry.weekly_downloads;

    switch (aEntry.type) {
      case "persona":
      case "statictheme":
        addon.type = "theme";
        break;

      case "language":
        addon.type = "locale";
        break;

      default:
        addon.type = aEntry.type;
        break;
    }

    if (Array.isArray(aEntry.authors)) {
      let authors = aEntry.authors.map(
        author =>
          new lazy.AddonManagerPrivate.AddonAuthor(author.name, author.url)
      );
      if (authors.length) {
        addon.creator = authors[0];
        addon.developers = authors.slice(1);
      }
    }

    if (typeof aEntry.previews == "object") {
      addon.screenshots = aEntry.previews.map(shot => {
        let safeSize = orig =>
          Array.isArray(orig) && orig.length >= 2 ? orig : [null, null];
        let imageSize = safeSize(shot.image_size);
        let thumbSize = safeSize(shot.thumbnail_size);
        return new lazy.AddonManagerPrivate.AddonScreenshot(
          shot.image_url,
          imageSize[0],
          imageSize[1],
          shot.thumbnail_url,
          thumbSize[0],
          thumbSize[1],
          shot.caption
        );
      });
    }

    addon.contributionURL = aEntry.contributions_url;

    if (typeof aEntry.ratings == "object") {
      addon.averageRating = Math.min(5, aEntry.ratings.average);
      addon.reviewCount = aEntry.ratings.text_count;
    }

    addon.reviewURL = aEntry.ratings_url;
    if (aEntry.last_updated) {
      addon.updateDate = new Date(aEntry.last_updated);
    }

    addon.icons = aEntry.icons || {};

    return addon;
  },

  // Create url from preference, returning null if preference does not exist
  _formatURLPref(aPreference, aSubstitutions = {}) {
    let url = Services.prefs.getCharPref(aPreference, "");
    if (!url) {
      logger.warn("_formatURLPref: Couldn't get pref: " + aPreference);
      return null;
    }

    url = url.replace(/%([A-Z_]+)%/g, function (aMatch, aKey) {
      return aKey in aSubstitutions
        ? encodeURIComponent(aSubstitutions[aKey])
        : aMatch;
    });

    return Services.urlFormatter.formatURL(url);
  },

  flush() {
    return AddonDatabase.flush();
  },

  async getAvailableLangpacks() {
    // This should be the API endpoint documented at:
    // http://addons-server.readthedocs.io/en/latest/topics/api/addons.html#language-tools
    let url = this._formatURLPref(PREF_GET_LANGPACKS);

    let response = await fetch(url, { credentials: "omit" });
    if (!response.ok) {
      throw new Error("fetching available language packs failed");
    }

    let data = await response.json();

    let result = [];
    for (let entry of data.results) {
      if (
        !entry.current_compatible_version ||
        !entry.current_compatible_version.files
      ) {
        continue;
      }

      for (let file of entry.current_compatible_version.files) {
        if (
          file.platform == "all" ||
          file.platform == Services.appinfo.OS.toLowerCase()
        ) {
          result.push({
            target_locale: entry.target_locale,
            url: file.url,
            hash: file.hash,
          });
        }
      }
    }

    return result;
  },
};

var AddonDatabase = {
  connectionPromise: null,
  _loaded: false,
  _saveTask: null,
  _blockerAdded: false,

  // the in-memory database
  DB: BLANK_DB(),

  /**
   * A getter to retrieve the path to the DB
   */
  get jsonFile() {
    return PathUtils.join(
      Services.dirsvc.get("ProfD", Ci.nsIFile).path,
      FILE_DATABASE
    );
  },

  /**
   * Asynchronously opens a new connection to the database file.
   *
   * @return {Promise} a promise that resolves to the database.
   */
  openConnection() {
    if (!this.connectionPromise) {
      this.connectionPromise = (async () => {
        let inputDB, schema;

        try {
          let data = await IOUtils.readUTF8(this.jsonFile);
          inputDB = JSON.parse(data);

          if (
            !inputDB.hasOwnProperty("addons") ||
            !Array.isArray(inputDB.addons)
          ) {
            throw new Error("No addons array.");
          }

          if (!inputDB.hasOwnProperty("schema")) {
            throw new Error("No schema specified.");
          }

          schema = parseInt(inputDB.schema, 10);

          if (!Number.isInteger(schema) || schema < DB_MIN_JSON_SCHEMA) {
            throw new Error("Invalid schema value.");
          }
        } catch (e) {
          if (e.name == "NotFoundError") {
            logger.debug("No " + FILE_DATABASE + " found.");
          } else {
            logger.error(
              `Malformed ${FILE_DATABASE}: ${e} - resetting to empty`
            );
          }

          // Create a blank addons.json file
          this.save();

          Services.prefs.setIntPref(PREF_GETADDONS_DB_SCHEMA, DB_SCHEMA);
          this._loaded = true;
          return this.DB;
        }

        Services.prefs.setIntPref(PREF_GETADDONS_DB_SCHEMA, DB_SCHEMA);

        // Convert the addon objects as necessary
        // and store them in our in-memory copy of the database.
        for (let addon of inputDB.addons) {
          let id = addon.id;

          let entry = this._parseAddon(addon);
          this.DB.addons.set(id, entry);
        }

        this._loaded = true;
        return this.DB;
      })();
    }

    return this.connectionPromise;
  },

  /**
   * Asynchronously shuts down the database connection and releases all
   * cached objects
   *
   * @param  aCallback
   *         An optional callback to call once complete
   * @param  aSkipFlush
   *         An optional boolean to skip flushing data to disk. Useful
   *         when the database is going to be deleted afterwards.
   */
  shutdown(aSkipFlush) {
    if (!this.connectionPromise) {
      return Promise.resolve();
    }

    this.connectionPromise = null;
    this._loaded = false;

    if (aSkipFlush) {
      return Promise.resolve();
    }

    return this.flush();
  },

  /**
   * Asynchronously deletes the database, shutting down the connection
   * first if initialized
   *
   * @param  aCallback
   *         An optional callback to call once complete
   * @return Promise{null} resolves when the database has been deleted
   */
  delete(aCallback) {
    this.DB = BLANK_DB();

    if (this._saveTask) {
      this._saveTask.disarm();
      this._saveTask = null;
    }

    // shutdown(true) never rejects
    this._deleting = this.shutdown(true)
      .then(() => IOUtils.remove(this.jsonFile))
      .catch(error =>
        logger.error(
          "Unable to delete Addon Repository file " + this.jsonFile,
          error
        )
      )
      .then(() => (this._deleting = null))
      .then(aCallback);

    return this._deleting;
  },

  async _saveNow() {
    let json = {
      schema: this.DB.schema,
      addons: Array.from(this.DB.addons.values()),
    };

    await IOUtils.writeUTF8(this.jsonFile, JSON.stringify(json), {
      tmpPath: `${this.jsonFile}.tmp`,
    });
  },

  save() {
    if (!this._saveTask) {
      this._saveTask = new lazy.DeferredTask(
        () => this._saveNow(),
        DB_BATCH_TIMEOUT_MS
      );

      if (!this._blockerAdded) {
        lazy.AsyncShutdown.profileBeforeChange.addBlocker(
          "Flush AddonRepository",
          () => this.flush()
        );
        this._blockerAdded = true;
      }
    }
    this._saveTask.arm();
  },

  /**
   * Flush any pending I/O on the addons.json file
   * @return: Promise{null}
   *          Resolves when the pending I/O (writing out or deleting
   *          addons.json) completes
   */
  flush() {
    if (this._deleting) {
      return this._deleting;
    }

    if (this._saveTask) {
      let promise = this._saveTask.finalize();
      this._saveTask = null;
      return promise;
    }

    return Promise.resolve();
  },

  /**
   * Get an individual addon entry from the in-memory cache.
   * Note: calling this function before the database is read will
   * return undefined.
   *
   * @param {string} aId The id of the addon to retrieve.
   */
  getAddon(aId) {
    return this.DB.addons.get(aId);
  },

  /**
   * Asynchronously repopulates the database so it only contains the
   * specified add-ons
   *
   * @param {array<AddonSearchResult>} aAddons
   *              Add-ons to repopulate the database with.
   */
  repopulate(aAddons) {
    this.DB = BLANK_DB();
    this._update(aAddons);

    let now = Math.round(Date.now() / 1000);
    logger.debug(
      "Cache repopulated, setting " + PREF_METADATA_LASTUPDATE + " to " + now
    );
    Services.prefs.setIntPref(PREF_METADATA_LASTUPDATE, now);
  },

  /**
   * Asynchronously insert new addons into the database.
   *
   * @param {array<AddonSearchResult>} aAddons
   *              Add-ons to insert/update in the database
   */
  async update(aAddons) {
    await this.openConnection();

    this._update(aAddons);
  },

  /**
   * Merge the given addons into the database.
   *
   * @param {array<AddonSearchResult>} aAddons
   *              Add-ons to insert/update in the database
   */
  _update(aAddons) {
    for (let addon of aAddons) {
      this.DB.addons.set(addon.id, this._parseAddon(addon));
    }

    this.save();
  },

  /*
   * Creates an AddonSearchResult by parsing an object structure
   * retrieved from the DB JSON representation.
   *
   * @param  aObj
   *         The object to parse
   * @return Returns an AddonSearchResult object.
   */
  _parseAddon(aObj) {
    if (aObj instanceof AddonSearchResult) {
      return aObj;
    }

    let id = aObj.id;
    if (!aObj.id) {
      return null;
    }

    let addon = new AddonSearchResult(id);

    for (let expectedProperty of Object.keys(AddonSearchResult.prototype)) {
      if (
        !(expectedProperty in aObj) ||
        typeof aObj[expectedProperty] === "function"
      ) {
        continue;
      }

      let value = aObj[expectedProperty];

      try {
        switch (expectedProperty) {
          case "sourceURI":
            addon.sourceURI = value ? lazy.NetUtil.newURI(value) : null;
            break;

          case "creator":
            addon.creator = value ? this._makeDeveloper(value) : null;
            break;

          case "updateDate":
            addon.updateDate = value ? new Date(value) : null;
            break;

          case "developers":
            if (!addon.developers) {
              addon.developers = [];
            }
            for (let developer of value) {
              addon.developers.push(this._makeDeveloper(developer));
            }
            break;

          case "screenshots":
            if (!addon.screenshots) {
              addon.screenshots = [];
            }
            for (let screenshot of value) {
              addon.screenshots.push(this._makeScreenshot(screenshot));
            }
            break;

          case "icons":
            if (!addon.icons) {
              addon.icons = {};
            }
            for (let size of Object.keys(aObj.icons)) {
              addon.icons[size] = aObj.icons[size];
            }
            break;

          case "iconURL":
            break;

          default:
            addon[expectedProperty] = value;
        }
      } catch (ex) {
        logger.warn(
          "Error in parsing property value for " + expectedProperty + " | " + ex
        );
      }

      // delete property from obj to indicate we've already
      // handled it. The remaining public properties will
      // be stored separately and just passed through to
      // be written back to the DB.
      delete aObj[expectedProperty];
    }

    // Copy remaining properties to a separate object
    // to prevent accidental access on downgraded versions.
    // The properties will be merged in the same object
    // prior to being written back through toJSON.
    for (let remainingProperty of Object.keys(aObj)) {
      switch (typeof aObj[remainingProperty]) {
        case "boolean":
        case "number":
        case "string":
        case "object":
          // these types are accepted
          break;
        default:
          continue;
      }

      if (!remainingProperty.startsWith("_")) {
        addon._unsupportedProperties[remainingProperty] =
          aObj[remainingProperty];
      }
    }

    return addon;
  },

  /**
   * Make a developer object from a vanilla
   * JS object from the JSON database
   *
   * @param  aObj
   *         The JS object to use
   * @return The created developer
   */
  _makeDeveloper(aObj) {
    let name = aObj.name;
    let url = aObj.url;
    return new lazy.AddonManagerPrivate.AddonAuthor(name, url);
  },

  /**
   * Make a screenshot object from a vanilla
   * JS object from the JSON database
   *
   * @param  aObj
   *         The JS object to use
   * @return The created screenshot
   */
  _makeScreenshot(aObj) {
    let url = aObj.url;
    let width = aObj.width;
    let height = aObj.height;
    let thumbnailURL = aObj.thumbnailURL;
    let thumbnailWidth = aObj.thumbnailWidth;
    let thumbnailHeight = aObj.thumbnailHeight;
    let caption = aObj.caption;
    return new lazy.AddonManagerPrivate.AddonScreenshot(
      url,
      width,
      height,
      thumbnailURL,
      thumbnailWidth,
      thumbnailHeight,
      caption
    );
  },
};