diff options
Diffstat (limited to 'services/sync/modules/engines/history.sys.mjs')
| -rw-r--r-- | services/sync/modules/engines/history.sys.mjs | 654 |
1 files changed, 654 insertions, 0 deletions
diff --git a/services/sync/modules/engines/history.sys.mjs b/services/sync/modules/engines/history.sys.mjs new file mode 100644 index 0000000000..44014e4d9e --- /dev/null +++ b/services/sync/modules/engines/history.sys.mjs @@ -0,0 +1,654 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const HISTORY_TTL = 5184000; // 60 days in milliseconds +const THIRTY_DAYS_IN_MS = 2592000000; // 30 days in milliseconds +// Sync may bring new fields from other clients, not yet understood by our engine. +// Unknown fields outside these fields are aggregated into 'unknownFields' and +// safely synced to prevent data loss. +const VALID_HISTORY_FIELDS = ["id", "title", "histUri", "visits"]; +const VALID_VISIT_FIELDS = ["date", "type", "transition"]; + +import { Async } from "resource://services-common/async.sys.mjs"; +import { CommonUtils } from "resource://services-common/utils.sys.mjs"; + +import { + MAX_HISTORY_DOWNLOAD, + MAX_HISTORY_UPLOAD, + SCORE_INCREMENT_SMALL, + SCORE_INCREMENT_XLARGE, +} from "resource://services-sync/constants.sys.mjs"; + +import { + Store, + SyncEngine, + LegacyTracker, +} from "resource://services-sync/engines.sys.mjs"; +import { CryptoWrapper } from "resource://services-sync/record.sys.mjs"; +import { Utils } from "resource://services-sync/util.sys.mjs"; + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +export function HistoryRec(collection, id) { + CryptoWrapper.call(this, collection, id); +} + +HistoryRec.prototype = { + _logName: "Sync.Record.History", + ttl: HISTORY_TTL, +}; +Object.setPrototypeOf(HistoryRec.prototype, CryptoWrapper.prototype); + +Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]); + +export function HistoryEngine(service) { + SyncEngine.call(this, "History", service); +} + +HistoryEngine.prototype = { + _recordObj: HistoryRec, + _storeObj: HistoryStore, + _trackerObj: HistoryTracker, + downloadLimit: MAX_HISTORY_DOWNLOAD, + + syncPriority: 7, + + async getSyncID() { + return lazy.PlacesSyncUtils.history.getSyncId(); + }, + + async ensureCurrentSyncID(newSyncID) { + this._log.debug( + "Checking if server sync ID ${newSyncID} matches existing", + { newSyncID } + ); + await lazy.PlacesSyncUtils.history.ensureCurrentSyncId(newSyncID); + return newSyncID; + }, + + async resetSyncID() { + // First, delete the collection on the server. It's fine if we're + // interrupted here: on the next sync, we'll detect that our old sync ID is + // now stale, and start over as a first sync. + await this._deleteServerCollection(); + // Then, reset our local sync ID. + return this.resetLocalSyncID(); + }, + + async resetLocalSyncID() { + let newSyncID = await lazy.PlacesSyncUtils.history.resetSyncId(); + this._log.debug("Assigned new sync ID ${newSyncID}", { newSyncID }); + return newSyncID; + }, + + async getLastSync() { + let lastSync = await lazy.PlacesSyncUtils.history.getLastSync(); + return lastSync; + }, + + async setLastSync(lastSync) { + await lazy.PlacesSyncUtils.history.setLastSync(lastSync); + }, + + shouldSyncURL(url) { + return !url.startsWith("file:"); + }, + + async pullNewChanges() { + const changedIDs = await this._tracker.getChangedIDs(); + let modifiedGUIDs = Object.keys(changedIDs); + if (!modifiedGUIDs.length) { + return {}; + } + + let guidsToRemove = + await lazy.PlacesSyncUtils.history.determineNonSyncableGuids( + modifiedGUIDs + ); + await this._tracker.removeChangedID(...guidsToRemove); + return changedIDs; + }, + + async _resetClient() { + await super._resetClient(); + await lazy.PlacesSyncUtils.history.reset(); + }, +}; +Object.setPrototypeOf(HistoryEngine.prototype, SyncEngine.prototype); + +function HistoryStore(name, engine) { + Store.call(this, name, engine); +} + +HistoryStore.prototype = { + // We try and only update this many visits at one time. + MAX_VISITS_PER_INSERT: 500, + + // Some helper functions to handle GUIDs + async setGUID(uri, guid) { + if (!guid) { + guid = Utils.makeGUID(); + } + + try { + await lazy.PlacesSyncUtils.history.changeGuid(uri, guid); + } catch (e) { + this._log.error("Error setting GUID ${guid} for URI ${uri}", guid, uri); + } + + return guid; + }, + + async GUIDForUri(uri, create) { + // Use the existing GUID if it exists + let guid; + try { + guid = await lazy.PlacesSyncUtils.history.fetchGuidForURL(uri); + } catch (e) { + this._log.error("Error fetching GUID for URL ${uri}", uri); + } + + // If the URI has an existing GUID, return it. + if (guid) { + return guid; + } + + // If the URI doesn't have a GUID and we were indicated to create one. + if (create) { + return this.setGUID(uri); + } + + // If the URI doesn't have a GUID and we didn't create one for it. + return null; + }, + + async changeItemID(oldID, newID) { + let info = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(oldID); + if (!info) { + throw new Error(`Can't change ID for nonexistent history entry ${oldID}`); + } + this.setGUID(info.url, newID); + }, + + async getAllIDs() { + let urls = await lazy.PlacesSyncUtils.history.getAllURLs({ + since: new Date(Date.now() - THIRTY_DAYS_IN_MS), + limit: MAX_HISTORY_UPLOAD, + }); + + let urlsByGUID = {}; + for (let url of urls) { + if (!this.engine.shouldSyncURL(url)) { + continue; + } + let guid = await this.GUIDForUri(url, true); + urlsByGUID[guid] = url; + } + return urlsByGUID; + }, + + async applyIncomingBatch(records, countTelemetry) { + // Convert incoming records to mozIPlaceInfo objects which are applied as + // either history additions or removals. + let failed = []; + let toAdd = []; + let toRemove = []; + let pageGuidsWithUnknownFields = new Map(); + let visitTimesWithUnknownFields = new Map(); + await Async.yieldingForEach(records, async record => { + if (record.deleted) { + toRemove.push(record); + } else { + try { + let pageInfo = await this._recordToPlaceInfo(record); + if (pageInfo) { + toAdd.push(pageInfo); + + // Pull any unknown fields that may have come from other clients + let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields( + record.cleartext, + VALID_HISTORY_FIELDS + ); + if (unknownFields) { + pageGuidsWithUnknownFields.set(pageInfo.guid, { unknownFields }); + } + + // Visits themselves could also contain unknown fields + for (const visit of pageInfo.visits) { + let unknownVisitFields = + lazy.PlacesSyncUtils.extractUnknownFields( + visit, + VALID_VISIT_FIELDS + ); + if (unknownVisitFields) { + // Visits don't have an id at the time of sync so we'll need + // to use the time instead until it's inserted in the DB + visitTimesWithUnknownFields.set(visit.date.getTime(), { + unknownVisitFields, + }); + } + } + } + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to create a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + } + }); + if (toAdd.length || toRemove.length) { + if (toRemove.length) { + // PlacesUtils.history.remove takes an array of visits to remove, + // but the error semantics are tricky - a single "bad" entry will cause + // an exception before anything is removed. So we do remove them one at + // a time. + await Async.yieldingForEach(toRemove, async record => { + try { + await this.remove(record); + } catch (ex) { + if (Async.isShutdownException(ex)) { + throw ex; + } + this._log.error("Failed to delete a place info", ex); + this._log.trace("The record that failed", record); + failed.push(record.id); + countTelemetry.addIncomingFailedReason(ex.message); + } + }); + } + for (let chunk of this._generateChunks(toAdd)) { + // Per bug 1415560, we ignore any exceptions returned by insertMany + // as they are likely to be spurious. We do supply an onError handler + // and log the exceptions seen there as they are likely to be + // informative, but we still never abort the sync based on them. + let unknownFieldsToInsert = []; + try { + await lazy.PlacesUtils.history.insertMany( + chunk, + result => { + const placeToUpdate = pageGuidsWithUnknownFields.get(result.guid); + // Extract the placeId from this result so we can add the unknownFields + // to the proper table + if (placeToUpdate) { + unknownFieldsToInsert.push({ + placeId: result.placeId, + unknownFields: placeToUpdate.unknownFields, + }); + } + // same for visits + result.visits.forEach(visit => { + let visitToUpdate = visitTimesWithUnknownFields.get( + visit.date.getTime() + ); + if (visitToUpdate) { + unknownFieldsToInsert.push({ + visitId: visit.visitId, + unknownFields: visitToUpdate.unknownVisitFields, + }); + } + }); + }, + failedVisit => { + this._log.info( + "Failed to insert a history record", + failedVisit.guid + ); + this._log.trace("The record that failed", failedVisit); + failed.push(failedVisit.guid); + } + ); + } catch (ex) { + this._log.info("Failed to insert history records", ex); + countTelemetry.addIncomingFailedReason(ex.message); + } + + // All the top level places or visits that had unknown fields are sent + // to be added to the appropiate tables + await lazy.PlacesSyncUtils.history.updateUnknownFieldsBatch( + unknownFieldsToInsert + ); + } + } + + return failed; + }, + + /** + * Returns a generator that splits records into sanely sized chunks suitable + * for passing to places to prevent places doing bad things at shutdown. + */ + *_generateChunks(records) { + // We chunk based on the number of *visits* inside each record. However, + // we do not split a single record into multiple records, because at some + // time in the future, we intend to ensure these records are ordered by + // lastModified, and advance the engine's timestamp as we process them, + // meaning we can resume exactly where we left off next sync - although + // currently that's not done, so we will retry the entire batch next sync + // if interrupted. + // ie, this means that if a single record has more than MAX_VISITS_PER_INSERT + // visits, we will call insertMany() with exactly 1 record, but with + // more than MAX_VISITS_PER_INSERT visits. + let curIndex = 0; + this._log.debug(`adding ${records.length} records to history`); + while (curIndex < records.length) { + Async.checkAppReady(); // may throw if we are shutting down. + let toAdd = []; // what we are going to insert. + let count = 0; // a counter which tells us when toAdd is full. + do { + let record = records[curIndex]; + curIndex += 1; + toAdd.push(record); + count += record.visits.length; + } while ( + curIndex < records.length && + count + records[curIndex].visits.length <= this.MAX_VISITS_PER_INSERT + ); + this._log.trace(`adding ${toAdd.length} items in this chunk`); + yield toAdd; + } + }, + + /* An internal helper to determine if we can add an entry to places. + Exists primarily so tests can override it. + */ + _canAddURI(uri) { + return lazy.PlacesUtils.history.canAddURI(uri); + }, + + /** + * Converts a Sync history record to a mozIPlaceInfo. + * + * Throws if an invalid record is encountered (invalid URI, etc.), + * returns a new PageInfo object if the record is to be applied, null + * otherwise (no visits to add, etc.), + */ + async _recordToPlaceInfo(record) { + // Sort out invalid URIs and ones Places just simply doesn't want. + record.url = lazy.PlacesUtils.normalizeToURLOrGUID(record.histUri); + record.uri = CommonUtils.makeURI(record.histUri); + + if (!Utils.checkGUID(record.id)) { + this._log.warn("Encountered record with invalid GUID: " + record.id); + return null; + } + record.guid = record.id; + + if ( + !this._canAddURI(record.uri) || + !this.engine.shouldSyncURL(record.uri.spec) + ) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": can't add this URI." + ); + return null; + } + + // We dupe visits by date and type. So an incoming visit that has + // the same timestamp and type as a local one won't get applied. + // To avoid creating new objects, we rewrite the query result so we + // can simply check for containment below. + let curVisitsAsArray = []; + let curVisits = new Set(); + try { + curVisitsAsArray = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + } + let oldestAllowed = + lazy.PlacesSyncUtils.bookmarks.EARLIEST_BOOKMARK_TIMESTAMP; + if (curVisitsAsArray.length == 20) { + let oldestVisit = curVisitsAsArray[curVisitsAsArray.length - 1]; + oldestAllowed = lazy.PlacesSyncUtils.history.clampVisitDate( + lazy.PlacesUtils.toDate(oldestVisit.date).getTime() + ); + } + + let i, k; + for (i = 0; i < curVisitsAsArray.length; i++) { + // Same logic as used in the loop below to generate visitKey. + let { date, type } = curVisitsAsArray[i]; + let dateObj = lazy.PlacesUtils.toDate(date); + let millis = lazy.PlacesSyncUtils.history + .clampVisitDate(dateObj) + .getTime(); + curVisits.add(`${millis},${type}`); + } + + // Walk through the visits, make sure we have sound data, and eliminate + // dupes. The latter is done by rewriting the array in-place. + for (i = 0, k = 0; i < record.visits.length; i++) { + let visit = (record.visits[k] = record.visits[i]); + + if ( + !visit.date || + typeof visit.date != "number" || + !Number.isInteger(visit.date) + ) { + this._log.warn( + "Encountered record with invalid visit date: " + visit.date + ); + continue; + } + + if ( + !visit.type || + !Object.values(lazy.PlacesUtils.history.TRANSITIONS).includes( + visit.type + ) + ) { + this._log.warn( + "Encountered record with invalid visit type: " + + visit.type + + "; ignoring." + ); + continue; + } + + // Dates need to be integers. Future and far past dates are clamped to the + // current date and earliest sensible date, respectively. + let originalVisitDate = lazy.PlacesUtils.toDate(Math.round(visit.date)); + visit.date = + lazy.PlacesSyncUtils.history.clampVisitDate(originalVisitDate); + + if (visit.date.getTime() < oldestAllowed) { + // Visit is older than the oldest visit we have, and we have so many + // visits for this uri that we hit our limit when inserting. + continue; + } + let visitKey = `${visit.date.getTime()},${visit.type}`; + if (curVisits.has(visitKey)) { + // Visit is a dupe, don't increment 'k' so the element will be + // overwritten. + continue; + } + + // Note the visit key, so that we don't add duplicate visits with + // clamped timestamps. + curVisits.add(visitKey); + + visit.transition = visit.type; + k += 1; + } + record.visits.length = k; // truncate array + + // No update if there aren't any visits to apply. + // History wants at least one visit. + // In any case, the only thing we could change would be the title + // and that shouldn't change without a visit. + if (!record.visits.length) { + this._log.trace( + "Ignoring record " + + record.id + + " with URI " + + record.uri.spec + + ": no visits to add." + ); + return null; + } + + // PageInfo is validated using validateItemProperties which does a shallow + // copy of the properties. Since record uses getters some of the properties + // are not copied over. Thus we create and return a new object. + let pageInfo = { + title: record.title, + url: record.url, + guid: record.guid, + visits: record.visits, + }; + + return pageInfo; + }, + + async remove(record) { + this._log.trace("Removing page: " + record.id); + let removed = await lazy.PlacesUtils.history.remove(record.id); + if (removed) { + this._log.trace("Removed page: " + record.id); + } else { + this._log.debug("Page already removed: " + record.id); + } + }, + + async itemExists(id) { + return !!(await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id)); + }, + + async createRecord(id, collection) { + let foo = await lazy.PlacesSyncUtils.history.fetchURLInfoForGuid(id); + let record = new HistoryRec(collection, id); + if (foo) { + record.histUri = foo.url; + record.title = foo.title; + record.sortindex = foo.frecency; + + // If we had any unknown fields, ensure we put it back on the + // top-level record + if (foo.unknownFields) { + let unknownFields = JSON.parse(foo.unknownFields); + Object.assign(record.cleartext, unknownFields); + } + + try { + record.visits = await lazy.PlacesSyncUtils.history.fetchVisitsForURL( + record.histUri + ); + } catch (e) { + this._log.error( + "Error while fetching visits for URL ${record.histUri}", + record.histUri + ); + record.visits = []; + } + } else { + record.deleted = true; + } + + return record; + }, + + async wipe() { + return lazy.PlacesSyncUtils.history.wipe(); + }, +}; +Object.setPrototypeOf(HistoryStore.prototype, Store.prototype); + +function HistoryTracker(name, engine) { + LegacyTracker.call(this, name, engine); +} +HistoryTracker.prototype = { + onStart() { + this._log.info("Adding Places observer."); + this._placesObserver = new PlacesWeakCallbackWrapper( + this.handlePlacesEvents.bind(this) + ); + PlacesObservers.addListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + }, + + onStop() { + this._log.info("Removing Places observer."); + if (this._placesObserver) { + PlacesObservers.removeListener( + ["page-visited", "history-cleared", "page-removed"], + this._placesObserver + ); + } + }, + + QueryInterface: ChromeUtils.generateQI(["nsISupportsWeakReference"]), + + handlePlacesEvents(aEvents) { + this.asyncObserver.enqueueCall(() => this._handlePlacesEvents(aEvents)); + }, + + async _handlePlacesEvents(aEvents) { + if (this.ignoreAll) { + this._log.trace( + "ignoreAll: ignoring visits [" + + aEvents.map(v => v.guid).join(",") + + "]" + ); + return; + } + for (let event of aEvents) { + switch (event.type) { + case "page-visited": { + this._log.trace("'page-visited': " + event.url); + if ( + this.engine.shouldSyncURL(event.url) && + (await this.addChangedID(event.pageGuid)) + ) { + this.score += SCORE_INCREMENT_SMALL; + } + break; + } + case "history-cleared": { + this._log.trace("history-cleared"); + // Note that we're going to trigger a sync, but none of the cleared + // pages are tracked, so the deletions will not be propagated. + // See Bug 578694. + this.score += SCORE_INCREMENT_XLARGE; + break; + } + case "page-removed": { + if (event.reason === PlacesVisitRemoved.REASON_EXPIRED) { + return; + } + + this._log.trace( + "page-removed: " + event.url + ", reason " + event.reason + ); + const added = await this.addChangedID(event.pageGuid); + if (added) { + this.score += event.isRemovedFromStore + ? SCORE_INCREMENT_XLARGE + : SCORE_INCREMENT_SMALL; + } + break; + } + } + } + }, +}; +Object.setPrototypeOf(HistoryTracker.prototype, LegacyTracker.prototype); |