diff options
Diffstat (limited to 'comm/mailnews/extensions/newsblog/Feed.jsm')
-rw-r--r-- | comm/mailnews/extensions/newsblog/Feed.jsm | 700 |
1 files changed, 700 insertions, 0 deletions
diff --git a/comm/mailnews/extensions/newsblog/Feed.jsm b/comm/mailnews/extensions/newsblog/Feed.jsm new file mode 100644 index 0000000000..8b899308d5 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/Feed.jsm @@ -0,0 +1,700 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["Feed"]; + +const lazy = {}; + +ChromeUtils.defineModuleGetter( + lazy, + "FeedParser", + "resource:///modules/FeedParser.jsm" +); +ChromeUtils.defineModuleGetter( + lazy, + "FeedUtils", + "resource:///modules/FeedUtils.jsm" +); + +// Cache for all of the feeds currently being downloaded, indexed by URL, +// so the load event listener can access the Feed objects after it finishes +// downloading the feed. +var FeedCache = { + mFeeds: {}, + + putFeed(aFeed) { + this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed; + }, + + getFeed(aUrl) { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) { + return this.mFeeds[index]; + } + + return null; + }, + + removeFeed(aUrl) { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) { + delete this.mFeeds[index]; + } + }, + + normalizeHost(aUrl) { + try { + let normalizedUrl = Services.io.newURI(aUrl); + let newHost = normalizedUrl.host.toLowerCase(); + normalizedUrl = normalizedUrl.mutate().setHost(newHost).finalize(); + return normalizedUrl.spec; + } catch (ex) { + return aUrl; + } + }, +}; + +/** + * A Feed object. If aFolder is the account root folder, a new subfolder + * for the feed url is created otherwise the url will be subscribed to the + * existing aFolder, upon successful download() completion. + * + * @class + * @param {string} aFeedUrl - feed url. + * @param {nsIMsgFolder} aFolder - folder containing or to contain the feed + * subscription. + */ +function Feed(aFeedUrl, aFolder) { + this.url = aFeedUrl; + this.server = aFolder.server; + if (!aFolder.isServer) { + this.mFolder = aFolder; + } +} + +Feed.prototype = { + url: null, + description: null, + author: null, + request: null, + server: null, + downloadCallback: null, + resource: null, + itemsToStore: [], + itemsStored: 0, + fileSize: 0, + mFolder: null, + mInvalidFeed: false, + mFeedType: null, + mLastModified: null, + + get folder() { + return this.mFolder; + }, + + set folder(aFolder) { + this.mFolder = aFolder; + }, + + get name() { + // Used for the feed's title in Subscribe dialog and opml export. + let name = this.title || this.description || this.url; + /* eslint-disable-next-line no-control-regex */ + return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, ""); + }, + + get folderName() { + if (this.mFolderName) { + return this.mFolderName; + } + + // Get a unique sanitized name. Use title or description as a base; + // these are mandatory by spec. Length of 80 is plenty. + let folderName = (this.title || this.description || "").substr(0, 80); + let defaultName = + lazy.FeedUtils.strings.GetStringFromName("ImportFeedsNew"); + return (this.mFolderName = lazy.FeedUtils.getSanitizedFolderName( + this.server.rootMsgFolder, + folderName, + defaultName, + true + )); + }, + + download(aParseItems, aCallback) { + // May be null. + this.downloadCallback = aCallback; + + // Whether or not to parse items when downloading and parsing the feed. + // Defaults to true, but setting to false is useful for obtaining + // just the title of the feed when the user subscribes to it. + this.parseItems = aParseItems == null || aParseItems; + + // Before we do anything, make sure the url is an http url. This is just + // a sanity check so we don't try opening mailto urls, imap urls, etc. that + // the user may have tried to subscribe to as an rss feed. + if (!lazy.FeedUtils.isValidScheme(this.url)) { + // Simulate an invalid feed error. + lazy.FeedUtils.log.info( + "Feed.download: invalid protocol for - " + this.url + ); + this.onParseError(this); + return; + } + + // Before we try to download the feed, make sure we aren't already + // processing the feed by looking up the url in our feed cache. + if (FeedCache.getFeed(this.url)) { + if (this.downloadCallback) { + this.downloadCallback.downloaded( + this, + lazy.FeedUtils.kNewsBlogFeedIsBusy + ); + } + + // Return, the feed is already in use. + return; + } + + if (Services.io.offline) { + // If offline and don't want to go online, just add the feed subscription; + // it can be verified later (the folder name will be the url if not adding + // to an existing folder). Only for subscribe actions; passive biff and + // active get new messages are handled prior to getting here. + let win = Services.wm.getMostRecentWindow("mail:3pane"); + if (!win.MailOfflineMgr.getNewMail()) { + this.storeNextItem(); + return; + } + } + + this.request = new XMLHttpRequest(); + // Must set onProgress before calling open. + this.request.onprogress = this.onProgress; + this.request.open("GET", this.url, true); + this.request.channel.loadFlags |= + Ci.nsIRequest.LOAD_BYPASS_CACHE | Ci.nsIRequest.INHIBIT_CACHING; + + // Some servers, if sent If-Modified-Since, will send 304 if subsequently + // not sent If-Modified-Since, as in the case of an unsubscribe and new + // subscribe. Send start of century date to force a download; some servers + // will 304 on older dates (such as epoch 1970). + let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT"; + this.request.setRequestHeader("If-Modified-Since", lastModified); + + // Only order what you're going to eat... + this.request.responseType = "document"; + this.request.overrideMimeType("text/xml"); + this.request.setRequestHeader("Accept", lazy.FeedUtils.REQUEST_ACCEPT); + this.request.timeout = lazy.FeedUtils.REQUEST_TIMEOUT; + this.request.onload = this.onDownloaded; + this.request.onreadystatechange = this.onReadyStateChange; + this.request.onerror = this.onDownloadError; + this.request.ontimeout = this.onDownloadError; + FeedCache.putFeed(this); + this.request.send(null); + }, + + onReadyStateChange(aEvent) { + // Once a server responds with data, reset the timeout to allow potentially + // large files to complete the download. + let request = aEvent.target; + if (request.timeout && request.readyState == request.LOADING) { + request.timeout = 0; + } + }, + + onDownloaded(aEvent) { + let request = aEvent.target; + let isHttp = request.channel.originalURI.scheme.startsWith("http"); + let url = request.channel.originalURI.spec; + if (isHttp && (request.status < 200 || request.status >= 300)) { + Feed.prototype.onDownloadError(aEvent); + return; + } + + lazy.FeedUtils.log.debug( + "Feed.onDownloaded: got a download, fileSize:url - " + + aEvent.loaded + + " : " + + url + ); + let feed = FeedCache.getFeed(url); + if (!feed) { + throw new Error( + "Feed.onDownloaded: error - couldn't retrieve feed from cache" + ); + } + + // If the server sends a Last-Modified header, store the property on the + // feed so we can use it when making future requests, to avoid downloading + // and parsing feeds that have not changed. Don't update if merely checking + // the url, as for subscribe move/copy, as a subsequent refresh may get a 304. + // Save the response and persist it only upon successful completion of the + // refresh cycle (i.e. not if the request is cancelled). + let lastModifiedHeader = request.getResponseHeader("Last-Modified"); + feed.mLastModified = + lastModifiedHeader && feed.parseItems ? lastModifiedHeader : null; + + feed.fileSize = aEvent.loaded; + + // The download callback is called asynchronously when parse() is done. + feed.parse(); + }, + + onProgress(aEvent) { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + + if (feed.downloadCallback) { + feed.downloadCallback.onProgress( + feed, + aEvent.loaded, + aEvent.total, + aEvent.lengthComputable + ); + } + }, + + onDownloadError(aEvent) { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + if (feed.downloadCallback) { + // Generic network or 'not found' error initially. + let error = lazy.FeedUtils.kNewsBlogRequestFailure; + // Certain errors should disable the feed. + let disable = false; + + if (request.status == 304) { + // If the http status code is 304, the feed has not been modified + // since we last downloaded it and does not need to be parsed. + error = lazy.FeedUtils.kNewsBlogNoNewItems; + } else { + let [errType, errName] = + lazy.FeedUtils.createTCPErrorFromFailedXHR(request); + lazy.FeedUtils.log.info( + "Feed.onDownloaded: request errType:errName:statusCode - " + + errType + + ":" + + errName + + ":" + + request.status + ); + if (errType == "SecurityCertificate") { + // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT + // overridable security certificate errors. + error = lazy.FeedUtils.kNewsBlogBadCertError; + } + + if (request.status == 401 || request.status == 403) { + // Unauthorized or Forbidden. + error = lazy.FeedUtils.kNewsBlogNoAuthError; + } + + if ( + request.status != 0 || + error == lazy.FeedUtils.kNewsBlogBadCertError || + errName == "DomainNotFoundError" + ) { + disable = true; + } + } + + feed.downloadCallback.downloaded(feed, error, disable); + } + + FeedCache.removeFeed(url); + }, + + onParseError(aFeed) { + if (!aFeed) { + return; + } + + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded( + aFeed, + lazy.FeedUtils.kNewsBlogInvalidFeed, + true + ); + } + + FeedCache.removeFeed(aFeed.url); + }, + + onUrlChange(aFeed, aOldUrl) { + if (!aFeed) { + return; + } + + // Simulate a cancel after a url update; next cycle will check the new url. + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded(aFeed, lazy.FeedUtils.kNewsBlogCancel); + } + + FeedCache.removeFeed(aOldUrl); + }, + + // nsIUrlListener methods for getDatabaseWithReparse(). + OnStartRunningUrl(aUrl) {}, + OnStopRunningUrl(aUrl, aExitCode) { + if (Components.isSuccessCode(aExitCode)) { + lazy.FeedUtils.log.debug( + "Feed.OnStopRunningUrl: rebuilt msgDatabase for " + + this.folder.name + + " - " + + this.folder.filePath.path + ); + } else { + lazy.FeedUtils.log.error( + "Feed.OnStopRunningUrl: rebuild msgDatabase failed, " + + "error " + + aExitCode + + ", for " + + this.folder.name + + " - " + + this.folder.filePath.path + ); + } + // Continue. + this.storeNextItem(); + }, + + get title() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "title", + "" + ); + }, + + set title(aNewTitle) { + if (!aNewTitle) { + return; + } + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "title", + aNewTitle + ); + }, + + get lastModified() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "lastModified", + "" + ); + }, + + set lastModified(aLastModified) { + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "lastModified", + aLastModified + ); + }, + + get quickMode() { + let defaultValue = this.server.getBoolValue("quickMode"); + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "quickMode", + defaultValue + ); + }, + + set quickMode(aNewQuickMode) { + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "quickMode", + aNewQuickMode + ); + }, + + get options() { + let options = lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "options", + null + ); + if (options && options.version == lazy.FeedUtils._optionsDefault.version) { + return options; + } + + let newOptions = lazy.FeedUtils.newOptions(options); + this.options = newOptions; + return newOptions; + }, + + set options(aOptions) { + let newOptions = aOptions ? aOptions : lazy.FeedUtils.optionsTemplate; + lazy.FeedUtils.setSubscriptionAttr( + this.url, + this.server, + "options", + newOptions + ); + }, + + get link() { + return lazy.FeedUtils.getSubscriptionAttr( + this.url, + this.server, + "link", + "" + ); + }, + + set link(aNewLink) { + if (!aNewLink) { + return; + } + lazy.FeedUtils.setSubscriptionAttr(this.url, this.server, "link", aNewLink); + }, + + parse() { + // Create a feed parser which will parse the feed. + let parser = new lazy.FeedParser(); + this.itemsToStore = parser.parseFeed(this, this.request.responseXML); + parser = null; + + if (this.mInvalidFeed) { + this.request = null; + this.mInvalidFeed = false; + return; + } + + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + + // At this point, if we have items to potentially store and an existing + // folder, ensure the folder's msgDatabase is openable for new message + // processing. If not, reparse with an async nsIUrlListener |this| to + // continue once the reparse is complete. + if ( + this.itemsToStore.length > 0 && + this.folder && + !lazy.FeedUtils.isMsgDatabaseOpenable(this.folder, true, this) + ) { + return; + } + + // We have an msgDatabase; storeNextItem() will iterate through the parsed + // items, storing each one. + this.storeNextItem(); + }, + + /** + * Clear the 'valid' field of all feeditems associated with this feed. + * + * @returns {void} + */ + invalidateItems() { + let ds = lazy.FeedUtils.getItemsDS(this.server); + for (let id in ds.data) { + let item = ds.data[id]; + if (item.feedURLs.includes(this.url)) { + item.valid = false; + lazy.FeedUtils.log.trace("Feed.invalidateItems: item - " + id); + } + } + ds.saveSoon(); + }, + + /** + * Discards invalid items (in the feed item store) associated with the + * feed. There's a delay - invalid items are kept around for a set time + * before being purged. + * + * @param {Boolean} aDeleteFeed - is the feed being deleted (bypasses + * the delay time). + * @returns {void} + */ + removeInvalidItems(aDeleteFeed) { + let ds = lazy.FeedUtils.getItemsDS(this.server); + lazy.FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url); + + let currentTime = new Date().getTime(); + for (let id in ds.data) { + let item = ds.data[id]; + // skip valid items and ones not part of this feed. + if (!item.feedURLs.includes(this.url) || item.valid) { + continue; + } + let lastSeenTime = item.lastSeenTime || 0; + + if ( + currentTime - lastSeenTime < lazy.FeedUtils.INVALID_ITEM_PURGE_DELAY && + !aDeleteFeed + ) { + // Don't immediately purge items in active feeds; do so for deleted feeds. + continue; + } + + lazy.FeedUtils.log.trace("Feed.removeInvalidItems: item - " + id); + // Detach the item from this feed (it could be shared by multiple feeds). + item.feedURLs = item.feedURLs.filter(url => url != this.url); + if (item.feedURLs.length > 0) { + lazy.FeedUtils.log.debug( + "Feed.removeInvalidItems: " + + id + + " is from more than one feed; only the reference to" + + " this feed removed" + ); + } else { + delete ds.data[id]; + } + } + ds.saveSoon(); + }, + + createFolder() { + if (this.folder) { + return; + } + + try { + this.folder = this.server.rootMsgFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .createLocalSubfolder(this.folderName); + } catch (ex) { + // An error creating. + lazy.FeedUtils.log.info( + "Feed.createFolder: error creating folder - '" + + this.folderName + + "' in parent folder " + + this.server.rootMsgFolder.filePath.path + + " -- " + + ex + ); + // But its remnants are still there, clean up. + let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName); + this.server.rootMsgFolder.propagateDelete(xfolder, true); + } + }, + + // Gets the next item from itemsToStore and forces that item to be stored + // to the folder. If more items are left to be stored, fires a timer for + // the next one, otherwise triggers a download done notification to the UI. + storeNextItem() { + if (lazy.FeedUtils.CANCEL_REQUESTED) { + lazy.FeedUtils.CANCEL_REQUESTED = false; + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogCancel); + return; + } + + if (this.itemsToStore.length == 0) { + let code = lazy.FeedUtils.kNewsBlogSuccess; + this.createFolder(); + if (!this.folder) { + code = lazy.FeedUtils.kNewsBlogFileError; + } + + this.cleanupParsingState(this, code); + return; + } + + let item = this.itemsToStore[this.itemsToStoreIndex]; + + if (item.store()) { + this.itemsStored++; + } + + if (!this.folder) { + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogFileError); + return; + } + + this.itemsToStoreIndex++; + + // If the listener is tracking progress for each item, report it here. + if ( + item.feed.downloadCallback && + item.feed.downloadCallback.onFeedItemStored + ) { + item.feed.downloadCallback.onFeedItemStored( + item.feed, + this.itemsToStoreIndex, + this.itemsToStore.length + ); + } + + // Eventually we'll report individual progress here. + + if (this.itemsToStoreIndex < this.itemsToStore.length) { + if (!this.storeItemsTimer) { + this.storeItemsTimer = Cc["@mozilla.org/timer;1"].createInstance( + Ci.nsITimer + ); + } + + this.storeItemsTimer.initWithCallback( + this, + 50, + Ci.nsITimer.TYPE_ONE_SHOT + ); + } else { + // We have just finished downloading one or more feed items into the + // destination folder; if the folder is still listed as having new + // messages in it, then we should set the biff state on the folder so the + // right RDF UI changes happen in the folder pane to indicate new mail. + if (item.feed.folder.hasNewMessages) { + item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail; + // Run the bayesian spam filter, if enabled. + item.feed.folder.callFilterPlugins(null); + } + + this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogSuccess); + } + }, + + cleanupParsingState(aFeed, aCode) { + // Now that we are done parsing the feed, remove the feed from the cache. + FeedCache.removeFeed(aFeed.url); + + if (aFeed.parseItems) { + // Do this only if we're in parse/store mode. + aFeed.removeInvalidItems(false); + + if (aCode == lazy.FeedUtils.kNewsBlogSuccess && aFeed.mLastModified) { + aFeed.lastModified = aFeed.mLastModified; + } + + // Flush any feed item changes to disk. + let ds = lazy.FeedUtils.getItemsDS(aFeed.server); + ds.saveSoon(); + lazy.FeedUtils.log.debug( + "Feed.cleanupParsingState: items stored - " + this.itemsStored + ); + } + + // Force the xml http request to go away. This helps reduce some nasty + // assertions on shut down. + this.request = null; + this.itemsToStore = []; + this.itemsToStoreIndex = 0; + this.storeItemsTimer = null; + + if (aFeed.downloadCallback) { + aFeed.downloadCallback.downloaded(aFeed, aCode); + } + }, + + // nsITimerCallback + notify(aTimer) { + this.storeNextItem(); + }, +}; |