summaryrefslogtreecommitdiffstats
path: root/comm/mailnews/extensions/newsblog/Feed.jsm
diff options
context:
space:
mode:
Diffstat (limited to 'comm/mailnews/extensions/newsblog/Feed.jsm')
-rw-r--r--comm/mailnews/extensions/newsblog/Feed.jsm700
1 files changed, 700 insertions, 0 deletions
diff --git a/comm/mailnews/extensions/newsblog/Feed.jsm b/comm/mailnews/extensions/newsblog/Feed.jsm
new file mode 100644
index 0000000000..8b899308d5
--- /dev/null
+++ b/comm/mailnews/extensions/newsblog/Feed.jsm
@@ -0,0 +1,700 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const EXPORTED_SYMBOLS = ["Feed"];
+
+const lazy = {};
+
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "FeedParser",
+ "resource:///modules/FeedParser.jsm"
+);
+ChromeUtils.defineModuleGetter(
+ lazy,
+ "FeedUtils",
+ "resource:///modules/FeedUtils.jsm"
+);
+
+// Cache for all of the feeds currently being downloaded, indexed by URL,
+// so the load event listener can access the Feed objects after it finishes
+// downloading the feed.
+var FeedCache = {
+ mFeeds: {},
+
+ putFeed(aFeed) {
+ this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed;
+ },
+
+ getFeed(aUrl) {
+ let index = this.normalizeHost(aUrl);
+ if (index in this.mFeeds) {
+ return this.mFeeds[index];
+ }
+
+ return null;
+ },
+
+ removeFeed(aUrl) {
+ let index = this.normalizeHost(aUrl);
+ if (index in this.mFeeds) {
+ delete this.mFeeds[index];
+ }
+ },
+
+ normalizeHost(aUrl) {
+ try {
+ let normalizedUrl = Services.io.newURI(aUrl);
+ let newHost = normalizedUrl.host.toLowerCase();
+ normalizedUrl = normalizedUrl.mutate().setHost(newHost).finalize();
+ return normalizedUrl.spec;
+ } catch (ex) {
+ return aUrl;
+ }
+ },
+};
+
+/**
+ * A Feed object. If aFolder is the account root folder, a new subfolder
+ * for the feed url is created otherwise the url will be subscribed to the
+ * existing aFolder, upon successful download() completion.
+ *
+ * @class
+ * @param {string} aFeedUrl - feed url.
+ * @param {nsIMsgFolder} aFolder - folder containing or to contain the feed
+ * subscription.
+ */
+function Feed(aFeedUrl, aFolder) {
+ this.url = aFeedUrl;
+ this.server = aFolder.server;
+ if (!aFolder.isServer) {
+ this.mFolder = aFolder;
+ }
+}
+
+Feed.prototype = {
+ url: null,
+ description: null,
+ author: null,
+ request: null,
+ server: null,
+ downloadCallback: null,
+ resource: null,
+ itemsToStore: [],
+ itemsStored: 0,
+ fileSize: 0,
+ mFolder: null,
+ mInvalidFeed: false,
+ mFeedType: null,
+ mLastModified: null,
+
+ get folder() {
+ return this.mFolder;
+ },
+
+ set folder(aFolder) {
+ this.mFolder = aFolder;
+ },
+
+ get name() {
+ // Used for the feed's title in Subscribe dialog and opml export.
+ let name = this.title || this.description || this.url;
+ /* eslint-disable-next-line no-control-regex */
+ return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, "");
+ },
+
+ get folderName() {
+ if (this.mFolderName) {
+ return this.mFolderName;
+ }
+
+ // Get a unique sanitized name. Use title or description as a base;
+ // these are mandatory by spec. Length of 80 is plenty.
+ let folderName = (this.title || this.description || "").substr(0, 80);
+ let defaultName =
+ lazy.FeedUtils.strings.GetStringFromName("ImportFeedsNew");
+ return (this.mFolderName = lazy.FeedUtils.getSanitizedFolderName(
+ this.server.rootMsgFolder,
+ folderName,
+ defaultName,
+ true
+ ));
+ },
+
+ download(aParseItems, aCallback) {
+ // May be null.
+ this.downloadCallback = aCallback;
+
+ // Whether or not to parse items when downloading and parsing the feed.
+ // Defaults to true, but setting to false is useful for obtaining
+ // just the title of the feed when the user subscribes to it.
+ this.parseItems = aParseItems == null || aParseItems;
+
+ // Before we do anything, make sure the url is an http url. This is just
+ // a sanity check so we don't try opening mailto urls, imap urls, etc. that
+ // the user may have tried to subscribe to as an rss feed.
+ if (!lazy.FeedUtils.isValidScheme(this.url)) {
+ // Simulate an invalid feed error.
+ lazy.FeedUtils.log.info(
+ "Feed.download: invalid protocol for - " + this.url
+ );
+ this.onParseError(this);
+ return;
+ }
+
+ // Before we try to download the feed, make sure we aren't already
+ // processing the feed by looking up the url in our feed cache.
+ if (FeedCache.getFeed(this.url)) {
+ if (this.downloadCallback) {
+ this.downloadCallback.downloaded(
+ this,
+ lazy.FeedUtils.kNewsBlogFeedIsBusy
+ );
+ }
+
+ // Return, the feed is already in use.
+ return;
+ }
+
+ if (Services.io.offline) {
+ // If offline and don't want to go online, just add the feed subscription;
+ // it can be verified later (the folder name will be the url if not adding
+ // to an existing folder). Only for subscribe actions; passive biff and
+ // active get new messages are handled prior to getting here.
+ let win = Services.wm.getMostRecentWindow("mail:3pane");
+ if (!win.MailOfflineMgr.getNewMail()) {
+ this.storeNextItem();
+ return;
+ }
+ }
+
+ this.request = new XMLHttpRequest();
+ // Must set onProgress before calling open.
+ this.request.onprogress = this.onProgress;
+ this.request.open("GET", this.url, true);
+ this.request.channel.loadFlags |=
+ Ci.nsIRequest.LOAD_BYPASS_CACHE | Ci.nsIRequest.INHIBIT_CACHING;
+
+ // Some servers, if sent If-Modified-Since, will send 304 if subsequently
+ // not sent If-Modified-Since, as in the case of an unsubscribe and new
+ // subscribe. Send start of century date to force a download; some servers
+ // will 304 on older dates (such as epoch 1970).
+ let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT";
+ this.request.setRequestHeader("If-Modified-Since", lastModified);
+
+ // Only order what you're going to eat...
+ this.request.responseType = "document";
+ this.request.overrideMimeType("text/xml");
+ this.request.setRequestHeader("Accept", lazy.FeedUtils.REQUEST_ACCEPT);
+ this.request.timeout = lazy.FeedUtils.REQUEST_TIMEOUT;
+ this.request.onload = this.onDownloaded;
+ this.request.onreadystatechange = this.onReadyStateChange;
+ this.request.onerror = this.onDownloadError;
+ this.request.ontimeout = this.onDownloadError;
+ FeedCache.putFeed(this);
+ this.request.send(null);
+ },
+
+ onReadyStateChange(aEvent) {
+ // Once a server responds with data, reset the timeout to allow potentially
+ // large files to complete the download.
+ let request = aEvent.target;
+ if (request.timeout && request.readyState == request.LOADING) {
+ request.timeout = 0;
+ }
+ },
+
+ onDownloaded(aEvent) {
+ let request = aEvent.target;
+ let isHttp = request.channel.originalURI.scheme.startsWith("http");
+ let url = request.channel.originalURI.spec;
+ if (isHttp && (request.status < 200 || request.status >= 300)) {
+ Feed.prototype.onDownloadError(aEvent);
+ return;
+ }
+
+ lazy.FeedUtils.log.debug(
+ "Feed.onDownloaded: got a download, fileSize:url - " +
+ aEvent.loaded +
+ " : " +
+ url
+ );
+ let feed = FeedCache.getFeed(url);
+ if (!feed) {
+ throw new Error(
+ "Feed.onDownloaded: error - couldn't retrieve feed from cache"
+ );
+ }
+
+ // If the server sends a Last-Modified header, store the property on the
+ // feed so we can use it when making future requests, to avoid downloading
+ // and parsing feeds that have not changed. Don't update if merely checking
+ // the url, as for subscribe move/copy, as a subsequent refresh may get a 304.
+ // Save the response and persist it only upon successful completion of the
+ // refresh cycle (i.e. not if the request is cancelled).
+ let lastModifiedHeader = request.getResponseHeader("Last-Modified");
+ feed.mLastModified =
+ lastModifiedHeader && feed.parseItems ? lastModifiedHeader : null;
+
+ feed.fileSize = aEvent.loaded;
+
+ // The download callback is called asynchronously when parse() is done.
+ feed.parse();
+ },
+
+ onProgress(aEvent) {
+ let request = aEvent.target;
+ let url = request.channel.originalURI.spec;
+ let feed = FeedCache.getFeed(url);
+
+ if (feed.downloadCallback) {
+ feed.downloadCallback.onProgress(
+ feed,
+ aEvent.loaded,
+ aEvent.total,
+ aEvent.lengthComputable
+ );
+ }
+ },
+
+ onDownloadError(aEvent) {
+ let request = aEvent.target;
+ let url = request.channel.originalURI.spec;
+ let feed = FeedCache.getFeed(url);
+ if (feed.downloadCallback) {
+ // Generic network or 'not found' error initially.
+ let error = lazy.FeedUtils.kNewsBlogRequestFailure;
+ // Certain errors should disable the feed.
+ let disable = false;
+
+ if (request.status == 304) {
+ // If the http status code is 304, the feed has not been modified
+ // since we last downloaded it and does not need to be parsed.
+ error = lazy.FeedUtils.kNewsBlogNoNewItems;
+ } else {
+ let [errType, errName] =
+ lazy.FeedUtils.createTCPErrorFromFailedXHR(request);
+ lazy.FeedUtils.log.info(
+ "Feed.onDownloaded: request errType:errName:statusCode - " +
+ errType +
+ ":" +
+ errName +
+ ":" +
+ request.status
+ );
+ if (errType == "SecurityCertificate") {
+ // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT
+ // overridable security certificate errors.
+ error = lazy.FeedUtils.kNewsBlogBadCertError;
+ }
+
+ if (request.status == 401 || request.status == 403) {
+ // Unauthorized or Forbidden.
+ error = lazy.FeedUtils.kNewsBlogNoAuthError;
+ }
+
+ if (
+ request.status != 0 ||
+ error == lazy.FeedUtils.kNewsBlogBadCertError ||
+ errName == "DomainNotFoundError"
+ ) {
+ disable = true;
+ }
+ }
+
+ feed.downloadCallback.downloaded(feed, error, disable);
+ }
+
+ FeedCache.removeFeed(url);
+ },
+
+ onParseError(aFeed) {
+ if (!aFeed) {
+ return;
+ }
+
+ aFeed.mInvalidFeed = true;
+ if (aFeed.downloadCallback) {
+ aFeed.downloadCallback.downloaded(
+ aFeed,
+ lazy.FeedUtils.kNewsBlogInvalidFeed,
+ true
+ );
+ }
+
+ FeedCache.removeFeed(aFeed.url);
+ },
+
+ onUrlChange(aFeed, aOldUrl) {
+ if (!aFeed) {
+ return;
+ }
+
+ // Simulate a cancel after a url update; next cycle will check the new url.
+ aFeed.mInvalidFeed = true;
+ if (aFeed.downloadCallback) {
+ aFeed.downloadCallback.downloaded(aFeed, lazy.FeedUtils.kNewsBlogCancel);
+ }
+
+ FeedCache.removeFeed(aOldUrl);
+ },
+
+ // nsIUrlListener methods for getDatabaseWithReparse().
+ OnStartRunningUrl(aUrl) {},
+ OnStopRunningUrl(aUrl, aExitCode) {
+ if (Components.isSuccessCode(aExitCode)) {
+ lazy.FeedUtils.log.debug(
+ "Feed.OnStopRunningUrl: rebuilt msgDatabase for " +
+ this.folder.name +
+ " - " +
+ this.folder.filePath.path
+ );
+ } else {
+ lazy.FeedUtils.log.error(
+ "Feed.OnStopRunningUrl: rebuild msgDatabase failed, " +
+ "error " +
+ aExitCode +
+ ", for " +
+ this.folder.name +
+ " - " +
+ this.folder.filePath.path
+ );
+ }
+ // Continue.
+ this.storeNextItem();
+ },
+
+ get title() {
+ return lazy.FeedUtils.getSubscriptionAttr(
+ this.url,
+ this.server,
+ "title",
+ ""
+ );
+ },
+
+ set title(aNewTitle) {
+ if (!aNewTitle) {
+ return;
+ }
+ lazy.FeedUtils.setSubscriptionAttr(
+ this.url,
+ this.server,
+ "title",
+ aNewTitle
+ );
+ },
+
+ get lastModified() {
+ return lazy.FeedUtils.getSubscriptionAttr(
+ this.url,
+ this.server,
+ "lastModified",
+ ""
+ );
+ },
+
+ set lastModified(aLastModified) {
+ lazy.FeedUtils.setSubscriptionAttr(
+ this.url,
+ this.server,
+ "lastModified",
+ aLastModified
+ );
+ },
+
+ get quickMode() {
+ let defaultValue = this.server.getBoolValue("quickMode");
+ return lazy.FeedUtils.getSubscriptionAttr(
+ this.url,
+ this.server,
+ "quickMode",
+ defaultValue
+ );
+ },
+
+ set quickMode(aNewQuickMode) {
+ lazy.FeedUtils.setSubscriptionAttr(
+ this.url,
+ this.server,
+ "quickMode",
+ aNewQuickMode
+ );
+ },
+
+ get options() {
+ let options = lazy.FeedUtils.getSubscriptionAttr(
+ this.url,
+ this.server,
+ "options",
+ null
+ );
+ if (options && options.version == lazy.FeedUtils._optionsDefault.version) {
+ return options;
+ }
+
+ let newOptions = lazy.FeedUtils.newOptions(options);
+ this.options = newOptions;
+ return newOptions;
+ },
+
+ set options(aOptions) {
+ let newOptions = aOptions ? aOptions : lazy.FeedUtils.optionsTemplate;
+ lazy.FeedUtils.setSubscriptionAttr(
+ this.url,
+ this.server,
+ "options",
+ newOptions
+ );
+ },
+
+ get link() {
+ return lazy.FeedUtils.getSubscriptionAttr(
+ this.url,
+ this.server,
+ "link",
+ ""
+ );
+ },
+
+ set link(aNewLink) {
+ if (!aNewLink) {
+ return;
+ }
+ lazy.FeedUtils.setSubscriptionAttr(this.url, this.server, "link", aNewLink);
+ },
+
+ parse() {
+ // Create a feed parser which will parse the feed.
+ let parser = new lazy.FeedParser();
+ this.itemsToStore = parser.parseFeed(this, this.request.responseXML);
+ parser = null;
+
+ if (this.mInvalidFeed) {
+ this.request = null;
+ this.mInvalidFeed = false;
+ return;
+ }
+
+ this.itemsToStoreIndex = 0;
+ this.itemsStored = 0;
+
+ // At this point, if we have items to potentially store and an existing
+ // folder, ensure the folder's msgDatabase is openable for new message
+ // processing. If not, reparse with an async nsIUrlListener |this| to
+ // continue once the reparse is complete.
+ if (
+ this.itemsToStore.length > 0 &&
+ this.folder &&
+ !lazy.FeedUtils.isMsgDatabaseOpenable(this.folder, true, this)
+ ) {
+ return;
+ }
+
+ // We have an msgDatabase; storeNextItem() will iterate through the parsed
+ // items, storing each one.
+ this.storeNextItem();
+ },
+
+ /**
+ * Clear the 'valid' field of all feeditems associated with this feed.
+ *
+ * @returns {void}
+ */
+ invalidateItems() {
+ let ds = lazy.FeedUtils.getItemsDS(this.server);
+ for (let id in ds.data) {
+ let item = ds.data[id];
+ if (item.feedURLs.includes(this.url)) {
+ item.valid = false;
+ lazy.FeedUtils.log.trace("Feed.invalidateItems: item - " + id);
+ }
+ }
+ ds.saveSoon();
+ },
+
+ /**
+ * Discards invalid items (in the feed item store) associated with the
+ * feed. There's a delay - invalid items are kept around for a set time
+ * before being purged.
+ *
+ * @param {Boolean} aDeleteFeed - is the feed being deleted (bypasses
+ * the delay time).
+ * @returns {void}
+ */
+ removeInvalidItems(aDeleteFeed) {
+ let ds = lazy.FeedUtils.getItemsDS(this.server);
+ lazy.FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url);
+
+ let currentTime = new Date().getTime();
+ for (let id in ds.data) {
+ let item = ds.data[id];
+ // skip valid items and ones not part of this feed.
+ if (!item.feedURLs.includes(this.url) || item.valid) {
+ continue;
+ }
+ let lastSeenTime = item.lastSeenTime || 0;
+
+ if (
+ currentTime - lastSeenTime < lazy.FeedUtils.INVALID_ITEM_PURGE_DELAY &&
+ !aDeleteFeed
+ ) {
+ // Don't immediately purge items in active feeds; do so for deleted feeds.
+ continue;
+ }
+
+ lazy.FeedUtils.log.trace("Feed.removeInvalidItems: item - " + id);
+ // Detach the item from this feed (it could be shared by multiple feeds).
+ item.feedURLs = item.feedURLs.filter(url => url != this.url);
+ if (item.feedURLs.length > 0) {
+ lazy.FeedUtils.log.debug(
+ "Feed.removeInvalidItems: " +
+ id +
+ " is from more than one feed; only the reference to" +
+ " this feed removed"
+ );
+ } else {
+ delete ds.data[id];
+ }
+ }
+ ds.saveSoon();
+ },
+
+ createFolder() {
+ if (this.folder) {
+ return;
+ }
+
+ try {
+ this.folder = this.server.rootMsgFolder
+ .QueryInterface(Ci.nsIMsgLocalMailFolder)
+ .createLocalSubfolder(this.folderName);
+ } catch (ex) {
+ // An error creating.
+ lazy.FeedUtils.log.info(
+ "Feed.createFolder: error creating folder - '" +
+ this.folderName +
+ "' in parent folder " +
+ this.server.rootMsgFolder.filePath.path +
+ " -- " +
+ ex
+ );
+ // But its remnants are still there, clean up.
+ let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName);
+ this.server.rootMsgFolder.propagateDelete(xfolder, true);
+ }
+ },
+
+ // Gets the next item from itemsToStore and forces that item to be stored
+ // to the folder. If more items are left to be stored, fires a timer for
+ // the next one, otherwise triggers a download done notification to the UI.
+ storeNextItem() {
+ if (lazy.FeedUtils.CANCEL_REQUESTED) {
+ lazy.FeedUtils.CANCEL_REQUESTED = false;
+ this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogCancel);
+ return;
+ }
+
+ if (this.itemsToStore.length == 0) {
+ let code = lazy.FeedUtils.kNewsBlogSuccess;
+ this.createFolder();
+ if (!this.folder) {
+ code = lazy.FeedUtils.kNewsBlogFileError;
+ }
+
+ this.cleanupParsingState(this, code);
+ return;
+ }
+
+ let item = this.itemsToStore[this.itemsToStoreIndex];
+
+ if (item.store()) {
+ this.itemsStored++;
+ }
+
+ if (!this.folder) {
+ this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogFileError);
+ return;
+ }
+
+ this.itemsToStoreIndex++;
+
+ // If the listener is tracking progress for each item, report it here.
+ if (
+ item.feed.downloadCallback &&
+ item.feed.downloadCallback.onFeedItemStored
+ ) {
+ item.feed.downloadCallback.onFeedItemStored(
+ item.feed,
+ this.itemsToStoreIndex,
+ this.itemsToStore.length
+ );
+ }
+
+ // Eventually we'll report individual progress here.
+
+ if (this.itemsToStoreIndex < this.itemsToStore.length) {
+ if (!this.storeItemsTimer) {
+ this.storeItemsTimer = Cc["@mozilla.org/timer;1"].createInstance(
+ Ci.nsITimer
+ );
+ }
+
+ this.storeItemsTimer.initWithCallback(
+ this,
+ 50,
+ Ci.nsITimer.TYPE_ONE_SHOT
+ );
+ } else {
+ // We have just finished downloading one or more feed items into the
+ // destination folder; if the folder is still listed as having new
+ // messages in it, then we should set the biff state on the folder so the
+ // right RDF UI changes happen in the folder pane to indicate new mail.
+ if (item.feed.folder.hasNewMessages) {
+ item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail;
+ // Run the bayesian spam filter, if enabled.
+ item.feed.folder.callFilterPlugins(null);
+ }
+
+ this.cleanupParsingState(this, lazy.FeedUtils.kNewsBlogSuccess);
+ }
+ },
+
+ cleanupParsingState(aFeed, aCode) {
+ // Now that we are done parsing the feed, remove the feed from the cache.
+ FeedCache.removeFeed(aFeed.url);
+
+ if (aFeed.parseItems) {
+ // Do this only if we're in parse/store mode.
+ aFeed.removeInvalidItems(false);
+
+ if (aCode == lazy.FeedUtils.kNewsBlogSuccess && aFeed.mLastModified) {
+ aFeed.lastModified = aFeed.mLastModified;
+ }
+
+ // Flush any feed item changes to disk.
+ let ds = lazy.FeedUtils.getItemsDS(aFeed.server);
+ ds.saveSoon();
+ lazy.FeedUtils.log.debug(
+ "Feed.cleanupParsingState: items stored - " + this.itemsStored
+ );
+ }
+
+ // Force the xml http request to go away. This helps reduce some nasty
+ // assertions on shut down.
+ this.request = null;
+ this.itemsToStore = [];
+ this.itemsToStoreIndex = 0;
+ this.storeItemsTimer = null;
+
+ if (aFeed.downloadCallback) {
+ aFeed.downloadCallback.downloaded(aFeed, aCode);
+ }
+ },
+
+ // nsITimerCallback
+ notify(aTimer) {
+ this.storeNextItem();
+ },
+};
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-13 22:31:05 +0000