diff options
Diffstat (limited to 'comm/mailnews/extensions/newsblog/FeedUtils.jsm')
| -rw-r--r-- | comm/mailnews/extensions/newsblog/FeedUtils.jsm | 2136 |
1 files changed, 2136 insertions, 0 deletions
diff --git a/comm/mailnews/extensions/newsblog/FeedUtils.jsm b/comm/mailnews/extensions/newsblog/FeedUtils.jsm new file mode 100644 index 0000000000..b881c358d8 --- /dev/null +++ b/comm/mailnews/extensions/newsblog/FeedUtils.jsm @@ -0,0 +1,2136 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const EXPORTED_SYMBOLS = ["FeedUtils"]; + +const { XPCOMUtils } = ChromeUtils.importESModule( + "resource://gre/modules/XPCOMUtils.sys.mjs" +); +const { MailServices } = ChromeUtils.import( + "resource:///modules/MailServices.jsm" +); + +const lazy = {}; + +ChromeUtils.defineESModuleGetters(lazy, { + JSONFile: "resource://gre/modules/JSONFile.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", +}); + +XPCOMUtils.defineLazyModuleGetters(lazy, { + Feed: "resource:///modules/Feed.jsm", + jsmime: "resource:///modules/jsmime.jsm", + MailUtils: "resource:///modules/MailUtils.jsm", +}); + +var FeedUtils = { + MOZ_PARSERERROR_NS: "http://www.mozilla.org/newlayout/xml/parsererror.xml", + + RDF_SYNTAX_NS: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + RDF_SYNTAX_TYPE: "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", + RSS_090_NS: "http://my.netscape.com/rdf/simple/0.9/", + + RSS_NS: "http://purl.org/rss/1.0/", + + RSS_CONTENT_NS: "http://purl.org/rss/1.0/modules/content/", + + RSS_SY_NS: "http://purl.org/rss/1.0/modules/syndication/", + RSS_SY_UNITS: ["hourly", "daily", "weekly", "monthly", "yearly"], + kBiffUnitsMinutes: "min", + kBiffUnitsDays: "d", + + DC_NS: "http://purl.org/dc/elements/1.1/", + + MRSS_NS: "http://search.yahoo.com/mrss/", + FEEDBURNER_NS: "http://rssnamespace.org/feedburner/ext/1.0", + ITUNES_NS: "http://www.itunes.com/dtds/podcast-1.0.dtd", + + FZ_NS: "urn:forumzilla:", + FZ_ITEM_NS: "urn:feeditem:", + + // Atom constants + ATOM_03_NS: "http://purl.org/atom/ns#", + ATOM_IETF_NS: "http://www.w3.org/2005/Atom", + ATOM_THREAD_NS: "http://purl.org/syndication/thread/1.0", + + // Accept content mimetype preferences for feeds. + REQUEST_ACCEPT: + "application/atom+xml," + + "application/rss+xml;q=0.9," + + "application/rdf+xml;q=0.8," + + "application/xml;q=0.7,text/xml;q=0.7," + + "*/*;q=0.1", + // Timeout for nonresponse to request, 30 seconds. + REQUEST_TIMEOUT: 30 * 1000, + + MILLISECONDS_PER_DAY: 24 * 60 * 60 * 1000, + + // Maximum number of concurrent in progress feeds, across all accounts. + kMaxConcurrentFeeds: 25, + get MAX_CONCURRENT_FEEDS() { + let pref = "rss.max_concurrent_feeds"; + if (Services.prefs.prefHasUserValue(pref)) { + return Services.prefs.getIntPref(pref); + } + + Services.prefs.setIntPref(pref, FeedUtils.kMaxConcurrentFeeds); + return FeedUtils.kMaxConcurrentFeeds; + }, + + // The amount of time, specified in milliseconds, to leave an item in the + // feeditems cache after the item has disappeared from the publisher's + // file. The default delay is one day. + kInvalidItemPurgeDelayDays: 1, + get INVALID_ITEM_PURGE_DELAY() { + let pref = "rss.invalid_item_purge_delay_days"; + if (Services.prefs.prefHasUserValue(pref)) { + return Services.prefs.getIntPref(pref) * this.MILLISECONDS_PER_DAY; + } + + Services.prefs.setIntPref(pref, FeedUtils.kInvalidItemPurgeDelayDays); + return FeedUtils.kInvalidItemPurgeDelayDays * this.MILLISECONDS_PER_DAY; + }, + + // Polling interval to check individual feed update interval preference. + kBiffPollMinutes: 1, + kNewsBlogSuccess: 0, + // Usually means there was an error trying to parse the feed. + kNewsBlogInvalidFeed: 1, + // Generic networking failure when trying to download the feed. + kNewsBlogRequestFailure: 2, + kNewsBlogFeedIsBusy: 3, + // For 304 Not Modified; There are no new articles for this feed. + kNewsBlogNoNewItems: 4, + kNewsBlogCancel: 5, + kNewsBlogFileError: 6, + // Invalid certificate, for overridable user exception errors. + kNewsBlogBadCertError: 7, + // For 401 Unauthorized or 403 Forbidden. + kNewsBlogNoAuthError: 8, + + CANCEL_REQUESTED: false, + AUTOTAG: "~AUTOTAG", + + FEED_ACCOUNT_TYPES: ["rss"], + + /** + * Get all rss account servers rootFolders. + * + * @returns {nsIMsgIncomingServer}[] - Array of servers (empty array if none). + */ + getAllRssServerRootFolders() { + let rssRootFolders = []; + for (let server of MailServices.accounts.allServers) { + if (server && server.type == "rss") { + rssRootFolders.push(server.rootFolder); + } + } + + // By default, Tb sorts by hostname, ie Feeds, Feeds-1, and not by alpha + // prettyName. Do the same as a stock install to match folderpane order. + rssRootFolders.sort(function (a, b) { + return a.hostname > b.hostname; + }); + + return rssRootFolders; + }, + + /** + * Create rss account. + * + * @param {String} aName - Optional account name to override default. + * @returns {nsIMsgAccount} - The creaged account. + */ + createRssAccount(aName) { + let userName = "nobody"; + let hostName = "Feeds"; + let hostNamePref = hostName; + let server; + let serverType = "rss"; + let defaultName = FeedUtils.strings.GetStringFromName("feeds-accountname"); + let i = 2; + while (MailServices.accounts.findServer(userName, hostName, serverType)) { + // If "Feeds" exists, try "Feeds-2", then "Feeds-3", etc. + hostName = hostNamePref + "-" + i++; + } + + server = MailServices.accounts.createIncomingServer( + userName, + hostName, + serverType + ); + server.biffMinutes = FeedUtils.kBiffPollMinutes; + server.prettyName = aName ? aName : defaultName; + server.valid = true; + let account = MailServices.accounts.createAccount(); + account.incomingServer = server; + // Initialize the feed_options now. + this.getOptionsAcct(server); + + // Ensure the Trash folder db (.msf) is created otherwise folder/message + // deletes will throw until restart creates it. + server.msgStore.discoverSubFolders(server.rootMsgFolder, false); + + // Save new accounts in case of a crash. + try { + MailServices.accounts.saveAccountInfo(); + } catch (ex) { + this.log.error( + "FeedUtils.createRssAccount: error on saveAccountInfo - " + ex + ); + } + + this.log.debug( + "FeedUtils.createRssAccount: " + + account.incomingServer.rootFolder.prettyName + ); + + return account; + }, + + /** + * Helper routine that checks our subscriptions list array and returns + * true if the url is already in our list. This is used to prevent the + * user from subscribing to the same feed multiple times for the same server. + * + * @param {string} aUrl - The url. + * @param {nsIMsgIncomingServer} aServer - Account server. + * @returns {Boolean} - true if exists else false. + */ + feedAlreadyExists(aUrl, aServer) { + let ds = this.getSubscriptionsDS(aServer); + let sub = ds.data.find(x => x.url == aUrl); + if (sub === undefined) { + return false; + } + let folder = sub.destFolder; + this.log.info( + "FeedUtils.feedAlreadyExists: feed url " + + aUrl + + " subscribed in folder url " + + decodeURI(folder) + ); + + return true; + }, + + /** + * Download a feed url on biff or get new messages. + * + * @param {nsIMsgFolder} aFolder - The folder. + * @param {nsIUrlListener} aUrlListener - Feed url. + * @param {Boolean} aIsBiff - true if biff, false if manual get. + * @param {nsIDOMWindow} aMsgWindow - The window. + * + * @returns {Promise<void>} When all feeds downloading have been set off. + */ + async downloadFeed(aFolder, aUrlListener, aIsBiff, aMsgWindow) { + FeedUtils.log.debug( + "downloadFeed: account isBiff:isOffline - " + + aIsBiff + + " : " + + Services.io.offline + ); + // User set. + if (Services.io.offline) { + return; + } + + // No network connection. Unfortunately, this is only set if the event is + // received by Tb from the OS (ie it must already be running) and doesn't + // necessarily mean connectivity to the internet, only the nearest network + // point. But it's something. + if (!Services.io.connectivity) { + FeedUtils.log.warn("downloadFeed: network connection unavailable"); + return; + } + + // We don't yet support the ability to check for new articles while we are + // in the middle of subscribing to a feed. For now, abort the check for + // new feeds. + if (FeedUtils.progressNotifier.mSubscribeMode) { + FeedUtils.log.warn( + "downloadFeed: Aborting RSS New Mail Check. " + + "Feed subscription in progress\n" + ); + return; + } + + let forceDownload = !aIsBiff; + let inStartup = false; + if (aFolder.isServer) { + // The lastUpdateTime is |null| only at session startup/initialization. + // Note: feed processing does not impact startup, as the biff poll + // will go off in about kBiffPollMinutes (1) and process each feed + // according to its own lastUpdatTime/update frequency. + if (FeedUtils.getStatus(aFolder, aFolder.URI).lastUpdateTime === null) { + inStartup = true; + } + + FeedUtils.setStatus(aFolder, aFolder.URI, "lastUpdateTime", Date.now()); + } + + let allFolders = aFolder.descendants; + if (!aFolder.isServer) { + // Add the base folder; it does not get returned by .descendants. Do not + // add the account folder as it doesn't have the feedUrl property or even + // a msgDatabase necessarily. + allFolders.unshift(aFolder); + } + + let folder; + async function* feeder() { + for (let i = 0; i < allFolders.length; i++) { + folder = allFolders[i]; + FeedUtils.log.debug( + "downloadFeed: START x/# folderName:folderPath - " + + (i + 1) + + "/" + + allFolders.length + + " " + + folder.name + + " : " + + folder.filePath.path + ); + + let feedUrlArray = FeedUtils.getFeedUrlsInFolder(folder); + // Continue if there are no feedUrls for the folder in the feeds + // database. All folders in Trash are skipped. + if (!feedUrlArray) { + continue; + } + + FeedUtils.log.debug( + "downloadFeed: CONTINUE foldername:urlArray - " + + folder.name + + " : " + + feedUrlArray + ); + + // We need to kick off a download for each feed. + let now = Date.now(); + for (let url of feedUrlArray) { + // Check whether this feed should be updated; if forceDownload is true + // skip the per feed check. + let status = FeedUtils.getStatus(folder, url); + if (!forceDownload) { + // Also skip if user paused, or error paused (but not inStartup; + // go check the feed again), or update interval hasn't expired. + if ( + status.enabled === false || + (status.enabled === null && !inStartup) || + now - status.lastUpdateTime < status.updateMinutes * 60000 + ) { + FeedUtils.log.debug( + "downloadFeed: SKIP feed, " + + "aIsBiff:enabled:minsSinceLastUpdate::url - " + + aIsBiff + + " : " + + status.enabled + + " : " + + Math.round((now - status.lastUpdateTime) / 60) / 1000 + + " :: " + + url + ); + continue; + } + } + // Update feed icons only once every 24h, tops. + if ( + forceDownload || + now - status.lastUpdateTime >= 24 * 60 * 60 * 1000 + ) { + await FeedUtils.getFavicon(folder, url); + } + + // Create a feed object. + let feed = new lazy.Feed(url, folder); + + // init can be called multiple times. Checks if it should actually + // init itself. + FeedUtils.progressNotifier.init(aMsgWindow, false); + + // Bump our pending feed download count. From now on, all feeds will + // be resolved and finish with progressNotifier.downloaded(). Any + // early returns must call downloaded() so mNumPendingFeedDownloads + // is decremented and notification/status feedback is reset. + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + + // If the current active count exceeds the max desired, exit from + // the current poll cycle. Only throttle for a background biff; for + // a user manual get messages, do them all. + if ( + aIsBiff && + FeedUtils.progressNotifier.mNumPendingFeedDownloads > + FeedUtils.MAX_CONCURRENT_FEEDS + ) { + FeedUtils.log.debug( + "downloadFeed: RETURN active feeds count is greater " + + "than the max - " + + FeedUtils.MAX_CONCURRENT_FEEDS + ); + FeedUtils.progressNotifier.downloaded( + feed, + FeedUtils.kNewsBlogFeedIsBusy + ); + return; + } + + // Set status info and download. + FeedUtils.log.debug("downloadFeed: DOWNLOAD feed url - " + url); + FeedUtils.setStatus( + folder, + url, + "code", + FeedUtils.kNewsBlogFeedIsBusy + ); + feed.download(true, FeedUtils.progressNotifier); + + Services.tm.mainThread.dispatch(function () { + try { + let done = getFeed.next().done; + if (done) { + // Finished with all feeds in base aFolder and its subfolders. + FeedUtils.log.debug( + "downloadFeed: Finished with folder - " + aFolder.name + ); + folder = null; + allFolders = null; + } + } catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded( + feed, + FeedUtils.kNewsBlogFeedIsBusy + ); + } + }, Ci.nsIThread.DISPATCH_NORMAL); + + yield undefined; + } + } + } + + let getFeed = await feeder(); + try { + let done = getFeed.next().done; + if (done) { + // Nothing to do. + FeedUtils.log.debug( + "downloadFeed: Nothing to do in folder - " + aFolder.name + ); + folder = null; + allFolders = null; + } + } catch (ex) { + FeedUtils.log.error("downloadFeed: error - " + ex); + FeedUtils.progressNotifier.downloaded( + { folder: aFolder, url: "" }, + FeedUtils.kNewsBlogFeedIsBusy + ); + } + }, + + /** + * Subscribe a new feed url. + * + * @param {String} aUrl - Feed url. + * @param {nsIMsgFolder} aFolder - Folder. + * + * @returns {void} + */ + subscribeToFeed(aUrl, aFolder) { + // We don't support the ability to subscribe to several feeds at once yet. + // For now, abort the subscription if we are already in the middle of + // subscribing to a feed via drag and drop. + if (FeedUtils.progressNotifier.mNumPendingFeedDownloads > 0) { + FeedUtils.log.warn( + "subscribeToFeed: Aborting RSS subscription. " + + "Feed downloads already in progress\n" + ); + return; + } + + // If aFolder is null, then use the root folder for the first RSS account. + if (!aFolder) { + aFolder = FeedUtils.getAllRssServerRootFolders()[0]; + } + + // If the user has no Feeds account yet, create one. + if (!aFolder) { + aFolder = FeedUtils.createRssAccount().incomingServer.rootFolder; + } + + // If aUrl is a feed url, then it is either of the form + // feed://example.org/feed.xml or feed:https://example.org/feed.xml. + // Replace feed:// with http:// per the spec, then strip off feed: + // for the second case. + aUrl = aUrl.replace(/^feed:\x2f\x2f/i, "http://"); + aUrl = aUrl.replace(/^feed:/i, ""); + + let msgWindow = Services.wm.getMostRecentWindow("mail:3pane").msgWindow; + + // Make sure we aren't already subscribed to this feed before we attempt + // to subscribe to it. + if (FeedUtils.feedAlreadyExists(aUrl, aFolder.server)) { + msgWindow.statusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName("subscribe-feedAlreadySubscribed") + ); + return; + } + + let feed = new lazy.Feed(aUrl, aFolder); + // Default setting for new feeds per account settings. + feed.quickMode = feed.server.getBoolValue("quickMode"); + feed.options = FeedUtils.getOptionsAcct(feed.server); + + FeedUtils.progressNotifier.init(msgWindow, true); + FeedUtils.progressNotifier.mNumPendingFeedDownloads++; + feed.download(true, FeedUtils.progressNotifier); + }, + + /** + * Enable or disable updates for all subscriptions in a folder, or all + * subscriptions in an account if the folder is the account folder. + * A folder's subfolders' feeds are not included. + * + * @param {nsIMsgFolder} aFolder - Folder or account folder (server). + * @param {boolean} aPause - To pause or not to pause. + * @param {Boolean} aBiffNow - If aPause is false, and aBiffNow is true + * do the biff immediately. + * @returns {void} + */ + pauseFeedFolderUpdates(aFolder, aPause, aBiffNow) { + if (aFolder.isServer) { + let serverFolder = aFolder.server.rootFolder; + // Remove server from biff first. If enabling biff, this will make the + // latest biffMinutes take effect now rather than waiting for the timer + // to expire. + aFolder.server.doBiff = false; + if (!aPause) { + aFolder.server.doBiff = true; + } + + FeedUtils.setStatus(serverFolder, serverFolder.URI, "enabled", !aPause); + if (!aPause && aBiffNow) { + aFolder.server.performBiff(null); + } + + return; + } + + let feedUrls = FeedUtils.getFeedUrlsInFolder(aFolder); + if (!feedUrls) { + return; + } + + for (let feedUrl of feedUrls) { + let feed = new lazy.Feed(feedUrl, aFolder); + let options = feed.options; + options.updates.enabled = !aPause; + feed.options = options; + FeedUtils.setStatus(aFolder, feedUrl, "enabled", !aPause); + FeedUtils.log.debug( + "pauseFeedFolderUpdates: enabled:url " + !aPause + ": " + feedUrl + ); + } + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + let curItem = win.FeedSubscriptions.mView.currentItem; + win.FeedSubscriptions.refreshSubscriptionView(); + if (curItem.container) { + win.FeedSubscriptions.selectFolder(curItem.folder); + } else { + let feed = new lazy.Feed(curItem.url, curItem.parentFolder); + win.FeedSubscriptions.selectFeed(feed); + } + } + }, + + /** + * Add a feed record to the feeds database and update the folder's feedUrl + * property. + * + * @param {Feed} aFeed - Our feed object. + * + * @returns {void} + */ + addFeed(aFeed) { + // Find or create subscription entry. + let ds = this.getSubscriptionsDS(aFeed.server); + let sub = ds.data.find(x => x.url == aFeed.url); + if (sub === undefined) { + sub = {}; + ds.data.push(sub); + } + sub.url = aFeed.url; + sub.destFolder = aFeed.folder.URI; + if (aFeed.title) { + sub.title = aFeed.title; + } + ds.saveSoon(); + + // Update folderpane. + Services.obs.notifyObservers(aFeed.folder, "folder-properties-changed"); + }, + + /** + * Delete a feed record from the feeds database and update the folder's + * feedUrl property. + * + * @param {Feed} aFeed - Our feed object. + * + * @returns {void} + */ + deleteFeed(aFeed) { + // Remove items associated with this feed from the items db. + aFeed.invalidateItems(); + aFeed.removeInvalidItems(true); + + // Remove the entry in the subscriptions db. + let ds = this.getSubscriptionsDS(aFeed.server); + ds.data = ds.data.filter(x => x.url != aFeed.url); + ds.saveSoon(); + + // Update folderpane. + Services.obs.notifyObservers(aFeed.folder, "folder-properties-changed"); + }, + + /** + * Change an existing feed's url. + * + * @param {Feed} aFeed - The feed object. + * @param {string} aNewUrl - New url. + * + * @returns {Boolean} - true if successful, else false. + */ + changeUrlForFeed(aFeed, aNewUrl) { + if (!aFeed || !aFeed.folder || !aNewUrl) { + return false; + } + + if (this.feedAlreadyExists(aNewUrl, aFeed.server)) { + this.log.info( + "FeedUtils.changeUrlForFeed: new feed url " + + aNewUrl + + " already subscribed in account " + + aFeed.server.prettyName + ); + return false; + } + + let title = aFeed.title; + let link = aFeed.link; + let quickMode = aFeed.quickMode; + let options = aFeed.options; + + this.deleteFeed(aFeed); + aFeed.url = aNewUrl; + aFeed.title = title; + aFeed.link = link; + aFeed.quickMode = quickMode; + aFeed.options = options; + this.addFeed(aFeed); + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + win.FeedSubscriptions.refreshSubscriptionView(aFeed.folder, aNewUrl); + } + + return true; + }, + + /** + * Determine if a message is a feed message. Prior to Tb15, a message had to + * be in an rss acount type folder. In Tb15 and later, a flag is set on the + * message itself upon initial store; the message can be moved to any folder. + * + * @param {nsIMsgDBHdr} aMsgHdr - The message. + * + * @returns {Boolean} - true if message is a feed, false if not. + */ + isFeedMessage(aMsgHdr) { + return Boolean( + aMsgHdr instanceof Ci.nsIMsgDBHdr && + (aMsgHdr.flags & Ci.nsMsgMessageFlags.FeedMsg || + this.isFeedFolder(aMsgHdr.folder)) + ); + }, + + /** + * Determine if a folder is a feed acount folder. Trash or a folder in Trash + * should be checked with FeedUtils.isInTrash() if required. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {Boolean} - true if folder's server.type is in FEED_ACCOUNT_TYPES, + * false if not. + */ + isFeedFolder(aFolder) { + return Boolean( + aFolder instanceof Ci.nsIMsgFolder && + this.FEED_ACCOUNT_TYPES.includes(aFolder.server.type) + ); + }, + + /** + * Get the list of feed urls for a folder. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {String}[] - Array of urls, or null if none. + */ + getFeedUrlsInFolder(aFolder) { + if ( + !aFolder || + aFolder.isServer || + aFolder.server.type != "rss" || + aFolder.getFlag(Ci.nsMsgFolderFlags.Trash) || + aFolder.getFlag(Ci.nsMsgFolderFlags.Virtual) || + !aFolder.filePath.exists() + ) { + // There are never any feedUrls in the account/non-feed/trash/virtual + // folders or in a ghost folder (nonexistent on disk yet found in + // aFolder.subFolders). + return null; + } + + let feedUrlArray = []; + + // Get the list from the feeds database. + try { + let ds = this.getSubscriptionsDS(aFolder.server); + for (const sub of ds.data) { + if (sub.destFolder == aFolder.URI) { + feedUrlArray.push(sub.url); + } + } + } catch (ex) { + this.log.error("getFeedUrlsInFolder: feeds db error - " + ex); + this.log.error( + "getFeedUrlsInFolder: feeds db error for account - " + + aFolder.server.serverURI + + " : " + + aFolder.server.prettyName + ); + } + + return feedUrlArray.length ? feedUrlArray : null; + }, + + /** + * Check if the folder's msgDatabase is openable, reparse if desired. + * + * @param {nsIMsgFolder} aFolder - The folder. + * @param {boolean} aReparse - Reparse if true. + * @param {nsIUrlListener} aUrlListener - Object implementing nsIUrlListener. + * + * @returns {Boolean} - true if msgDb is available, else false + */ + isMsgDatabaseOpenable(aFolder, aReparse, aUrlListener) { + let msgDb; + try { + msgDb = Cc["@mozilla.org/msgDatabase/msgDBService;1"] + .getService(Ci.nsIMsgDBService) + .openFolderDB(aFolder, true); + } catch (ex) {} + + if (msgDb) { + return true; + } + + if (!aReparse) { + return false; + } + + // Force a reparse. + FeedUtils.log.debug( + "checkMsgDb: rebuild msgDatabase for " + + aFolder.name + + " - " + + aFolder.filePath.path + ); + try { + // Ignore error returns. + aFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .getDatabaseWithReparse(aUrlListener, null); + } catch (ex) {} + + return false; + }, + + /** + * Return properties for nsITreeView getCellProperties, for a tree row item in + * folderpane or subscribe dialog tree. + * + * @param {nsIMsgFolder} aFolder - Folder or a feed url's parent folder. + * @param {string} aFeedUrl - Feed url for a feed row, null for folder. + * + * @returns {string} - Space separated properties. + */ + getFolderProperties(aFolder, aFeedUrl) { + let folder = aFolder; + let feedUrls = aFeedUrl ? [aFeedUrl] : this.getFeedUrlsInFolder(aFolder); + if (!feedUrls && !folder.isServer) { + return ""; + } + + let serverEnabled = this.getStatus( + folder.server.rootFolder, + folder.server.rootFolder.URI + ).enabled; + if (folder.isServer) { + return !serverEnabled ? " isPaused" : ""; + } + + let properties = aFeedUrl ? " isFeed-true" : " isFeedFolder-true"; + let hasError, + isBusy, + numPaused = 0; + for (let feedUrl of feedUrls) { + let feedStatus = this.getStatus(folder, feedUrl); + if ( + feedStatus.code == FeedUtils.kNewsBlogInvalidFeed || + feedStatus.code == FeedUtils.kNewsBlogRequestFailure || + feedStatus.code == FeedUtils.kNewsBlogBadCertError || + feedStatus.code == FeedUtils.kNewsBlogNoAuthError + ) { + hasError = true; + } + if (feedStatus.code == FeedUtils.kNewsBlogFeedIsBusy) { + isBusy = true; + } + if (!feedStatus.enabled) { + numPaused++; + } + } + + properties += hasError ? " hasError" : ""; + properties += isBusy ? " isBusy" : ""; + properties += numPaused == feedUrls.length ? " isPaused" : ""; + + return properties; + }, + + /** + * Get a cached feed or folder status. + * + * @param {nsIMsgFolder} aFolder - Folder. + * @param {string} aUrl - Url key (feed url or folder URI). + * + * @returns {String} aValue - The value. + */ + getStatus(aFolder, aUrl) { + if (!aFolder || !aUrl) { + return null; + } + + let serverKey = aFolder.server.serverURI; + if (!this[serverKey]) { + this[serverKey] = {}; + } + + if (!this[serverKey][aUrl]) { + // Seed the status object. + this[serverKey][aUrl] = {}; + this[serverKey][aUrl].status = this.statusTemplate; + if (FeedUtils.isValidScheme(aUrl)) { + // Seed persisted status properties for feed urls. + let feed = new lazy.Feed(aUrl, aFolder); + this[serverKey][aUrl].status.enabled = feed.options.updates.enabled; + this[serverKey][aUrl].status.updateMinutes = + feed.options.updates.updateMinutes; + this[serverKey][aUrl].status.lastUpdateTime = + feed.options.updates.lastUpdateTime; + feed = null; + } else { + // Seed persisted status properties for servers. + let optionsAcct = FeedUtils.getOptionsAcct(aFolder.server); + this[serverKey][aUrl].status.enabled = optionsAcct.doBiff; + } + FeedUtils.log.debug("getStatus: seed url - " + aUrl); + } + + return this[serverKey][aUrl].status; + }, + + /** + * Update a feed or folder status and refresh folderpane. + * + * @param {nsIMsgFolder} aFolder - Folder. + * @param {string} aUrl - Url key (feed url or folder URI). + * @param {string} aProperty - Url status property. + * @param {string} aValue - Value. + * + * @returns {void} + */ + setStatus(aFolder, aUrl, aProperty, aValue) { + if (!aFolder || !aUrl || !aProperty) { + return; + } + + if ( + !this[aFolder.server.serverURI] || + !this[aFolder.server.serverURI][aUrl] + ) { + // Not yet seeded, so do it. + this.getStatus(aFolder, aUrl); + } + + this[aFolder.server.serverURI][aUrl].status[aProperty] = aValue; + + Services.obs.notifyObservers(aFolder, "folder-properties-changed"); + + let win = Services.wm.getMostRecentWindow("Mail:News-BlogSubscriptions"); + if (win) { + win.FeedSubscriptions.mView.tree.invalidate(); + } + }, + + /** + * Get the favicon for a feed folder subscription url (first one) or a feed + * message url. The favicon service caches it in memory if places history is + * not enabled. + * + * @param {?nsIMsgFolder} folder - The feed folder or null if url set. + * @param {?string} feedURL - A url (feed, message, other) or null if folder set. + * + * @returns {Promise<string>} - The favicon url or empty string. + */ + async getFavicon(folder, feedURL) { + if ( + !Services.prefs.getBoolPref("browser.chrome.site_icons") || + !Services.prefs.getBoolPref("browser.chrome.favicons") || + !Services.prefs.getBoolPref("places.history.enabled") + ) { + return ""; + } + + let url = feedURL; + if (!url) { + // Get the proposed iconUrl from the folder's first subscribed feed's + // <link>. + url = this.getFeedUrlsInFolder(folder)[0]; + if (!url) { + return ""; + } + feedURL = url; + } + + if (folder) { + let feed = new lazy.Feed(url, folder); + url = feed.link && feed.link.startsWith("http") ? feed.link : url; + } + + /** + * Convert a Blob to data URL. + * @param {Blob} blob - Blob to convert. + * @returns {string} data URL. + */ + let blobToBase64 = blob => { + return new Promise((resolve, reject) => { + let reader = new FileReader(); + reader.onloadend = () => { + if (reader.result.endsWith("base64,")) { + reject(new Error(`Invalid blob encountered.`)); + return; + } + resolve(reader.result); + }; + reader.readAsDataURL(blob); + }); + }; + + /** + * Try getting favicon from url. + * @param {string} url - The favicon url. + * @returns {Blob} - Existing favicon. + */ + let fetchFavicon = async url => { + let response = await fetch(url); + if (!response.ok) { + throw new Error(`No favicon for url ${url}`); + } + if (!/^image\//i.test(response.headers.get("Content-Type"))) { + throw new Error(`Non-image favicon for ${url}`); + } + return response.blob(); + }; + + /** + * Try getting favicon from the a html page. + * @param {string} page - The page url to check. + * @returns {Blob} - Found favicon. + */ + let discoverFaviconURL = async page => { + let response = await fetch(page); + if (!response.ok) { + throw new Error(`No favicon for page ${page}`); + } + if (!/^text\/html/i.test(response.headers.get("Content-Type"))) { + throw new Error(`No page to get favicon from for ${page}`); + } + let doc = new DOMParser().parseFromString( + await response.text(), + "text/html" + ); + let iconLink = doc.querySelector( + `link[href][rel~='icon']:is([sizes~='any'],[sizes~='16x16' i],[sizes~='32x32' i],:not([sizes])` + ); + if (!iconLink) { + throw new Error(`No iconLink discovered for page=${page}`); + } + if (/^https?:/.test(iconLink.href)) { + return iconLink.href; + } + if (iconLink.href.at(0) != "/") { + iconLink.href = "/" + iconLink.href; + } + return new URL(page).origin + iconLink.href; + }; + + let uri = Services.io.newURI(url); + let iconURL = await fetchFavicon(uri.prePath + "/favicon.ico") + .then(blobToBase64) + .catch(e => { + return discoverFaviconURL(url) + .catch(() => discoverFaviconURL(uri.prePath)) + .then(fetchFavicon) + .then(blobToBase64) + .catch(() => ""); + }); + + if (!iconURL) { + return ""; + } + + // setAndFetchFaviconForPage needs the url to be in the database first. + await lazy.PlacesUtils.history.insert({ + url: feedURL, + visits: [ + { + date: new Date(), + }, + ], + }); + return new Promise(resolve => { + // All good. Now store iconURL for future usage. + lazy.PlacesUtils.favicons.setAndFetchFaviconForPage( + Services.io.newURI(feedURL), + Services.io.newURI(iconURL), + true, + lazy.PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, + faviconURI => { + resolve(faviconURI.spec); + }, + Services.scriptSecurityManager.getSystemPrincipal() + ); + }); + }, + + /** + * Update the feeds database for rename and move/copy folder name changes. + * + * @param {nsIMsgFolder} aFolder - The folder, new if rename or target of + * move/copy folder (new parent). + * @param {nsIMsgFolder} aOrigFolder - Original folder. + * @param {String} aAction - "move" or "copy" or "rename". + * + * @returns {void} + */ + updateSubscriptionsDS(aFolder, aOrigFolder, aAction) { + this.log.debug( + "FeedUtils.updateSubscriptionsDS: " + + "\nfolder changed - " + + aAction + + "\nnew folder - " + + aFolder.filePath.path + + "\norig folder - " + + aOrigFolder.filePath.path + ); + + this.log.debug( + `updateSubscriptions(${aFolder.name}, ${aOrigFolder.name}, ${aAction})` + ); + + if (aFolder.server.type != "rss" || FeedUtils.isInTrash(aOrigFolder)) { + // Target not a feed account folder; nothing to do, or move/rename in + // trash; no subscriptions already. + return; + } + + let newFolder = aFolder; + let newParentURI = aFolder.URI; + let origParentURI = aOrigFolder.URI; + if (aAction == "move" || aAction == "copy") { + // Get the new folder. Don't process the entire parent (new dest folder)! + newFolder = aFolder.getChildNamed(aOrigFolder.name); + origParentURI = aOrigFolder.parent + ? aOrigFolder.parent.URI + : aOrigFolder.rootFolder.URI; + } + + this.updateFolderChangeInFeedsDS(newFolder, aOrigFolder, null, null); + + // There may be subfolders, but we only get a single notification; iterate + // over all descendent folders of the folder whose location has changed. + for (let newSubFolder of newFolder.descendants) { + FeedUtils.updateFolderChangeInFeedsDS( + newSubFolder, + aOrigFolder, + newParentURI, + origParentURI + ); + } + }, + + /** + * Update the feeds database with the new folder's or subfolder's location + * for rename and move/copy name changes. The feeds subscriptions db is + * also synced on cross account folder copies. Note that if a copied folder's + * url exists in the new account, its active subscription will be switched to + * the folder being copied, to enforce the one unique url per account design. + * + * @param {nsIMsgFolder} aFolder - New folder. + * @param {nsIMsgFolder} aOrigFolder - Original folder. + * @param {string} aNewAncestorURI - For subfolders, ancestor new folder. + * @param {String} aOrigAncestorURI - For subfolders, ancestor original folder. + * + * @returns {void} + */ + updateFolderChangeInFeedsDS( + aFolder, + aOrigFolder, + aNewAncestorURI, + aOrigAncestorURI + ) { + this.log.debug( + "updateFolderChangeInFeedsDS: " + + "\naFolder - " + + aFolder.URI + + "\naOrigFolder - " + + aOrigFolder.URI + + "\naOrigAncestor - " + + aOrigAncestorURI + + "\naNewAncestor - " + + aNewAncestorURI + ); + + // Get the original folder's URI. + let folderURI = aFolder.URI; + let origURI = + aNewAncestorURI && aOrigAncestorURI + ? folderURI.replace(aNewAncestorURI, aOrigAncestorURI) + : aOrigFolder.URI; + this.log.debug("updateFolderChangeInFeedsDS: urls origURI - " + origURI); + + // Get affected feed subscriptions - all the ones in the original folder. + let origDS = this.getSubscriptionsDS(aOrigFolder.server); + let affectedSubs = origDS.data.filter(sub => sub.destFolder == origURI); + if (affectedSubs.length == 0) { + this.log.debug("updateFolderChangeInFeedsDS: no feedUrls in this folder"); + return; + } + + if (this.isInTrash(aFolder)) { + // Moving to trash. Unsubscribe. + affectedSubs.forEach(function (sub) { + let feed = new lazy.Feed(sub.url, aFolder); + FeedUtils.deleteFeed(feed); + }); + // note: deleteFeed() calls saveSoon(), so we don't need to. + } else if (aFolder.server == aOrigFolder.server) { + // Staying in same account - just update destFolder as required + for (let sub of affectedSubs) { + sub.destFolder = folderURI; + } + origDS.saveSoon(); + } else { + // Moving between accounts. + let destDS = this.getSubscriptionsDS(aFolder.server); + for (let sub of affectedSubs) { + // Move to the new subscription db (replacing any existing entry). + origDS.data = origDS.data.filter(x => x.url != sub.url); + destDS.data = destDS.data.filter(x => x.url != sub.url); + sub.destFolder = folderURI; + destDS.data.push(sub); + } + + origDS.saveSoon(); + destDS.saveSoon(); + } + }, + + /** + * When subscribing to feeds by dnd on, or adding a url to, the account + * folder (only), or creating folder structure via opml import, a subfolder is + * autocreated and thus the derived/given name must be sanitized to prevent + * filesystem errors. Hashing invalid chars based on OS rather than filesystem + * is not strictly correct. + * + * @param {nsIMsgFolder} aParentFolder - Parent folder. + * @param {String} aProposedName - Proposed name. + * @param {String} aDefaultName - Default name if proposed sanitizes to + * blank, caller ensures sane value. + * @param {Boolean} aUnique - If true, return a unique indexed name. + * + * @returns {String} - Sanitized unique name. + */ + getSanitizedFolderName(aParentFolder, aProposedName, aDefaultName, aUnique) { + // Clean up the name for the strictest fs (fat) and to ensure portability. + // 1) Replace line breaks and tabs '\n\r\t' with a space. + // 2) Remove nonprintable ascii. + // 3) Remove invalid win chars '* | \ / : < > ? "'. + // 4) Remove all '.' as starting/ending with one is trouble on osx/win. + // 5) No leading/trailing spaces. + /* eslint-disable no-control-regex */ + let folderName = aProposedName + .replace(/[\n\r\t]+/g, " ") + .replace(/[\x00-\x1F]+/g, "") + .replace(/[*|\\\/:<>?"]+/g, "") + .replace(/[\.]+/g, "") + .trim(); + /* eslint-enable no-control-regex */ + + // Prefix with __ if name is: + // 1) a reserved win filename. + // 2) an undeletable/unrenameable special folder name (bug 259184). + if ( + folderName + .toUpperCase() + .match(/^COM\d$|^LPT\d$|^CON$|PRN$|^AUX$|^NUL$|^CLOCK\$/) || + folderName + .toUpperCase() + .match(/^INBOX$|^OUTBOX$|^UNSENT MESSAGES$|^TRASH$/) + ) { + folderName = "__" + folderName; + } + + // Use a default if no name is found. + if (!folderName) { + folderName = aDefaultName; + } + + if (!aUnique) { + return folderName; + } + + // Now ensure the folder name is not a dupe; if so append index. + let folderNameBase = folderName; + let i = 2; + while (aParentFolder.containsChildNamed(folderName)) { + folderName = folderNameBase + "-" + i++; + } + + return folderName; + }, + + /** + * This object contains feed/account status info. + */ + _statusDefault: { + // Derived from persisted value. + enabled: null, + // Last update result code, a kNewsBlog* value. + code: 0, + updateMinutes: null, + // JS Date; startup state is null indicating no update since startup. + lastUpdateTime: null, + }, + + get statusTemplate() { + // Copy the object. + return JSON.parse(JSON.stringify(this._statusDefault)); + }, + + /** + * This object will contain all persisted feed specific properties. + */ + _optionsDefault: { + version: 2, + updates: { + enabled: true, + // User set. + updateMinutes: 100, + // User set: "min"=minutes, "d"=days + updateUnits: "min", + // JS Date. + lastUpdateTime: null, + // The last time a new message was stored. JS Date. + lastDownloadTime: null, + // Publisher recommended from the feed. + updatePeriod: null, + updateFrequency: 1, + updateBase: null, + }, + // Autotag and <category> handling options. + category: { + enabled: false, + prefixEnabled: false, + prefix: null, + }, + }, + + get optionsTemplate() { + // Copy the object. + return JSON.parse(JSON.stringify(this._optionsDefault)); + }, + + getOptionsAcct(aServer) { + let optionsAcct; + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + let check_new_mail = "mail.server." + aServer.key + ".check_new_mail"; + let check_time = "mail.server." + aServer.key + ".check_time"; + + // Biff enabled or not. Make sure pref exists. + if (!Services.prefs.prefHasUserValue(check_new_mail)) { + Services.prefs.setBoolPref(check_new_mail, true); + } + + // System polling interval. Make sure pref exists. + if (!Services.prefs.prefHasUserValue(check_time)) { + Services.prefs.setIntPref(check_time, FeedUtils.kBiffPollMinutes); + } + + try { + optionsAcct = JSON.parse(Services.prefs.getCharPref(optionsAcctPref)); + // Add the server specific biff enabled state. + optionsAcct.doBiff = Services.prefs.getBoolPref(check_new_mail); + } catch (ex) {} + + if (optionsAcct && optionsAcct.version == this._optionsDefault.version) { + return optionsAcct; + } + + // Init account updates options if new or upgrading to version in + // |_optionsDefault.version|. + if (!optionsAcct || optionsAcct.version < this._optionsDefault.version) { + this.initAcct(aServer); + } + + let newOptions = this.newOptions(optionsAcct); + this.setOptionsAcct(aServer, newOptions); + newOptions.doBiff = Services.prefs.getBoolPref(check_new_mail); + return newOptions; + }, + + setOptionsAcct(aServer, aOptions) { + let optionsAcctPref = "mail.server." + aServer.key + ".feed_options"; + aOptions = aOptions || this.optionsTemplate; + Services.prefs.setCharPref(optionsAcctPref, JSON.stringify(aOptions)); + }, + + initAcct(aServer) { + let serverPrefStr = "mail.server." + aServer.key; + // System polling interval. Effective after current interval expires on + // change; no user facing ui. + Services.prefs.setIntPref( + serverPrefStr + ".check_time", + FeedUtils.kBiffPollMinutes + ); + + // If this pref is false, polling on biff is disabled and account updates + // are paused; ui in account server settings and folderpane context menu + // (Pause All Updates). Checking Enable updates or unchecking Pause takes + // effect immediately. Here on startup, we just ensure the polling interval + // above is reset immediately. + let doBiff = Services.prefs.getBoolPref(serverPrefStr + ".check_new_mail"); + FeedUtils.log.debug( + "initAcct: " + aServer.prettyName + " doBiff - " + doBiff + ); + this.pauseFeedFolderUpdates(aServer.rootFolder, !doBiff, false); + }, + + newOptions(aCurrentOptions) { + if (!aCurrentOptions) { + return this.optionsTemplate; + } + + // Options version has changed; meld current template with existing + // aCurrentOptions settings, removing items gone from the template while + // preserving user settings for retained template items. + let newOptions = this.optionsTemplate; + this.Mixins.meld(aCurrentOptions, false, true).into(newOptions); + newOptions.version = this.optionsTemplate.version; + return newOptions; + }, + + /** + * A generalized recursive melder of two objects. Getters/setters not included. + */ + Mixins: { + meld(source, keep, replace) { + function meldin(source, target, keep, replace) { + for (let attribute in source) { + // Recurse for objects. + if ( + typeof source[attribute] == "object" && + typeof target[attribute] == "object" + ) { + meldin(source[attribute], target[attribute], keep, replace); + } else { + // Use attribute values from source for the target, unless + // replace is false. + if (attribute in target && !replace) { + continue; + } + // Don't copy attribute from source to target if it is not in the + // target, unless keep is true. + if (!(attribute in target) && !keep) { + continue; + } + + target[attribute] = source[attribute]; + } + } + } + return { + source, + into(target) { + meldin(this.source, target, keep, replace); + }, + }; + }, + }, + + /** + * Returns a reference to the feed subscriptions store for the given server + * (the feeds.json data). + * + * @param {nsIMsgIncomingServer} aServer - server to fetch item data for. + * @returns {JSONFile} - a JSONFile holding the array of feed subscriptions + * in its data field. + */ + getSubscriptionsDS(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI].FeedsDS) { + return this[aServer.serverURI].FeedsDS; + } + + let rssServer = aServer.QueryInterface(Ci.nsIRssIncomingServer); + let feedsFile = rssServer.subscriptionsPath; // Path to feeds.json + let exists = feedsFile.exists(); + let ds = new lazy.JSONFile({ + path: feedsFile.path, + backupTo: feedsFile.path + ".backup", + }); + ds.ensureDataReady(); + if (!this[aServer.serverURI]) { + this[aServer.serverURI] = {}; + } + this[aServer.serverURI].FeedsDS = ds; + if (!exists) { + // No feeds.json, so we need to initialise. + ds.data = []; + } + return ds; + }, + + /** + * Fetch an attribute for a subscribed feed. + * + * @param {string} feedURL - URL of the feed. + * @param {nsIMsgIncomingServer} server - Server holding the subscription. + * @param {String} attrName - Name of attribute to fetch. + * @param {undefined} defaultValue - Value to return if not found. + * + * @returns {undefined} - the fetched value (defaultValue if the + * subscription or attribute doesn't exist). + */ + getSubscriptionAttr(feedURL, server, attrName, defaultValue) { + let ds = this.getSubscriptionsDS(server); + let sub = ds.data.find(feed => feed.url == feedURL); + if (sub === undefined || sub[attrName] === undefined) { + return defaultValue; + } + return sub[attrName]; + }, + + /** + * Set an attribute for a feed in the subscriptions store. + * NOTE: If the feed is not already in the store, it will be + * added. + * + * @param {string} feedURL - URL of the feed. + * @param {nsIMsgIncomingServer} server - server holding subscription. + * @param {String} attrName - Name of attribute to fetch. + * @param {undefined} value - Value to store. + * + * @returns {void} + */ + setSubscriptionAttr(feedURL, server, attrName, value) { + let ds = this.getSubscriptionsDS(server); + let sub = ds.data.find(feed => feed.url == feedURL); + if (sub === undefined) { + // Add a new entry. + sub = { url: feedURL }; + ds.data.push(sub); + } + sub[attrName] = value; + ds.saveSoon(); + }, + + /** + * Returns a reference to the feeditems store for the given server + * (the feeditems.json data). + * + * @param {nsIMsgIncomingServer} aServer - server to fetch item data for. + * @returns {JSONFile} - JSONFile with data field containing a collection + * of feeditems indexed by item url. + */ + getItemsDS(aServer) { + if (this[aServer.serverURI] && this[aServer.serverURI].FeedItemsDS) { + return this[aServer.serverURI].FeedItemsDS; + } + + let rssServer = aServer.QueryInterface(Ci.nsIRssIncomingServer); + let itemsFile = rssServer.feedItemsPath; // Path to feeditems.json + let exists = itemsFile.exists(); + let ds = new lazy.JSONFile({ + path: itemsFile.path, + backupTo: itemsFile.path + ".backup", + }); + ds.ensureDataReady(); + if (!this[aServer.serverURI]) { + this[aServer.serverURI] = {}; + } + this[aServer.serverURI].FeedItemsDS = ds; + if (!exists) { + // No feeditems.json, need to initialise our data. + ds.data = {}; + } + return ds; + }, + + /** + * Dragging something from somewhere. It may be a nice x-moz-url or from a + * browser or app that provides a less nice dataTransfer object in the event. + * Extract the url and if it passes the scheme test, try to subscribe. + * + * @param {nsISupports} aDataTransfer - The dnd event's dataTransfer. + * + * @returns {nsIURI} or null - A uri if valid, null if none. + */ + getFeedUriFromDataTransfer(aDataTransfer) { + let dt = aDataTransfer; + let types = ["text/x-moz-url-data", "text/x-moz-url"]; + let validUri = false; + let uri; + + if (dt.getData(types[0])) { + // The url is the data. + uri = Services.io.newURI(dt.mozGetDataAt(types[0], 0)); + validUri = this.isValidScheme(uri); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + + " : " + + types[0] + + " : " + + uri.spec + ); + } else if (dt.getData(types[1])) { + // The url is the first part of the data, the second part is random. + uri = Services.io.newURI(dt.mozGetDataAt(types[1], 0).split("\n")[0]); + validUri = this.isValidScheme(uri); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:type:value - " + + dt.dropEffect + + " : " + + types[0] + + " : " + + uri.spec + ); + } else { + // Go through the types and see if there's a url; get the first one. + for (let i = 0; i < dt.types.length; i++) { + let spec = dt.mozGetDataAt(dt.types[i], 0); + this.log.trace( + "getFeedUriFromDataTransfer: dropEffect:index:type:value - " + + dt.dropEffect + + " : " + + i + + " : " + + dt.types[i] + + " : " + + spec + ); + try { + uri = Services.io.newURI(spec); + validUri = this.isValidScheme(uri); + } catch (ex) {} + + if (validUri) { + break; + } + } + } + + return validUri ? uri : null; + }, + + /** + * Returns security/certificate/network error details for an XMLHTTPRequest. + * + * @param {XMLHTTPRequest} xhr - The xhr request. + * + * @returns {Array}[{String} errType or null, {String} errName or null] + * - Array with 2 error codes, (nulls if not determined). + */ + createTCPErrorFromFailedXHR(xhr) { + let status = xhr.channel.QueryInterface(Ci.nsIRequest).status; + + let errType = null; + let errName = null; + if ((status & 0xff0000) === 0x5a0000) { + // Security module. + const nsINSSErrorsService = Ci.nsINSSErrorsService; + let nssErrorsService = + Cc["@mozilla.org/nss_errors_service;1"].getService(nsINSSErrorsService); + let errorClass; + + // getErrorClass()) will throw a generic NS_ERROR_FAILURE if the error + // code is somehow not in the set of covered errors. + try { + errorClass = nssErrorsService.getErrorClass(status); + } catch (ex) { + // Catch security protocol exception. + errorClass = "SecurityProtocol"; + } + + if (errorClass == nsINSSErrorsService.ERROR_CLASS_BAD_CERT) { + errType = "SecurityCertificate"; + } else { + errType = "SecurityProtocol"; + } + + // NSS_SEC errors (happen below the base value because of negative vals). + if ( + (status & 0xffff) < + Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE) + ) { + // The bases are actually negative, so in our positive numeric space, + // we need to subtract the base off our value. + let nssErr = + Math.abs(nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (status & 0xffff); + + switch (nssErr) { + case 11: // SEC_ERROR_EXPIRED_CERTIFICATE, sec(11) + errName = "SecurityExpiredCertificateError"; + break; + case 12: // SEC_ERROR_REVOKED_CERTIFICATE, sec(12) + errName = "SecurityRevokedCertificateError"; + break; + + // Per bsmith, we will be unable to tell these errors apart very soon, + // so it makes sense to just folder them all together already. + case 13: // SEC_ERROR_UNKNOWN_ISSUER, sec(13) + case 20: // SEC_ERROR_UNTRUSTED_ISSUER, sec(20) + case 21: // SEC_ERROR_UNTRUSTED_CERT, sec(21) + case 36: // SEC_ERROR_CA_CERT_INVALID, sec(36) + errName = "SecurityUntrustedCertificateIssuerError"; + break; + case 90: // SEC_ERROR_INADEQUATE_KEY_USAGE, sec(90) + errName = "SecurityInadequateKeyUsageError"; + break; + case 176: // SEC_ERROR_CERT_SIGNATURE_ALGORITHM_DISABLED, sec(176) + errName = "SecurityCertificateSignatureAlgorithmDisabledError"; + break; + default: + errName = "SecurityError"; + break; + } + } else { + // Calculating the difference. + let sslErr = + Math.abs(nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (status & 0xffff); + + switch (sslErr) { + case 3: // SSL_ERROR_NO_CERTIFICATE, ssl(3) + errName = "SecurityNoCertificateError"; + break; + case 4: // SSL_ERROR_BAD_CERTIFICATE, ssl(4) + errName = "SecurityBadCertificateError"; + break; + case 8: // SSL_ERROR_UNSUPPORTED_CERTIFICATE_TYPE, ssl(8) + errName = "SecurityUnsupportedCertificateTypeError"; + break; + case 9: // SSL_ERROR_UNSUPPORTED_VERSION, ssl(9) + errName = "SecurityUnsupportedTLSVersionError"; + break; + case 12: // SSL_ERROR_BAD_CERT_DOMAIN, ssl(12) + errName = "SecurityCertificateDomainMismatchError"; + break; + default: + errName = "SecurityError"; + break; + } + } + } else { + errType = "Network"; + switch (status) { + // Connect to host:port failed. + case 0x804b000c: // NS_ERROR_CONNECTION_REFUSED, network(13) + errName = "ConnectionRefusedError"; + break; + // network timeout error. + case 0x804b000e: // NS_ERROR_NET_TIMEOUT, network(14) + errName = "NetworkTimeoutError"; + break; + // Hostname lookup failed. + case 0x804b001e: // NS_ERROR_UNKNOWN_HOST, network(30) + errName = "DomainNotFoundError"; + break; + case 0x804b0047: // NS_ERROR_NET_INTERRUPT, network(71) + errName = "NetworkInterruptError"; + break; + default: + errName = "NetworkError"; + break; + } + } + + return [errType, errName]; + }, + + /** + * Returns if a uri/url is valid to subscribe. + * + * @param {nsIURI} aUri or {String} aUrl - The Uri/Url. + * + * @returns {boolean} - true if a valid scheme, false if not. + */ + _validSchemes: ["http", "https", "file"], + isValidScheme(aUri) { + if (!(aUri instanceof Ci.nsIURI)) { + try { + aUri = Services.io.newURI(aUri); + } catch (ex) { + return false; + } + } + + return this._validSchemes.includes(aUri.scheme); + }, + + /** + * Is a folder Trash or in Trash. + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {Boolean} - true if folder is Trash else false. + */ + isInTrash(aFolder) { + let trashFolder = aFolder.rootFolder.getFolderWithFlags( + Ci.nsMsgFolderFlags.Trash + ); + if ( + trashFolder && + (trashFolder == aFolder || trashFolder.isAncestorOf(aFolder)) + ) { + return true; + } + + return false; + }, + + /** + * Return a folder path string constructed from individual folder UTF8 names + * stored as properties (not possible hashes used to construct disk foldername). + * + * @param {nsIMsgFolder} aFolder - The folder. + * + * @returns {String} prettyName or null - Name or null if not a disk folder. + */ + getFolderPrettyPath(aFolder) { + let msgFolder = lazy.MailUtils.getExistingFolder(aFolder.URI); + if (!msgFolder) { + // Not a real folder uri. + return null; + } + + if (msgFolder.URI == msgFolder.server.serverURI) { + return msgFolder.server.prettyName; + } + + // Server part first. + let pathParts = [msgFolder.server.prettyName]; + let rawPathParts = msgFolder.URI.split(msgFolder.server.serverURI + "/"); + let folderURI = msgFolder.server.serverURI; + rawPathParts = rawPathParts[1].split("/"); + for (let i = 0; i < rawPathParts.length - 1; i++) { + // Two or more folders deep parts here. + folderURI += "/" + rawPathParts[i]; + msgFolder = lazy.MailUtils.getExistingFolder(folderURI); + pathParts.push(msgFolder.name); + } + + // Leaf folder last. + pathParts.push(aFolder.name); + return pathParts.join("/"); + }, + + /** + * Date validator for feeds. + * + * @param {string} aDate - Date string. + * + * @returns {Boolean} - true if passes regex test, false if not. + */ + isValidRFC822Date(aDate) { + const FZ_RFC822_RE = + "^(((Mon)|(Tue)|(Wed)|(Thu)|(Fri)|(Sat)|(Sun)), *)?\\d\\d?" + + " +((Jan)|(Feb)|(Mar)|(Apr)|(May)|(Jun)|(Jul)|(Aug)|(Sep)|(Oct)|(Nov)|(Dec))" + + " +\\d\\d(\\d\\d)? +\\d\\d:\\d\\d(:\\d\\d)? +(([+-]?\\d\\d\\d\\d)|(UT)|(GMT)" + + "|(EST)|(EDT)|(CST)|(CDT)|(MST)|(MDT)|(PST)|(PDT)|\\w)$"; + let regex = new RegExp(FZ_RFC822_RE); + return regex.test(aDate); + }, + + /** + * Create rfc5322 date. + * + * @param {string} aDateString - Optional date string; if null or invalid + * date, get the current datetime. + * + * @returns {String} - An rfc5322 date string. + */ + getValidRFC5322Date(aDateString) { + let d = new Date(aDateString || new Date().getTime()); + d = isNaN(d.getTime()) ? new Date() : d; + return lazy.jsmime.headeremitter + .emitStructuredHeader("Date", d, {}) + .substring(6) + .trim(); + }, + + /** + * Progress glue code. Acts as a go between the RSS back end and the mail + * window front end determined by the aMsgWindow parameter passed into + * nsINewsBlogFeedDownloader. + */ + progressNotifier: { + mSubscribeMode: false, + mMsgWindow: null, + mStatusFeedback: null, + mFeeds: {}, + // Keeps track of the total number of feeds we have been asked to download. + // This number may not reflect the # of entries in our mFeeds array because + // not all feeds may have reported in for the first time. + mNumPendingFeedDownloads: 0, + + /** + * @param {?nsIMsgWindow} aMsgWindow - Associated aMsgWindow if any. + * @param {boolean} aSubscribeMode - Whether we're in subscribe mode. + * @returns {void} + */ + init(aMsgWindow, aSubscribeMode) { + if (this.mNumPendingFeedDownloads == 0) { + // If we aren't already in the middle of downloading feed items. + this.mStatusFeedback = aMsgWindow ? aMsgWindow.statusFeedback : null; + this.mSubscribeMode = aSubscribeMode; + this.mMsgWindow = aMsgWindow; + + if (this.mStatusFeedback) { + this.mStatusFeedback.startMeteors(); + this.mStatusFeedback.showStatusString( + FeedUtils.strings.GetStringFromName( + aSubscribeMode + ? "subscribe-validating-feed" + : "newsblog-getNewMsgsCheck" + ) + ); + } + } + }, + + /** + * Called on final success or error resolution of a feed download and + * parsing. If aDisable is true, the error shouldn't be retried continually + * and the url should be verified by the user. A bad html response code or + * cert error will cause the url to be disabled, while general network + * connectivity errors applying to all urls will not. + * + * @param {Feed} feed - The Feed object, or a synthetic object that must + * contain members {nsIMsgFolder folder, String url}. + * @param {Integer} aErrorCode - The resolution code, a kNewsBlog* value. + * @param {Boolean} aDisable - If true, disable/pause the feed. + * + * @returns {void} + */ + downloaded(feed, aErrorCode, aDisable) { + let folderName = feed.folder + ? feed.folder.name + : feed.server.rootFolder.prettyName; + FeedUtils.log.debug( + "downloaded: " + + (this.mSubscribeMode ? "Subscribe " : "Update ") + + "errorCode:folderName:feedUrl - " + + aErrorCode + + " : " + + folderName + + " : " + + feed.url + ); + if (this.mSubscribeMode) { + if (aErrorCode == FeedUtils.kNewsBlogSuccess) { + // Add the feed to the databases. + FeedUtils.addFeed(feed); + + // Nice touch: notify so the window ca select the folder that now + // contains the newly subscribed feed. + // This is particularly nice if we just finished subscribing + // to a feed URL that the operating system gave us. + Services.obs.notifyObservers(feed.folder, "folder-subscribed"); + + // Check for an existing feed subscriptions window and update it. + let subscriptionsWindow = Services.wm.getMostRecentWindow( + "Mail:News-BlogSubscriptions" + ); + if (subscriptionsWindow) { + subscriptionsWindow.FeedSubscriptions.FolderListener.folderAdded( + feed.folder + ); + } + } else if (feed && feed.url && feed.server) { + // Non success. Remove intermediate traces from the feeds database. + FeedUtils.deleteFeed(feed); + } + } + + if (aErrorCode != FeedUtils.kNewsBlogFeedIsBusy) { + if ( + aErrorCode == FeedUtils.kNewsBlogSuccess || + aErrorCode == FeedUtils.kNewsBlogNoNewItems + ) { + // Update lastUpdateTime only if successful normal processing. + let options = feed.options; + let now = Date.now(); + options.updates.lastUpdateTime = now; + if (feed.itemsStored) { + options.updates.lastDownloadTime = now; + } + + // If a previously disabled due to error feed is successful, set + // enabled state on, as that was the desired user setting. + if (options.updates.enabled == null) { + options.updates.enabled = true; + FeedUtils.setStatus(feed.folder, feed.url, "enabled", true); + } + + feed.options = options; + FeedUtils.setStatus(feed.folder, feed.url, "lastUpdateTime", now); + } else if (aDisable) { + if ( + Services.prefs.getBoolPref("rss.disable_feeds_on_update_failure") + ) { + // Do not keep retrying feeds with error states. Set persisted state + // to |null| to indicate error disable (and not user disable), but + // only if the feed is user enabled. + let options = feed.options; + if (options.updates.enabled) { + options.updates.enabled = null; + } + + feed.options = options; + FeedUtils.setStatus(feed.folder, feed.url, "enabled", false); + FeedUtils.log.warn( + "downloaded: updates disabled due to error, " + + "check the url - " + + feed.url + ); + } else { + FeedUtils.log.warn( + "downloaded: update failed, check the url - " + feed.url + ); + } + } + + if (!this.mSubscribeMode) { + FeedUtils.setStatus(feed.folder, feed.url, "code", aErrorCode); + + if ( + feed.folder && + !FeedUtils.getFolderProperties(feed.folder).includes("isBusy") + ) { + // Free msgDatabase after new mail biff is set; if busy let the next + // result do the freeing. Otherwise new messages won't be indicated. + // This feed may belong to a folder with multiple other feeds, some + // of which may not yet be finished, so free only if the folder is + // no longer busy. + feed.folder.msgDatabase = null; + FeedUtils.log.debug( + "downloaded: msgDatabase freed - " + feed.folder.name + ); + } + } + } + + let message = ""; + switch (aErrorCode) { + case FeedUtils.kNewsBlogSuccess: + case FeedUtils.kNewsBlogFeedIsBusy: + message = ""; + break; + case FeedUtils.kNewsBlogNoNewItems: + message = + feed.url + + ". " + + FeedUtils.strings.GetStringFromName( + "newsblog-noNewArticlesForFeed" + ); + break; + case FeedUtils.kNewsBlogInvalidFeed: + message = FeedUtils.strings.formatStringFromName( + "newsblog-feedNotValid", + [feed.url] + ); + break; + case FeedUtils.kNewsBlogRequestFailure: + message = FeedUtils.strings.formatStringFromName( + "newsblog-networkError", + [feed.url] + ); + break; + case FeedUtils.kNewsBlogFileError: + message = FeedUtils.strings.GetStringFromName( + "subscribe-errorOpeningFile" + ); + break; + case FeedUtils.kNewsBlogBadCertError: + let host = Services.io.newURI(feed.url).host; + message = FeedUtils.strings.formatStringFromName( + "newsblog-badCertError", + [host] + ); + break; + case FeedUtils.kNewsBlogNoAuthError: + message = FeedUtils.strings.formatStringFromName( + "newsblog-noAuthError", + [feed.url] + ); + break; + } + + if (message) { + let location = + FeedUtils.getFolderPrettyPath(feed.folder || feed.server.rootFolder) + + " -> "; + FeedUtils.log.info( + "downloaded: " + + (this.mSubscribeMode ? "Subscribe: " : "Update: ") + + location + + message + ); + } + + if (this.mStatusFeedback) { + this.mStatusFeedback.showStatusString(message); + this.mStatusFeedback.stopMeteors(); + } + + this.mNumPendingFeedDownloads--; + + if (this.mNumPendingFeedDownloads == 0) { + this.mFeeds = {}; + this.mSubscribeMode = false; + FeedUtils.log.debug("downloaded: all pending downloads finished"); + + // Should we do this on a timer so the text sticks around for a little + // while? It doesn't look like we do it on a timer for newsgroups so + // we'll follow that model. Don't clear the status text if we just + // dumped an error to the status bar! + if (aErrorCode == FeedUtils.kNewsBlogSuccess && this.mStatusFeedback) { + this.mStatusFeedback.showStatusString(""); + } + } + + feed = null; + }, + + /** + * This gets called after the RSS parser finishes storing a feed item to + * disk. aCurrentFeedItems is an integer corresponding to how many feed + * items have been downloaded so far. aMaxFeedItems is an integer + * corresponding to the total number of feed items to download. + * + * @param {Feed} feed - The Feed object. + * @param {Integer} aCurrentFeedItems - Number downloaded so far. + * @param {Integer} aMaxFeedItems - Total number to download. + * + * @returns {void} + */ + onFeedItemStored(feed, aCurrentFeedItems, aMaxFeedItems) { + // We currently don't do anything here. Eventually we may add status + // text about the number of new feed articles received. + + if (this.mSubscribeMode && this.mStatusFeedback) { + // If we are subscribing to a feed, show feed download progress. + this.mStatusFeedback.showStatusString( + FeedUtils.strings.formatStringFromName("subscribe-gettingFeedItems", [ + aCurrentFeedItems, + aMaxFeedItems, + ]) + ); + this.onProgress(feed, aCurrentFeedItems, aMaxFeedItems); + } + }, + + onProgress(feed, aProgress, aProgressMax, aLengthComputable) { + if (feed.url in this.mFeeds) { + // Have we already seen this feed? + this.mFeeds[feed.url].currentProgress = aProgress; + } else { + this.mFeeds[feed.url] = { + currentProgress: aProgress, + maxProgress: aProgressMax, + }; + } + + this.updateProgressBar(); + }, + + updateProgressBar() { + let currentProgress = 0; + let maxProgress = 0; + for (let index in this.mFeeds) { + currentProgress += this.mFeeds[index].currentProgress; + maxProgress += this.mFeeds[index].maxProgress; + } + + // If we start seeing weird "jumping" behavior where the progress bar + // goes below a threshold then above it again, then we can factor a + // fudge factor here based on the number of feeds that have not reported + // yet and the avg progress we've already received for existing feeds. + // Fortunately the progressmeter is on a timer and only updates every so + // often. For the most part all of our request have initial progress + // before the UI actually picks up a progress value. + if (this.mStatusFeedback) { + let progress = (currentProgress * 100) / maxProgress; + this.mStatusFeedback.showProgress(progress); + } + }, + }, +}; + +XPCOMUtils.defineLazyGetter(FeedUtils, "log", function () { + return console.createInstance({ + prefix: "feeds", + maxLogLevelPref: "feeds.loglevel", + }); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "strings", function () { + return Services.strings.createBundle( + "chrome://messenger-newsblog/locale/newsblog.properties" + ); +}); + +XPCOMUtils.defineLazyGetter(FeedUtils, "stringsPrefs", function () { + return Services.strings.createBundle( + "chrome://messenger/locale/prefs.properties" + ); +}); |