diff options
Diffstat (limited to 'mailnews/extensions/newsblog/content/Feed.js')
| -rw-r--r-- | mailnews/extensions/newsblog/content/Feed.js | 620 |
1 files changed, 620 insertions, 0 deletions
diff --git a/mailnews/extensions/newsblog/content/Feed.js b/mailnews/extensions/newsblog/content/Feed.js new file mode 100644 index 000000000..7e47260a8 --- /dev/null +++ b/mailnews/extensions/newsblog/content/Feed.js @@ -0,0 +1,620 @@ +/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Cache for all of the feeds currently being downloaded, indexed by URL, +// so the load event listener can access the Feed objects after it finishes +// downloading the feed. +var FeedCache = +{ + mFeeds: {}, + + putFeed: function (aFeed) + { + this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed; + }, + + getFeed: function (aUrl) + { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) + return this.mFeeds[index]; + + return null; + }, + + removeFeed: function (aUrl) + { + let index = this.normalizeHost(aUrl); + if (index in this.mFeeds) + delete this.mFeeds[index]; + }, + + normalizeHost: function (aUrl) + { + try + { + let normalizedUrl = Services.io.newURI(aUrl, null, null); + normalizedUrl.host = normalizedUrl.host.toLowerCase(); + return normalizedUrl.spec + } + catch (ex) + { + return aUrl; + } + } +}; + +function Feed(aResource, aRSSServer) +{ + this.resource = aResource.QueryInterface(Ci.nsIRDFResource); + this.server = aRSSServer; +} + +Feed.prototype = +{ + description: null, + author: null, + request: null, + server: null, + downloadCallback: null, + resource: null, + items: new Array(), + itemsStored: 0, + mFolder: null, + mInvalidFeed: false, + mFeedType: null, + mLastModified: null, + + get folder() + { + return this.mFolder; + }, + + set folder (aFolder) + { + this.mFolder = aFolder; + }, + + get name() + { + // Used for the feed's title in Subcribe dialog and opml export. + let name = this.title || this.description || this.url; + return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, ""); + }, + + get folderName() + { + if (this.mFolderName) + return this.mFolderName; + + // Get a unique sanitized name. Use title or description as a base; + // these are mandatory by spec. Length of 80 is plenty. + let folderName = (this.title || this.description || "").substr(0,80); + let defaultName = FeedUtils.strings.GetStringFromName("ImportFeedsNew"); + return this.mFolderName = FeedUtils.getSanitizedFolderName(this.server.rootMsgFolder, + folderName, + defaultName, + true); + }, + + download: function(aParseItems, aCallback) + { + // May be null. + this.downloadCallback = aCallback; + + // Whether or not to parse items when downloading and parsing the feed. + // Defaults to true, but setting to false is useful for obtaining + // just the title of the feed when the user subscribes to it. + this.parseItems = aParseItems == null ? true : aParseItems ? true : false; + + // Before we do anything, make sure the url is an http url. This is just + // a sanity check so we don't try opening mailto urls, imap urls, etc. that + // the user may have tried to subscribe to as an rss feed. + if (!FeedUtils.isValidScheme(this.url)) + { + // Simulate an invalid feed error. + FeedUtils.log.info("Feed.download: invalid protocol for - " + this.url); + this.onParseError(this); + return; + } + + // Before we try to download the feed, make sure we aren't already + // processing the feed by looking up the url in our feed cache. + if (FeedCache.getFeed(this.url)) + { + if (this.downloadCallback) + this.downloadCallback.downloaded(this, FeedUtils.kNewsBlogFeedIsBusy); + // Return, the feed is already in use. + return; + } + + if (Services.io.offline) { + // If offline and don't want to go online, just add the feed subscription; + // it can be verified later (the folder name will be the url if not adding + // to an existing folder). Only for subscribe actions; passive biff and + // active get new messages are handled prior to getting here. + let win = Services.wm.getMostRecentWindow("mail:3pane"); + if (!win.MailOfflineMgr.getNewMail()) { + this.storeNextItem(); + return; + } + } + + this.request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]. + createInstance(Ci.nsIXMLHttpRequest); + // Must set onProgress before calling open. + this.request.onprogress = this.onProgress; + this.request.open("GET", this.url, true); + this.request.channel.loadFlags |= Ci.nsIRequest.LOAD_BYPASS_CACHE | + Ci.nsIRequest.INHIBIT_CACHING; + + // Some servers, if sent If-Modified-Since, will send 304 if subsequently + // not sent If-Modified-Since, as in the case of an unsubscribe and new + // subscribe. Send start of century date to force a download; some servers + // will 304 on older dates (such as epoch 1970). + let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT"; + this.request.setRequestHeader("If-Modified-Since", lastModified); + + // Only order what you're going to eat... + this.request.responseType = "document"; + this.request.overrideMimeType("text/xml"); + this.request.setRequestHeader("Accept", FeedUtils.REQUEST_ACCEPT); + this.request.timeout = FeedUtils.REQUEST_TIMEOUT; + this.request.onload = this.onDownloaded; + this.request.onerror = this.onDownloadError; + this.request.ontimeout = this.onDownloadError; + FeedCache.putFeed(this); + this.request.send(null); + }, + + onDownloaded: function(aEvent) + { + let request = aEvent.target; + let isHttp = request.channel.originalURI.scheme.startsWith("http"); + let url = request.channel.originalURI.spec; + if (isHttp && (request.status < 200 || request.status >= 300)) + { + Feed.prototype.onDownloadError(aEvent); + return; + } + + FeedUtils.log.debug("Feed.onDownloaded: got a download - " + url); + let feed = FeedCache.getFeed(url); + if (!feed) + throw new Error("Feed.onDownloaded: error - couldn't retrieve feed " + + "from cache"); + + // If the server sends a Last-Modified header, store the property on the + // feed so we can use it when making future requests, to avoid downloading + // and parsing feeds that have not changed. Don't update if merely checking + // the url, as for subscribe move/copy, as a subsequent refresh may get a 304. + // Save the response and persist it only upon successful completion of the + // refresh cycle (i.e. not if the request is cancelled). + let lastModifiedHeader = request.getResponseHeader("Last-Modified"); + feed.mLastModified = (lastModifiedHeader && feed.parseItems) ? + lastModifiedHeader : null; + + // The download callback is called asynchronously when parse() is done. + feed.parse(); + }, + + onProgress: function(aEvent) + { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + + if (feed.downloadCallback) + feed.downloadCallback.onProgress(feed, aEvent.loaded, aEvent.total, + aEvent.lengthComputable); + }, + + onDownloadError: function(aEvent) + { + let request = aEvent.target; + let url = request.channel.originalURI.spec; + let feed = FeedCache.getFeed(url); + if (feed.downloadCallback) + { + // Generic network or 'not found' error initially. + let error = FeedUtils.kNewsBlogRequestFailure; + + if (request.status == 304) { + // If the http status code is 304, the feed has not been modified + // since we last downloaded it and does not need to be parsed. + error = FeedUtils.kNewsBlogNoNewItems; + } + else { + let [errType, errName] = FeedUtils.createTCPErrorFromFailedXHR(request); + FeedUtils.log.info("Feed.onDownloaded: request errType:errName:statusCode - " + + errType + ":" + errName + ":" + request.status); + if (errType == "SecurityCertificate") + // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT + // overrideable security certificate errors. + error = FeedUtils.kNewsBlogBadCertError; + + if (request.status == 401 || request.status == 403) + // Unauthorized or Forbidden. + error = FeedUtils.kNewsBlogNoAuthError; + } + + feed.downloadCallback.downloaded(feed, error); + } + + FeedCache.removeFeed(url); + }, + + onParseError: function(aFeed) + { + if (!aFeed) + return; + + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogInvalidFeed); + + FeedCache.removeFeed(aFeed.url); + }, + + onUrlChange: function(aFeed, aOldUrl) + { + if (!aFeed) + return; + + // Simulate a cancel after a url update; next cycle will check the new url. + aFeed.mInvalidFeed = true; + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogCancel); + + FeedCache.removeFeed(aOldUrl); + }, + + get url() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let url = ds.GetTarget(this.resource, FeedUtils.DC_IDENTIFIER, true); + if (url) + url = url.QueryInterface(Ci.nsIRDFLiteral).Value; + else + url = this.resource.ValueUTF8; + + return url; + }, + + get title() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true); + if (title) + title = title.QueryInterface(Ci.nsIRDFLiteral).Value; + + return title; + }, + + set title (aNewTitle) + { + if (!aNewTitle) + return; + + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewTitle = FeedUtils.rdf.GetLiteral(aNewTitle); + let old_title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true); + if (old_title) + ds.Change(this.resource, FeedUtils.DC_TITLE, old_title, aNewTitle); + else + ds.Assert(this.resource, FeedUtils.DC_TITLE, aNewTitle, true); + }, + + get lastModified() + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let lastModified = ds.GetTarget(this.resource, + FeedUtils.DC_LASTMODIFIED, + true); + if (lastModified) + lastModified = lastModified.QueryInterface(Ci.nsIRDFLiteral).Value; + + return lastModified; + }, + + set lastModified(aLastModified) + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + aLastModified = FeedUtils.rdf.GetLiteral(aLastModified); + let old_lastmodified = ds.GetTarget(this.resource, + FeedUtils.DC_LASTMODIFIED, + true); + if (old_lastmodified) + ds.Change(this.resource, FeedUtils.DC_LASTMODIFIED, + old_lastmodified, aLastModified); + else + ds.Assert(this.resource, FeedUtils.DC_LASTMODIFIED, aLastModified, true); + }, + + get quickMode () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let quickMode = ds.GetTarget(this.resource, FeedUtils.FZ_QUICKMODE, true); + if (quickMode) + { + quickMode = quickMode.QueryInterface(Ci.nsIRDFLiteral); + quickMode = quickMode.Value == "true"; + } + + return quickMode; + }, + + set quickMode (aNewQuickMode) + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewQuickMode = FeedUtils.rdf.GetLiteral(aNewQuickMode); + let old_quickMode = ds.GetTarget(this.resource, + FeedUtils.FZ_QUICKMODE, + true); + if (old_quickMode) + ds.Change(this.resource, FeedUtils.FZ_QUICKMODE, + old_quickMode, aNewQuickMode); + else + ds.Assert(this.resource, FeedUtils.FZ_QUICKMODE, + aNewQuickMode, true); + }, + + get options () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let options = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true); + if (options) + return JSON.parse(options.QueryInterface(Ci.nsIRDFLiteral).Value); + + return null; + }, + + set options (aOptions) + { + let newOptions = aOptions ? FeedUtils.newOptions(aOptions) : + FeedUtils._optionsDefault; + let ds = FeedUtils.getSubscriptionsDS(this.server); + newOptions = FeedUtils.rdf.GetLiteral(JSON.stringify(newOptions)); + let oldOptions = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true); + if (oldOptions) + ds.Change(this.resource, FeedUtils.FZ_OPTIONS, oldOptions, newOptions); + else + ds.Assert(this.resource, FeedUtils.FZ_OPTIONS, newOptions, true); + }, + + categoryPrefs: function () + { + let categoryPrefsAcct = FeedUtils.getOptionsAcct(this.server).category; + if (!this.options) + return categoryPrefsAcct; + + return this.options.category; + }, + + get link () + { + let ds = FeedUtils.getSubscriptionsDS(this.server); + let link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true); + if (link) + link = link.QueryInterface(Ci.nsIRDFLiteral).Value; + + return link; + }, + + set link (aNewLink) + { + if (!aNewLink) + return; + + let ds = FeedUtils.getSubscriptionsDS(this.server); + aNewLink = FeedUtils.rdf.GetLiteral(aNewLink); + let old_link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true); + if (old_link) + ds.Change(this.resource, FeedUtils.RSS_LINK, old_link, aNewLink); + else + ds.Assert(this.resource, FeedUtils.RSS_LINK, aNewLink, true); + }, + + parse: function() + { + // Create a feed parser which will parse the feed. + let parser = new FeedParser(); + this.itemsToStore = parser.parseFeed(this, this.request.responseXML); + parser = null; + + if (this.mInvalidFeed) + { + this.request = null; + this.mInvalidFeed = false; + return; + } + + // storeNextItem() will iterate through the parsed items, storing each one. + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + this.storeNextItem(); + }, + + invalidateItems: function () + { + let ds = FeedUtils.getItemsDS(this.server); + FeedUtils.log.debug("Feed.invalidateItems: for url - " + this.url); + let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true); + let item; + + while (items.hasMoreElements()) + { + item = items.getNext(); + item = item.QueryInterface(Ci.nsIRDFResource); + FeedUtils.log.trace("Feed.invalidateItems: item - " + item.Value); + let valid = ds.GetTarget(item, FeedUtils.FZ_VALID, true); + if (valid) + ds.Unassert(item, FeedUtils.FZ_VALID, valid, true); + } + }, + + removeInvalidItems: function(aDeleteFeed) + { + let ds = FeedUtils.getItemsDS(this.server); + FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url); + let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true); + let item; + let currentTime = new Date().getTime(); + while (items.hasMoreElements()) + { + item = items.getNext(); + item = item.QueryInterface(Ci.nsIRDFResource); + + if (ds.HasAssertion(item, FeedUtils.FZ_VALID, + FeedUtils.RDF_LITERAL_TRUE, true)) + continue; + + let lastSeenTime = ds.GetTarget(item, FeedUtils.FZ_LAST_SEEN_TIMESTAMP, true); + if (lastSeenTime) + lastSeenTime = parseInt(lastSeenTime.QueryInterface(Ci.nsIRDFLiteral).Value) + else + lastSeenTime = 0; + + if ((currentTime - lastSeenTime) < FeedUtils.INVALID_ITEM_PURGE_DELAY && + !aDeleteFeed) + // Don't immediately purge items in active feeds; do so for deleted feeds. + continue; + + FeedUtils.log.trace("Feed.removeInvalidItems: item - " + item.Value); + ds.Unassert(item, FeedUtils.FZ_FEED, this.resource, true); + if (ds.hasArcOut(item, FeedUtils.FZ_FEED)) + FeedUtils.log.debug("Feed.removeInvalidItems: " + item.Value + + " is from more than one feed; only the reference to" + + " this feed removed"); + else + FeedUtils.removeAssertions(ds, item); + } + }, + + createFolder: function() + { + if (this.folder) + return; + + try { + this.folder = this.server.rootMsgFolder + .QueryInterface(Ci.nsIMsgLocalMailFolder) + .createLocalSubfolder(this.folderName); + } + catch (ex) { + // An error creating. + FeedUtils.log.info("Feed.createFolder: error creating folder - '"+ + this.folderName+"' in parent folder "+ + this.server.rootMsgFolder.filePath.path + " -- "+ex); + // But its remnants are still there, clean up. + let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName); + this.server.rootMsgFolder.propagateDelete(xfolder, true, null); + } + }, + + // Gets the next item from itemsToStore and forces that item to be stored + // to the folder. If more items are left to be stored, fires a timer for + // the next one, otherwise triggers a download done notification to the UI. + storeNextItem: function() + { + if (FeedUtils.CANCEL_REQUESTED) + { + FeedUtils.CANCEL_REQUESTED = false; + this.cleanupParsingState(this, FeedUtils.kNewsBlogCancel); + return; + } + + if (!this.itemsToStore || !this.itemsToStore.length) + { + let code = FeedUtils.kNewsBlogSuccess; + this.createFolder(); + if (!this.folder) + code = FeedUtils.kNewsBlogFileError; + this.cleanupParsingState(this, code); + return; + } + + let item = this.itemsToStore[this.itemsToStoreIndex]; + + if (item.store()) + this.itemsStored++; + + if (!this.folder) + { + this.cleanupParsingState(this, FeedUtils.kNewsBlogFileError); + return; + } + + this.itemsToStoreIndex++; + + // If the listener is tracking progress for each item, report it here. + if (item.feed.downloadCallback && item.feed.downloadCallback.onFeedItemStored) + item.feed.downloadCallback.onFeedItemStored(item.feed, + this.itemsToStoreIndex, + this.itemsToStore.length); + + // Eventually we'll report individual progress here. + + if (this.itemsToStoreIndex < this.itemsToStore.length) + { + if (!this.storeItemsTimer) + this.storeItemsTimer = Cc["@mozilla.org/timer;1"]. + createInstance(Ci.nsITimer); + this.storeItemsTimer.initWithCallback(this, 50, Ci.nsITimer.TYPE_ONE_SHOT); + } + else + { + // We have just finished downloading one or more feed items into the + // destination folder; if the folder is still listed as having new + // messages in it, then we should set the biff state on the folder so the + // right RDF UI changes happen in the folder pane to indicate new mail. + if (item.feed.folder.hasNewMessages) + { + item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail; + // Run the bayesian spam filter, if enabled. + item.feed.folder.callFilterPlugins(null); + } + + this.cleanupParsingState(this, FeedUtils.kNewsBlogSuccess); + } + }, + + cleanupParsingState: function(aFeed, aCode) + { + // Now that we are done parsing the feed, remove the feed from the cache. + FeedCache.removeFeed(aFeed.url); + + if (aFeed.parseItems) + { + // Do this only if we're in parse/store mode. + aFeed.removeInvalidItems(false); + + if (aCode == FeedUtils.kNewsBlogSuccess && aFeed.mLastModified) + aFeed.lastModified = aFeed.mLastModified; + + // Flush any feed item changes to disk. + let ds = FeedUtils.getItemsDS(aFeed.server); + ds.Flush(); + FeedUtils.log.debug("Feed.cleanupParsingState: items stored - " + this.itemsStored); + } + + // Force the xml http request to go away. This helps reduce some nasty + // assertions on shut down. + this.request = null; + this.itemsToStore = ""; + this.itemsToStoreIndex = 0; + this.itemsStored = 0; + this.storeItemsTimer = null; + + if (aFeed.downloadCallback) + aFeed.downloadCallback.downloaded(aFeed, aCode); + }, + + // nsITimerCallback + notify: function(aTimer) + { + this.storeNextItem(); + } +}; |