path: root/mailnews/extensions/newsblog/content/Feed.js

/* -*- Mode: JavaScript; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Cache for all of the feeds currently being downloaded, indexed by URL,
// so the load event listener can access the Feed objects after it finishes
// downloading the feed.
var FeedCache =
{
  mFeeds: {},

  putFeed: function (aFeed)
  {
    this.mFeeds[this.normalizeHost(aFeed.url)] = aFeed;
  },

  getFeed: function (aUrl)
  {
    let index = this.normalizeHost(aUrl);
    if (index in this.mFeeds)
      return this.mFeeds[index];

    return null;
  },

  removeFeed: function (aUrl)
  {
    let index = this.normalizeHost(aUrl);
    if (index in this.mFeeds)
      delete this.mFeeds[index];
  },

  normalizeHost: function (aUrl)
  {
    try
    {
      let normalizedUrl = Services.io.newURI(aUrl, null, null);
      normalizedUrl.host = normalizedUrl.host.toLowerCase();
      return normalizedUrl.spec
    }
    catch (ex)
    {
      return aUrl;
    }
  }
};

function Feed(aResource, aRSSServer)
{
  this.resource = aResource.QueryInterface(Ci.nsIRDFResource);
  this.server = aRSSServer;
}

Feed.prototype = 
{
  description: null,
  author: null,
  request: null,
  server: null,
  downloadCallback: null,
  resource: null,
  items: new Array(),
  itemsStored: 0,
  mFolder: null,
  mInvalidFeed: false,
  mFeedType: null,
  mLastModified: null,

  get folder()
  {
    return this.mFolder;
  },

  set folder (aFolder)
  {
    this.mFolder = aFolder;
  },

  get name()
  {
    // Used for the feed's title in Subcribe dialog and opml export.
    let name = this.title || this.description || this.url;
    return name.replace(/[\n\r\t]+/g, " ").replace(/[\x00-\x1F]+/g, "");
  },

  get folderName()
  {
    if (this.mFolderName)
      return this.mFolderName;

    // Get a unique sanitized name. Use title or description as a base;
    // these are mandatory by spec. Length of 80 is plenty.
    let folderName = (this.title || this.description || "").substr(0,80);
    let defaultName = FeedUtils.strings.GetStringFromName("ImportFeedsNew");
    return this.mFolderName = FeedUtils.getSanitizedFolderName(this.server.rootMsgFolder,
                                                               folderName,
                                                               defaultName,
                                                               true);
  },

  download: function(aParseItems, aCallback)
  {
     // May be null.
    this.downloadCallback = aCallback;

    // Whether or not to parse items when downloading and parsing the feed.
    // Defaults to true, but setting to false is useful for obtaining
    // just the title of the feed when the user subscribes to it.
    this.parseItems = aParseItems == null ? true : aParseItems ? true : false;

    // Before we do anything, make sure the url is an http url.  This is just
    // a sanity check so we don't try opening mailto urls, imap urls, etc. that
    // the user may have tried to subscribe to as an rss feed.
    if (!FeedUtils.isValidScheme(this.url))
    {
       // Simulate an invalid feed error.
      FeedUtils.log.info("Feed.download: invalid protocol for - " + this.url);
      this.onParseError(this);
      return;
    }

    // Before we try to download the feed, make sure we aren't already
    // processing the feed by looking up the url in our feed cache.
    if (FeedCache.getFeed(this.url))
    {
      if (this.downloadCallback)
        this.downloadCallback.downloaded(this, FeedUtils.kNewsBlogFeedIsBusy);
      // Return, the feed is already in use.
      return;
    }

    if (Services.io.offline) {
      // If offline and don't want to go online, just add the feed subscription;
      // it can be verified later (the folder name will be the url if not adding
      // to an existing folder). Only for subscribe actions; passive biff and
      // active get new messages are handled prior to getting here.
      let win = Services.wm.getMostRecentWindow("mail:3pane");
      if (!win.MailOfflineMgr.getNewMail()) {
        this.storeNextItem();
        return;
      }
    }

    this.request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].
                   createInstance(Ci.nsIXMLHttpRequest);
    // Must set onProgress before calling open.
    this.request.onprogress = this.onProgress;
    this.request.open("GET", this.url, true);
    this.request.channel.loadFlags |= Ci.nsIRequest.LOAD_BYPASS_CACHE |
                                      Ci.nsIRequest.INHIBIT_CACHING;

    // Some servers, if sent If-Modified-Since, will send 304 if subsequently
    // not sent If-Modified-Since, as in the case of an unsubscribe and new
    // subscribe.  Send start of century date to force a download; some servers
    // will 304 on older dates (such as epoch 1970).
    let lastModified = this.lastModified || "Sat, 01 Jan 2000 00:00:00 GMT";
    this.request.setRequestHeader("If-Modified-Since", lastModified);

    // Only order what you're going to eat...
    this.request.responseType = "document";
    this.request.overrideMimeType("text/xml");
    this.request.setRequestHeader("Accept", FeedUtils.REQUEST_ACCEPT);
    this.request.timeout = FeedUtils.REQUEST_TIMEOUT;
    this.request.onload = this.onDownloaded;
    this.request.onerror = this.onDownloadError;
    this.request.ontimeout = this.onDownloadError;
    FeedCache.putFeed(this);
    this.request.send(null);
  },

  onDownloaded: function(aEvent)
  {
    let request = aEvent.target;
    let isHttp = request.channel.originalURI.scheme.startsWith("http");
    let url = request.channel.originalURI.spec;
    if (isHttp && (request.status < 200 || request.status >= 300))
    {
      Feed.prototype.onDownloadError(aEvent);
      return;
    }

    FeedUtils.log.debug("Feed.onDownloaded: got a download - " + url);
    let feed = FeedCache.getFeed(url);
    if (!feed)
      throw new Error("Feed.onDownloaded: error - couldn't retrieve feed " +
                      "from cache");

    // If the server sends a Last-Modified header, store the property on the
    // feed so we can use it when making future requests, to avoid downloading
    // and parsing feeds that have not changed.  Don't update if merely checking
    // the url, as for subscribe move/copy, as a subsequent refresh may get a 304.
    // Save the response and persist it only upon successful completion of the
    // refresh cycle (i.e. not if the request is cancelled).
    let lastModifiedHeader = request.getResponseHeader("Last-Modified");
    feed.mLastModified = (lastModifiedHeader && feed.parseItems) ?
                           lastModifiedHeader : null;

    // The download callback is called asynchronously when parse() is done.
    feed.parse();
  },
  
  onProgress: function(aEvent) 
  {
    let request = aEvent.target;
    let url = request.channel.originalURI.spec;
    let feed = FeedCache.getFeed(url);

    if (feed.downloadCallback)
      feed.downloadCallback.onProgress(feed, aEvent.loaded, aEvent.total,
                                       aEvent.lengthComputable);
  },

  onDownloadError: function(aEvent)
  {
    let request = aEvent.target;
    let url = request.channel.originalURI.spec;
    let feed = FeedCache.getFeed(url);
    if (feed.downloadCallback) 
    {
      // Generic network or 'not found' error initially.
      let error = FeedUtils.kNewsBlogRequestFailure;

      if (request.status == 304) {
        // If the http status code is 304, the feed has not been modified
        // since we last downloaded it and does not need to be parsed.
        error = FeedUtils.kNewsBlogNoNewItems;
      }
      else {
        let [errType, errName] = FeedUtils.createTCPErrorFromFailedXHR(request);
        FeedUtils.log.info("Feed.onDownloaded: request errType:errName:statusCode - " +
                           errType + ":" + errName + ":" + request.status);
        if (errType == "SecurityCertificate")
          // This is the code for nsINSSErrorsService.ERROR_CLASS_BAD_CERT
          // overrideable security certificate errors.
          error = FeedUtils.kNewsBlogBadCertError;

        if (request.status == 401 || request.status == 403)
          // Unauthorized or Forbidden.
          error = FeedUtils.kNewsBlogNoAuthError;
      }

      feed.downloadCallback.downloaded(feed, error);
    }

    FeedCache.removeFeed(url);
  },

  onParseError: function(aFeed)
  {
    if (!aFeed)
      return;

    aFeed.mInvalidFeed = true;
    if (aFeed.downloadCallback)
      aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogInvalidFeed);

    FeedCache.removeFeed(aFeed.url);
  },

  onUrlChange: function(aFeed, aOldUrl)
  {
    if (!aFeed)
      return;

    // Simulate a cancel after a url update; next cycle will check the new url.
    aFeed.mInvalidFeed = true;
    if (aFeed.downloadCallback)
      aFeed.downloadCallback.downloaded(aFeed, FeedUtils.kNewsBlogCancel);

    FeedCache.removeFeed(aOldUrl);
  },

  get url()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let url = ds.GetTarget(this.resource, FeedUtils.DC_IDENTIFIER, true);
    if (url)
      url = url.QueryInterface(Ci.nsIRDFLiteral).Value;
    else
      url = this.resource.ValueUTF8;

    return url;
  },

  get title()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true);
    if (title)
      title = title.QueryInterface(Ci.nsIRDFLiteral).Value;

    return title;
  },

  set title (aNewTitle)
  {
    if (!aNewTitle)
      return;

    let ds = FeedUtils.getSubscriptionsDS(this.server);
    aNewTitle = FeedUtils.rdf.GetLiteral(aNewTitle);
    let old_title = ds.GetTarget(this.resource, FeedUtils.DC_TITLE, true);
    if (old_title)
        ds.Change(this.resource, FeedUtils.DC_TITLE, old_title, aNewTitle);
    else
        ds.Assert(this.resource, FeedUtils.DC_TITLE, aNewTitle, true);
  },

  get lastModified()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let lastModified = ds.GetTarget(this.resource,
                                    FeedUtils.DC_LASTMODIFIED,
                                    true);
    if (lastModified)
      lastModified = lastModified.QueryInterface(Ci.nsIRDFLiteral).Value;

    return lastModified;
  },

  set lastModified(aLastModified)
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    aLastModified = FeedUtils.rdf.GetLiteral(aLastModified);
    let old_lastmodified = ds.GetTarget(this.resource,
                                        FeedUtils.DC_LASTMODIFIED,
                                        true);
    if (old_lastmodified)
      ds.Change(this.resource, FeedUtils.DC_LASTMODIFIED,
                old_lastmodified, aLastModified);
    else
      ds.Assert(this.resource, FeedUtils.DC_LASTMODIFIED, aLastModified, true);
  },

  get quickMode ()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let quickMode = ds.GetTarget(this.resource, FeedUtils.FZ_QUICKMODE, true);
    if (quickMode)
    {
      quickMode = quickMode.QueryInterface(Ci.nsIRDFLiteral);
      quickMode = quickMode.Value == "true";
    }

    return quickMode;
  },

  set quickMode (aNewQuickMode)
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    aNewQuickMode = FeedUtils.rdf.GetLiteral(aNewQuickMode);
    let old_quickMode = ds.GetTarget(this.resource, 
                                     FeedUtils.FZ_QUICKMODE,
                                     true);
    if (old_quickMode)
      ds.Change(this.resource, FeedUtils.FZ_QUICKMODE,
                old_quickMode, aNewQuickMode);
    else
      ds.Assert(this.resource, FeedUtils.FZ_QUICKMODE,
                aNewQuickMode, true);
  },

  get options ()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let options = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true);
    if (options)
      return JSON.parse(options.QueryInterface(Ci.nsIRDFLiteral).Value);

    return null;
  },

  set options (aOptions)
  {
    let newOptions = aOptions ? FeedUtils.newOptions(aOptions) :
                                FeedUtils._optionsDefault;
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    newOptions = FeedUtils.rdf.GetLiteral(JSON.stringify(newOptions));
    let oldOptions = ds.GetTarget(this.resource, FeedUtils.FZ_OPTIONS, true);
    if (oldOptions)
      ds.Change(this.resource, FeedUtils.FZ_OPTIONS, oldOptions, newOptions);
    else
      ds.Assert(this.resource, FeedUtils.FZ_OPTIONS, newOptions, true);
  },

  categoryPrefs: function ()
  {
    let categoryPrefsAcct = FeedUtils.getOptionsAcct(this.server).category;
    if (!this.options)
      return categoryPrefsAcct;

    return this.options.category;
  },

  get link ()
  {
    let ds = FeedUtils.getSubscriptionsDS(this.server);
    let link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true);
    if (link)
      link = link.QueryInterface(Ci.nsIRDFLiteral).Value;

    return link;
  },

  set link (aNewLink)
  {
    if (!aNewLink)
      return;

    let ds = FeedUtils.getSubscriptionsDS(this.server);
    aNewLink = FeedUtils.rdf.GetLiteral(aNewLink);
    let old_link = ds.GetTarget(this.resource, FeedUtils.RSS_LINK, true);
    if (old_link)
      ds.Change(this.resource, FeedUtils.RSS_LINK, old_link, aNewLink);
    else
      ds.Assert(this.resource, FeedUtils.RSS_LINK, aNewLink, true);
  },

  parse: function()
  {
    // Create a feed parser which will parse the feed.
    let parser = new FeedParser();
    this.itemsToStore = parser.parseFeed(this, this.request.responseXML);
    parser = null;

    if (this.mInvalidFeed)
    {
      this.request = null;
      this.mInvalidFeed = false;
      return;
    }

    // storeNextItem() will iterate through the parsed items, storing each one.
    this.itemsToStoreIndex = 0;
    this.itemsStored = 0;
    this.storeNextItem();
  },

  invalidateItems: function ()
  {
    let ds = FeedUtils.getItemsDS(this.server);
    FeedUtils.log.debug("Feed.invalidateItems: for url - " + this.url);
    let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true);
    let item;

    while (items.hasMoreElements())
    {
      item = items.getNext();
      item = item.QueryInterface(Ci.nsIRDFResource);
      FeedUtils.log.trace("Feed.invalidateItems: item - " + item.Value);
      let valid = ds.GetTarget(item, FeedUtils.FZ_VALID, true);
      if (valid)
        ds.Unassert(item, FeedUtils.FZ_VALID, valid, true);
    }
  },

  removeInvalidItems: function(aDeleteFeed)
  {
    let ds = FeedUtils.getItemsDS(this.server);
    FeedUtils.log.debug("Feed.removeInvalidItems: for url - " + this.url);
    let items = ds.GetSources(FeedUtils.FZ_FEED, this.resource, true);
    let item;
    let currentTime = new Date().getTime();
    while (items.hasMoreElements())
    {
      item = items.getNext();
      item = item.QueryInterface(Ci.nsIRDFResource);

      if (ds.HasAssertion(item, FeedUtils.FZ_VALID,
                          FeedUtils.RDF_LITERAL_TRUE, true))
        continue;

      let lastSeenTime = ds.GetTarget(item, FeedUtils.FZ_LAST_SEEN_TIMESTAMP, true);
      if (lastSeenTime)
        lastSeenTime = parseInt(lastSeenTime.QueryInterface(Ci.nsIRDFLiteral).Value)
      else
        lastSeenTime = 0;

      if ((currentTime - lastSeenTime) < FeedUtils.INVALID_ITEM_PURGE_DELAY &&
          !aDeleteFeed)
        // Don't immediately purge items in active feeds; do so for deleted feeds.
        continue;

      FeedUtils.log.trace("Feed.removeInvalidItems: item - " + item.Value);
      ds.Unassert(item, FeedUtils.FZ_FEED, this.resource, true);
      if (ds.hasArcOut(item, FeedUtils.FZ_FEED))
        FeedUtils.log.debug("Feed.removeInvalidItems: " + item.Value +
                            " is from more than one feed; only the reference to" +
                            " this feed removed");
      else
        FeedUtils.removeAssertions(ds, item);
    }
  },

  createFolder: function()
  {
    if (this.folder)
      return;

    try {
      this.folder = this.server.rootMsgFolder
                               .QueryInterface(Ci.nsIMsgLocalMailFolder)
                               .createLocalSubfolder(this.folderName);
    }
    catch (ex) {
      // An error creating.
      FeedUtils.log.info("Feed.createFolder: error creating folder - '"+
                          this.folderName+"' in parent folder "+
                          this.server.rootMsgFolder.filePath.path + " -- "+ex);
      // But its remnants are still there, clean up.
      let xfolder = this.server.rootMsgFolder.getChildNamed(this.folderName);
      this.server.rootMsgFolder.propagateDelete(xfolder, true, null);
    }
  },

  // Gets the next item from itemsToStore and forces that item to be stored
  // to the folder.  If more items are left to be stored, fires a timer for
  // the next one, otherwise triggers a download done notification to the UI.
  storeNextItem: function()
  {
    if (FeedUtils.CANCEL_REQUESTED)
    {
      FeedUtils.CANCEL_REQUESTED = false;
      this.cleanupParsingState(this, FeedUtils.kNewsBlogCancel);
      return;
    }

    if (!this.itemsToStore || !this.itemsToStore.length)
    {
      let code = FeedUtils.kNewsBlogSuccess;
      this.createFolder();
      if (!this.folder)
        code = FeedUtils.kNewsBlogFileError;
      this.cleanupParsingState(this, code);
      return;
    }

    let item = this.itemsToStore[this.itemsToStoreIndex];

    if (item.store())
      this.itemsStored++;

    if (!this.folder)
    {
      this.cleanupParsingState(this, FeedUtils.kNewsBlogFileError);
      return;
    }

    this.itemsToStoreIndex++;

    // If the listener is tracking progress for each item, report it here.
    if (item.feed.downloadCallback && item.feed.downloadCallback.onFeedItemStored)
      item.feed.downloadCallback.onFeedItemStored(item.feed,
                                                  this.itemsToStoreIndex,
                                                  this.itemsToStore.length);

    // Eventually we'll report individual progress here.

    if (this.itemsToStoreIndex < this.itemsToStore.length)
    {
      if (!this.storeItemsTimer)
        this.storeItemsTimer = Cc["@mozilla.org/timer;1"].
                               createInstance(Ci.nsITimer);
      this.storeItemsTimer.initWithCallback(this, 50, Ci.nsITimer.TYPE_ONE_SHOT);
    }
    else
    {
      // We have just finished downloading one or more feed items into the
      // destination folder; if the folder is still listed as having new
      // messages in it, then we should set the biff state on the folder so the
      // right RDF UI changes happen in the folder pane to indicate new mail.
      if (item.feed.folder.hasNewMessages)
      {
        item.feed.folder.biffState = Ci.nsIMsgFolder.nsMsgBiffState_NewMail;
        // Run the bayesian spam filter, if enabled.
        item.feed.folder.callFilterPlugins(null);
      }

      this.cleanupParsingState(this, FeedUtils.kNewsBlogSuccess);
    }
  },

  cleanupParsingState: function(aFeed, aCode)
  {
    // Now that we are done parsing the feed, remove the feed from the cache.
    FeedCache.removeFeed(aFeed.url);

    if (aFeed.parseItems)
    {
      // Do this only if we're in parse/store mode.
      aFeed.removeInvalidItems(false);

      if (aCode == FeedUtils.kNewsBlogSuccess && aFeed.mLastModified)
        aFeed.lastModified = aFeed.mLastModified;

      // Flush any feed item changes to disk.
      let ds = FeedUtils.getItemsDS(aFeed.server);
      ds.Flush();
      FeedUtils.log.debug("Feed.cleanupParsingState: items stored - " + this.itemsStored);
    }

    // Force the xml http request to go away.  This helps reduce some nasty
    // assertions on shut down.
    this.request = null;
    this.itemsToStore = "";
    this.itemsToStoreIndex = 0;
    this.itemsStored = 0;
    this.storeItemsTimer = null;

    if (aFeed.downloadCallback)
      aFeed.downloadCallback.downloaded(aFeed, aCode);
  },

  // nsITimerCallback
  notify: function(aTimer)
  {
    this.storeNextItem();
  }
};