From 76772ddded6e8b488f22efde0629a67b61276a10 Mon Sep 17 00:00:00 2001 From: Kip Gebhardt Date: Tue, 3 Mar 2015 15:58:56 -0800 Subject: [PATCH] Fixing plugin issue that appeared under high load. Adding 'source' to plugins --- examples/config/feed.js | 5 ++++ examples/config/feed_with_plugins.js | 5 ++++ lib/RssBraider.js | 29 ++++++++----------- lib/plugins/add_media_thumbnail.js | 2 +- lib/plugins/content_encoded.js | 3 +- lib/plugins/strip_elements_from_powerpress.js | 2 +- lib/plugins/wfw_slash_comments.js | 2 +- 7 files changed, 26 insertions(+), 22 deletions(-) diff --git a/examples/config/feed.js b/examples/config/feed.js index f8e0f14..1d790db 100644 --- a/examples/config/feed.js +++ b/examples/config/feed.js @@ -16,6 +16,11 @@ var feed = { "name" : "NPR Health", "count" : 1, "file_path" : __dirname + "/../feed_xml/npr_health.xml", + }, + { + "name" : "NPR Sports", + "count" : 1, + "feed_url" : "http://www.npr.org/rss/rss.php?id=1055" } ] }; diff --git a/examples/config/feed_with_plugins.js b/examples/config/feed_with_plugins.js index d4d8ecd..76a1563 100644 --- a/examples/config/feed_with_plugins.js +++ b/examples/config/feed_with_plugins.js @@ -24,6 +24,11 @@ var feed = { "name" : "NPR Health", "count" : 1, "file_path" : __dirname + "/../feed_xml/npr_health.xml", + }, + { + "name" : "NPR Sports", + "count" : 1, + "feed_url" : "http://www.npr.org/rss/rss.php?id=1055" } ] }; diff --git a/lib/RssBraider.js b/lib/RssBraider.js index 971b380..073cb53 100644 --- a/lib/RssBraider.js +++ b/lib/RssBraider.js @@ -53,14 +53,9 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) format = 'rss'; } var self = this, - feed = this.feeds[feed_name], + feed = self.feeds[feed_name], feed_articles = []; - // set these for the request - self.feed_name = feed_name; - self.format = format; - self.feed = feed; - // logger.info("DEBUG processFeed: feed is set to " + feed_name); if (!feed || !feed.sources || feed.sources.length < 1) { @@ -68,11 +63,12 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) } async.each(feed.sources, function(source, callback) { - var count = source.count || feed.default_count || 10, + var count = source.count || feed.default_count || 10, // Number of articles url = source.feed_url || null, file_path = source.file_path || null, source_articles = []; + logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name); // todo: Check if source.file is set and set up a fs stream read var feedparser = new FeedParser(); if (url) { @@ -97,6 +93,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) filestream.pipe(feedparser); } else { logger.error("url or file_path not defined for feed: " + source.name); + return callback(); } feedparser.on('error', function(error) { @@ -113,7 +110,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) if (source.feed_url) { item.source_url = source.feed_url; } - var article = self.processItem(item, source); + // Process Item/Article + var article = self.processItem(item, source, feed_name); if (article) { source_articles.push(article); } @@ -155,7 +153,6 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) var newfeed = new RSS(options, feed_articles); - var ret_string; switch (format.toLowerCase()) { case 'json': @@ -176,7 +173,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) }; // Accepts a feed-parser item and builds a node-rss itemOptions object -RssBraider.prototype.processItem = function (item, source) { +RssBraider.prototype.processItem = function (item, source, feed_name) { var self = this; if (!item) { @@ -199,19 +196,17 @@ RssBraider.prototype.processItem = function (item, source) { // Run the plugins specified by the "plugins" section of the // feed config file to build out any custom elements or // do transforms - self.runPlugins(item, itemOptions, source); + self.runPlugins(item, itemOptions, source, feed_name); - // logger.info("returning for item.guid:" + item.guid); return itemOptions; }; -RssBraider.prototype.runPlugins = function (item, itemOptions, source) { +RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) { var self = this, - feed = self.feed, - feed_plugins = feed.plugins || []; - + feed = self.feeds[feed_name] || {}, + plugins_list = feed.plugins || []; // Process the item through the desired feed plugins - feed_plugins.forEach(function(plugin_name){ + plugins_list.forEach(function(plugin_name){ if (self.plugins[plugin_name]) { // logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title); self.plugins[plugin_name](item, itemOptions, source); diff --git a/lib/plugins/add_media_thumbnail.js b/lib/plugins/add_media_thumbnail.js index f24d916..8a482e7 100644 --- a/lib/plugins/add_media_thumbnail.js +++ b/lib/plugins/add_media_thumbnail.js @@ -5,7 +5,7 @@ // else // 'media:thumbnail' var _ = require('lodash'); -module.exports = function (item, itemOptions) { +module.exports = function (item, itemOptions, source) { if (!item || !itemOptions) { return; } diff --git a/lib/plugins/content_encoded.js b/lib/plugins/content_encoded.js index 663450b..d9b477f 100644 --- a/lib/plugins/content_encoded.js +++ b/lib/plugins/content_encoded.js @@ -3,7 +3,7 @@ // // Stewart let the news slip during a taping of his show today.]]> // -module.exports = function (item, itemOptions) { +module.exports = function (item, itemOptions, source) { if (!item || !itemOptions) { return; } @@ -17,5 +17,4 @@ module.exports = function (item, itemOptions) { } ); } - return; }; \ No newline at end of file diff --git a/lib/plugins/strip_elements_from_powerpress.js b/lib/plugins/strip_elements_from_powerpress.js index cfeb2a5..00931da 100644 --- a/lib/plugins/strip_elements_from_powerpress.js +++ b/lib/plugins/strip_elements_from_powerpress.js @@ -1,4 +1,4 @@ -module.exports = function (item, itemOptions) { +module.exports = function (item, itemOptions, source) { if (!item || !itemOptions) { return; } diff --git a/lib/plugins/wfw_slash_comments.js b/lib/plugins/wfw_slash_comments.js index bc7e2f8..c8a96e9 100644 --- a/lib/plugins/wfw_slash_comments.js +++ b/lib/plugins/wfw_slash_comments.js @@ -1,4 +1,4 @@ -module.exports = function (item, itemOptions) { +module.exports = function (item, itemOptions, source) { if (!item || !itemOptions) { return; }