From f1ff14c5dce8c240e23659bbe63231ff7021ba88 Mon Sep 17 00:00:00 2001
From: Ron Cordell <ron.cordell@berkeley.edu>
Date: Sun, 21 Aug 2016 15:47:21 -0700
Subject: [PATCH] Working pipeline

---
 mongodb/roles/mongod/tasks/main.yml           |   5 +
 rest/.idea/workspace.xml                      | 268 ++++++++----------
 rest/{rest.py => application.py}              |  11 +-
 rest/requirements.txt                         |  38 +++
 twitter/sentimentConsumer.py                  |   2 +-
 twitter/sentiments/sentimentProcessor.py      |   3 +-
 .../twitter_utils/twitterStreamListener.py    |   5 +-
 7 files changed, 170 insertions(+), 162 deletions(-)
 rename rest/{rest.py => application.py} (94%)
 create mode 100644 rest/requirements.txt

diff --git a/mongodb/roles/mongod/tasks/main.yml b/mongodb/roles/mongod/tasks/main.yml
index 3a2135d..4bf30a3 100644
--- a/mongodb/roles/mongod/tasks/main.yml
+++ b/mongodb/roles/mongod/tasks/main.yml
@@ -33,3 +33,8 @@
   sudo: yes
   service: name=mongod state=started
 
+- name: Create the tweet sentiment collection
+  shell: mongo tweets --eval "db.createCollection('sentiment')
+
+- name: Create the sentiment geospatial index
+  shell: mongo tweets --eval 'db.sentiment.createIndex({ coordinates: "2dsphere" })'
diff --git a/rest/.idea/workspace.xml b/rest/.idea/workspace.xml
index 5367619..cecd40a 100644
--- a/rest/.idea/workspace.xml
+++ b/rest/.idea/workspace.xml
@@ -5,15 +5,11 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="66aa4bd1-f896-4686-8e50-cb43c2b5549f" name="Default" comment="">
-      <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/../twitter/sentimentConsumer.py" />
-      <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/../twitter/sentiments/__init__.py" />
-      <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../kafka/Vagrantfile" afterPath="$PROJECT_DIR$/../kafka/Vagrantfile" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../kafka/roles/zookeeper/templates/zoo.cfg.j2" afterPath="$PROJECT_DIR$/../kafka/roles/zookeeper/templates/zoo.cfg.j2" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../mongodb/Vagrantfile" afterPath="$PROJECT_DIR$/../mongodb/Vagrantfile" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml" afterPath="$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml" />
       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../twitter/tweet_serializers/kafkaTweetSerializer.py" afterPath="$PROJECT_DIR$/../twitter/tweet_serializers/kafkaTweetSerializer.py" />
-      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../twitter/twitter.py" afterPath="$PROJECT_DIR$/../twitter/twitter.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/rest.py" afterPath="$PROJECT_DIR$/rest.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../twitter/sentimentConsumer.py" afterPath="$PROJECT_DIR$/../twitter/sentimentConsumer.py" />
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py" afterPath="$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py" />
       <change type="MODIFICATION" beforePath="$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py" afterPath="$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py" />
     </list>
     <ignored path="rest.iws" />
@@ -27,8 +23,8 @@
     <option name="LAST_RESOLUTION" value="IGNORE" />
   </component>
   <component name="CoverageDataManager">
-    <SUITE FILE_PATH="coverage/rest$sentimentConsumer.coverage" NAME="sentimentConsumer Coverage Results" MODIFIED="1471811465489" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/../twitter" />
-    <SUITE FILE_PATH="coverage/rest$twitter_kafka.coverage" NAME="twitter kafka Coverage Results" MODIFIED="1471811296587" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/../twitter" />
+    <SUITE FILE_PATH="coverage/rest$sentimentConsumer.coverage" NAME="sentimentConsumer Coverage Results" MODIFIED="1471815345832" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/../twitter" />
+    <SUITE FILE_PATH="coverage/rest$twitter_kafka.coverage" NAME="twitter kafka Coverage Results" MODIFIED="1471815993876" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/../twitter" />
     <SUITE FILE_PATH="coverage/rest$twitter_locations.coverage" NAME="twitter locations Coverage Results" MODIFIED="1471735506050" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/../twitter" />
     <SUITE FILE_PATH="coverage/rest$rest.coverage" NAME="rest Coverage Results" MODIFIED="1471721182207" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="" />
     <SUITE FILE_PATH="coverage/rest$.coverage" NAME=" Coverage Results" MODIFIED="1471810528675" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="" />
@@ -43,6 +39,18 @@
   </component>
   <component name="FileEditorManager">
     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+      <file leaf-file-name="rest.py" pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/rest.py">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="870">
+              <caret line="58" column="0" selection-start-line="58" selection-start-column="0" selection-end-line="58" selection-end-column="0" />
+              <folding>
+                <element signature="e#0#23#0" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
       <file leaf-file-name="kafkaTweetSerializer.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/../twitter/tweet_serializers/kafkaTweetSerializer.py">
           <provider selected="true" editor-type-id="text-editor">
@@ -83,8 +91,8 @@
       <file leaf-file-name="twitterStreamListener.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="621">
-              <caret line="49" column="0" selection-start-line="49" selection-start-column="0" selection-end-line="49" selection-end-column="0" />
+            <state relative-caret-position="291">
+              <caret line="27" column="40" selection-start-line="27" selection-start-column="40" selection-end-line="27" selection-end-column="40" />
               <folding>
                 <element signature="e#0#43#0" expanded="true" />
               </folding>
@@ -92,11 +100,21 @@
           </provider>
         </entry>
       </file>
-      <file leaf-file-name="sentimentConsumer.py" pinned="false" current-in-tab="true">
+      <file leaf-file-name="main.yml" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="600">
+              <caret line="40" column="0" selection-start-line="40" selection-start-column="0" selection-end-line="40" selection-end-column="0" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
+      <file leaf-file-name="sentimentConsumer.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/../twitter/sentimentConsumer.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="855">
-              <caret line="57" column="0" selection-start-line="57" selection-start-column="0" selection-end-line="57" selection-end-column="0" />
+            <state relative-caret-position="690">
+              <caret line="46" column="0" selection-start-line="46" selection-start-column="0" selection-end-line="46" selection-end-column="0" />
               <folding>
                 <element signature="e#19#34#0" expanded="true" />
               </folding>
@@ -104,11 +122,21 @@
           </provider>
         </entry>
       </file>
+      <file leaf-file-name="server.properties.j2" pinned="false" current-in-tab="false">
+        <entry file="file://$PROJECT_DIR$/../kafka/roles/kafka/templates/server.properties.j2">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="629">
+              <caret line="104" column="14" selection-start-line="104" selection-start-column="14" selection-end-line="104" selection-end-column="14" />
+              <folding />
+            </state>
+          </provider>
+        </entry>
+      </file>
       <file leaf-file-name="sentimentProcessor.py" pinned="false" current-in-tab="false">
         <entry file="file://$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="285">
-              <caret line="19" column="0" selection-start-line="19" selection-start-column="0" selection-end-line="19" selection-end-column="0" />
+            <state relative-caret-position="75">
+              <caret line="5" column="27" selection-start-line="5" selection-start-column="27" selection-end-line="5" selection-end-column="27" />
               <folding>
                 <element signature="e#0#29#0" expanded="true" />
               </folding>
@@ -132,7 +160,6 @@
   <component name="IdeDocumentHistory">
     <option name="CHANGED_PATHS">
       <list>
-        <option value="$PROJECT_DIR$/rest.py" />
         <option value="$PROJECT_DIR$/../twitter/tweet_serializers/mongoTweetSerializer.py" />
         <option value="$PROJECT_DIR$/../twitter/tweet_serializers/chunkedTweetSerializer.py" />
         <option value="$PROJECT_DIR$/../twitter/tweet_serializers/s3ChunkedTweetSerializer.py" />
@@ -157,16 +184,17 @@
         <option value="$PROJECT_DIR$/../mongodb/mongodb.yml" />
         <option value="$PROJECT_DIR$/../mongodb/roles/common/tasks/main.yml" />
         <option value="$PROJECT_DIR$/../mongodb/roles/mongod/templates/mongod.conf.j2" />
-        <option value="$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml" />
         <option value="$PROJECT_DIR$/../twitter/mongoConsumer.py" />
         <option value="$PROJECT_DIR$/../kafka/Vagrantfile" />
         <option value="$PROJECT_DIR$/../kafka/roles/zookeeper/templates/zoo.cfg.j2" />
         <option value="$PROJECT_DIR$/../twitter/twitter.py" />
-        <option value="$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py" />
         <option value="$PROJECT_DIR$/../twitter/tweet_serializers/kafkaTweetSerializer.py" />
         <option value="$PROJECT_DIR$/../mongodb/Vagrantfile" />
         <option value="$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py" />
         <option value="$PROJECT_DIR$/../twitter/sentimentConsumer.py" />
+        <option value="$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py" />
+        <option value="$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml" />
+        <option value="$PROJECT_DIR$/rest.py" />
       </list>
     </option>
   </component>
@@ -177,8 +205,8 @@
     <sorting>DEFINITION_ORDER</sorting>
   </component>
   <component name="ProjectFrameBounds">
-    <option name="x" value="1174" />
-    <option name="y" value="23" />
+    <option name="x" value="229" />
+    <option name="y" value="43" />
     <option name="width" value="1906" />
     <option name="height" value="1777" />
   </component>
@@ -222,70 +250,8 @@
               <option name="myItemId" value="rest" />
               <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
             </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="twitter" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-          </PATH>
-          <PATH>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="rest" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="twitter" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="twitter_utils" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-          </PATH>
-          <PATH>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="rest" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="twitter" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="tweet_serializers" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-          </PATH>
-          <PATH>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="rest" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="twitter" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="sentiments" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-          </PATH>
-          <PATH>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="rest" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="mongodb" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
-            </PATH_ELEMENT>
-          </PATH>
-          <PATH>
             <PATH_ELEMENT>
               <option name="myItemId" value="rest" />
-              <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
-            </PATH_ELEMENT>
-            <PATH_ELEMENT>
-              <option name="myItemId" value="kafka" />
               <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
             </PATH_ELEMENT>
           </PATH>
@@ -306,7 +272,7 @@
       <recent name="$PROJECT_DIR$/../mongodb" />
     </key>
   </component>
-  <component name="RunManager" selected="Python.twitter kafka">
+  <component name="RunManager" selected="Python.sentimentConsumer">
     <configuration default="true" type="BashConfigurationType" factoryName="Bash">
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="INTERPRETER_PATH" value="/bin/bash" />
@@ -620,7 +586,7 @@
       <module name="rest" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" enabled="false" sample_coverage="true" runner="coverage.py" />
       <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../twitter/sentimentConsumer.py" />
-      <option name="PARAMETERS" value="--redishost sentiment-analysis.wezjx7.0001.usw1.cache.amazonaws.com --redisport 6379 --kafkahost ec2-54-183-36-33.us-west-1.compute.amazonaws.com --kafkaport 9092 --mongohost c2-54-193-5-118.us-west-1.compute.amazonaws.com --mongoport 27017" />
+      <option name="PARAMETERS" value="--redishost sentiment-analysis.wezjx7.0001.usw1.cache.amazonaws.com --redisport 6379 --kafkahost ec2-54-183-36-33.us-west-1.compute.amazonaws.com --kafkaport 9092 --mongohost ec2-54-193-5-118.us-west-1.compute.amazonaws.com --mongoport 27017" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <method />
     </configuration>
@@ -646,23 +612,23 @@
     <servers />
   </component>
   <component name="ToolWindowManager">
-    <frame x="1174" y="23" width="1906" height="1777" extended-state="0" />
-    <editor active="false" />
+    <frame x="229" y="43" width="1906" height="1777" extended-state="0" />
+    <editor active="true" />
     <layout>
       <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
       <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
       <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32997033" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
-      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32997033" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32997033" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
       <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32997033" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
       <window_info id="Mongo Explorer" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32997033" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
-      <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25804722" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25804722" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
       <window_info id="The R Graphics" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Packages" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
       <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
-      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
       <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
       <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
       <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
@@ -708,13 +674,9 @@
   </component>
   <component name="XDebuggerManager">
     <breakpoint-manager>
-      <breakpoints>
-        <line-breakpoint enabled="true" type="python-line">
-          <url>file://$PROJECT_DIR$/../twitter/sentimentConsumer.py</url>
-          <line>42</line>
-          <option name="timeStamp" value="15" />
-        </line-breakpoint>
-      </breakpoints>
+      <breakpoints-dialog>
+        <breakpoints-dialog />
+      </breakpoints-dialog>
       <default-breakpoints>
         <breakpoint type="python-exception">
           <properties notifyOnTerminate="true" exception="BaseException">
@@ -722,7 +684,7 @@
           </properties>
         </breakpoint>
       </default-breakpoints>
-      <option name="time" value="17" />
+      <option name="time" value="19" />
     </breakpoint-manager>
     <watches-manager />
   </component>
@@ -750,7 +712,7 @@
         <state relative-caret-position="465">
           <caret line="34" column="40" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
           <folding>
-            <element signature="e#0#23#0" expanded="false" />
+            <element signature="e#0#23#0" expanded="true" />
           </folding>
         </state>
       </provider>
@@ -807,16 +769,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/rest.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="0">
-          <caret line="34" column="40" selection-start-line="34" selection-start-column="40" selection-end-line="34" selection-end-column="40" />
-          <folding>
-            <element signature="e#0#23#0" expanded="false" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/../twitter/tweet_serializers/mongoTweetSerializer.py" />
     <entry file="file://$PROJECT_DIR$/../twitter/tweet_serializers/s3ChunkedTweetSerializer.py" />
     <entry file="file://$PROJECT_DIR$/../twitter/tweet_serializers/chunkedTweetSerializer.py" />
@@ -942,14 +894,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="240">
-          <caret line="16" column="16" selection-start-line="16" selection-start-column="16" selection-end-line="16" selection-end-column="16" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/../twitter/tweet_serializers/consoleTweetSerializer.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="255">
@@ -998,14 +942,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/../kafka/roles/kafka/templates/server.properties.j2">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="705">
-          <caret line="47" column="58" selection-start-line="47" selection-start-column="58" selection-end-line="47" selection-end-column="58" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/../kafka/roles/zookeeper/tasks/main.yml">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="780">
@@ -1014,14 +950,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/sentiment_analysis/lib/python2.7/site-packages/kafka/vendor/selectors34.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="1283">
-          <caret line="614" column="60" selection-start-line="614" selection-start-column="60" selection-end-line="614" selection-end-column="60" />
-          <folding />
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$USER_HOME$/sentiment_analysis/lib/python2.7/site-packages/tweepy/streaming.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="-138">
@@ -1051,16 +979,6 @@
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py">
-      <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="621">
-          <caret line="49" column="0" selection-start-line="49" selection-start-column="0" selection-end-line="49" selection-end-column="0" />
-          <folding>
-            <element signature="e#0#43#0" expanded="true" />
-          </folding>
-        </state>
-      </provider>
-    </entry>
     <entry file="file://$PROJECT_DIR$/../twitter/twitter.py">
       <provider selected="true" editor-type-id="text-editor">
         <state relative-caret-position="878">
@@ -1081,30 +999,74 @@
         </state>
       </provider>
     </entry>
+    <entry file="file://$USER_HOME$/sentiment_analysis/lib/python2.7/site-packages/redis/connection.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="402">
+          <caret line="481" column="0" selection-start-line="481" selection-start-column="0" selection-end-line="481" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../kafka/roles/kafka/templates/server.properties.j2">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="629">
+          <caret line="104" column="14" selection-start-line="104" selection-start-column="14" selection-end-line="104" selection-end-column="14" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../twitter/sentimentConsumer.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="690">
+          <caret line="46" column="0" selection-start-line="46" selection-start-column="0" selection-end-line="46" selection-end-column="0" />
+          <folding>
+            <element signature="e#19#34#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
     <entry file="file://$PROJECT_DIR$/../twitter/sentiments/sentimentProcessor.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="285">
-          <caret line="19" column="0" selection-start-line="19" selection-start-column="0" selection-end-line="19" selection-end-column="0" />
+        <state relative-caret-position="75">
+          <caret line="5" column="27" selection-start-line="5" selection-start-column="27" selection-end-line="5" selection-end-column="27" />
           <folding>
             <element signature="e#0#29#0" expanded="true" />
           </folding>
         </state>
       </provider>
     </entry>
-    <entry file="file://$USER_HOME$/sentiment_analysis/lib/python2.7/site-packages/redis/connection.py">
+    <entry file="file://$USER_HOME$/sentiment_analysis/lib/python2.7/site-packages/kafka/vendor/selectors34.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="402">
-          <caret line="481" column="0" selection-start-line="481" selection-start-column="0" selection-end-line="481" selection-end-column="0" />
+        <state relative-caret-position="788">
+          <caret line="603" column="0" selection-start-line="603" selection-start-column="0" selection-end-line="603" selection-end-column="0" />
           <folding />
         </state>
       </provider>
     </entry>
-    <entry file="file://$PROJECT_DIR$/../twitter/sentimentConsumer.py">
+    <entry file="file://$PROJECT_DIR$/../twitter/twitter_utils/twitterStreamListener.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="855">
-          <caret line="57" column="0" selection-start-line="57" selection-start-column="0" selection-end-line="57" selection-end-column="0" />
+        <state relative-caret-position="291">
+          <caret line="27" column="40" selection-start-line="27" selection-start-column="40" selection-end-line="27" selection-end-column="40" />
           <folding>
-            <element signature="e#19#34#0" expanded="true" />
+            <element signature="e#0#43#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/../mongodb/roles/mongod/tasks/main.yml">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="600">
+          <caret line="40" column="0" selection-start-line="40" selection-start-column="0" selection-end-line="40" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/rest.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="870">
+          <caret line="58" column="0" selection-start-line="58" selection-start-column="0" selection-end-line="58" selection-end-column="0" />
+          <folding>
+            <element signature="e#0#23#0" expanded="true" />
           </folding>
         </state>
       </provider>
diff --git a/rest/rest.py b/rest/application.py
similarity index 94%
rename from rest/rest.py
rename to rest/application.py
index 1960a29..f46401b 100644
--- a/rest/rest.py
+++ b/rest/application.py
@@ -2,6 +2,7 @@
 from flask_restful import Resource, Api
 import unittest
 import json
+from pymongo import MongoClient
 
 
 """
@@ -12,7 +13,7 @@
         longitude
         radius (meters)
 
-    GET http://localhost:5000/sentiment?lat=40.7128N?lon=74.0059W?radius=5000
+    GET http://localhost:5000/sentiment?lat=40.7128?lon=74.0059?radius=5000
 
     The response is a JSON object with the following structure
 
@@ -52,8 +53,8 @@ def get(self):
 api.add_resource(Sentiment, '/')
 
 # run the app server
-# if __name__ == '__main__':
-#    app.run(debug=True)
+if __name__ == '__main__':
+    app.run(debug=True)
 
 
 """
@@ -104,5 +105,5 @@ def test_bad_path(self):
 
 
 # Run the unit tests
-if __name__ == '__main__':
-    unittest.main()
+#if __name__ == '__main__':
+#    unittest.main()
diff --git a/rest/requirements.txt b/rest/requirements.txt
new file mode 100644
index 0000000..a6ef560
--- /dev/null
+++ b/rest/requirements.txt
@@ -0,0 +1,38 @@
+aniso8601==1.1.0
+awsebcli==3.7.7
+blessed==1.9.5
+botocore==1.4.48
+cement==2.8.2
+click==6.6
+colorama==0.3.7
+docker-py==1.7.2
+dockerpty==0.4.1
+docopt==0.6.2
+docutils==0.12
+Flask==0.11.1
+Flask-RESTful==0.3.5
+itsdangerous==0.24
+Jinja2==2.8
+jmespath==0.9.0
+kafka==1.3.1
+MarkupSafe==0.23
+nltk==3.2.1
+oauthlib==1.1.2
+pathspec==0.3.4
+pymongo==3.3.0
+python-dateutil==2.5.3
+pytz==2016.6.1
+PyYAML==3.11
+redis==2.10.5
+requests==2.9.1
+requests-oauthlib==0.6.2
+rq==0.6.0
+semantic-version==2.5.0
+six==1.10.0
+termcolor==1.1.0
+textblob==0.11.1
+texttable==0.8.4
+tweepy==3.5.0
+wcwidth==0.1.7
+websocket-client==0.37.0
+Werkzeug==0.11.10
diff --git a/twitter/sentimentConsumer.py b/twitter/sentimentConsumer.py
index 1e01d79..ed7d299 100644
--- a/twitter/sentimentConsumer.py
+++ b/twitter/sentimentConsumer.py
@@ -51,7 +51,7 @@
 
 for message in consumer:
     try:
-        job = q.enqueue(process_tweet, args=(message, args.mongohost, args.mongoport, 'tweets'))
+        job = q.enqueue(process_tweet, args=(message.value, args.mongohost, args.mongoport, 'tweets'))
 
     except KeyboardInterrupt:
         consumer.close()
diff --git a/twitter/sentiments/sentimentProcessor.py b/twitter/sentiments/sentimentProcessor.py
index bb191eb..49ee580 100644
--- a/twitter/sentiments/sentimentProcessor.py
+++ b/twitter/sentiments/sentimentProcessor.py
@@ -3,9 +3,10 @@
 
 
 def process_tweet(tweet, host, port, db):
+    print ('processing...')
     if 'text' in tweet:
         print 'processing...'
-        mongo_client = MongoClient(host=host, port=port, db=db)
+        mongo_client = MongoClient(host=host, port=port)
         mongodb = mongo_client[db]
         tweet_text_blob = TextBlob(tweet['text'])
         result = dict(text=tweet['text'],
diff --git a/twitter/twitter_utils/twitterStreamListener.py b/twitter/twitter_utils/twitterStreamListener.py
index dc32f56..24298da 100644
--- a/twitter/twitter_utils/twitterStreamListener.py
+++ b/twitter/twitter_utils/twitterStreamListener.py
@@ -28,8 +28,9 @@ def on_data(self, data):
             if not self.caughtInterrupt:
                 tweet = json.loads(data)
                 if self.geo_only and 'coordinates' in tweet and tweet['lang'] == 'en':
-                    self.serializer.write(tweet)
-                return True
+                    if tweet['coordinates'] is not None:
+                        self.serializer.write(tweet)
+                        return True
             else:
                 self.serializer.end()
                 return False