Merge pull request #24 from manik96m/MapBoxScore

Map box score

ApplicationController.py

@@ -10,18 +10,36 @@
 
 
 class EmotionScore(Resource):
-    def get(self):
-        return {}
+    emotionController = EmotionController()
+    def get(self, text):
+        return self.emotionController.scoreForText(text)
 
 
 class EmotionScoreList(Resource):
     emotionController = EmotionController()
 
-    def get(self, location='CANADA'):
-        return self.emotionController.start()
+    def get(self, location):
+        return self.emotionController.start(location)
+
+class TweetScoreList(Resource):
+    emotionController = EmotionController()
 
+    def get(self, hashtag):
+        return self.emotionController.getTweetsScores(hashtag)
+
+class ProvinceScoreList(Resource):
+    emotionController = EmotionController()
 
-api.add_resource(EmotionScore, "/emotion/score")
+    def get(self, lat, long):
+        return self.emotionController.scoreforProvince(lat, long)
+
+
+api.add_resource(EmotionScore, "/emotion/score/<string:text>")
 api.add_resource(EmotionScoreList, "/emotion/score/location/<string:location>")
+api.add_resource(TweetScoreList, "/tweet/score/<string:hashtag>")
+api.add_resource(ProvinceScoreList, "/province/score/<string:lat>,<string:long>")
+
+
 
-app.run()
+if __name__ == '__main__':
+    app.run(debug=True)

CalculateSentiment.py

@@ -1 +1 @@
-"""Created on Oct 21 2020@author: ashlychinnuvarghese, tincythomas"""from configparser import ConfigParserfrom ibm_watson import ToneAnalyzerV3# @UnresolvedImportfrom ibm_cloud_sdk_core.authenticators import IAMAuthenticator# @UnresolvedImportimport jsonclass CalculateSentiments:    def __init__(self):        configur = ConfigParser()        configur.read("config.ini")        self.__apikey = configur.get('toneanalyser','apikey')        self.__url = configur.get('toneanalyser','apiurl')        self.__authenticator = IAMAuthenticator(self.__apikey)        self.__toneAnalyser = ToneAnalyzerV3(version=configur.get('toneanalyser','apiversion'), authenticator=self.__authenticator)        self.__toneAnalyser.set_service_url(self.__url)        self.__analyticScore = 0;        self.__tentativeScore = 0;        self.__confidentScore = 0;        self.__joyScore = 0;        self.__sadnessScore = 0;        self.__fearScore = 0;        self.__angerScore = 0;    def __tweetEmotion(self,tweet):        response = self.__toneAnalyser.tone(tweet,content_type='text/plain')        #json_object = json.dumps(response.result)                        return response    def __calculateScores(self,response):        self.__analyticScore = 0;        self.__tentativeScore = 0;        self.__confidentScore = 0;        self.__joyScore = 0;        self.__sadnessScore = 0;        self.__fearScore = 0;        self.__angerScore = 0;        for row in response.result["document_tone"]["tones"]:            #print(row)                        if(row['tone_id']=='analytical'):                self.__analyticScore+=row['score']            if(row['tone_id']=='tentative'):                self.__tentativeScore+=row['score']            if(row['tone_id']=='confident'):                self.__confidentScore+=row['score']            if(row['tone_id']=='joy'):                self.__joyScore+=row['score']            if(row['tone_id']=='sadness'):                self.__sadnessScore+=row['score']            if(row['tone_id']=='fear'):                self.__fearScore+=row['score']            if(row['tone_id']=='anger'):                self.__angerScore+=row['score']    def getScoresforTweets(self,location,tweet):        data = {}        totalTweet = ''        for ind in tweet.index:            totalTweet += str(tweet['text'][ind])        response = self.__tweetEmotion(str(totalTweet).encode("utf-8"))        self.__calculateScores(response)        data =  {            "provinceName": location,            "analyticScore": round(self.__analyticScore*100, 0),            "tentativeScore": round(self.__tentativeScore*100, 0),            "confidentScore": round(self.__confidentScore*100, 0),            "joyScore": round(self.__joyScore*100, 0),            "sadnessScore": round(self.__sadnessScore*100, 0),            "fearScore": round(self.__fearScore*100, 0),            "angerScore": round(self.__angerScore*100, 0)        }        #print('Total tweets :',tweet.size)        print('EMOTIONAL ANALYSIS')        print('analyticScore :',round(self.__analyticScore*100, 0))        print('tentativeScore :',round(self.__tentativeScore*100, 0))        print('confidentScore :',round(self.__confidentScore*100, 0))        print('joyScore :',round(self.__joyScore*100, 0))        print('sadnessScore :',round(self.__sadnessScore*100, 0))        print('fearScore :',round(self.__fearScore*100, 0))        print('angerScore :',round(self.__angerScore*100, 0))        return  data    def getScoreForText(self,text):        data = {}        response = self.__tweetEmotion(str((text).encode("utf-8")))        self.__calculateScores(response)        data["text"] =  {            "type": "text",            "analyticScore": round(self.__analyticScore*100, 0),            "tentativeScore": round(self.__tentativeScore*100, 0),            "confidentScore": round(self.__confidentScore*100, 0),            "joyScore": round(self.__joyScore*100, 0),            "sadnessScore": round(self.__sadnessScore*100, 0),            "fearScore": round(self.__fearScore*100, 0),            "angerScore": round(self.__angerScore*100, 0)        }        return data                                                
+"""Created on Oct 21 2020@author: ashlychinnuvarghese, tincythomas"""from ibm_watson import ToneAnalyzerV3# @UnresolvedImportfrom ibm_cloud_sdk_core.authenticators import IAMAuthenticatorfrom dotenv import load_dotenvload_dotenv()import osclass CalculateSentiments:    def __init__(self):        self.__apikey = os.environ.get("apikey")        self.__url = os.environ.get("apiurl")        self.__authenticator = IAMAuthenticator(self.__apikey)        self.__toneAnalyser = ToneAnalyzerV3(version=os.environ.get("apiversion"), authenticator=self.__authenticator)        self.__toneAnalyser.set_service_url(self.__url)        self.__analyticScore = 0;        self.__tentativeScore = 0;        self.__confidentScore = 0;        self.__joyScore = 0;        self.__sadnessScore = 0;        self.__fearScore = 0;        self.__angerScore = 0;        self.__flag = False;        self.count = 1    def __tweetEmotion(self,tweet):        response = self.__toneAnalyser.tone(tweet,content_type='text/plain')        return response    def __calculateScores(self,response):        self.__analyticScore = 0;        self.__tentativeScore = 0;        self.__confidentScore = 0;        self.__joyScore = 0;        self.__sadnessScore = 0;        self.__fearScore = 0;        self.__angerScore = 0;        for row in response.result["document_tone"]["tones"]:            if(row['tone_id']=='analytical'):                self.__analyticScore+=row['score']            if(row['tone_id']=='tentative'):                self.__tentativeScore+=row['score']            if(row['tone_id']=='confident'):                self.__confidentScore+=row['score']            if(row['tone_id']=='joy'):                self.__joyScore+=row['score']            if(row['tone_id']=='sadness'):                self.__sadnessScore+=row['score']            if(row['tone_id']=='fear'):                self.__fearScore+=row['score']            if(row['tone_id']=='anger'):                self.__angerScore+=row['score']        if(self.__analyticScore==0 and self.__tentativeScore==0 and self.__confidentScore==0 and                self.__joyScore==0 and self.__sadnessScore==0 and self.__fearScore==0 and self.__angerScore==0):            print('Inside sentences_tone')            self.__flag = True;            for row in response.result["sentences_tone"]:                for tone in row['tones']:                    self.count = self.count + 1                    if(tone['tone_id']=='analytical'):                        self.__analyticScore+=tone['score']                    if(tone['tone_id']=='tentative'):                        self.__tentativeScore+=tone['score']                    if(tone['tone_id']=='confident'):                        self.__confidentScore+=tone['score']                    if(tone['tone_id']=='joy'):                        self.__joyScore+=tone['score']                    if(tone['tone_id']=='sadness'):                        self.__sadnessScore+=tone['score']                    if(tone['tone_id']=='fear'):                        self.__fearScore+=tone['score']                    if(tone['tone_id']=='anger'):                        self.__angerScore+=tone['score']        print(self.count)    def getScoresforTweets(self,location,tweet):        print('Inside getScoresforTweets')        data = {}        totalTweet = ''        for ind in tweet.index:            totalTweet += str(tweet['text'][ind])        #print(totalTweet)        response = self.__tweetEmotion(str(totalTweet).encode("utf-8"))        self.__calculateScores(response)        data =  {            "provinceName": location,            "analyticScore": round((self.__analyticScore*100)/ (self.count if self.__flag==True else 1) , 0),            "tentativeScore": round((self.__tentativeScore*100)/ (self.count if self.__flag==True else 1), 0),            "confidentScore": round((self.__confidentScore*100)/ (self.count if self.__flag==True else 1), 0),            "joyScore": round((self.__joyScore*100)/ (self.count if self.__flag==True else 1), 0),            "sadnessScore": round((self.__sadnessScore*100)/ (self.count if self.__flag==True else 1), 0),            "fearScore": round((self.__fearScore*100)/ (self.count if self.__flag==True else 1), 0),            "angerScore": round((self.__angerScore*100)/ (self.count if self.__flag==True else 1), 0)        }        print('Total tweets :',tweet.size)        print('EMOTIONAL ANALYSIS')        print('analyticScore :',round(self.__analyticScore*100, 0))        print('tentativeScore :',round(self.__tentativeScore*100, 0))        print('confidentScore :',round(self.__confidentScore*100, 0))        print('joyScore :',round(self.__joyScore*100, 0))        print('sadnessScore :',round(self.__sadnessScore*100, 0))        print('fearScore :',round(self.__fearScore*100, 0))        print('angerScore :',round(self.__angerScore*100, 0))        return  data    def getScoreForText(self,text):        data = {}        response = self.__tweetEmotion(str((text).encode("utf-8")))        self.__calculateScores(response)        data["text"] =  {            "type": text,            "analyticScore": round(self.__analyticScore*100, 0),            "tentativeScore": round(self.__tentativeScore*100, 0),            "confidentScore": round(self.__confidentScore*100, 0),            "joyScore": round(self.__joyScore*100, 0),            "sadnessScore": round(self.__sadnessScore*100, 0),            "fearScore": round(self.__fearScore*100, 0),            "angerScore": round(self.__angerScore*100, 0)        }        return data

EmotionController.py

@@ -5,36 +5,105 @@
 """
 from ExtractTweet import *
 from CalculateSentiment import *
-from Canada import *
-from India import *
-
+import queue
+from threading import Thread
+import time
 
 class EmotionController:
-    def __init__(self):
-        self.emotions = CalculateSentiments()
 
-    def start(self, location='canada'):
+    def start(self, location):
+        print('Start---------')
+        start = int(time.time())
         data = {}
 
-        tweet = ExtractTweet()
-        if(location.casefold() == 'canada'):
-            hotSpots = Canada()
-        if(location.casefold() == 'india'):
-            hotSpots = India()
+        print('After taking HorSpots')
+        que = queue.Queue()
+        threads_list = list()
+        hotSpotsValues=""
+        for uniqueHotspot in hotSpotsValues:
+            tweet = ExtractTweet()
+            print('Getting tweets for hotspots')
+            th = Thread(target=lambda q, arg1: q.put(tweet.getTweets(uniqueHotspot,location)), args=(que, '' ))
+            th.start()
+            threads_list.append(th)
+            print('Ending tweets for hotspots')
 
-        hotSpotsValues = hotSpots.getHotSpots()
 
-        for uniqueHotspot in hotSpotsValues:
-            locationTweet = tweet.getTweets(uniqueHotspot,location)
-            if(locationTweet.empty==False):
-                print(uniqueHotspot)
-                data[uniqueHotspot] = self.emotions.getScoresforTweets(
-                    uniqueHotspot, locationTweet)
+        # Join all the threads
+        for t in threads_list:
+            print('Joining tweets for hotspots',t)
+            t.join();
+
+        # Check thread's return value
+        tweets = {}
+        while not que.empty():
+            print('Storing return value from tweets')
+            province,locationTweet  = que.get()
+            tweets[province] = locationTweet
+
+        que = queue.Queue()
+        threads_list = list()
+        for location in tweets:
+            self.emotions = CalculateSentiments()
+            print('Starting IBM Tone Analyser')
+            th = Thread(target=lambda q, arg1: q.put(self.emotions.getScoresforTweets(
+                location, tweets[location])), args=(que, '' ))
+            th.start()
+            threads_list.append(th)
+            print('Ending IBM Tone Analyser')
+
+        # Join all the threads
+        for t in threads_list:
+            print('Joining IBM Tone Analyser ', t)
+            t.join()
+
+        # Check thread's return value
+        while not que.empty():
+            print('Return Values from IBM Tone Analyser')
+            location,result  = que.get()
+            print("data----",result)
+            data[location] = result
 
-        print(data)
+        stop = int(time.time())
+        print("Function(Emotion Controller) a is running at time: " + str(stop-start) + " seconds.")
         return data
 
     def scoreForText(self, text):
         data = self.emotions.getScoreForText(text)
-        print(data)
         return data
+
+    def scoreforProvince(self, lat, long):
+        data = {}
+        emotions = CalculateSentiments()
+        tweet = ExtractTweet()
+        province = str(lat)+ str(long)
+        locationTweet = tweet.getTweetsLatLong(lat, long)
+        if(locationTweet.empty==False):
+            data = emotions.getScoresforTweets(province, locationTweet)
+        else:
+            data = {
+                "provinceName": province,
+                "analyticScore": 0,
+                "tentativeScore": 0,
+                "confidentScore": 0,
+                "joyScore": 0,
+                "sadnessScore": 0,
+                "fearScore": 0,
+                "angerScore": 0
+            }
+        return data
+
+    def getTweetsScores(self, hashtag):
+        print('Inside getTweetsScores')
+        emotions = CalculateSentiments()
+        tweet = ExtractTweet()
+        result = tweet.getTweetsHashTag(hashtag)
+
+        tweets = {}
+        i = 0
+        for tweet in result:
+            data = emotions.getScoreForText(tweet.text)
+            tweets[i] = data
+            i+=1
+
+        return tweets

ExtractTweet.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/.env python3
 # -*- coding: utf-8 -*-
 """
 Created on Thu Oct 22 07:32:12 2020
@@ -8,44 +8,45 @@
 
 import pandas as pd
 import tweepy as tw
-from opencage.geocoder import OpenCageGeocode
-from configparser import ConfigParser
-
+from dotenv import load_dotenv
+load_dotenv()
+import os
 
 class ExtractTweet:
     def __init__(self):
-        configur = ConfigParser()
-        configur.read("config.ini")
-        #OpenGeoCoder Credentials
-        self.geocoderkey = configur.get('tweepy','geocoderkey')
-        self.geocoder = OpenCageGeocode(self.geocoderkey)
-        ####TWEEPY API Credentials
-        self.consumer_key = configur.get('tweepy','consumer_key')
-        self.consumer_secret = configur.get('tweepy','consumer_secret')
-        self.access_token = configur.get('tweepy','access_token')
-        self.access_token_secret = configur.get('tweepy','access_token_secret')
+        self.consumer_key = os.environ.get("consumer_key")
+        self.consumer_secret = os.environ.get("consumer_secret")
+        self.access_token = os.environ.get("access_token")
+        self.access_token_secret = os.environ.get("access_token_secret")
         self.auth = tw.OAuthHandler(self.consumer_key, self.consumer_secret)
         self.auth.set_access_token(self.access_token, self.access_token_secret)
         self.api = tw.API(self.auth,wait_on_rate_limit=True)
         self.search_words = "#covid19"
-        #date_since = "2020-03-01"
         self.new_search = self.search_words + " -filter:retweets"
 
 
     def getTweets(self,province,country):
-        location_geo=str(province)+','+ str(country)
-        print(location_geo)
-        results = self.geocoder.geocode(location_geo)
+        print('Tweet Start')
+        tweets = tw.Cursor(self.api.search,q=self.new_search,lang="en").items(20)
+        tweetsviatweepy = [[tweet.text.encode('utf-8'), tweet.user.location] for tweet in tweets]
+        tweet_text = pd.DataFrame(data=tweetsviatweepy,
+                                  columns=['text', "user_location"])
+        print('Tweet Stop')
+        return province,tweet_text
+
+    def getTweetsHashTag(self, hashtag):
+        self.new_search = hashtag + " -filter:retweets"
+        tweets = tw.Cursor(self.api.search,q=self.new_search,lang="en",
+                           since="2017-04-03",full_text = True).items(20)
+        return tweets
 
-        print(u'%f;%f;%s;%s' % (results[0]['geometry']['lat'],
-                                results[0]['geometry']['lng'],
-                                results[0]['components']['country_code'],
-                                results[0]['annotations']['timezone']['name']))
-        geo=str(results[0]['geometry']['lat']) +','+str(results[0]['geometry']['lng'])+ ','+'100km'
+    def getTweetsLatLong(self, lat, long):
+        print('Tweet Start')
+        geo=str(str(lat)+','+ str(long))+ ','+'500km'
         print(geo)
         tweets = tw.Cursor(self.api.search,q=self.new_search,lang="en",geocode=geo).items(20)
         tweetsviatweepy = [[tweet.text.encode('utf-8'), tweet.user.location] for tweet in tweets]
         tweet_text = pd.DataFrame(data=tweetsviatweepy,
                                   columns=['text', "user_location"])
-        print(tweet_text)
-        return tweet_text
+        print('Tweet Stop')
+        return tweet_text

Procfile

@@ -1 +1 @@
-worker: python ApplicationController.py
+web: gunicorn ApplicationController:app