DSE203_Final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from time import time\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Clustering covid topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "topicMap = dict()\n",
    "topicMap['generic'] = [\"corona\", \"#corona\", \"coronavirus\", \"#coronavirus\", \"covid\", \"#covid\", \"covid19\", \"#covid19\", \"covid-19\", \"#covid-19\", \"sarscov2\", \"#sarscov2\", \"sars cov2\", \"sars cov 2\", \"covid_19\", \"#covid_19\", \"pandemic\", \"#pandemic\" ,\"#coronaupdate\"]\n",
    "topicMap['covid_count'] = [\"#ncov\", \"ncov\", \"#ncov2019\", \"ncov2019\", \"2019-ncov\", \"#2019-ncov\", \"#2019ncov\", \"2019ncov\"]\n",
    "topicMap['government_norms'] =  [\"#lockdown\", \"lockdown\"]\n",
    "topicMap['safety_guidelines'] =  [\"social distancing\", \"#socialdistancing\", \"wash ur hands\", \"wash your hands\", \"#washurhands\", \"#washyourhands\" , \"wearamask\", \"#wearamask\", \"wear a mask\"]\n",
    "topicMap['public_tackling_covid'] = [\"quarantine\", \"#quarantine\", \"#selfisolating\", \"self isolating\" , \"flatten the curve\", \"flattening the curve\", \"#flatteningthecurve\", \"#flattenthecurve\"]\n",
    "topicMap['vaccines_and_medicines'] = [\"vaccine\", \"vaccines\", \"#vaccine\", \"#vaccines\", \"corona vaccine\", \"corona vaccines\", \"#coronavaccine\", \"#coronavaccines\"]\n",
    "topicMap['herd_immunity'] = [\"herd immunity\", \"#herdimmunity\"]\n",
    "topicMap['working_from_home'] = [\"#stayathome\", \"#stayhome\",\"#wfh\", \"work from home\", \"#workfromhome\", \"working from home\", \"#workingfromhome\", \"#stayhomestaysafe\", \"#hometasking\"]\n",
    "topicMap['protective_equipment'] = [\"n95\" , \"ppe\", \"#ppe\", \"#n95\", \"hand sanitizer\", \"#handsanitizer\" ,\"#faceshields\", \"#masks4all\", \"face shield\", \"#faceshield\", \"face shields\"] \n",
    "topicMap['people_not_following_measures'] = [\"#covidiots\", \"covidiots\"]\n",
    "topicMap['comparison_with_flu']  = [\"pneumonia\", \"#pneumonia\"]\n",
    "topicMap['covid_nick_names'] = [\"chinese virus\", \"#chinesevirus\", \"wuhan virus\", \"#wuhanvirus\", \"kung flu\", \"#kungflu\"]\n",
    "topicMap['health_workers'] = [\"health worker\", \"#healthworker\", \"health workers\", \"#healthworkers\", \"#frontlineheroes\", \"#coronawarriors\"]\n",
    "topicMap['home_schooling'] = [\"#homeschool\", \"#homeschooling\"]\n",
    "                                    \n",
    "pickle.dump(topicMap, open( \"topics.pkl\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Fetching Topic by Topic "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_data_dict(topic):\n",
    "    data_dict = dict()\n",
    "    query ='text:' + ','.join(topic)\n",
    "    data_dict['q'] = query\n",
    "    data_dict['rows'] = 1000\n",
    "    return data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed batch Number 1\n",
      "Completed batch Number 2\n",
      "Completed batch Number 3\n",
      "Completed batch Number 4\n",
      "Completed batch Number 5\n",
      "Completed batch Number 6\n",
      "Completed batch Number 7\n",
      "Completed batch Number 8\n",
      "Completed batch Number 9\n",
      "Completed batch Number 10\n",
      "Completed batch Number 11\n",
      "Completed batch Number 12\n",
      "Completed batch Number 13\n",
      "Completed batch Number 14\n",
      "Completed batch Number 15\n",
      "Completed batch Number 16\n",
      "Completed batch Number 17\n",
      "Completed batch Number 18\n",
      "Completed batch Number 19\n",
      "Completed batch Number 20\n",
      "Completed batch Number 21\n",
      "Completed batch Number 22\n",
      "Completed batch Number 23\n",
      "Completed batch Number 24\n",
      "Completed batch Number 25\n",
      "Completed batch Number 26\n",
      "Completed batch Number 27\n",
      "Completed batch Number 28\n",
      "Completed batch Number 29\n",
      "Completed batch Number 30\n",
      "Completed batch Number 31\n",
      "Completed batch Number 32\n",
      "Completed batch Number 33\n",
      "Completed batch Number 34\n",
      "Completed batch Number 35\n",
      "Completed batch Number 36\n",
      "Completed batch Number 37\n",
      "Completed batch Number 38\n",
      "Completed batch Number 39\n",
      "Completed batch Number 40\n",
      "Completed batch Number 41\n",
      "Completed batch Number 42\n",
      "Completed batch Number 43\n",
      "Completed batch Number 44\n",
      "Completed batch Number 45\n",
      "Completed batch Number 46\n",
      "Completed batch Number 47\n",
      "Completed batch Number 48\n",
      "Completed batch Number 49\n",
      "Completed batch Number 50\n",
      "Completed batch Number 51\n",
      "Completed batch Number 52\n",
      "Completed batch Number 53\n",
      "Completed batch Number 54\n",
      "Completed batch Number 55\n",
      "Completed batch Number 56\n",
      "Completed batch Number 57\n",
      "Completed batch Number 58\n",
      "Completed batch Number 59\n",
      "Completed batch Number 60\n",
      "Completed batch Number 61\n",
      "Completed batch Number 62\n",
      "Completed batch Number 63\n",
      "Completed batch Number 64\n",
      "Completed batch Number 65\n",
      "Completed batch Number 66\n",
      "Completed batch Number 67\n",
      "Completed batch Number 68\n",
      "Completed batch Number 69\n",
      "Completed batch Number 70\n",
      "Completed batch Number 71\n",
      "Completed batch Number 72\n",
      "Completed batch Number 73\n",
      "Completed batch Number 74\n",
      "Completed batch Number 75\n",
      "Completed batch Number 76\n",
      "Completed batch Number 77\n",
      "Completed batch Number 78\n",
      "Completed batch Number 79\n",
      "Completed batch Number 80\n",
      "Completed batch Number 81\n",
      "Completed batch Number 82\n",
      "Completed batch Number 83\n",
      "Completed batch Number 84\n",
      "Completed batch Number 85\n",
      "Completed batch Number 86\n",
      "Completed batch Number 87\n",
      "Completed batch Number 88\n",
      "Completed batch Number 89\n",
      "Completed batch Number 90\n",
      "Completed batch Number 91\n",
      "Completed batch Number 92\n",
      "Completed batch Number 93\n",
      "Completed batch Number 94\n",
      "Completed batch Number 95\n",
      "Completed batch Number 96\n",
      "Completed batch Number 97\n",
      "Completed batch Number 98\n",
      "Completed batch Number 99\n",
      "Completed batch Number 100\n",
      "Completed batch Number 101\n",
      "Completed batch Number 102\n",
      "Completed batch Number 103\n",
      "Completed batch Number 104\n",
      "Completed batch Number 105\n",
      "Completed batch Number 106\n",
      "Completed batch Number 107\n",
      "Completed batch Number 108\n",
      "Completed batch Number 109\n",
      "Completed batch Number 110\n",
      "Completed batch Number 111\n",
      "Completed batch Number 112\n",
      "Completed batch Number 113\n",
      "Completed batch Number 114\n",
      "Completed batch Number 115\n",
      "Completed batch Number 116\n",
      "Completed batch Number 117\n",
      "Completed batch Number 118\n",
      "Completed batch Number 119\n",
      "Completed batch Number 120\n",
      "Completed batch Number 121\n",
      "Completed batch Number 122\n",
      "Completed batch Number 123\n",
      "Completed batch Number 124\n",
      "Completed batch Number 125\n",
      "Completed batch Number 126\n",
      "Completed batch Number 127\n",
      "Completed batch Number 128\n",
      "Completed batch Number 129\n",
      "Completed batch Number 130\n",
      "Completed batch Number 131\n",
      "Completed batch Number 132\n",
      "Completed batch Number 133\n",
      "Completed batch Number 134\n",
      "Completed batch Number 135\n",
      "Completed batch Number 136\n",
      "Completed batch Number 137\n",
      "Completed batch Number 138\n",
      "Completed batch Number 139\n",
      "Completed batch Number 140\n",
      "Completed batch Number 141\n",
      "Completed batch Number 142\n",
      "Completed batch Number 143\n",
      "Completed batch Number 144\n",
      "Completed batch Number 145\n",
      "Completed batch Number 146\n",
      "Completed batch Number 147\n",
      "Completed batch Number 148\n",
      "Completed batch Number 149\n",
      "Completed batch Number 150\n",
      "Time taken for entire batch rows: 16.153252\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "start = time()\n",
    "\n",
    "batch = 0\n",
    "\n",
    "url = 'http://awesome-hw.sdsc.edu:8983/solr/political_data/select'\n",
    "\n",
    "for topic in topicMap.keys():\n",
    "    if topic == 'people_not_following_measures':\n",
    "        for i in range(0,150000,1000):\n",
    "            data_dict = build_data_dict(topicMap[topic])\n",
    "            data_dict['wt'] = 'json'\n",
    "            data_dict['start'] = i\n",
    "            r = requests.post(url,data = data_dict)\n",
    "            if not os.path.exists('political_tweets/' + topic):\n",
    "                os.mkdir('political_tweets/' + topic)\n",
    "            if eval(r.content)['response']['docs']:\n",
    "                with open('political_tweets/' + topic +'/tweets_'+str(batch)+ '.json','wb') as f:\n",
    "                    f.write(r.content)\n",
    "            batch += 1 \n",
    "            print(\"Completed batch Number %d\"%batch)\n",
    "\n",
    "end = time()\n",
    "\n",
    "print('Time taken for entire batch rows: %f'%(end-start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For each topic create create labels and add emoji attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "import ipywidgets as widgets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Twitter setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/akash/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/akash/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/akash/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#!pip install tweepy\n",
    "import os\n",
    "import tweepy as tw\n",
    "import text2emotion as te\n",
    "\n",
    "##  Keys\n",
    "consumer_key= '9DthIWnJQ7ZMzL2jEsBgPYsu4'\n",
    "consumer_secret= 'BarxE5K10dENEYMIavMoYC2iG4OeGJKhz9KD8ymvjxHLvnOZYI'\n",
    "access_token= '1196931775114514433-VDVfqYwRHgl5g6TdxYDnL2NjUum6GL'\n",
    "access_token_secret= 'kiyWDHE7ItO6YLtlgsynu558kYeokYs0ShgcEudVGdTAq'\n",
    "\n",
    "auth = tw.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "api = tw.API(auth, wait_on_rate_limit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_tweet_object(twobject, twdf, usrdf, pstdf, tgdf, mntndf, rtwtsdf, qtsdf, cntnsdf, rplsdf,topic):\n",
    "    tid=twobject[\"id\"]\n",
    "    \n",
    "    \n",
    "    if \"extended_tweet\" in twobject.keys():\n",
    "        twtext=twobject[\"extended_tweet\"][\"full_text\"]\n",
    "        thashtags=twobject[\"extended_tweet\"][\"entities\"][\"hashtags\"]\n",
    "        umentions=twobject[\"extended_tweet\"][\"entities\"][\"user_mentions\"]\n",
    "    else:\n",
    "        twtext=twobject[\"text\"]\n",
    "        thashtags=twobject[\"entities\"][\"hashtags\"]\n",
    "        umentions=twobject[\"entities\"][\"user_mentions\"]\n",
    "    \n",
    "    emotion_dict = fxget_emotionscore(twtext)        \n",
    "    twdf.loc[tid,\"happy\"]= emotion_dict['Happy']\n",
    "    twdf.loc[tid,\"angry\"]=emotion_dict['Angry']\n",
    "    twdf.loc[tid,\"surprise\"]=emotion_dict['Surprise']\n",
    "    twdf.loc[tid,\"sad\"]=emotion_dict['Sad']\n",
    "    twdf.loc[tid,\"fear\"]=emotion_dict['Fear']\n",
    "    twdf.loc[tid,\"topic\"]=topic\n",
    "    \n",
    "    if tid not in twdf.keys():\n",
    "        twdf.loc[tid,\"text\"]=twtext\n",
    "        twdf.loc[tid,\"date\"]=pd.to_datetime(twobject[\"created_at\"])\n",
    "        for key in replykeys:\n",
    "            twdf.loc[tid,key] = twobject[key]\n",
    "            \n",
    "        #set these fields to false by default\n",
    "        twdf.loc[tid,\"retweet\"] = False\n",
    "        twdf.loc[tid,\"quote\"] = False\n",
    "        twdf.loc[tid,\"reply\"] = False\n",
    "    else:\n",
    "        for key in replykeys:\n",
    "            twdf.loc[tid,key] = max(twdf.loc[key], twobject[key])\n",
    "   \n",
    "    uid=twobject[\"user\"][\"id\"]\n",
    "\n",
    "    if uid not in usrdf.index:\n",
    "        usrdf.loc[uid,\"screen_name\"]=twobject[\"user\"][\"screen_name\"]\n",
    "        usrdf.loc[uid,\"created\"]=pd.to_datetime(twobject[\"user\"][\"created_at\"])\n",
    "        usrdf.loc[uid,\"description\"]=twobject[\"user\"][\"description\"]\n",
    "        usrdf.loc[uid,\"location\"]=twobject[\"user\"][\"location\"]\n",
    "\n",
    "    #keep track of maximum number of friends found through database\n",
    "    ufriends=twobject[\"user\"][\"friends_count\"]\n",
    "    \n",
    "    if uid in usrdf.index:\n",
    "        usrdf.loc[uid,\"friends_count\"] = \\\n",
    "            max(ufriends,usrdf.loc[uid,\"friends_count\"])\n",
    "    else:\n",
    "        usrdf.loc[uid,\"friends_count\"] = ufriends\n",
    "\n",
    "    np=len(pstdf)\n",
    "    pstdf.loc[np,\"uID\"] = uid\n",
    "    pstdf.loc[np,\"tID\"] = tid\n",
    "\n",
    "    #extract hashtags\n",
    "    nh=len(tgdf)\n",
    "    for p,ht in enumerate(thashtags):\n",
    "        #twitter hashtags are not case sensitive, so they are converted to lowercase\n",
    "        tgdf.loc[nh+p,\"hashtag\"] = ht[\"text\"].lower()\n",
    "        tgdf.loc[nh+p,\"tID\"] = tid\n",
    "\n",
    "    #extract user mentions\n",
    "    nu=len(mentionsdf)\n",
    "\n",
    "    for p,ment in enumerate(umentions):\n",
    "        mntndf.loc[nu+p,\"tID\"] = tid\n",
    "        mntndf.loc[nu+p,\"uID\"] = ment[\"id\"]\n",
    "        \n",
    "    #extract retweets\n",
    "    if \"retweeted_status\" in twobject.keys():\n",
    "        nr=len(rtwtsdf)\n",
    "        rtwtsdf.loc[nr,\"tID\"] = tid\n",
    "        tid_re=twobject[\"retweeted_status\"][\"id\"]\n",
    "        rtwtsdf.loc[nr,\"tID_re\"] = tid_re\n",
    "        rtwtsdf.loc[nr,\"topic\"]=topic\n",
    "        twdf.loc[tid,\"retweet\"] = True\n",
    "    \n",
    "    if \"quoted_status\" in twobject.keys():\n",
    "        nq=len(qtsdf)\n",
    "        qtsdf.loc[nq,\"tID\"] = tid\n",
    "        qtsdf.loc[nq,\"tID_qo\"] = twobject[\"quoted_status\"][\"id\"]\n",
    "        qtsdf.loc[nq,\"topic\"]=topic\n",
    "        twdf.loc[tid,\"quote\"] = True\n",
    " \n",
    "    urls=twobject[\"entities\"][\"urls\"]\n",
    "    for url in urls:\n",
    "        if \"twitter.com/i/web/status/\" not in url[\"expanded_url\"]:\n",
    "            nc=len(cntnsdf)\n",
    "            cntnsdf.loc[nc,\"url\"] = url[\"url\"]\n",
    "            cntnsdf.loc[nc,\"tid\"] = tid\n",
    "            cntnsdf.loc[nc,\"expanded_url\"] = url[\"expanded_url\"]\n",
    "            \n",
    "        #else:\n",
    "        #    tid_ln=int(url[\"expanded_url\"].split(\"twitter.com/i/web/status/\")[1])\n",
    "        #    nl=len(linkedtweetsdf)\n",
    "        #    linkedtweetsdf.loc[nl,\"tid\"] = tid\n",
    "        #    linkedtweetsdf.loc[nl,\"tid_ln\"] = tid_ln\n",
    "    \n",
    "    #if \"in_reply_to_status_id\" in twobject.keys():\n",
    "    if type(twobject[\"in_reply_to_status_id\"]) != type(None):\n",
    "\n",
    "        nl=len(rplsdf)\n",
    "        rplsdf.loc[nl,\"tID\"] = tid\n",
    "        rplsdf.loc[nl,\"tID_rp\"] = twobject[\"in_reply_to_status_id\"]\n",
    "        rplsdf.loc[nl,\"uID_rp\"] = twobject[\"in_reply_to_user_id\"]\n",
    "        rplsdf.loc[nl,\"topic\"]=topic\n",
    "        twdf.loc[tid, \"reply\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_all_tweets(jsonfile,tweetdf,userdf,postdf,mentionsdf, retweetsdf,quotesdf,tagsdf,containsdf,repliesdf,topic):\n",
    "    tweet=json.load(open(jsonfile, \"rb\"))\n",
    "    alltweets=tweet[\"response\"][\"docs\"]\n",
    "    ntweet=len(alltweets)\n",
    "   \n",
    "    for n in range(ntweet):\n",
    "        atweet=alltweets[n]\n",
    "        fulltweet=eval(atweet[\"fulltweets\"])\n",
    "        \n",
    "        parse_tweet_object(fulltweet,\n",
    "                           tweetdf, userdf, postdf, tagsdf, mentionsdf, \n",
    "                           retweetsdf, quotesdf, containsdf, repliesdf,topic)\n",
    "        \n",
    "        if \"quoted_status\" in fulltweet.keys():\n",
    "            parse_tweet_object(fulltweet[\"quoted_status\"],\n",
    "                               tweetdf, userdf, postdf, tagsdf, mentionsdf, \n",
    "                               retweetsdf, quotesdf, containsdf, repliesdf,topic)\n",
    "            \n",
    "        if \"retweet_status\" in fulltweet.keys():\n",
    "            parse_tweet_object(fulltweet[\"retweet_status\"],\n",
    "                               tweetdf, userdf, postdf, tagsdf, mentionsdf, \n",
    "                               retweetsdf, quotesdf, containsdf, repliesdf,topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3c1575838fe400491558b5c046d108b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntProgress(value=0, max=20)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading political_tweets/people_not_following_measures/tweets_6.json ...  (100.00%)  done.\n"
     ]
    }
   ],
   "source": [
    "replykeys=[\"reply_count\", \"retweet_count\", \"quote_count\", \"favorite_count\"]\n",
    "\n",
    "for i in range(1,len(all_topics) - 1):\n",
    "    tweetfiles=glob(\"political_tweets/\" + all_topics[i] + \"/tweets_*.json\")\n",
    "    tweetdf=pd.DataFrame(columns=[\"text\", \"date\",\"happy\",\"angry\",\"surprise\",\"sad\",\"fear\",\"topic\"] + replykeys + [\"retweet\", \"quote\", \"reply\"])\n",
    "    userdf=pd.DataFrame(columns=[\"screen_name\", \"created\",\"description\",\n",
    "                            \"location\", \"friends_count\"])\n",
    "    #these may be reduntant as the hashtag and url serves as primary key\n",
    "    #hashtags=pd.DataFrame(columns=[\"hashtag\"])\n",
    "    #url=pd.DataFrame(columns=\"url\")\n",
    "    \n",
    "    #these relations will be edges in the graph\n",
    "    postdf=pd.DataFrame(columns=[\"uID\",\"tID\",\"topic\"])\n",
    "    \n",
    "    # tweets mentioning users\n",
    "    mentionsdf=pd.DataFrame(columns=[\"tID\",\"uID\"])\n",
    "    retweetsdf=pd.DataFrame(columns=[\"tID\",\"tID_re\",\"topic\"])\n",
    "    quotesdf=pd.DataFrame(columns=[\"tID\", \"tID_qo\",\"topic\"])\n",
    "    tagsdf=pd.DataFrame(columns=[\"hashtag\",\"tID\"])\n",
    "    containsdf=pd.DataFrame(columns=[\"url\",\"tID\", \"expanded_url\"])\n",
    "    repliesdf=pd.DataFrame(columns=[\"tID\",\"tID_rp\", \"uID_rp\",\"topic\"])\n",
    "    \n",
    "    #linkedtweetsdf=pd.DataFrame(columns=[\"tID\",\"tID_ln\"])\n",
    "    #for file in tweetfiles:\n",
    "    \n",
    "    #parse_tweet_info(file)\n",
    "    from ipywidgets import IntProgress\n",
    "    from IPython.display import display\n",
    "\n",
    "    wb = IntProgress(min=0, max=len(tweetfiles)) \n",
    "    display(wb)\n",
    "\n",
    "    for n,file in enumerate(tweetfiles):\n",
    "        pcent = float(n+1)/len(tweetfiles) * 100.\n",
    "        print(\"\\rReading %s ...  (%4.2f%%) \" % (file, pcent), end=\"\")\n",
    "        parse_all_tweets(file,tweetdf,userdf,postdf,mentionsdf, retweetsdf,quotesdf,tagsdf,containsdf,repliesdf,all_topics[i])\n",
    "        wb.value = n+1\n",
    "        \n",
    "    alldf={\"tweets\": tweetdf, \"users\": userdf, \"posts\": postdf, \"mentions\": mentionsdf, \"retweets\": retweetsdf,\n",
    "      \"quotes\": quotesdf, \"tags\": tagsdf, \"contains\": containsdf,\"replies\":repliesdf}\n",
    "    \n",
    "    if not os.path.exists('political_tweets/pickles'):\n",
    "        os.mkdir('political_tweets/pickles')\n",
    "    pickle.dump(alldf, open(\"political_tweets/pickles/\" + all_topics[i] + \".pkl\", \"wb\"))\n",
    "    \n",
    "    print(\" done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}