collect_reddit_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import re\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pushshift_data(data_type, **kwargs):\n",
    "    \"\"\"\n",
    "    Gets data from the pushshift api.\n",
    " \n",
    "    data_type can be 'comment' or 'submission'\n",
    "    The rest of the args are interpreted as payload.\n",
    " \n",
    "    Read more: https://github.com/pushshift/api\n",
    "    \"\"\"\n",
    " \n",
    "    base_url = f\"https://api.pushshift.io/reddit/search/{data_type}/\"\n",
    "    payload = kwargs\n",
    "    request = requests.get(base_url, params=payload)\n",
    "    return request.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_type=\"comment\"     # give me comments, use \"submission\" to publish something\n",
    "query=\"inspiring|inspired|inspire|uplifted|uplifting|uplift\"          # Add your query\n",
    "# duration=\"30d\"          # Select the timeframe. Epoch value or Integer + \"s,m,h,d\" (i.e. \"second\", \"minute\", \"hour\", \"day\")\n",
    "size=1000               # maximum 1000 comments\n",
    "sort_type=\"score\"       # Sort by score (Accepted: \"score\", \"num_comments\", \"created_utc\")\n",
    "sort=\"desc\"             # sort descending\n",
    "aggs=\"subreddit\"        #\"author\", \"link_id\", \"created_utc\", \"subreddit\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = get_pushshift_data(data_type=data_type,\n",
    "                          q=query,\n",
    "                          size=size,\n",
    "                          aggs=aggs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[\"data\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_body</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score_com</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The movie car was the 4th generation of Imprez...</td>\n",
       "      <td>t1_hbt3zn5</td>\n",
       "      <td>t5_2xrd1</td>\n",
       "      <td>1</td>\n",
       "      <td>gtaonline</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Enjoy anime, manga, or KPOP?\\n\\nHow about cand...</td>\n",
       "      <td>t3_pipwpa</td>\n",
       "      <td>t5_2qr34</td>\n",
       "      <td>1</td>\n",
       "      <td>smallbusiness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>They'd just gotten back from seeing *Friends w...</td>\n",
       "      <td>t1_hbtw0fb</td>\n",
       "      <td>t5_3aimx</td>\n",
       "      <td>1</td>\n",
       "      <td>IdiotsInCars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&amp;gt; half ass-setup\\n\\n***\\n\\n^(Bleep-bloop, I...</td>\n",
       "      <td>t1_hbuh482</td>\n",
       "      <td>t5_scu5z</td>\n",
       "      <td>1</td>\n",
       "      <td>fo76FilthyCasuals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Funny and real fact. This show inspired me to ...</td>\n",
       "      <td>t3_pipmvc</td>\n",
       "      <td>t5_2sd8v</td>\n",
       "      <td>1</td>\n",
       "      <td>PersonOfInterest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>A lot of the earlier couples are wonderful, an...</td>\n",
       "      <td>t1_hbu8n7f</td>\n",
       "      <td>t5_3o6nf</td>\n",
       "      <td>1</td>\n",
       "      <td>90dayfianceuncensored</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>Connections: Evil Time Traveling Doppelgängers...</td>\n",
       "      <td>t3_pj6qn2</td>\n",
       "      <td>t5_2sj683</td>\n",
       "      <td>1</td>\n",
       "      <td>DeathBattleMatchups</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>At first, I thought it was going to be a rage ...</td>\n",
       "      <td>t3_pj3mt5</td>\n",
       "      <td>t5_3c048</td>\n",
       "      <td>1</td>\n",
       "      <td>ConanExiles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>A short story heavily inspired by \"The Pedestr...</td>\n",
       "      <td>t3_pj6ltg</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>The stories were inspired by his \"thirst\" for ...</td>\n",
       "      <td>t1_hbubsbo</td>\n",
       "      <td>t5_qir9n</td>\n",
       "      <td>1</td>\n",
       "      <td>HolUp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         comment_body          id  \\\n",
       "0   The movie car was the 4th generation of Imprez...  t1_hbt3zn5   \n",
       "1   Enjoy anime, manga, or KPOP?\\n\\nHow about cand...   t3_pipwpa   \n",
       "2   They'd just gotten back from seeing *Friends w...  t1_hbtw0fb   \n",
       "3   &gt; half ass-setup\\n\\n***\\n\\n^(Bleep-bloop, I...  t1_hbuh482   \n",
       "4   Funny and real fact. This show inspired me to ...   t3_pipmvc   \n",
       "..                                                ...         ...   \n",
       "95  A lot of the earlier couples are wonderful, an...  t1_hbu8n7f   \n",
       "96  Connections: Evil Time Traveling Doppelgängers...   t3_pj6qn2   \n",
       "97  At first, I thought it was going to be a rage ...   t3_pj3mt5   \n",
       "98  A short story heavily inspired by \"The Pedestr...   t3_pj6ltg   \n",
       "99  The stories were inspired by his \"thirst\" for ...  t1_hbubsbo   \n",
       "\n",
       "   subreddit_id  score_com              subreddit  \n",
       "0      t5_2xrd1          1              gtaonline  \n",
       "1      t5_2qr34          1          smallbusiness  \n",
       "2      t5_3aimx          1           IdiotsInCars  \n",
       "3      t5_scu5z          1      fo76FilthyCasuals  \n",
       "4      t5_2sd8v          1       PersonOfInterest  \n",
       "..          ...        ...                    ...  \n",
       "95     t5_3o6nf          1  90dayfianceuncensored  \n",
       "96    t5_2sj683          1    DeathBattleMatchups  \n",
       "97     t5_3c048          1            ConanExiles  \n",
       "98     t5_2qh1i          1              AskReddit  \n",
       "99     t5_qir9n          1                  HolUp  \n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "column_values = ['body','parent_id','subreddit_id', 'score', 'subreddit']\n",
    "df_comments = pd.DataFrame.from_records(data)#[0:10]\n",
    "df_comments= pd.DataFrame(df_comments, columns=column_values)\n",
    "df_comments = df_comments.rename(columns={'body': \"comment_body\",'parent_id': \"id\", \"score\":\"score_com\"})\n",
    "df_comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetchObjects(**kwargs):\n",
    "    # Default paramaters for API query\n",
    "    params = {\n",
    "        \"sort_type\":\"created_utc\",\n",
    "        \"sort\":\"asc\",\n",
    "        \"size\":10000,\n",
    "        \"query\":\"inspiring|inspired|inspire|uplifted|uplifting\",\n",
    "        }\n",
    "\n",
    "    # Add additional paramters based on function arguments\n",
    "    for key,value in kwargs.items():\n",
    "        params[key] = value\n",
    "\n",
    "    # Perform an API request\n",
    "    base_url = f\"https://api.pushshift.io/reddit/search/comment/?q={query}\"\n",
    "    r = requests.get(base_url, params)\n",
    "\n",
    "\n",
    "    # Check the status code, if successful, process the data\n",
    "    if r.status_code == 200:\n",
    "        response = json.loads(r.text)\n",
    "        data = response['data']\n",
    "        sorted_data_by_id = sorted(data, key=lambda x: int(x['id'],36))\n",
    "        return sorted_data_by_id\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_reddit_data(**kwargs):\n",
    "    # Speficify the start timestamp\n",
    "    max_created_utc = 1356998400  # 01/01/2013 @ 12:00am (UTC)\n",
    "    max_id = 0\n",
    "    \n",
    "    for key,value in kwargs.items():\n",
    "        if key == \"columns\":\n",
    "            column_values = value\n",
    "    \n",
    "    df_all=pd.DataFrame(columns=column_values)\n",
    "    while 1:\n",
    "        nothing_processed = True\n",
    "        # Call the recursive function\n",
    "        objects = fetchObjects(**kwargs,after=max_created_utc)\n",
    "        df_new = pd.DataFrame(objects, columns = column_values)\n",
    "        df_all = pd.concat([df_all, df_new])\n",
    "#         if len(df_all) == 50000: #for more data\n",
    "        if len(df_all) == 1000:\n",
    "            return df_all.reset_index(drop=True).sort_values(by=['score'], ascending=False)\n",
    "        \n",
    "        # Loop the returned data, ordered by date\n",
    "        for object in objects:\n",
    "            id = int(object['id'],36)\n",
    "            if id > max_id:\n",
    "                nothing_processed = False\n",
    "                created_utc = object['created_utc']\n",
    "                max_id = id\n",
    "                if created_utc > max_created_utc: max_created_utc = created_utc\n",
    "\n",
    "        # Exit if nothing happened\n",
    "        if nothing_processed: return df_all.reset_index(drop=True).sort_values(by=['score'], ascending=False)\n",
    "        max_created_utc -= 1\n",
    "\n",
    "        # Sleep a little before the next recursive function call\n",
    "        time.sleep(.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>this has inspired me to make an early birthday...</td>\n",
       "      <td>t3_15rb1k</td>\n",
       "      <td>t5_2qh33</td>\n",
       "      <td>397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>You've just inspired my newest mission: get be...</td>\n",
       "      <td>t1_c7oxgnx</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>D- Demonstrate Value \\nE- Engage Physically \\n...</td>\n",
       "      <td>t1_c7pmkea</td>\n",
       "      <td>t5_2qh33</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>&amp;gt; I don't understand what your objection to...</td>\n",
       "      <td>t1_c7owury</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>I honestly felt a little...uncomfortable when ...</td>\n",
       "      <td>t1_c7p3ith</td>\n",
       "      <td>t5_2qh0u</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>hmmm, i don't know, it does kind of sound like...</td>\n",
       "      <td>t1_c7p8c6s</td>\n",
       "      <td>t5_2rske</td>\n",
       "      <td>-4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>I *also* cook 95% of my food from scratch, and...</td>\n",
       "      <td>t1_c7pceo4</td>\n",
       "      <td>t5_2qhpm</td>\n",
       "      <td>-5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>I'm sorry, but what nonsense. \\n\\nScience is a...</td>\n",
       "      <td>t3_15rbot</td>\n",
       "      <td>t5_2qh2p</td>\n",
       "      <td>-6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>You have to consider where they're coming from...</td>\n",
       "      <td>t1_c7p92lf</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Absolutely nothing. We have a shitty culture a...</td>\n",
       "      <td>t3_15r4rr</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  body   parent_id  \\\n",
       "149  this has inspired me to make an early birthday...   t3_15rb1k   \n",
       "37   You've just inspired my newest mission: get be...  t1_c7oxgnx   \n",
       "883  D- Demonstrate Value \\nE- Engage Physically \\n...  t1_c7pmkea   \n",
       "349  &gt; I don't understand what your objection to...  t1_c7owury   \n",
       "175  I honestly felt a little...uncomfortable when ...  t1_c7p3ith   \n",
       "..                                                 ...         ...   \n",
       "487  hmmm, i don't know, it does kind of sound like...  t1_c7p8c6s   \n",
       "494  I *also* cook 95% of my food from scratch, and...  t1_c7pceo4   \n",
       "220  I'm sorry, but what nonsense. \\n\\nScience is a...   t3_15rbot   \n",
       "498  You have to consider where they're coming from...  t1_c7p92lf   \n",
       "101  Absolutely nothing. We have a shitty culture a...   t3_15r4rr   \n",
       "\n",
       "    subreddit_id score  \n",
       "149     t5_2qh33   397  \n",
       "37      t5_2qh1i   227  \n",
       "883     t5_2qh33   204  \n",
       "349     t5_2qh13    74  \n",
       "175     t5_2qh0u    71  \n",
       "..           ...   ...  \n",
       "487     t5_2rske    -4  \n",
       "494     t5_2qhpm    -5  \n",
       "220     t5_2qh2p    -6  \n",
       "498     t5_2qh1i    -7  \n",
       "101     t5_2qh1i   -28  \n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_values = ['body','parent_id','subreddit_id', 'score']\n",
    "df_comments = extract_reddit_data(columns = column_values)\n",
    "df_comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_body</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score_com</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>this has inspired me to make an early birthday...</td>\n",
       "      <td>15rb1k</td>\n",
       "      <td>t5_2qh33</td>\n",
       "      <td>397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>You've just inspired my newest mission: get be...</td>\n",
       "      <td>c7oxgnx</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>D- Demonstrate Value \\nE- Engage Physically \\n...</td>\n",
       "      <td>c7pmkea</td>\n",
       "      <td>t5_2qh33</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>&amp;gt; I don't understand what your objection to...</td>\n",
       "      <td>c7owury</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>I honestly felt a little...uncomfortable when ...</td>\n",
       "      <td>c7p3ith</td>\n",
       "      <td>t5_2qh0u</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>hmmm, i don't know, it does kind of sound like...</td>\n",
       "      <td>c7p8c6s</td>\n",
       "      <td>t5_2rske</td>\n",
       "      <td>-4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>I *also* cook 95% of my food from scratch, and...</td>\n",
       "      <td>c7pceo4</td>\n",
       "      <td>t5_2qhpm</td>\n",
       "      <td>-5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>I'm sorry, but what nonsense. \\n\\nScience is a...</td>\n",
       "      <td>15rbot</td>\n",
       "      <td>t5_2qh2p</td>\n",
       "      <td>-6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>You have to consider where they're coming from...</td>\n",
       "      <td>c7p92lf</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Absolutely nothing. We have a shitty culture a...</td>\n",
       "      <td>15r4rr</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          comment_body       id subreddit_id  \\\n",
       "149  this has inspired me to make an early birthday...   15rb1k     t5_2qh33   \n",
       "37   You've just inspired my newest mission: get be...  c7oxgnx     t5_2qh1i   \n",
       "883  D- Demonstrate Value \\nE- Engage Physically \\n...  c7pmkea     t5_2qh33   \n",
       "349  &gt; I don't understand what your objection to...  c7owury     t5_2qh13   \n",
       "175  I honestly felt a little...uncomfortable when ...  c7p3ith     t5_2qh0u   \n",
       "..                                                 ...      ...          ...   \n",
       "487  hmmm, i don't know, it does kind of sound like...  c7p8c6s     t5_2rske   \n",
       "494  I *also* cook 95% of my food from scratch, and...  c7pceo4     t5_2qhpm   \n",
       "220  I'm sorry, but what nonsense. \\n\\nScience is a...   15rbot     t5_2qh2p   \n",
       "498  You have to consider where they're coming from...  c7p92lf     t5_2qh1i   \n",
       "101  Absolutely nothing. We have a shitty culture a...   15r4rr     t5_2qh1i   \n",
       "\n",
       "    score_com  \n",
       "149       397  \n",
       "37        227  \n",
       "883       204  \n",
       "349        74  \n",
       "175        71  \n",
       "..        ...  \n",
       "487        -4  \n",
       "494        -5  \n",
       "220        -6  \n",
       "498        -7  \n",
       "101       -28  \n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_parent_id = [] #rename ids from parent_id column\n",
    "for id in list(df_comments.parent_id):\n",
    "    new_parent_id.append(id.split(\"_\")[1])\n",
    "df_comments.parent_id = new_parent_id\n",
    "df_comments = df_comments.rename(columns={'body': \"comment_body\",'parent_id': \"id\", \"score\":\"score_com\"})\n",
    "df_comments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "53\n",
      "372\n",
      "53\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>score</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Deep fried hot dog stars with cheese sauce.</td>\n",
       "      <td>2026</td>\n",
       "      <td></td>\n",
       "      <td>http://imgur.com/a/VObuF</td>\n",
       "      <td>14v5iw</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>DJ Shadow kicked off decks in Miami - \"I don't...</td>\n",
       "      <td>1861</td>\n",
       "      <td></td>\n",
       "      <td>https://www.youtube.com/watch?v=g3sO0Se-NT0</td>\n",
       "      <td>15q2g1</td>\n",
       "      <td>Music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Can a group of KSP fans but an inexperienced p...</td>\n",
       "      <td>1793</td>\n",
       "      <td></td>\n",
       "      <td>http://imgur.com/a/sW3Tz</td>\n",
       "      <td>15qf0e</td>\n",
       "      <td>KerbalSpaceProgram</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>What is your favorite product that has been di...</td>\n",
       "      <td>1750</td>\n",
       "      <td>Simple question.  What product did you love th...</td>\n",
       "      <td>http://www.reddit.com/r/AskReddit/comments/15q...</td>\n",
       "      <td>15qmfc</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>White kid singing the blues like it's nobody's...</td>\n",
       "      <td>1682</td>\n",
       "      <td></td>\n",
       "      <td>http://youtu.be/F1pwnb0NpQw</td>\n",
       "      <td>15mmna</td>\n",
       "      <td>videos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>How do games affect us?</td>\n",
       "      <td>0</td>\n",
       "      <td>The consensus on reddit seems to be that video...</td>\n",
       "      <td>http://www.reddit.com/r/truegaming/comments/15...</td>\n",
       "      <td>15onta</td>\n",
       "      <td>truegaming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>If this redditor snaps... 11</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>http://imgur.com/a/i0DVB</td>\n",
       "      <td>15qmm5</td>\n",
       "      <td>GunsAreCool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Parents of disabled children, if you had the c...</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/AskReddit/comments/15q...</td>\n",
       "      <td>15qwfn</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>[Meta]Open Dialogue with the Mods</td>\n",
       "      <td>0</td>\n",
       "      <td>Ok guys so here goes...\\n\\nAfter [this No. 1 p...</td>\n",
       "      <td>http://www.reddit.com/r/childfree/comments/15q...</td>\n",
       "      <td>15qtp8</td>\n",
       "      <td>childfree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>A deep drawing from an 8th grader</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>http://imgur.com/MMug6</td>\n",
       "      <td>15qxg0</td>\n",
       "      <td>pics</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                title score  \\\n",
       "60        Deep fried hot dog stars with cheese sauce.  2026   \n",
       "97  DJ Shadow kicked off decks in Miami - \"I don't...  1861   \n",
       "30  Can a group of KSP fans but an inexperienced p...  1793   \n",
       "77  What is your favorite product that has been di...  1750   \n",
       "98  White kid singing the blues like it's nobody's...  1682   \n",
       "..                                                ...   ...   \n",
       "44                            How do games affect us?     0   \n",
       "76                       If this redditor snaps... 11     0   \n",
       "71  Parents of disabled children, if you had the c...     0   \n",
       "66                  [Meta]Open Dialogue with the Mods     0   \n",
       "64                  A deep drawing from an 8th grader     0   \n",
       "\n",
       "                                             selftext  \\\n",
       "60                                                      \n",
       "97                                                      \n",
       "30                                                      \n",
       "77  Simple question.  What product did you love th...   \n",
       "98                                                      \n",
       "..                                                ...   \n",
       "44  The consensus on reddit seems to be that video...   \n",
       "76                                                      \n",
       "71                                                      \n",
       "66  Ok guys so here goes...\\n\\nAfter [this No. 1 p...   \n",
       "64                                                      \n",
       "\n",
       "                                                  url      id  \\\n",
       "60                           http://imgur.com/a/VObuF  14v5iw   \n",
       "97        https://www.youtube.com/watch?v=g3sO0Se-NT0  15q2g1   \n",
       "30                           http://imgur.com/a/sW3Tz  15qf0e   \n",
       "77  http://www.reddit.com/r/AskReddit/comments/15q...  15qmfc   \n",
       "98                        http://youtu.be/F1pwnb0NpQw  15mmna   \n",
       "..                                                ...     ...   \n",
       "44  http://www.reddit.com/r/truegaming/comments/15...  15onta   \n",
       "76                           http://imgur.com/a/i0DVB  15qmm5   \n",
       "71  http://www.reddit.com/r/AskReddit/comments/15q...  15qwfn   \n",
       "66  http://www.reddit.com/r/childfree/comments/15q...  15qtp8   \n",
       "64                             http://imgur.com/MMug6  15qxg0   \n",
       "\n",
       "             subreddit  \n",
       "60                food  \n",
       "97               Music  \n",
       "30  KerbalSpaceProgram  \n",
       "77           AskReddit  \n",
       "98              videos  \n",
       "..                 ...  \n",
       "44          truegaming  \n",
       "76         GunsAreCool  \n",
       "71           AskReddit  \n",
       "66           childfree  \n",
       "64                pics  \n",
       "\n",
       "[106 rows x 6 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "# max 500 ids per request, need to call multiple times\n",
    "max_n = 500\n",
    "list_submission_ids = list(set(list(df_comments[\"id\"])))\n",
    "\n",
    "list_submission_ids_split = [list_submission_ids[i:i + max_n] for i in range(0, len(list_submission_ids), max_n)]  \n",
    "\n",
    "df_submissions = pd.DataFrame(columns = column_values)\n",
    "for list_submission_ids in list_submission_ids_split:\n",
    "    \n",
    "    print(len(list_submission_ids))\n",
    "\n",
    "    str_list_ids = \",\".join(list_submission_ids)\n",
    "\n",
    "    url = f\"https://api.pushshift.io/reddit/search/submission/?ids=\" + str_list_ids \n",
    "    request = requests.get(url)\n",
    "    try:\n",
    "        json_response = request.json()\n",
    "        data=json_response[\"data\"]\n",
    "        print(len(data))\n",
    "        df_submissions_part = pd.DataFrame.from_records(data)\n",
    "        column_values = ['title', 'score', 'selftext', 'url', 'id', \"subreddit\"]\n",
    "        df_submissions_part = pd.DataFrame(df_submissions_part, columns = column_values)\n",
    "        df_submissions = pd.concat([df_submissions, df_submissions_part])\n",
    "    except Exception as e:\n",
    "        print(\"Exception: \" + str(e))\n",
    "    \n",
    "df_submissions = df_submissions.reset_index(drop=True).sort_values(by=['score'], ascending=False)\n",
    "df_submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'What are your favorite blogs/tumblrs? \\n | What are your favorite natural hair, life, fashion, cultural, food educational blogs? | 9 | blackladies'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = 1\n",
    "df_submissions[\"title\"][idx] + \" | \" + df_submissions[\"selftext\"][idx] + \" | \"  + str(df_submissions[\"score\"][idx]) + \" | \" + df_submissions[\"subreddit\"][idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>food</td>\n",
       "      <td>14v5iw</td>\n",
       "      <td>Made them for Christmas, with the cheese sauce...</td>\n",
       "      <td>1</td>\n",
       "      <td>Deep fried hot dog stars with cheese sauce.</td>\n",
       "      <td></td>\n",
       "      <td>2026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Music</td>\n",
       "      <td>15q2g1</td>\n",
       "      <td>Dj shadow is my favorite artist and a top-tier...</td>\n",
       "      <td>1</td>\n",
       "      <td>DJ Shadow kicked off decks in Miami - \"I don't...</td>\n",
       "      <td></td>\n",
       "      <td>1861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>KerbalSpaceProgram</td>\n",
       "      <td>15qf0e</td>\n",
       "      <td>I'm inspired :D</td>\n",
       "      <td>4</td>\n",
       "      <td>Can a group of KSP fans but an inexperienced p...</td>\n",
       "      <td></td>\n",
       "      <td>1793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>15qmfc</td>\n",
       "      <td>I can't find evidence that these ever existed ...</td>\n",
       "      <td>1</td>\n",
       "      <td>What is your favorite product that has been di...</td>\n",
       "      <td>Simple question.  What product did you love th...</td>\n",
       "      <td>1750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>videos</td>\n",
       "      <td>15mmna</td>\n",
       "      <td>This video inspired/spoke to me thanks.</td>\n",
       "      <td>0</td>\n",
       "      <td>White kid singing the blues like it's nobody's...</td>\n",
       "      <td></td>\n",
       "      <td>1682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>truegaming</td>\n",
       "      <td>15onta</td>\n",
       "      <td>People take things away from games on an indiv...</td>\n",
       "      <td>2</td>\n",
       "      <td>How do games affect us?</td>\n",
       "      <td>The consensus on reddit seems to be that video...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>GunsAreCool</td>\n",
       "      <td>15qmm5</td>\n",
       "      <td>That was my post...and after getting slammed f...</td>\n",
       "      <td>2</td>\n",
       "      <td>If this redditor snaps... 11</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>15qwfn</td>\n",
       "      <td>I used to volunteer a lot with the special Oly...</td>\n",
       "      <td>1</td>\n",
       "      <td>Parents of disabled children, if you had the c...</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>childfree</td>\n",
       "      <td>15qtp8</td>\n",
       "      <td>I believe the minority hasn't spoken up here. ...</td>\n",
       "      <td>-2</td>\n",
       "      <td>[Meta]Open Dialogue with the Mods</td>\n",
       "      <td>Ok guys so here goes...\\n\\nAfter [this No. 1 p...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>pics</td>\n",
       "      <td>15qxg0</td>\n",
       "      <td>Age | User | Title | Reddit | Cmnt | Points\\n:...</td>\n",
       "      <td>1</td>\n",
       "      <td>A deep drawing from an 8th grader</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              subreddit      id  \\\n",
       "0                  food  14v5iw   \n",
       "1                 Music  15q2g1   \n",
       "2    KerbalSpaceProgram  15qf0e   \n",
       "3             AskReddit  15qmfc   \n",
       "4                videos  15mmna   \n",
       "..                  ...     ...   \n",
       "106          truegaming  15onta   \n",
       "107         GunsAreCool  15qmm5   \n",
       "108           AskReddit  15qwfn   \n",
       "109           childfree  15qtp8   \n",
       "110                pics  15qxg0   \n",
       "\n",
       "                                          comment_body score_com  \\\n",
       "0    Made them for Christmas, with the cheese sauce...         1   \n",
       "1    Dj shadow is my favorite artist and a top-tier...         1   \n",
       "2                                      I'm inspired :D         4   \n",
       "3    I can't find evidence that these ever existed ...         1   \n",
       "4              This video inspired/spoke to me thanks.         0   \n",
       "..                                                 ...       ...   \n",
       "106  People take things away from games on an indiv...         2   \n",
       "107  That was my post...and after getting slammed f...         2   \n",
       "108  I used to volunteer a lot with the special Oly...         1   \n",
       "109  I believe the minority hasn't spoken up here. ...        -2   \n",
       "110  Age | User | Title | Reddit | Cmnt | Points\\n:...         1   \n",
       "\n",
       "                                                 title  \\\n",
       "0          Deep fried hot dog stars with cheese sauce.   \n",
       "1    DJ Shadow kicked off decks in Miami - \"I don't...   \n",
       "2    Can a group of KSP fans but an inexperienced p...   \n",
       "3    What is your favorite product that has been di...   \n",
       "4    White kid singing the blues like it's nobody's...   \n",
       "..                                                 ...   \n",
       "106                            How do games affect us?   \n",
       "107                       If this redditor snaps... 11   \n",
       "108  Parents of disabled children, if you had the c...   \n",
       "109                  [Meta]Open Dialogue with the Mods   \n",
       "110                  A deep drawing from an 8th grader   \n",
       "\n",
       "                                              selftext score  \n",
       "0                                                       2026  \n",
       "1                                                       1861  \n",
       "2                                                       1793  \n",
       "3    Simple question.  What product did you love th...  1750  \n",
       "4                                                       1682  \n",
       "..                                                 ...   ...  \n",
       "106  The consensus on reddit seems to be that video...     0  \n",
       "107                                                        0  \n",
       "108                                                        0  \n",
       "109  Ok guys so here goes...\\n\\nAfter [this No. 1 p...     0  \n",
       "110                                                        0  \n",
       "\n",
       "[111 rows x 7 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged = pd.merge(df_submissions, df_comments, on=\"id\", how='inner')\n",
    "column_values =[\"subreddit\", \"id\", \"comment_body\", \"score_com\", \"title\", \"selftext\", \"score\"]\n",
    "merged= pd.DataFrame(merged, columns=column_values)\n",
    "merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged.transpose().to_json('data/comments_submission_all.json', index = 'true') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean data: "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove comments and posts that have negative scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pics</td>\n",
       "      <td>7m28mc</td>\n",
       "      <td>It's these kind of posts that restore some of ...</td>\n",
       "      <td>1</td>\n",
       "      <td>She took me in 15 years ago and no one makes m...</td>\n",
       "      <td></td>\n",
       "      <td>171795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pics</td>\n",
       "      <td>6f0lr7</td>\n",
       "      <td>I understand trying to uplift people with the ...</td>\n",
       "      <td>1</td>\n",
       "      <td>You can destroy a country. But you can't destr...</td>\n",
       "      <td></td>\n",
       "      <td>129553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pics</td>\n",
       "      <td>808ddp</td>\n",
       "      <td>Hope you put this on r/uplift.</td>\n",
       "      <td>1</td>\n",
       "      <td>Wife is a kindergarten teacher. A couple of th...</td>\n",
       "      <td></td>\n",
       "      <td>119527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>6po7fz</td>\n",
       "      <td>I wish we could do this on a global scale, upl...</td>\n",
       "      <td>2</td>\n",
       "      <td>TIL when a millionaire gave everyone in a Flor...</td>\n",
       "      <td></td>\n",
       "      <td>117421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>aww</td>\n",
       "      <td>7i2z6g</td>\n",
       "      <td>It’s the smallest of gestures  that Can can gi...</td>\n",
       "      <td>1</td>\n",
       "      <td>Little boy just wants to hug the police officer</td>\n",
       "      <td></td>\n",
       "      <td>104777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20484</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>30qhb0</td>\n",
       "      <td>I saw my babysitter naked, once. And she used ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Redditors who have had babysitters/hired babys...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20485</th>\n",
       "      <td>TwoXChromosomes</td>\n",
       "      <td>2cqot7</td>\n",
       "      <td>I am a feminist, but I'll tell you some flaws ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Why aren't you a feminist?</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20486</th>\n",
       "      <td>AskMen</td>\n",
       "      <td>8uuymy</td>\n",
       "      <td>In established sci fi terms this is called \"u...</td>\n",
       "      <td>10</td>\n",
       "      <td>If humans get the capability to create brains ...</td>\n",
       "      <td>Or should we not give the rational mind to any...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20487</th>\n",
       "      <td>askscience</td>\n",
       "      <td>2gn537</td>\n",
       "      <td>The answer to this is complicated and also a q...</td>\n",
       "      <td>5</td>\n",
       "      <td>Are all mountains formed by tectonic plate mov...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20488</th>\n",
       "      <td>willis7737_news</td>\n",
       "      <td>4jterm</td>\n",
       "      <td>UK to increase training to Oman\\n\\n---\\n\\n&amp;gt;...</td>\n",
       "      <td>1</td>\n",
       "      <td>UK to increase training to Oman</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19704 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             subreddit      id  \\\n",
       "1                 pics  7m28mc   \n",
       "3                 pics  6f0lr7   \n",
       "4                 pics  808ddp   \n",
       "5        todayilearned  6po7fz   \n",
       "6                  aww  7i2z6g   \n",
       "...                ...     ...   \n",
       "20484        AskReddit  30qhb0   \n",
       "20485  TwoXChromosomes  2cqot7   \n",
       "20486           AskMen  8uuymy   \n",
       "20487       askscience  2gn537   \n",
       "20488  willis7737_news  4jterm   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "1      It's these kind of posts that restore some of ...         1   \n",
       "3      I understand trying to uplift people with the ...         1   \n",
       "4                        Hope you put this on r/uplift.          1   \n",
       "5      I wish we could do this on a global scale, upl...         2   \n",
       "6      It’s the smallest of gestures  that Can can gi...         1   \n",
       "...                                                  ...       ...   \n",
       "20484  I saw my babysitter naked, once. And she used ...         1   \n",
       "20485  I am a feminist, but I'll tell you some flaws ...         1   \n",
       "20486   In established sci fi terms this is called \"u...        10   \n",
       "20487  The answer to this is complicated and also a q...         5   \n",
       "20488  UK to increase training to Oman\\n\\n---\\n\\n&gt;...         1   \n",
       "\n",
       "                                                   title  \\\n",
       "1      She took me in 15 years ago and no one makes m...   \n",
       "3      You can destroy a country. But you can't destr...   \n",
       "4      Wife is a kindergarten teacher. A couple of th...   \n",
       "5      TIL when a millionaire gave everyone in a Flor...   \n",
       "6        Little boy just wants to hug the police officer   \n",
       "...                                                  ...   \n",
       "20484  Redditors who have had babysitters/hired babys...   \n",
       "20485                         Why aren't you a feminist?   \n",
       "20486  If humans get the capability to create brains ...   \n",
       "20487  Are all mountains formed by tectonic plate mov...   \n",
       "20488                    UK to increase training to Oman   \n",
       "\n",
       "                                                selftext   score  \n",
       "1                                                         171795  \n",
       "3                                                         129553  \n",
       "4                                                         119527  \n",
       "5                                                         117421  \n",
       "6                                                         104777  \n",
       "...                                                  ...     ...  \n",
       "20484                                                          1  \n",
       "20485                                                          1  \n",
       "20486  Or should we not give the rational mind to any...       1  \n",
       "20487                                                          1  \n",
       "20488                                                          1  \n",
       "\n",
       "[19704 rows x 7 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged = merged[~(merged['score_com'] <= 0)]  \n",
    "merged = merged[~(merged['score'] <= 0)]\n",
    "merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. remove links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pics</td>\n",
       "      <td>7m28mc</td>\n",
       "      <td>It's these kind of posts that restore some of ...</td>\n",
       "      <td>1</td>\n",
       "      <td>She took me in 15 years ago and no one makes m...</td>\n",
       "      <td></td>\n",
       "      <td>171795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pics</td>\n",
       "      <td>6f0lr7</td>\n",
       "      <td>I understand trying to uplift people with the ...</td>\n",
       "      <td>1</td>\n",
       "      <td>You can destroy a country. But you can't destr...</td>\n",
       "      <td></td>\n",
       "      <td>129553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pics</td>\n",
       "      <td>808ddp</td>\n",
       "      <td>Hope you put this on r/uplift.</td>\n",
       "      <td>1</td>\n",
       "      <td>Wife is a kindergarten teacher. A couple of th...</td>\n",
       "      <td></td>\n",
       "      <td>119527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>6po7fz</td>\n",
       "      <td>I wish we could do this on a global scale, upl...</td>\n",
       "      <td>2</td>\n",
       "      <td>TIL when a millionaire gave everyone in a Flor...</td>\n",
       "      <td></td>\n",
       "      <td>117421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>aww</td>\n",
       "      <td>7i2z6g</td>\n",
       "      <td>It’s the smallest of gestures that Can can giv...</td>\n",
       "      <td>1</td>\n",
       "      <td>Little boy just wants to hug the police officer</td>\n",
       "      <td></td>\n",
       "      <td>104777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20484</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>30qhb0</td>\n",
       "      <td>I saw my babysitter naked, once. And she used ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Redditors who have had babysitters/hired babys...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20485</th>\n",
       "      <td>TwoXChromosomes</td>\n",
       "      <td>2cqot7</td>\n",
       "      <td>I am a feminist, but I'll tell you some flaws ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Why aren't you a feminist?</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20486</th>\n",
       "      <td>AskMen</td>\n",
       "      <td>8uuymy</td>\n",
       "      <td>In established sci fi terms this is called \"u...</td>\n",
       "      <td>10</td>\n",
       "      <td>If humans get the capability to create brains ...</td>\n",
       "      <td>Or should we not give the rational mind to any...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20487</th>\n",
       "      <td>askscience</td>\n",
       "      <td>2gn537</td>\n",
       "      <td>The answer to this is complicated and also a q...</td>\n",
       "      <td>5</td>\n",
       "      <td>Are all mountains formed by tectonic plate mov...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20488</th>\n",
       "      <td>willis7737_news</td>\n",
       "      <td>4jterm</td>\n",
       "      <td>UK to increase training to Oman---&amp;gt;* The UK...</td>\n",
       "      <td>1</td>\n",
       "      <td>UK to increase training to Oman</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19704 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             subreddit      id  \\\n",
       "1                 pics  7m28mc   \n",
       "3                 pics  6f0lr7   \n",
       "4                 pics  808ddp   \n",
       "5        todayilearned  6po7fz   \n",
       "6                  aww  7i2z6g   \n",
       "...                ...     ...   \n",
       "20484        AskReddit  30qhb0   \n",
       "20485  TwoXChromosomes  2cqot7   \n",
       "20486           AskMen  8uuymy   \n",
       "20487       askscience  2gn537   \n",
       "20488  willis7737_news  4jterm   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "1      It's these kind of posts that restore some of ...         1   \n",
       "3      I understand trying to uplift people with the ...         1   \n",
       "4                         Hope you put this on r/uplift.         1   \n",
       "5      I wish we could do this on a global scale, upl...         2   \n",
       "6      It’s the smallest of gestures that Can can giv...         1   \n",
       "...                                                  ...       ...   \n",
       "20484  I saw my babysitter naked, once. And she used ...         1   \n",
       "20485  I am a feminist, but I'll tell you some flaws ...         1   \n",
       "20486   In established sci fi terms this is called \"u...        10   \n",
       "20487  The answer to this is complicated and also a q...         5   \n",
       "20488  UK to increase training to Oman---&gt;* The UK...         1   \n",
       "\n",
       "                                                   title  \\\n",
       "1      She took me in 15 years ago and no one makes m...   \n",
       "3      You can destroy a country. But you can't destr...   \n",
       "4      Wife is a kindergarten teacher. A couple of th...   \n",
       "5      TIL when a millionaire gave everyone in a Flor...   \n",
       "6        Little boy just wants to hug the police officer   \n",
       "...                                                  ...   \n",
       "20484  Redditors who have had babysitters/hired babys...   \n",
       "20485                         Why aren't you a feminist?   \n",
       "20486  If humans get the capability to create brains ...   \n",
       "20487  Are all mountains formed by tectonic plate mov...   \n",
       "20488                    UK to increase training to Oman   \n",
       "\n",
       "                                                selftext   score  \n",
       "1                                                         171795  \n",
       "3                                                         129553  \n",
       "4                                                         119527  \n",
       "5                                                         117421  \n",
       "6                                                         104777  \n",
       "...                                                  ...     ...  \n",
       "20484                                                          1  \n",
       "20485                                                          1  \n",
       "20486  Or should we not give the rational mind to any...       1  \n",
       "20487                                                          1  \n",
       "20488                                                          1  \n",
       "\n",
       "[19704 rows x 7 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "merged.dropna()\n",
    "\n",
    "for index in merged.index:\n",
    "    title = merged.loc[index,'title']\n",
    "    title = re.sub(r'http\\S+', '', title)\n",
    "    title = title.replace(\"\\n\", \"\")\n",
    "    title = re.sub('\\s{2,}', ' ', title)  # To remove more than one space\n",
    "    merged.loc[index,'title'] = title.rstrip() #remove new line\n",
    "\n",
    "for index in merged.index:\n",
    "    selftext = merged.loc[index,'selftext']\n",
    "    selftext = re.sub(r'http\\S+', '', selftext)\n",
    "    selftext = selftext.replace(\"\\n\", \"\")\n",
    "    selftext = re.sub('\\s{2,}', ' ', selftext)  # To remove more than one space\n",
    "    merged.loc[index,'selftext'] = selftext.rstrip() #remove new line\n",
    "\n",
    "for index in merged.index:\n",
    "    comment_body = merged.loc[index,'comment_body']\n",
    "    comment_body = re.sub(r'http\\S+', '', comment_body)\n",
    "    comment_body = comment_body.replace(\"\\n\", \"\")\n",
    "    comment_body = re.sub('\\s{2,}', ' ', comment_body)  # To remove more than one space\n",
    "    merged.loc[index,'comment_body'] = comment_body.rstrip() #remove new line\n",
    "\n",
    "\n",
    "merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. remove posts that contains string in comment: \"Askreddit is for thought-provoking, discussion-inspiring questions. Askreddit is not your research source. If the answer can be googled, or adequately answered in one word, it’s not right for this subreddit. \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \" is for thought-provoking, discussion-inspiring questions\"\n",
    "merged = merged[~merged.comment_body.str.contains(query)]\n",
    "merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at top rated comments from certain subreddits - those asking questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>8pks1u</td>\n",
       "      <td>The sympathy in this thread is comforting.The ...</td>\n",
       "      <td>3</td>\n",
       "      <td>Suicide Prevention Megathread</td>\n",
       "      <td>With the news today of the passing of the amaz...</td>\n",
       "      <td>104195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>80ow6w</td>\n",
       "      <td>Hello Bill! First thank you and your wife Meli...</td>\n",
       "      <td>1</td>\n",
       "      <td>I’m Bill Gates, co-chair of the Bill &amp;amp; Mel...</td>\n",
       "      <td>I’m excited to be back for my sixth AMA.Here’s...</td>\n",
       "      <td>94575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>84e0nc</td>\n",
       "      <td>I never had a great relationship with my fathe...</td>\n",
       "      <td>2</td>\n",
       "      <td>Daughters of reddit, what is something you wis...</td>\n",
       "      <td></td>\n",
       "      <td>66465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>8qbnls</td>\n",
       "      <td>I'm hispanic and since Trump became President,...</td>\n",
       "      <td>1</td>\n",
       "      <td>Since Donald Trump has been President of the U...</td>\n",
       "      <td></td>\n",
       "      <td>48541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>7e8r3x</td>\n",
       "      <td>I used to work for one of those bargain stores...</td>\n",
       "      <td>2</td>\n",
       "      <td>What is unethical as fuck, but is extremely co...</td>\n",
       "      <td></td>\n",
       "      <td>47881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20459</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>5kojl7</td>\n",
       "      <td>I think I'd continue my giftOf using my verse ...</td>\n",
       "      <td>1</td>\n",
       "      <td>If you died and became a ghost, who would you ...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20460</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>4ll6k1</td>\n",
       "      <td>One man saved ten women from a blazing forest ...</td>\n",
       "      <td>2</td>\n",
       "      <td>What's your most uplifting story, Reddit?</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20461</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>4ll6k1</td>\n",
       "      <td>One man saved ten women from a blazing forest ...</td>\n",
       "      <td>2</td>\n",
       "      <td>What's your most uplifting story, Reddit?</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20473</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>7c5ii4</td>\n",
       "      <td>Convection currents / uplift.</td>\n",
       "      <td>1</td>\n",
       "      <td>How come gravity doesn't pull down clouds?</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20484</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>30qhb0</td>\n",
       "      <td>I saw my babysitter naked, once. And she used ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Redditors who have had babysitters/hired babys...</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>911 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       subreddit      id                                       comment_body  \\\n",
       "7      AskReddit  8pks1u  The sympathy in this thread is comforting.The ...   \n",
       "12          IAmA  80ow6w  Hello Bill! First thank you and your wife Meli...   \n",
       "36     AskReddit  84e0nc  I never had a great relationship with my fathe...   \n",
       "78     AskReddit  8qbnls  I'm hispanic and since Trump became President,...   \n",
       "82     AskReddit  7e8r3x  I used to work for one of those bargain stores...   \n",
       "...          ...     ...                                                ...   \n",
       "20459  AskReddit  5kojl7  I think I'd continue my giftOf using my verse ...   \n",
       "20460  AskReddit  4ll6k1  One man saved ten women from a blazing forest ...   \n",
       "20461  AskReddit  4ll6k1  One man saved ten women from a blazing forest ...   \n",
       "20473  AskReddit  7c5ii4                      Convection currents / uplift.   \n",
       "20484  AskReddit  30qhb0  I saw my babysitter naked, once. And she used ...   \n",
       "\n",
       "      score_com                                              title  \\\n",
       "7             3                      Suicide Prevention Megathread   \n",
       "12            1  I’m Bill Gates, co-chair of the Bill &amp; Mel...   \n",
       "36            2  Daughters of reddit, what is something you wis...   \n",
       "78            1  Since Donald Trump has been President of the U...   \n",
       "82            2  What is unethical as fuck, but is extremely co...   \n",
       "...         ...                                                ...   \n",
       "20459         1  If you died and became a ghost, who would you ...   \n",
       "20460         2          What's your most uplifting story, Reddit?   \n",
       "20461         2          What's your most uplifting story, Reddit?   \n",
       "20473         1         How come gravity doesn't pull down clouds?   \n",
       "20484         1  Redditors who have had babysitters/hired babys...   \n",
       "\n",
       "                                                selftext   score  \n",
       "7      With the news today of the passing of the amaz...  104195  \n",
       "12     I’m excited to be back for my sixth AMA.Here’s...   94575  \n",
       "36                                                         66465  \n",
       "78                                                         48541  \n",
       "82                                                         47881  \n",
       "...                                                  ...     ...  \n",
       "20459                                                          1  \n",
       "20460                                                          1  \n",
       "20461                                                          1  \n",
       "20473                                                          1  \n",
       "20484                                                          1  \n",
       "\n",
       "[911 rows x 7 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged6 = merged[merged.subreddit.isin([\"AskReddit\", \"IAmA\"])]\n",
    "merged6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged6.transpose().to_json('data/comments_submission_ask.json', index = 'true') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at top rated comments from everywhere and filter by length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>15twl7</td>\n",
       "      <td>1. Up1. Harold and Kumar1. Star Trek (2009)Aft...</td>\n",
       "      <td>2676</td>\n",
       "      <td>What three completely unrelated movies can you...</td>\n",
       "      <td>EDIT: Front page, that is awesome! It is takin...</td>\n",
       "      <td>2100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>worldnews</td>\n",
       "      <td>17xe4c</td>\n",
       "      <td>Here's my translation: &amp;gt;Last night, a truly...</td>\n",
       "      <td>1583</td>\n",
       "      <td>North Korea, poised to conduct a nuclear test ...</td>\n",
       "      <td></td>\n",
       "      <td>2895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1512</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>170tzo</td>\n",
       "      <td>Bratz Dolls. They look anorexic, dress like ho...</td>\n",
       "      <td>1400</td>\n",
       "      <td>What is the creepiest thing that society accep...</td>\n",
       "      <td></td>\n",
       "      <td>2437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2257</th>\n",
       "      <td>funny</td>\n",
       "      <td>166a9n</td>\n",
       "      <td>It's always a bummer waiting to buy child slav...</td>\n",
       "      <td>1282</td>\n",
       "      <td>Go home, fashion. You're drunk.</td>\n",
       "      <td></td>\n",
       "      <td>1983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1021</th>\n",
       "      <td>pics</td>\n",
       "      <td>16fjjm</td>\n",
       "      <td>Do not stand at my grave and weep I am not th...</td>\n",
       "      <td>1221</td>\n",
       "      <td>Aaron Shwartz- Reddit Co-founder R.I.P</td>\n",
       "      <td></td>\n",
       "      <td>2740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8257</th>\n",
       "      <td>RedditLaqueristas</td>\n",
       "      <td>17fiju</td>\n",
       "      <td>Ok, I'm inspired. My Wallis has been collectin...</td>\n",
       "      <td>1</td>\n",
       "      <td>Butter London's Wallis is such an interesting ...</td>\n",
       "      <td></td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8252</th>\n",
       "      <td>leagueoflegends</td>\n",
       "      <td>18qy3w</td>\n",
       "      <td>oh man, seeing those screenshots made me ache ...</td>\n",
       "      <td>1</td>\n",
       "      <td>MinionInvaders - a fan-made game</td>\n",
       "      <td>Hello Reddit,I just updated MinionInvaders, a ...</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8250</th>\n",
       "      <td>progresspics</td>\n",
       "      <td>16klde</td>\n",
       "      <td>THE FUQ I just C!!! It's like a total metamorp...</td>\n",
       "      <td>1</td>\n",
       "      <td>M 1 year 5'7\" 225-145</td>\n",
       "      <td>Then Now Exactly 1 year apartEdit: Will Post r...</td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8245</th>\n",
       "      <td>Minecraft</td>\n",
       "      <td>17t8rt</td>\n",
       "      <td>I like the village and the Super Craft bros in...</td>\n",
       "      <td>1</td>\n",
       "      <td>New PvP map Im working on</td>\n",
       "      <td></td>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21559</th>\n",
       "      <td>NoFap</td>\n",
       "      <td>15u2ma</td>\n",
       "      <td>I thought I was coming into this post to see y...</td>\n",
       "      <td>1</td>\n",
       "      <td>A friend of mine failed nofap after two and a ...</td>\n",
       "      <td>When I decided to stop fapping over a year ago...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15670 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               subreddit      id  \\\n",
       "2007           AskReddit  15twl7   \n",
       "894            worldnews  17xe4c   \n",
       "1512           AskReddit  170tzo   \n",
       "2257               funny  166a9n   \n",
       "1021                pics  16fjjm   \n",
       "...                  ...     ...   \n",
       "8257   RedditLaqueristas  17fiju   \n",
       "8252     leagueoflegends  18qy3w   \n",
       "8250        progresspics  16klde   \n",
       "8245           Minecraft  17t8rt   \n",
       "21559              NoFap  15u2ma   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "2007   1. Up1. Harold and Kumar1. Star Trek (2009)Aft...      2676   \n",
       "894    Here's my translation: &gt;Last night, a truly...      1583   \n",
       "1512   Bratz Dolls. They look anorexic, dress like ho...      1400   \n",
       "2257   It's always a bummer waiting to buy child slav...      1282   \n",
       "1021    Do not stand at my grave and weep I am not th...      1221   \n",
       "...                                                  ...       ...   \n",
       "8257   Ok, I'm inspired. My Wallis has been collectin...         1   \n",
       "8252   oh man, seeing those screenshots made me ache ...         1   \n",
       "8250   THE FUQ I just C!!! It's like a total metamorp...         1   \n",
       "8245   I like the village and the Super Craft bros in...         1   \n",
       "21559  I thought I was coming into this post to see y...         1   \n",
       "\n",
       "                                                   title  \\\n",
       "2007   What three completely unrelated movies can you...   \n",
       "894    North Korea, poised to conduct a nuclear test ...   \n",
       "1512   What is the creepiest thing that society accep...   \n",
       "2257                     Go home, fashion. You're drunk.   \n",
       "1021              Aaron Shwartz- Reddit Co-founder R.I.P   \n",
       "...                                                  ...   \n",
       "8257   Butter London's Wallis is such an interesting ...   \n",
       "8252                    MinionInvaders - a fan-made game   \n",
       "8250                               M 1 year 5'7\" 225-145   \n",
       "8245                           New PvP map Im working on   \n",
       "21559  A friend of mine failed nofap after two and a ...   \n",
       "\n",
       "                                                selftext score  \n",
       "2007   EDIT: Front page, that is awesome! It is takin...  2100  \n",
       "894                                                       2895  \n",
       "1512                                                      2437  \n",
       "2257                                                      1983  \n",
       "1021                                                      2740  \n",
       "...                                                  ...   ...  \n",
       "8257                                                       154  \n",
       "8252   Hello Reddit,I just updated MinionInvaders, a ...   154  \n",
       "8250   Then Now Exactly 1 year apartEdit: Will Post r...   154  \n",
       "8245                                                       154  \n",
       "21559  When I decided to stop fapping over a year ago...     1  \n",
       "\n",
       "[15670 rows x 7 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged7 = merged[(merged['comment_body']).str.split().str.len().gt(10)]\n",
    "merged7 = merged7[(merged7['comment_body']).str.split().str.len().lt(201)]\n",
    "merged7 = merged7.sort_values(by=['score_com'], ascending=False)\n",
    "merged7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged7.transpose().to_json('data/comments_submission_ask_all.json', index = 'true') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. remove posts with title + selftext less than 10 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pics</td>\n",
       "      <td>7m28mc</td>\n",
       "      <td>It's these kind of posts that restore some of ...</td>\n",
       "      <td>1</td>\n",
       "      <td>She took me in 15 years ago and no one makes m...</td>\n",
       "      <td></td>\n",
       "      <td>171795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pics</td>\n",
       "      <td>6f0lr7</td>\n",
       "      <td>I understand trying to uplift people with the ...</td>\n",
       "      <td>1</td>\n",
       "      <td>You can destroy a country. But you can't destr...</td>\n",
       "      <td></td>\n",
       "      <td>129553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pics</td>\n",
       "      <td>808ddp</td>\n",
       "      <td>Hope you put this on r/uplift.</td>\n",
       "      <td>1</td>\n",
       "      <td>Wife is a kindergarten teacher. A couple of th...</td>\n",
       "      <td></td>\n",
       "      <td>119527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>6po7fz</td>\n",
       "      <td>I wish we could do this on a global scale, upl...</td>\n",
       "      <td>2</td>\n",
       "      <td>TIL when a millionaire gave everyone in a Flor...</td>\n",
       "      <td></td>\n",
       "      <td>117421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>8pks1u</td>\n",
       "      <td>The sympathy in this thread is comforting.The ...</td>\n",
       "      <td>3</td>\n",
       "      <td>Suicide Prevention Megathread</td>\n",
       "      <td>With the news today of the passing of the amaz...</td>\n",
       "      <td>104195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20479</th>\n",
       "      <td>DnD</td>\n",
       "      <td>7lc6f8</td>\n",
       "      <td>Using strictly PHB, whichever class can buy a ...</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5 What's the optimal and sub optimal stuff i...</td>\n",
       "      <td>I played 3.5 for years but honestly wasn't act...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20481</th>\n",
       "      <td>relationships</td>\n",
       "      <td>6wpt41</td>\n",
       "      <td>My parents divorced when I was very young and ...</td>\n",
       "      <td>2</td>\n",
       "      <td>My [24F] mother [51F] is addicted to gaming an...</td>\n",
       "      <td>**Background:**\\n\\nMy mother was successful ac...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20482</th>\n",
       "      <td>Drugs</td>\n",
       "      <td>27x3cd</td>\n",
       "      <td>It helped me insofar as it better allowed me t...</td>\n",
       "      <td>4</td>\n",
       "      <td>Using LSD or Psilocybin Mushrooms to find out ...</td>\n",
       "      <td>I'm an incoming senior to high school.\\n\\nI sm...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20483</th>\n",
       "      <td>offmychest</td>\n",
       "      <td>91nzgb</td>\n",
       "      <td>I can’t even begin to imagine how you feel. On...</td>\n",
       "      <td>2</td>\n",
       "      <td>my dad hates me</td>\n",
       "      <td>my mom called me to come over because she made...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20486</th>\n",
       "      <td>AskMen</td>\n",
       "      <td>8uuymy</td>\n",
       "      <td>In established sci fi terms this is called \"u...</td>\n",
       "      <td>10</td>\n",
       "      <td>If humans get the capability to create brains ...</td>\n",
       "      <td>Or should we not give the rational mind to any...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12390 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           subreddit      id  \\\n",
       "1               pics  7m28mc   \n",
       "3               pics  6f0lr7   \n",
       "4               pics  808ddp   \n",
       "5      todayilearned  6po7fz   \n",
       "7          AskReddit  8pks1u   \n",
       "...              ...     ...   \n",
       "20479            DnD  7lc6f8   \n",
       "20481  relationships  6wpt41   \n",
       "20482          Drugs  27x3cd   \n",
       "20483     offmychest  91nzgb   \n",
       "20486         AskMen  8uuymy   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "1      It's these kind of posts that restore some of ...         1   \n",
       "3      I understand trying to uplift people with the ...         1   \n",
       "4                         Hope you put this on r/uplift.         1   \n",
       "5      I wish we could do this on a global scale, upl...         2   \n",
       "7      The sympathy in this thread is comforting.The ...         3   \n",
       "...                                                  ...       ...   \n",
       "20479  Using strictly PHB, whichever class can buy a ...         2   \n",
       "20481  My parents divorced when I was very young and ...         2   \n",
       "20482  It helped me insofar as it better allowed me t...         4   \n",
       "20483  I can’t even begin to imagine how you feel. On...         2   \n",
       "20486   In established sci fi terms this is called \"u...        10   \n",
       "\n",
       "                                                   title  \\\n",
       "1      She took me in 15 years ago and no one makes m...   \n",
       "3      You can destroy a country. But you can't destr...   \n",
       "4      Wife is a kindergarten teacher. A couple of th...   \n",
       "5      TIL when a millionaire gave everyone in a Flor...   \n",
       "7                          Suicide Prevention Megathread   \n",
       "...                                                  ...   \n",
       "20479  3.5 What's the optimal and sub optimal stuff i...   \n",
       "20481  My [24F] mother [51F] is addicted to gaming an...   \n",
       "20482  Using LSD or Psilocybin Mushrooms to find out ...   \n",
       "20483                                    my dad hates me   \n",
       "20486  If humans get the capability to create brains ...   \n",
       "\n",
       "                                                selftext   score  \n",
       "1                                                         171795  \n",
       "3                                                         129553  \n",
       "4                                                         119527  \n",
       "5                                                         117421  \n",
       "7      With the news today of the passing of the amaz...  104195  \n",
       "...                                                  ...     ...  \n",
       "20479  I played 3.5 for years but honestly wasn't act...       1  \n",
       "20481  **Background:**\\n\\nMy mother was successful ac...       1  \n",
       "20482  I'm an incoming senior to high school.\\n\\nI sm...       1  \n",
       "20483  my mom called me to come over because she made...       1  \n",
       "20486  Or should we not give the rational mind to any...       1  \n",
       "\n",
       "[12390 rows x 7 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged2 = merged[(merged['title'] + merged['selftext']).str.split().str.len().gt(20)]\n",
    "merged2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4. remove posts with title + selftext greater than 100 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pics</td>\n",
       "      <td>7m28mc</td>\n",
       "      <td>It's these kind of posts that restore some of ...</td>\n",
       "      <td>1</td>\n",
       "      <td>She took me in 15 years ago and no one makes m...</td>\n",
       "      <td></td>\n",
       "      <td>171795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pics</td>\n",
       "      <td>6f0lr7</td>\n",
       "      <td>I understand trying to uplift people with the ...</td>\n",
       "      <td>1</td>\n",
       "      <td>You can destroy a country. But you can't destr...</td>\n",
       "      <td></td>\n",
       "      <td>129553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pics</td>\n",
       "      <td>808ddp</td>\n",
       "      <td>Hope you put this on r/uplift.</td>\n",
       "      <td>1</td>\n",
       "      <td>Wife is a kindergarten teacher. A couple of th...</td>\n",
       "      <td></td>\n",
       "      <td>119527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>6po7fz</td>\n",
       "      <td>I wish we could do this on a global scale, upl...</td>\n",
       "      <td>2</td>\n",
       "      <td>TIL when a millionaire gave everyone in a Flor...</td>\n",
       "      <td></td>\n",
       "      <td>117421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>8pks1u</td>\n",
       "      <td>The sympathy in this thread is comforting.The ...</td>\n",
       "      <td>3</td>\n",
       "      <td>Suicide Prevention Megathread</td>\n",
       "      <td>With the news today of the passing of the amaz...</td>\n",
       "      <td>104195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20476</th>\n",
       "      <td>Com320_ASU_Spring2014</td>\n",
       "      <td>24k2fv</td>\n",
       "      <td>I think that the world presented in Idiocracy ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Last Extra Credit Discussion Board</td>\n",
       "      <td>Class - if you want to earn a few more (maximu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20478</th>\n",
       "      <td>adventism</td>\n",
       "      <td>3i58sa</td>\n",
       "      <td>Does the music uplift and draw one to a higher...</td>\n",
       "      <td>1</td>\n",
       "      <td>Genuine Question about Music</td>\n",
       "      <td>For some backstory, I am not an adventist. I a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20479</th>\n",
       "      <td>DnD</td>\n",
       "      <td>7lc6f8</td>\n",
       "      <td>Using strictly PHB, whichever class can buy a ...</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5 What's the optimal and sub optimal stuff i...</td>\n",
       "      <td>I played 3.5 for years but honestly wasn't act...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20483</th>\n",
       "      <td>offmychest</td>\n",
       "      <td>91nzgb</td>\n",
       "      <td>I can’t even begin to imagine how you feel. On...</td>\n",
       "      <td>2</td>\n",
       "      <td>my dad hates me</td>\n",
       "      <td>my mom called me to come over because she made...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20486</th>\n",
       "      <td>AskMen</td>\n",
       "      <td>8uuymy</td>\n",
       "      <td>In established sci fi terms this is called \"u...</td>\n",
       "      <td>10</td>\n",
       "      <td>If humans get the capability to create brains ...</td>\n",
       "      <td>Or should we not give the rational mind to any...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9112 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   subreddit      id  \\\n",
       "1                       pics  7m28mc   \n",
       "3                       pics  6f0lr7   \n",
       "4                       pics  808ddp   \n",
       "5              todayilearned  6po7fz   \n",
       "7                  AskReddit  8pks1u   \n",
       "...                      ...     ...   \n",
       "20476  Com320_ASU_Spring2014  24k2fv   \n",
       "20478              adventism  3i58sa   \n",
       "20479                    DnD  7lc6f8   \n",
       "20483             offmychest  91nzgb   \n",
       "20486                 AskMen  8uuymy   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "1      It's these kind of posts that restore some of ...         1   \n",
       "3      I understand trying to uplift people with the ...         1   \n",
       "4                         Hope you put this on r/uplift.         1   \n",
       "5      I wish we could do this on a global scale, upl...         2   \n",
       "7      The sympathy in this thread is comforting.The ...         3   \n",
       "...                                                  ...       ...   \n",
       "20476  I think that the world presented in Idiocracy ...         1   \n",
       "20478  Does the music uplift and draw one to a higher...         1   \n",
       "20479  Using strictly PHB, whichever class can buy a ...         2   \n",
       "20483  I can’t even begin to imagine how you feel. On...         2   \n",
       "20486   In established sci fi terms this is called \"u...        10   \n",
       "\n",
       "                                                   title  \\\n",
       "1      She took me in 15 years ago and no one makes m...   \n",
       "3      You can destroy a country. But you can't destr...   \n",
       "4      Wife is a kindergarten teacher. A couple of th...   \n",
       "5      TIL when a millionaire gave everyone in a Flor...   \n",
       "7                          Suicide Prevention Megathread   \n",
       "...                                                  ...   \n",
       "20476                 Last Extra Credit Discussion Board   \n",
       "20478                       Genuine Question about Music   \n",
       "20479  3.5 What's the optimal and sub optimal stuff i...   \n",
       "20483                                    my dad hates me   \n",
       "20486  If humans get the capability to create brains ...   \n",
       "\n",
       "                                                selftext   score  \n",
       "1                                                         171795  \n",
       "3                                                         129553  \n",
       "4                                                         119527  \n",
       "5                                                         117421  \n",
       "7      With the news today of the passing of the amaz...  104195  \n",
       "...                                                  ...     ...  \n",
       "20476  Class - if you want to earn a few more (maximu...       1  \n",
       "20478  For some backstory, I am not an adventist. I a...       1  \n",
       "20479  I played 3.5 for years but honestly wasn't act...       1  \n",
       "20483  my mom called me to come over because she made...       1  \n",
       "20486  Or should we not give the rational mind to any...       1  \n",
       "\n",
       "[9112 rows x 7 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged3 = merged2[(merged2['title'] + merged2['selftext']).str.split().str.len().lt(201)]\n",
    "merged3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove posts from certain subreddits: pics, videos, Music .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>6po7fz</td>\n",
       "      <td>I wish we could do this on a global scale, upl...</td>\n",
       "      <td>2</td>\n",
       "      <td>TIL when a millionaire gave everyone in a Flor...</td>\n",
       "      <td></td>\n",
       "      <td>117421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>8pks1u</td>\n",
       "      <td>The sympathy in this thread is comforting.The ...</td>\n",
       "      <td>3</td>\n",
       "      <td>Suicide Prevention Megathread</td>\n",
       "      <td>With the news today of the passing of the amaz...</td>\n",
       "      <td>104195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>80ow6w</td>\n",
       "      <td>Hello Bill! First thank you and your wife Meli...</td>\n",
       "      <td>1</td>\n",
       "      <td>I’m Bill Gates, co-chair of the Bill &amp;amp; Mel...</td>\n",
       "      <td>I’m excited to be back for my sixth AMA.Here’s...</td>\n",
       "      <td>94575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>8bwtmt</td>\n",
       "      <td>I'd been feeling bloated all morning and, as m...</td>\n",
       "      <td>59</td>\n",
       "      <td>TIL on the set of The Princess Bride, André th...</td>\n",
       "      <td></td>\n",
       "      <td>91310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Showerthoughts</td>\n",
       "      <td>95m643</td>\n",
       "      <td>Reincarnation is real, Exhibit A : I think I'l...</td>\n",
       "      <td>1</td>\n",
       "      <td>If reincarnation is real, then maybe flies and...</td>\n",
       "      <td></td>\n",
       "      <td>74457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20476</th>\n",
       "      <td>Com320_ASU_Spring2014</td>\n",
       "      <td>24k2fv</td>\n",
       "      <td>I think that the world presented in Idiocracy ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Last Extra Credit Discussion Board</td>\n",
       "      <td>Class - if you want to earn a few more (maximu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20478</th>\n",
       "      <td>adventism</td>\n",
       "      <td>3i58sa</td>\n",
       "      <td>Does the music uplift and draw one to a higher...</td>\n",
       "      <td>1</td>\n",
       "      <td>Genuine Question about Music</td>\n",
       "      <td>For some backstory, I am not an adventist. I a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20479</th>\n",
       "      <td>DnD</td>\n",
       "      <td>7lc6f8</td>\n",
       "      <td>Using strictly PHB, whichever class can buy a ...</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5 What's the optimal and sub optimal stuff i...</td>\n",
       "      <td>I played 3.5 for years but honestly wasn't act...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20483</th>\n",
       "      <td>offmychest</td>\n",
       "      <td>91nzgb</td>\n",
       "      <td>I can’t even begin to imagine how you feel. On...</td>\n",
       "      <td>2</td>\n",
       "      <td>my dad hates me</td>\n",
       "      <td>my mom called me to come over because she made...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20486</th>\n",
       "      <td>AskMen</td>\n",
       "      <td>8uuymy</td>\n",
       "      <td>In established sci fi terms this is called \"u...</td>\n",
       "      <td>10</td>\n",
       "      <td>If humans get the capability to create brains ...</td>\n",
       "      <td>Or should we not give the rational mind to any...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9019 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   subreddit      id  \\\n",
       "5              todayilearned  6po7fz   \n",
       "7                  AskReddit  8pks1u   \n",
       "12                      IAmA  80ow6w   \n",
       "15             todayilearned  8bwtmt   \n",
       "30            Showerthoughts  95m643   \n",
       "...                      ...     ...   \n",
       "20476  Com320_ASU_Spring2014  24k2fv   \n",
       "20478              adventism  3i58sa   \n",
       "20479                    DnD  7lc6f8   \n",
       "20483             offmychest  91nzgb   \n",
       "20486                 AskMen  8uuymy   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "5      I wish we could do this on a global scale, upl...         2   \n",
       "7      The sympathy in this thread is comforting.The ...         3   \n",
       "12     Hello Bill! First thank you and your wife Meli...         1   \n",
       "15     I'd been feeling bloated all morning and, as m...        59   \n",
       "30     Reincarnation is real, Exhibit A : I think I'l...         1   \n",
       "...                                                  ...       ...   \n",
       "20476  I think that the world presented in Idiocracy ...         1   \n",
       "20478  Does the music uplift and draw one to a higher...         1   \n",
       "20479  Using strictly PHB, whichever class can buy a ...         2   \n",
       "20483  I can’t even begin to imagine how you feel. On...         2   \n",
       "20486   In established sci fi terms this is called \"u...        10   \n",
       "\n",
       "                                                   title  \\\n",
       "5      TIL when a millionaire gave everyone in a Flor...   \n",
       "7                          Suicide Prevention Megathread   \n",
       "12     I’m Bill Gates, co-chair of the Bill &amp; Mel...   \n",
       "15     TIL on the set of The Princess Bride, André th...   \n",
       "30     If reincarnation is real, then maybe flies and...   \n",
       "...                                                  ...   \n",
       "20476                 Last Extra Credit Discussion Board   \n",
       "20478                       Genuine Question about Music   \n",
       "20479  3.5 What's the optimal and sub optimal stuff i...   \n",
       "20483                                    my dad hates me   \n",
       "20486  If humans get the capability to create brains ...   \n",
       "\n",
       "                                                selftext   score  \n",
       "5                                                         117421  \n",
       "7      With the news today of the passing of the amaz...  104195  \n",
       "12     I’m excited to be back for my sixth AMA.Here’s...   94575  \n",
       "15                                                         91310  \n",
       "30                                                         74457  \n",
       "...                                                  ...     ...  \n",
       "20476  Class - if you want to earn a few more (maximu...       1  \n",
       "20478  For some backstory, I am not an adventist. I a...       1  \n",
       "20479  I played 3.5 for years but honestly wasn't act...       1  \n",
       "20483  my mom called me to come over because she made...       1  \n",
       "20486  Or should we not give the rational mind to any...       1  \n",
       "\n",
       "[9019 rows x 7 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_subreddits_to_remove = [\"pics\", \"movies\", \"Music\", \"videos\", \"trapmuzik\"]\n",
    "\n",
    "merged4 = merged3[~merged3.subreddit.isin(list_subreddits_to_remove)]\n",
    "\n",
    "merged5 = merged3[merged3.subreddit.isin(list_subreddits_to_remove)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged5.transpose().to_json('data/comments_submission_pics_movies_music_videos.json', index = 'true') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove Quotes , titles, ads .."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "subreddit: IAmA, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>16dsdl</td>\n",
       "      <td>Man, this has inspired me to create a new camp...</td>\n",
       "      <td>1</td>\n",
       "      <td>TIL that after needing 13 liters of blood for ...</td>\n",
       "      <td></td>\n",
       "      <td>8195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>todayilearned</td>\n",
       "      <td>16dsdl</td>\n",
       "      <td>this has inspired me to donate blood.</td>\n",
       "      <td>1</td>\n",
       "      <td>TIL that after needing 13 liters of blood for ...</td>\n",
       "      <td></td>\n",
       "      <td>8195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>gaming</td>\n",
       "      <td>17iw9q</td>\n",
       "      <td>Thanks for posting - I've shared it with my so...</td>\n",
       "      <td>2</td>\n",
       "      <td>So we built Kings Landing in Minecraft. Heres ...</td>\n",
       "      <td></td>\n",
       "      <td>6241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>You have done so much with your life, I'm 16, ...</td>\n",
       "      <td>18</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>What inspired you to run for governor?</td>\n",
       "      <td>4</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14213</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>17zly4</td>\n",
       "      <td>Being that Reddit is the source for original (...</td>\n",
       "      <td>1</td>\n",
       "      <td>For you creative writers: Any hints or product...</td>\n",
       "      <td>I consider myself an aspiring writer who is ac...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14215</th>\n",
       "      <td>trance</td>\n",
       "      <td>17gvzy</td>\n",
       "      <td>It's mostly just uplifting trance in a weird m...</td>\n",
       "      <td>3</td>\n",
       "      <td>Trance with an eerie vibe?</td>\n",
       "      <td>I want some more of this style of trance. Some...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14216</th>\n",
       "      <td>trance</td>\n",
       "      <td>17gvzy</td>\n",
       "      <td>It's mostly just uplifting trance in a weird m...</td>\n",
       "      <td>3</td>\n",
       "      <td>Trance with an eerie vibe?</td>\n",
       "      <td>I want some more of this style of trance. Some...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14219</th>\n",
       "      <td>explainlikeimfive</td>\n",
       "      <td>183auv</td>\n",
       "      <td>It's largely a myth, started by a book called ...</td>\n",
       "      <td>3</td>\n",
       "      <td>Please explain nanotechnology to me.</td>\n",
       "      <td>I have recently been researching about nanotec...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14224</th>\n",
       "      <td>mylittlepony</td>\n",
       "      <td>177q50</td>\n",
       "      <td>Hm... not sure about specific ponys but if you...</td>\n",
       "      <td>1</td>\n",
       "      <td>I've always had this random idea going off in ...</td>\n",
       "      <td>Whenever I listen to the more intense songs th...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5417 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               subreddit      id  \\\n",
       "128        todayilearned  16dsdl   \n",
       "129        todayilearned  16dsdl   \n",
       "134               gaming  17iw9q   \n",
       "135                 IAmA  16mq0g   \n",
       "136                 IAmA  16mq0g   \n",
       "...                  ...     ...   \n",
       "14213          AskReddit  17zly4   \n",
       "14215             trance  17gvzy   \n",
       "14216             trance  17gvzy   \n",
       "14219  explainlikeimfive  183auv   \n",
       "14224       mylittlepony  177q50   \n",
       "\n",
       "                                            comment_body score_com  \\\n",
       "128    Man, this has inspired me to create a new camp...         1   \n",
       "129                this has inspired me to donate blood.         1   \n",
       "134    Thanks for posting - I've shared it with my so...         2   \n",
       "135    You have done so much with your life, I'm 16, ...        18   \n",
       "136               What inspired you to run for governor?         4   \n",
       "...                                                  ...       ...   \n",
       "14213  Being that Reddit is the source for original (...         1   \n",
       "14215  It's mostly just uplifting trance in a weird m...         3   \n",
       "14216  It's mostly just uplifting trance in a weird m...         3   \n",
       "14219  It's largely a myth, started by a book called ...         3   \n",
       "14224  Hm... not sure about specific ponys but if you...         1   \n",
       "\n",
       "                                                   title  \\\n",
       "128    TIL that after needing 13 liters of blood for ...   \n",
       "129    TIL that after needing 13 liters of blood for ...   \n",
       "134    So we built Kings Landing in Minecraft. Heres ...   \n",
       "135                        IAmArnold... Ask me anything.   \n",
       "136                        IAmArnold... Ask me anything.   \n",
       "...                                                  ...   \n",
       "14213  For you creative writers: Any hints or product...   \n",
       "14215                         Trance with an eerie vibe?   \n",
       "14216                         Trance with an eerie vibe?   \n",
       "14219               Please explain nanotechnology to me.   \n",
       "14224  I've always had this random idea going off in ...   \n",
       "\n",
       "                                                selftext score  \n",
       "128                                                       8195  \n",
       "129                                                       8195  \n",
       "134                                                       6241  \n",
       "135    Former Mr. Olympia, Conan, Terminator, and Gov...  5651  \n",
       "136    Former Mr. Olympia, Conan, Terminator, and Gov...  5651  \n",
       "...                                                  ...   ...  \n",
       "14213  I consider myself an aspiring writer who is ac...     1  \n",
       "14215  I want some more of this style of trance. Some...     1  \n",
       "14216  I want some more of this style of trance. Some...     1  \n",
       "14219  I have recently been researching about nanotec...     1  \n",
       "14224  Whenever I listen to the more intense songs th...     1  \n",
       "\n",
       "[5417 rows x 7 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"IAmA\"\n",
    "merged4 = merged4[~merged4.subreddit.str.contains(query)]\n",
    "merged4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged4.transpose().to_json('data/comments_submission_all_round3.json', index = 'true') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interesting findings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. \"What fictional person has had the biggest impact on your life and how?\" title on AskReddit - has many good inspiring comments\n",
    "\n",
    "2. subreddits: pics, movies, videos have comments with inspiring movies "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis of topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('AskReddit', 876),\n",
       " ('IAmA', 328),\n",
       " ('todayilearned', 132),\n",
       " ('gaming', 94),\n",
       " ('atheism', 76),\n",
       " ('funny', 60),\n",
       " ('Random_Acts_Of_Amazon', 54),\n",
       " ('RedditLaqueristas', 51),\n",
       " ('progresspics', 49),\n",
       " ('trees', 47),\n",
       " ('WTF', 46),\n",
       " ('Christianity', 44),\n",
       " ('loseit', 40),\n",
       " ('keto', 37),\n",
       " ('politics', 36),\n",
       " ('leagueoflegends', 35),\n",
       " ('malefashionadvice', 35),\n",
       " ('DIY', 34),\n",
       " ('Games', 34),\n",
       " ('Minecraft', 31),\n",
       " ('GetMotivated', 28),\n",
       " ('writing', 28),\n",
       " ('books', 27),\n",
       " ('Guitar', 25),\n",
       " ('gamedev', 25),\n",
       " ('SketchDaily', 25),\n",
       " ('nfl', 24),\n",
       " ('MakeupAddiction', 24),\n",
       " ('comicbooks', 24),\n",
       " ('rpg', 24),\n",
       " ('AskHistorians', 24),\n",
       " ('aww', 23),\n",
       " ('tattoos', 22),\n",
       " ('NoFap', 22),\n",
       " ('exmormon', 22),\n",
       " ('Metal', 22),\n",
       " ('nba', 21),\n",
       " ('Fitness', 20),\n",
       " ('mylittlepony', 20),\n",
       " ('running', 20),\n",
       " ('SquaredCircle', 20),\n",
       " ('booksuggestions', 20),\n",
       " ('space', 19),\n",
       " ('CFB', 19),\n",
       " ('TwoXChromosomes', 19),\n",
       " ('MensRights', 19),\n",
       " ('AskWomen', 19),\n",
       " ('MLPLounge', 19),\n",
       " ('DebateReligion', 18),\n",
       " ('gonewild', 17),\n",
       " ('masseffect', 17),\n",
       " ('Psychonaut', 17),\n",
       " ('femalefashionadvice', 17),\n",
       " ('AskMen', 17),\n",
       " ('travel', 16),\n",
       " ('Art', 16),\n",
       " ('photography', 16),\n",
       " ('science', 15),\n",
       " ('pokemon', 15),\n",
       " ('hockey', 15),\n",
       " ('DotA2', 14),\n",
       " ('motorcycles', 14),\n",
       " ('india', 14),\n",
       " ('depression', 14),\n",
       " ('WeAreTheMusicMakers', 14),\n",
       " ('gaybros', 14),\n",
       " ('RandomActsOfPolish', 14),\n",
       " ('worldnews', 13),\n",
       " ('AdviceAnimals', 13),\n",
       " ('food', 13),\n",
       " ('NetflixBestOf', 13),\n",
       " ('truegaming', 13),\n",
       " ('knitting', 13),\n",
       " ('Fantasy', 13),\n",
       " ('DnD', 13),\n",
       " ('anime', 13),\n",
       " ('POLITIC', 13),\n",
       " ('technology', 12),\n",
       " ('starcraft', 12),\n",
       " ('UpliftingNews', 12),\n",
       " ('soccer', 12),\n",
       " ('lotr', 12),\n",
       " ('doctorwho', 12),\n",
       " ('Documentaries', 12),\n",
       " ('magicTCG', 12),\n",
       " ('bicycling', 12),\n",
       " ('IWantToLearn', 12),\n",
       " ('worldbuilding', 12),\n",
       " ('Drugs', 12),\n",
       " ('TrueFilm', 12),\n",
       " ('sex', 11),\n",
       " ('conspiracy', 11),\n",
       " ('explainlikeimfive', 11),\n",
       " ('vegan', 11),\n",
       " ('OkCupid', 11),\n",
       " ('edmproduction', 11),\n",
       " ('teenagers', 11),\n",
       " ('skyrim', 10),\n",
       " ('hiphopheads', 10),\n",
       " ('cars', 10),\n",
       " ('vinyl', 10),\n",
       " ('darksouls', 10),\n",
       " ('beards', 10),\n",
       " ('classicalmusic', 10),\n",
       " ('mindcrack', 10),\n",
       " ('stopdrinking', 10),\n",
       " ('ModerationLog', 10),\n",
       " ('KerbalSpaceProgram', 9),\n",
       " ('halo', 9),\n",
       " ('Homebrewing', 9),\n",
       " ('philosophy', 9),\n",
       " ('trance', 9),\n",
       " ('Guildwars2', 9),\n",
       " ('ADHD', 9),\n",
       " ('startrek', 8),\n",
       " ('MorbidReality', 8),\n",
       " ('circlejerk', 8),\n",
       " ('Paleo', 8),\n",
       " ('BabyBumps', 8),\n",
       " ('Filmmakers', 8),\n",
       " ('Poetry', 8),\n",
       " ('ShitRedditSays', 8),\n",
       " ('entwives', 8),\n",
       " ('Meditation', 8),\n",
       " ('xxfitness', 8),\n",
       " ('asoiaf', 8),\n",
       " ('postrock', 8),\n",
       " ('FoodPorn', 7),\n",
       " ('Android', 7),\n",
       " ('FancyFollicles', 7),\n",
       " ('news', 7),\n",
       " ('lgbt', 7),\n",
       " ('bestof', 7),\n",
       " ('cringe', 7),\n",
       " ('wow', 7),\n",
       " ('Warhammer', 7),\n",
       " ('yoga', 7),\n",
       " ('Design', 7),\n",
       " ('Military', 7),\n",
       " ('Buddhism', 7),\n",
       " ('homestuck', 7),\n",
       " ('boardgames', 7),\n",
       " ('Libertarian', 7),\n",
       " ('climbing', 7),\n",
       " ('howtonotgiveafuck', 7),\n",
       " ('australia', 7),\n",
       " ('electronicmusic', 7),\n",
       " ('LizzieBennet', 7),\n",
       " ('casualiama', 7),\n",
       " ('stopsmoking', 7),\n",
       " ('radiohead', 7),\n",
       " ('exjw', 7),\n",
       " ('tipofmytongue', 7),\n",
       " ('Catholicism', 7),\n",
       " ('vexillology', 7),\n",
       " ('fantasywriters', 7),\n",
       " ('cringepics', 6),\n",
       " ('TrueReddit', 6),\n",
       " ('baseball', 6),\n",
       " ('community', 6),\n",
       " ('Fallout', 6),\n",
       " ('guns', 6),\n",
       " ('offmychest', 6),\n",
       " ('seduction', 6),\n",
       " ('Patriots', 6),\n",
       " ('harrypotter', 6),\n",
       " ('MvC3', 6),\n",
       " ('transgender', 6),\n",
       " ('Bass', 6),\n",
       " ('redditgetsdrawn', 6),\n",
       " ('IndieGaming', 6),\n",
       " ('frugalmalefashion', 6),\n",
       " ('drums', 6),\n",
       " ('television', 6),\n",
       " ('printSF', 6),\n",
       " ('gameofthrones', 6),\n",
       " ('dwarffortress', 5),\n",
       " ('britishproblems', 5),\n",
       " ('canada', 5),\n",
       " ('AMA', 5),\n",
       " ('socialskills', 5),\n",
       " ('C25K', 5),\n",
       " ('programming', 5),\n",
       " ('CampingandHiking', 5),\n",
       " ('lego', 5),\n",
       " ('totalwar', 5),\n",
       " ('steampunk', 5),\n",
       " ('circlebroke', 5),\n",
       " ('crochet', 5),\n",
       " ('Cooking', 5),\n",
       " ('unitedkingdom', 5),\n",
       " ('GiftofGames', 5),\n",
       " ('Teachers', 5),\n",
       " ('graphic_design', 5),\n",
       " ('startups', 5),\n",
       " ('AskScienceFiction', 5),\n",
       " ('Nerf', 5),\n",
       " ('listentothis', 5),\n",
       " ('gainit', 5),\n",
       " ('ShouldIbuythisgame', 5),\n",
       " ('Warhammer40k', 5),\n",
       " ('amiugly', 5),\n",
       " ('tarot', 5),\n",
       " ('silenthill', 5),\n",
       " ('socialism', 5),\n",
       " ('gamingsuggestions', 5),\n",
       " ('d3hardcore', 5),\n",
       " ('DebateAChristian', 5),\n",
       " ('DoesAnybodyElse', 4),\n",
       " ('RoomPorn', 4),\n",
       " ('gifs', 4),\n",
       " ('QuotesPorn', 4),\n",
       " ('geek', 4),\n",
       " ('happy', 4),\n",
       " ('zelda', 4),\n",
       " ('scifi', 4),\n",
       " ('linux', 4),\n",
       " ('snowboarding', 4),\n",
       " ('fashion', 4),\n",
       " ('RandomActsOfGaming', 4),\n",
       " ('minimalism', 4),\n",
       " ('JoeRogan', 4),\n",
       " ('tolkienfans', 4),\n",
       " ('knives', 4),\n",
       " ('web_design', 4),\n",
       " ('Frisson', 4),\n",
       " ('webdev', 4),\n",
       " ('nerdcubed', 4),\n",
       " ('civ', 4),\n",
       " ('explainlikeIAmA', 4),\n",
       " ('SubredditDrama', 4),\n",
       " ('homestead', 4),\n",
       " ('beatles', 4),\n",
       " ('AskSocialScience', 4),\n",
       " ('asmr', 4),\n",
       " ('MechanicalKeyboards', 4),\n",
       " ('Futurology', 4),\n",
       " ('Eve', 4),\n",
       " ('environment', 4),\n",
       " ('DnB', 4),\n",
       " ('Jeep', 4),\n",
       " ('animation', 4),\n",
       " ('Gunners', 4),\n",
       " ('Illustration', 4),\n",
       " ('hookah', 4),\n",
       " ('LongDistance', 4),\n",
       " ('kpop', 4),\n",
       " ('audioengineering', 4),\n",
       " ('NovelThisMonth', 4),\n",
       " ('pinkfloyd', 4),\n",
       " ('rush', 4),\n",
       " ('beer', 4),\n",
       " ('actuallesbians', 4),\n",
       " ('Parenting', 4),\n",
       " ('rawdenim', 4),\n",
       " ('literature', 4),\n",
       " ('singing', 4),\n",
       " ('Stoicism', 4),\n",
       " ('Songwriters', 4),\n",
       " ('ExploreReligion', 4),\n",
       " ('ifyoulikeblank', 4),\n",
       " ('islam', 4),\n",
       " ('skateboarding', 4),\n",
       " ('katawashoujo', 4),\n",
       " ('LadyBoners', 3),\n",
       " ('JusticePorn', 3),\n",
       " ('history', 3),\n",
       " ('Frugal', 3),\n",
       " ('adventuretime', 3),\n",
       " ('batman', 3),\n",
       " ('InternetIsBeautiful', 3),\n",
       " ('somethingimade', 3),\n",
       " ('OceanSkys', 3),\n",
       " ('gamegrumps', 3),\n",
       " ('Cyberpunk', 3),\n",
       " ('Coffee', 3),\n",
       " ('InteriorDesign', 3),\n",
       " ('eatsandwiches', 3),\n",
       " ('dayz', 3),\n",
       " ('Watches', 3),\n",
       " ('drawing', 3),\n",
       " ('ainbow', 3),\n",
       " ('Justrolledintotheshop', 3),\n",
       " ('self', 3),\n",
       " ('Planetside', 3),\n",
       " ('Bioshock', 3),\n",
       " ('HistoryPorn', 3),\n",
       " ('FinalFantasy', 3),\n",
       " ('EDC', 3),\n",
       " ('architecture', 3),\n",
       " ('Ubuntu', 3),\n",
       " ('theketodiet', 3),\n",
       " ('AmericanHorrorStory', 3),\n",
       " ('androidthemes', 3),\n",
       " ('typography', 3),\n",
       " ('wicked_edge', 3),\n",
       " ('scuba', 3),\n",
       " ('cats', 3),\n",
       " ('Pathfinder_RPG', 3),\n",
       " ('ketorecipes', 3),\n",
       " ('Breadit', 3),\n",
       " ('ireland', 3),\n",
       " ('coheedandcambria', 3),\n",
       " ('recipes', 3),\n",
       " ('TumblrInAction', 3),\n",
       " ('photocritique', 3),\n",
       " ('HomeImprovement', 3),\n",
       " ('LetsTalkMusic', 3),\n",
       " ('lost', 3),\n",
       " ('pathofexile', 3),\n",
       " ('DJs', 3),\n",
       " ('creepy', 3),\n",
       " ('bjj', 3),\n",
       " ('Xcom', 3),\n",
       " ('bodybuilding', 3),\n",
       " ('LiverpoolFC', 3),\n",
       " ('electrohouse', 3),\n",
       " ('LifeProTips', 3),\n",
       " ('normalnudes', 3),\n",
       " ('TheLastAirbender', 3),\n",
       " ('ToolBand', 3),\n",
       " ('LGBTeens', 3),\n",
       " ('DCcomics', 3),\n",
       " ('Negareddit', 3),\n",
       " ('feminisms', 3),\n",
       " ('Anarcho_Capitalism', 3),\n",
       " ('askscience', 3),\n",
       " ('DebateAnAtheist', 3),\n",
       " ('randomactsofkindness', 3),\n",
       " ('blackops2', 3),\n",
       " ('ouya', 3),\n",
       " ('Lovecraft', 3),\n",
       " ('Judaism', 3),\n",
       " ('StarWars', 3),\n",
       " ('Anarchism', 3),\n",
       " ('punk', 3),\n",
       " ('electronic_cigarette', 3),\n",
       " ('gallifrey', 3),\n",
       " ('horror', 3),\n",
       " ('blackladies', 3),\n",
       " ('Sidehugs', 3),\n",
       " ('EatCheapAndHealthy', 3),\n",
       " ('Screenwriting', 3),\n",
       " ('WoT', 3),\n",
       " ('composer', 3),\n",
       " ('makinghiphop', 3),\n",
       " ('linguistics', 3),\n",
       " ('52weeksofcooking', 3),\n",
       " ('TeenMFA', 3),\n",
       " ('Philippines', 3),\n",
       " ('ForeverAlone', 3),\n",
       " ('OnePiece', 3),\n",
       " ('learnart', 3),\n",
       " ('violinist', 3),\n",
       " ('hiphop101', 3),\n",
       " ('DebateCommunism', 3),\n",
       " ('Random_Acts_of_Etsy', 3),\n",
       " ('airsoft', 3),\n",
       " ('Songwriting', 3),\n",
       " ('Standup', 3),\n",
       " ('mormondebate', 3),\n",
       " ('mildlyinteresting', 2),\n",
       " ('firefly', 2),\n",
       " ('CrazyIdeas', 2),\n",
       " ('tf2', 2),\n",
       " ('battlefield3', 2),\n",
       " ('nostalgia', 2),\n",
       " ('gonewildcurvy', 2),\n",
       " ('sewing', 2),\n",
       " ('rage', 2),\n",
       " ('Pyongyang', 2),\n",
       " ('TheHobbit', 2),\n",
       " ('lifehacks', 2),\n",
       " ('Dexter', 2),\n",
       " ('getdisciplined', 2),\n",
       " ('China', 2),\n",
       " ('Foodforthought', 2),\n",
       " ('thesims', 2),\n",
       " ('DepthHub', 2),\n",
       " ('DesignPorn', 2),\n",
       " ('Anxiety', 2),\n",
       " ('quotes', 2),\n",
       " ('reddevils', 2),\n",
       " ('BeardPorn', 2),\n",
       " ('skiing', 2),\n",
       " ('sailing', 2),\n",
       " ('bodyweightfitness', 2),\n",
       " ('crafts', 2),\n",
       " ('chicago', 2),\n",
       " ('gamernews', 2),\n",
       " ('progressive', 2),\n",
       " ('london', 2),\n",
       " ('sports', 2),\n",
       " ('gamemusic', 2),\n",
       " ('ShittyLifeProTips', 2),\n",
       " ('Pottery', 2),\n",
       " ('whatisthisthing', 2),\n",
       " ('CHIBears', 2),\n",
       " ('Autos', 2),\n",
       " ('aves', 2),\n",
       " ('simpleliving', 2),\n",
       " ('bikesgonewild', 2),\n",
       " ('smashbros', 2),\n",
       " ('glitch_art', 2),\n",
       " ('entertainment', 2),\n",
       " ('AmISexy', 2),\n",
       " ('productivity', 2),\n",
       " ('SCP', 2),\n",
       " ('evolutionReddit', 2),\n",
       " ('navy', 2),\n",
       " ('StonerEngineering', 2),\n",
       " ('CanadaPolitics', 2),\n",
       " ('collapse', 2),\n",
       " ('ultimate', 2),\n",
       " ('gonewildaudio', 2),\n",
       " ('weddingplanning', 2),\n",
       " ('bmx', 2),\n",
       " ('analog', 2),\n",
       " ('shittyadvice', 2),\n",
       " ('beadsprites', 2),\n",
       " ('eldertrees', 2),\n",
       " ('Aquariums', 2),\n",
       " ('formula1', 2),\n",
       " ('bleach', 2),\n",
       " ('PropagandaPosters', 2),\n",
       " ('metalgearsolid', 2),\n",
       " ('farcry', 2),\n",
       " ('nottheonion', 2),\n",
       " ('PostHardcore', 2),\n",
       " ('windows', 2),\n",
       " ('Israel', 2),\n",
       " ('Rateme', 2),\n",
       " ('electroswing', 2),\n",
       " ('TrueAtheism', 2),\n",
       " ('gaymers', 2),\n",
       " ('europe', 2),\n",
       " ('pcgaming', 2),\n",
       " ('modelmakers', 2),\n",
       " ('battlestations', 2),\n",
       " ('Minecraft360', 2),\n",
       " ('coys', 2),\n",
       " ('iosgaming', 2),\n",
       " ('Seahawks', 2),\n",
       " ('speedrun', 2),\n",
       " ('Warmachine', 2),\n",
       " ('vita', 2),\n",
       " ('glassheads', 2),\n",
       " ('DenverBroncos', 2),\n",
       " ('MCPE', 2),\n",
       " ('standupshots', 2),\n",
       " ('Colts', 2),\n",
       " ('dubstep', 2),\n",
       " ('musictheory', 2),\n",
       " ('religion', 2),\n",
       " ('Permaculture', 2),\n",
       " ('Marvel', 2),\n",
       " ('bindingofisaac', 2),\n",
       " ('SexPositive', 2),\n",
       " ('manga', 2),\n",
       " ('secretsanta', 2),\n",
       " ('ArtCrit', 2),\n",
       " ('TrueAskReddit', 2),\n",
       " ('conspiratard', 2),\n",
       " ('TheDarkTower', 2),\n",
       " ('exchristian', 2),\n",
       " ('ravens', 2),\n",
       " ('furry', 2),\n",
       " ('toronto', 2),\n",
       " ('rct', 2),\n",
       " ('survivorzero', 2),\n",
       " ('roguelikes', 2),\n",
       " ('leaves', 2),\n",
       " ('SRSsucks', 2),\n",
       " ('StLouis', 2),\n",
       " ('MMA', 2),\n",
       " ('askseddit', 2),\n",
       " ('rollerderby', 2),\n",
       " ('Wicca', 2),\n",
       " ('ottawa', 2),\n",
       " ('gamecollecting', 2),\n",
       " ('RandomActsOfChristmas', 2),\n",
       " ('bigboobproblems', 2),\n",
       " ('baltimore', 2),\n",
       " ('Civcraft', 2),\n",
       " ('washingtondc', 2),\n",
       " ('Cricket', 2),\n",
       " ('BDSMcommunity', 2),\n",
       " ('wiiu', 2),\n",
       " ('RepublicOfMusic', 2),\n",
       " ('RedHotChiliPeppers', 2),\n",
       " ('Cyberpunk_Music', 2),\n",
       " ('IndoorGarden', 2),\n",
       " ('Dance', 2),\n",
       " ('Monstercat', 2),\n",
       " ('magicskyfairy', 2),\n",
       " ('backpacking', 2),\n",
       " ('infp', 2),\n",
       " ('veganrecipes', 2),\n",
       " ('intj', 2),\n",
       " ('otaku', 2),\n",
       " ('WritingPrompts', 2),\n",
       " ('MLPtunes', 2),\n",
       " ('classicalguitar', 2),\n",
       " ('SRSMicroaggressions', 2),\n",
       " ('firstimpression', 2),\n",
       " ('yugioh', 2),\n",
       " ('PowerMetal', 2),\n",
       " ('Catacombs', 2),\n",
       " ('hacking', 2),\n",
       " ('bakchodi', 2),\n",
       " ('metacanada', 2),\n",
       " ('PennStateUniversity', 2),\n",
       " ('poi', 2),\n",
       " ('medicine', 2),\n",
       " ('INTP', 2),\n",
       " ('bostoncollege', 2),\n",
       " ('Vocaloid', 2),\n",
       " ('Adelaide', 2),\n",
       " ('martialarts', 2),\n",
       " ('peloton', 2),\n",
       " ('breakingbad', 2),\n",
       " ('RWF', 2),\n",
       " ('thewalkingdead', 2),\n",
       " ('MLS', 2),\n",
       " ('DharmaStudy', 2),\n",
       " ('asktransgender', 2),\n",
       " ('DrawForMe', 2),\n",
       " ('hinduism', 2),\n",
       " ('occult', 2),\n",
       " ('latterdaysaints', 2),\n",
       " ('askphilosophy', 2),\n",
       " ('CannabisExtracts', 2),\n",
       " ('GuildWarsDyeJob', 2),\n",
       " ('Aleague', 2),\n",
       " ('atheistvids', 2),\n",
       " ('FCJbookclub', 2),\n",
       " ('Baking', 2),\n",
       " ('mbti', 2),\n",
       " ('shittyaskscience', 2),\n",
       " ('custommagic', 2),\n",
       " ('tron', 2),\n",
       " ('mormon', 2),\n",
       " ('golf', 2),\n",
       " ('stonerrock', 2),\n",
       " ('cosplay', 2),\n",
       " ('ECEProfessionals', 2),\n",
       " ('nuzlocke', 2),\n",
       " ('ABraThatFits', 2),\n",
       " ('poledancing', 2),\n",
       " ('Jazz', 2),\n",
       " ('RandomActsofMakeup', 2),\n",
       " ('wedding', 2),\n",
       " ('tennis', 2),\n",
       " ('DarkKnightDiscussion', 2),\n",
       " ('BooksAMA', 2),\n",
       " ('artstore', 2),\n",
       " ('shakespeare', 2),\n",
       " ('Dota2Trade', 2),\n",
       " ('Cypher', 2),\n",
       " ('whatstheword', 2),\n",
       " ('Etsy', 2),\n",
       " ('Project52', 2),\n",
       " ('piano', 2),\n",
       " ('AbandonedPorn', 2),\n",
       " ('headphones', 2),\n",
       " ('musicians', 2),\n",
       " ('Calgary', 2),\n",
       " ('fatpeoplestories', 1),\n",
       " ('reactiongifs', 1),\n",
       " ('onetruegod', 1),\n",
       " ('YouShouldKnow', 1),\n",
       " ('EarthPorn', 1),\n",
       " ('blog', 1),\n",
       " ('MURICA', 1),\n",
       " ('PandR', 1),\n",
       " ('jailbreak', 1),\n",
       " ('wallpaper', 1),\n",
       " ('LearnUselessTalents', 1),\n",
       " ('compsci', 1),\n",
       " ('Graffiti', 1),\n",
       " ('RedditThroughHistory', 1),\n",
       " ('GreenDawn', 1),\n",
       " ('calvinandhobbes', 1),\n",
       " ('SpecArt', 1),\n",
       " ('Firearms', 1),\n",
       " ('texas', 1),\n",
       " ('tardcat', 1),\n",
       " ('woodworking', 1),\n",
       " ('Economics', 1),\n",
       " ('ExposurePorn', 1),\n",
       " ('Freethought', 1),\n",
       " ('fffffffuuuuuuuuuuuu', 1),\n",
       " ('biology', 1),\n",
       " ('TheWire', 1),\n",
       " ('pittsburgh', 1),\n",
       " ('GWCouples', 1),\n",
       " ('hentai', 1),\n",
       " ('Diablo', 1),\n",
       " ('DestructionPorn', 1),\n",
       " ('shutupandtakemymoney', 1),\n",
       " ('Entrepreneur', 1),\n",
       " ('ToasterRights', 1),\n",
       " ('DunderMifflin', 1),\n",
       " ('Survival', 1),\n",
       " ('Borderlands', 1),\n",
       " ('gadgets', 1),\n",
       " ('woahdude', 1),\n",
       " ('FractalPorn', 1),\n",
       " ('CrusaderKings', 1),\n",
       " ('comics', 1),\n",
       " ('nintendo', 1),\n",
       " ('MovieSuggestions', 1),\n",
       " ('chemistry', 1),\n",
       " ('InfrastructurePorn', 1),\n",
       " ('HumanPorn', 1),\n",
       " ('defaultgems', 1),\n",
       " ('relationships', 1),\n",
       " ('Blacksmith', 1),\n",
       " ('PrettyGirlsUglyFaces', 1),\n",
       " ('longboarding', 1),\n",
       " ('shittyfoodporn', 1),\n",
       " ('ThriftStoreHauls', 1),\n",
       " ('tonightsdinner', 1),\n",
       " ('TopGear', 1),\n",
       " ('fantasyfootball', 1),\n",
       " ('slowcooking', 1),\n",
       " ('agnostic', 1),\n",
       " ('ImGoingToHellForThis', 1),\n",
       " ('Marijuana', 1),\n",
       " ('answers', 1),\n",
       " ('feet', 1),\n",
       " ('blender', 1),\n",
       " ('RedditDayOf', 1),\n",
       " ('AFOL', 1),\n",
       " ('ArcherFX', 1),\n",
       " ('CemeteryPorn', 1),\n",
       " ('business', 1),\n",
       " ('aviation', 1),\n",
       " ('Anthropology', 1),\n",
       " ('sanfrancisco', 1),\n",
       " ('gunpolitics', 1),\n",
       " ('languagelearning', 1),\n",
       " ('newzealand', 1),\n",
       " ('subaru', 1),\n",
       " ('Mommit', 1),\n",
       " ('Ska', 1),\n",
       " ('tall', 1),\n",
       " ('datfeel', 1),\n",
       " ('declutter', 1),\n",
       " ('hardbodies', 1),\n",
       " ('runescape', 1),\n",
       " ('bakedart', 1),\n",
       " ('California', 1),\n",
       " ('nexus4', 1),\n",
       " ('IndianFood', 1),\n",
       " ('Physics', 1),\n",
       " ('Dreadlocks', 1),\n",
       " ('49ers', 1),\n",
       " ('windowsphone', 1),\n",
       " ('xbox360', 1),\n",
       " ('dresdenfiles', 1),\n",
       " ('MilitaryPorn', 1),\n",
       " ('evedreddit', 1),\n",
       " ('Rabbits', 1),\n",
       " ('BodyAcceptance', 1),\n",
       " ('NASCAR', 1),\n",
       " ('StandUpComedy', 1),\n",
       " ('SkyDiving', 1),\n",
       " ('futurebeats', 1),\n",
       " ('Coldplay', 1),\n",
       " ('bayarea', 1),\n",
       " ('sad', 1),\n",
       " ('origami', 1),\n",
       " ('surfing', 1),\n",
       " ('Bluegrass', 1),\n",
       " ('energy', 1),\n",
       " ('cancer', 1),\n",
       " ('skylanders', 1),\n",
       " ('sushi', 1),\n",
       " ('TheFacebookDelusion', 1),\n",
       " ('hiking', 1),\n",
       " ('Cinemagraphs', 1),\n",
       " ('CombatFootage', 1),\n",
       " ('daddit', 1),\n",
       " ('deadpool', 1),\n",
       " ('Nexus7', 1),\n",
       " ('SRSMythos', 1),\n",
       " ('Hardcore', 1),\n",
       " ('HalfLife', 1),\n",
       " ('poppunkers', 1),\n",
       " ('EvilLeagueOfEvil', 1),\n",
       " ('shittyama', 1),\n",
       " ('deadmau5', 1),\n",
       " ('Boise', 1),\n",
       " ('investing', 1),\n",
       " ('berkeley', 1),\n",
       " ('craftit', 1),\n",
       " ('engineering', 1),\n",
       " ('swtor', 1),\n",
       " ('WarshipPorn', 1),\n",
       " ('quilting', 1),\n",
       " ('InTheStudio', 1),\n",
       " ('souleater', 1),\n",
       " ('SRSGaming', 1),\n",
       " ('upvotegifs', 1),\n",
       " ('snackexchange', 1),\n",
       " ('myevilplan', 1),\n",
       " ('solotravel', 1),\n",
       " ('shroomers', 1),\n",
       " ('Bowyer', 1),\n",
       " ('happycrowds', 1),\n",
       " ('AllThingsTerran', 1),\n",
       " ('HongKong', 1),\n",
       " ('malefoodadvice', 1),\n",
       " ('SanJoseSharks', 1),\n",
       " ('lectures', 1),\n",
       " ('nocontext_wallpapers', 1),\n",
       " ('stlouisblues', 1),\n",
       " ('carporn', 1),\n",
       " ('assassinscreed', 1),\n",
       " ('TF2fashionadvice', 1),\n",
       " ('ChivalryGame', 1),\n",
       " ('Archaeology', 1),\n",
       " ('childfree', 1),\n",
       " ('Macklemore', 1),\n",
       " ('ArtHistory', 1),\n",
       " ('FanTheories', 1),\n",
       " ('ScottPilgrim', 1),\n",
       " ('ShitCosmoSays', 1),\n",
       " ('lakers', 1),\n",
       " ('MetalMemes', 1),\n",
       " ('GameDeals', 1),\n",
       " ('camping', 1),\n",
       " ('Basketball', 1),\n",
       " ('EnoughPaulSpam', 1),\n",
       " ('weightroom', 1),\n",
       " ('Foofighters', 1),\n",
       " ('blues', 1),\n",
       " ('birdsofprey', 1),\n",
       " ('AntiJokes', 1),\n",
       " ('TribeTwelve', 1),\n",
       " ('SF4', 1),\n",
       " ('fairytail', 1),\n",
       " ('montageparodies', 1),\n",
       " ('darknetplan', 1),\n",
       " ('northernireland', 1),\n",
       " ('education', 1),\n",
       " ('falcons', 1),\n",
       " ('oblivion', 1),\n",
       " ('occupywallstreet', 1),\n",
       " ('arma', 1),\n",
       " ('FUI', 1),\n",
       " ('capybara', 1),\n",
       " ('icecreamery', 1),\n",
       " ('warehouse13', 1),\n",
       " ('Weird', 1),\n",
       " ('NewsPorn', 1),\n",
       " ('BuyItForLife', 1),\n",
       " ('indie', 1),\n",
       " ('PointsPlus', 1),\n",
       " ('video', 1),\n",
       " ('chelseafc', 1),\n",
       " ('Paranormal', 1),\n",
       " ('exmuslim', 1),\n",
       " ('C_Programming', 1),\n",
       " ('Atlanta', 1),\n",
       " ('AskCulinary', 1),\n",
       " ('Dallas', 1),\n",
       " ('purebattlefield', 1),\n",
       " ('mathrock', 1),\n",
       " ('Albany', 1),\n",
       " ('maryland', 1),\n",
       " ('slumberparty', 1),\n",
       " ('apologizeplease', 1),\n",
       " ('Dreamtheater', 1),\n",
       " ('chicagofood', 1),\n",
       " ('NeutralPolitics', 1),\n",
       " ('MusicNews', 1),\n",
       " ('uktrees', 1),\n",
       " ('iphone', 1),\n",
       " ('arizona', 1),\n",
       " ('LV426', 1),\n",
       " ('OnceUponATime', 1),\n",
       " ('Avengers', 1),\n",
       " ('fountainpens', 1),\n",
       " ('mfacirclejerk', 1),\n",
       " ('baseballcirclejerk', 1),\n",
       " ('stephenking', 1),\n",
       " ('xxketo', 1),\n",
       " ('MAME', 1),\n",
       " ('allthingszerg', 1),\n",
       " ('r4r', 1),\n",
       " ('algorithms', 1),\n",
       " ('chiptunes', 1),\n",
       " ('JapaneseFood', 1),\n",
       " ('moped', 1),\n",
       " ('crappymusic', 1),\n",
       " ('JRPG', 1),\n",
       " ('Vegetarianism', 1),\n",
       " ('AskScienceDiscussion', 1),\n",
       " ('korea', 1),\n",
       " ('bourbon', 1),\n",
       " ('GoneWildPlus', 1),\n",
       " ('FlashGames', 1),\n",
       " ('folkmetal', 1),\n",
       " ('wesanderson', 1),\n",
       " ('cmu', 1),\n",
       " ('nookcolor', 1),\n",
       " ('penpals', 1),\n",
       " ('minerapocalypse', 1),\n",
       " ('led_zeppelin', 1),\n",
       " ('greatdanes', 1),\n",
       " ('AsianParentStories', 1),\n",
       " ('peoplewithbirdheads', 1),\n",
       " ('MECoOp', 1),\n",
       " ('InsightfulQuestions', 1),\n",
       " ('graphicnovels', 1),\n",
       " ('trailrunning', 1),\n",
       " ('beagle', 1),\n",
       " ('microgrowery', 1),\n",
       " ('feminineboys', 1),\n",
       " ('zombies', 1),\n",
       " ('OneY', 1),\n",
       " ('BackYardChickens', 1),\n",
       " ('selfharm', 1),\n",
       " ('raw', 1),\n",
       " ('NorthCarolina', 1),\n",
       " ('LucidDreaming', 1),\n",
       " ('SRSImages', 1),\n",
       " ('dogs', 1),\n",
       " ('AppHookup', 1),\n",
       " ('weightwatchers', 1),\n",
       " ('rugbyunion', 1),\n",
       " ('3DS', 1),\n",
       " ('cpp', 1),\n",
       " ('DaftPunk', 1),\n",
       " ('hypotheticalsituation', 1),\n",
       " ('premed', 1),\n",
       " ('socialanxiety', 1),\n",
       " ('TrueAnime', 1),\n",
       " ('treemusic', 1),\n",
       " ('teaching', 1),\n",
       " ('ACMilan', 1),\n",
       " ('virginients', 1),\n",
       " ('netflix', 1),\n",
       " ('allcapsmotivation', 1),\n",
       " ('unixporn', 1),\n",
       " ('moviescirclejerk', 1),\n",
       " ('BALLET', 1),\n",
       " ('amateurradio', 1),\n",
       " ('ps3bf3', 1),\n",
       " ('fsu', 1),\n",
       " ('TrueTrueReddit', 1),\n",
       " ('Yogscast', 1),\n",
       " ('SegaSaturn', 1),\n",
       " ('Naruto', 1),\n",
       " ('Zappa', 1),\n",
       " ('nyc', 1),\n",
       " ('MLHFIS', 1),\n",
       " ('CircLoLjerk', 1),\n",
       " ('themountaingoats', 1),\n",
       " ('SF_Book_Club', 1),\n",
       " ('Fifa13', 1),\n",
       " ('bitcointip', 1),\n",
       " ('braces', 1),\n",
       " ('invaderzim', 1),\n",
       " ('audiojerk', 1),\n",
       " ('selfhelp', 1),\n",
       " ('perth', 1),\n",
       " ('diypedals', 1),\n",
       " ('StonerPhilosophy', 1),\n",
       " ('futurama', 1),\n",
       " ('FaithInHumanity', 1),\n",
       " ('EnterShikari', 1),\n",
       " ('volleyball', 1),\n",
       " ('Portlandia', 1),\n",
       " ('redditsync', 1),\n",
       " ('orlando', 1),\n",
       " ('Gunpla', 1),\n",
       " ('AnimeSketch', 1),\n",
       " ('JamesBond', 1),\n",
       " ('USC', 1),\n",
       " ('AndroidGaming', 1),\n",
       " ('sociology', 1),\n",
       " ('CourseraFilm', 1),\n",
       " ('neilgaiman', 1),\n",
       " ('ucf', 1),\n",
       " ('PoliticalDiscussion', 1),\n",
       " ('gravityfalls', 1),\n",
       " ('anonymous', 1),\n",
       " ('boats', 1),\n",
       " ('weightlifting', 1),\n",
       " ('4hourbodyslowcarb', 1),\n",
       " ('opiates', 1),\n",
       " ('futurebeatproducers', 1),\n",
       " ('wildhockey', 1),\n",
       " ('UIUC', 1),\n",
       " ('goth', 1),\n",
       " ('fastfood', 1),\n",
       " ('transhumanism', 1),\n",
       " ('whitewater', 1),\n",
       " ('TheGirlSurvivalGuide', 1),\n",
       " ('elianscript', 1),\n",
       " ('rockford', 1),\n",
       " ('nosleep', 1),\n",
       " ('blackpower', 1),\n",
       " ('edmprodcirclejerk', 1),\n",
       " ('gratefuldead', 1),\n",
       " ('trueMusic', 1),\n",
       " ('AcademicPsychology', 1),\n",
       " ('EDH', 1),\n",
       " ('Redditropolis', 1),\n",
       " ('fitnesscirclejerk', 1),\n",
       " ('cincinnati', 1),\n",
       " ('brighton', 1),\n",
       " ('grandrapids', 1),\n",
       " ('canadaguns', 1),\n",
       " ('elderscrollsonline', 1),\n",
       " ('Drag', 1),\n",
       " ('genderqueer', 1),\n",
       " ('CompanyOfHeroes', 1),\n",
       " ('DeadSpace', 1),\n",
       " ('wine', 1),\n",
       " ('patientgamers', 1),\n",
       " ('comingout', 1),\n",
       " ('christiananarchism', 1),\n",
       " ('realdubstep', 1),\n",
       " ('eagles', 1),\n",
       " ('NaturalPregnancy', 1),\n",
       " ('DebateaCommunist', 1),\n",
       " ('counterstrike', 1),\n",
       " ('TransSpace', 1),\n",
       " ('UFOs', 1),\n",
       " ('phish', 1),\n",
       " ('payitforward', 1),\n",
       " ('fireemblem', 1),\n",
       " ('wollongong', 1),\n",
       " ('ukulele', 1),\n",
       " ('polyamory', 1),\n",
       " ('OneGameAMonth', 1),\n",
       " ('psbattleroyale', 1),\n",
       " ('SRSDiscussion', 1),\n",
       " ('Mountaineering', 1),\n",
       " ('sonic', 1),\n",
       " ('tattoo', 1),\n",
       " ('Ultralight', 1),\n",
       " ('EntrepreneurRideAlong', 1),\n",
       " ('adventuretrees', 1),\n",
       " ('angularjs', 1),\n",
       " ('editors', 1),\n",
       " ('oculus', 1),\n",
       " ('ADHDbuddies', 1),\n",
       " ('LeagueofLegendsMeta', 1),\n",
       " ('ytmnd', 1),\n",
       " ('MTB', 1),\n",
       " ('asianamerican', 1),\n",
       " ('spikes', 1),\n",
       " ('arabs', 1),\n",
       " ('Tulpas', 1),\n",
       " ('DeviantArt', 1),\n",
       " ('philadelphia', 1),\n",
       " ('SocialEngineering', 1),\n",
       " ('ZBrush', 1),\n",
       " ('Shadowrun', 1),\n",
       " ('hockeygoalies', 1),\n",
       " ('Hiphopcirclejerk', 1),\n",
       " ('Sherlock_Fanfiction', 1),\n",
       " ('mcgill', 1),\n",
       " ('ldscirclejerk', 1),\n",
       " ('Fantasy_Bookclub', 1),\n",
       " ('cellular_automata', 1),\n",
       " ('BanjoKazooie', 1),\n",
       " ('HistoryofIdeas', 1),\n",
       " ('gamereviews', 1),\n",
       " ('scrapbooking', 1),\n",
       " ('Fighters', 1),\n",
       " ('trains', 1),\n",
       " ('avesNYC', 1),\n",
       " ('hardstyle', 1),\n",
       " ('javascript', 1),\n",
       " ('CysticFibrosis', 1),\n",
       " ('SpaceMusic', 1),\n",
       " ('CityPorn', 1),\n",
       " ('buffy', 1),\n",
       " ('nin', 1),\n",
       " ('CrossStitch', 1),\n",
       " ('cassetteculture', 1),\n",
       " ('SRSAnime', 1),\n",
       " ('CircleMusic', 1),\n",
       " ('largeformat', 1),\n",
       " ('summonerschool', 1),\n",
       " ('ambien', 1),\n",
       " ('CSUS', 1),\n",
       " ('Parkour', 1),\n",
       " ('LoLFanArt', 1),\n",
       " ('doodleordie', 1),\n",
       " ('hcfactions', 1),\n",
       " ('marketing', 1),\n",
       " ('FrenchHouse', 1),\n",
       " ('Scotch', 1),\n",
       " ...]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "all_subreddits = list(merged4[\"subreddit\"])\n",
    "Counter(all_subreddits).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12822</th>\n",
       "      <td>AskReddit</td>\n",
       "      <td>184vmk</td>\n",
       "      <td>Some pretty inspiring posts here!</td>\n",
       "      <td>1</td>\n",
       "      <td>Who is your hero and why?</td>\n",
       "      <td>Is it your dad, a teacher or a random Good Sam...</td>\n",
       "      <td>4</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       subreddit      id                       comment_body score_com  \\\n",
       "12822  AskReddit  184vmk  Some pretty inspiring posts here!         1   \n",
       "\n",
       "                           title  \\\n",
       "12822  Who is your hero and why?   \n",
       "\n",
       "                                                selftext score  freq  \n",
       "12822  Is it your dad, a teacher or a random Good Sam...     4   511  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substring_list = [\"Reddit, what is the happiest fact you know? I need something to make me feel better.\", \n",
    "                  \"Hey Reddit, what is a harsh truth that helped you change your life? This article has changed my life in so many ways\",\n",
    "                \"What kind of teacher did / might have inspired you in high school? i.e. What kind of teacher would you want your children to have? What made your teacher awesome / what made your teacher terrible that you would NOT want to see? Who inspired you to be a better person and/or pursue whatever it is you're pursuing?\",\n",
    "                  \"Who is your hero and why? Is it your dad, a teacher or a random Good Samaritan that changed your life? There's got to be some interesting stories out there!\",\n",
    "                  \"Who inspired you and how? Which Person, no matter if famous or not famous, dead or alive has inspired the way you live/ your life and how?\",\n",
    "                    \"Reddit I'm feeling down, what are some subreddits that are sure to brighten by day?\",\n",
    "                  \"How do you help yourself get out of depression? We've all been there.Personally, I write down what I'm feeling. No matter what it is, I just write it down.That and always remembering, tomorrow the sun will rise.\",\n",
    "                  \"What is the most inspiring thing you have ever seen or heard? Lots of things are going on, and I need some inspiration! Hopefully some of you can get inspired from some of the responses!\",\n",
    "                  \"I'm a TA. What are some fun ice breakers or games I can dish out to my students on their first day? For the love of God, G-rated please and thank you.\",\n",
    "                 \"It is my cakeday today, and I would like to ask you redditors a question: What inspires you?\",\n",
    "                 \"When was the last time you felt inspired? Who or what inspired you?\",\n",
    "                 ]\n",
    "                  \n",
    "merged_sth = merged4[(merged4['title'] + \" \" + merged4['selftext']).str.contains(\"Who is your hero and why?\")]\n",
    "merged_sth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subreddit</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_body</th>\n",
       "      <th>score_com</th>\n",
       "      <th>title</th>\n",
       "      <th>selftext</th>\n",
       "      <th>score</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>You have done so much with your life, I'm 16, ...</td>\n",
       "      <td>18</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>What inspired you to run for governor?</td>\n",
       "      <td>4</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>Whats in the future for you? Do you still lift...</td>\n",
       "      <td>2</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>I aspire to be like you, your role in pumping ...</td>\n",
       "      <td>2</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>IAmA</td>\n",
       "      <td>16mq0g</td>\n",
       "      <td>Mr. Schwarzenegger... You have completely insp...</td>\n",
       "      <td>2</td>\n",
       "      <td>IAmArnold... Ask me anything.</td>\n",
       "      <td>Former Mr. Olympia, Conan, Terminator, and Gov...</td>\n",
       "      <td>5651</td>\n",
       "      <td>760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10453</th>\n",
       "      <td>wollongong</td>\n",
       "      <td>163trx</td>\n",
       "      <td>Not too bad. Finally submitted my PhD thesis, ...</td>\n",
       "      <td>3</td>\n",
       "      <td>This subreddit is way too dead. Let's talk and...</td>\n",
       "      <td>Hey. This subreddit doesn't get enough love. H...</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13388</th>\n",
       "      <td>woodworking</td>\n",
       "      <td>16yo9i</td>\n",
       "      <td>For a workbench, I highly recommend something ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Help set up my workshop</td>\n",
       "      <td>I recently moved across country and bought a n...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13339</th>\n",
       "      <td>writers</td>\n",
       "      <td>16gvlj</td>\n",
       "      <td>Two of the most important things are a good pl...</td>\n",
       "      <td>1</td>\n",
       "      <td>Writers of Reddit. I have a few questions for ...</td>\n",
       "      <td>Hello everyone! I'm an architecture student an...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6611</th>\n",
       "      <td>xbox360</td>\n",
       "      <td>16g9fk</td>\n",
       "      <td>Fallout New Vegas probably doesn't seem very b...</td>\n",
       "      <td>2</td>\n",
       "      <td>What are the most visually appealing Xbox 360 ...</td>\n",
       "      <td>What are some of the best visually, artwork, e...</td>\n",
       "      <td>77</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11506</th>\n",
       "      <td>yugioh</td>\n",
       "      <td>17767i</td>\n",
       "      <td>I'll keep playing Wind-Ups until I complete th...</td>\n",
       "      <td>1</td>\n",
       "      <td>What are you playing until March and beyond?</td>\n",
       "      <td>I know that right now I'm finishing Noble Knig...</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5417 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         subreddit      id                                       comment_body  \\\n",
       "135           IAmA  16mq0g  You have done so much with your life, I'm 16, ...   \n",
       "136           IAmA  16mq0g             What inspired you to run for governor?   \n",
       "137           IAmA  16mq0g  Whats in the future for you? Do you still lift...   \n",
       "138           IAmA  16mq0g  I aspire to be like you, your role in pumping ...   \n",
       "139           IAmA  16mq0g  Mr. Schwarzenegger... You have completely insp...   \n",
       "...            ...     ...                                                ...   \n",
       "10453   wollongong  163trx  Not too bad. Finally submitted my PhD thesis, ...   \n",
       "13388  woodworking  16yo9i  For a workbench, I highly recommend something ...   \n",
       "13339      writers  16gvlj  Two of the most important things are a good pl...   \n",
       "6611       xbox360  16g9fk  Fallout New Vegas probably doesn't seem very b...   \n",
       "11506       yugioh  17767i  I'll keep playing Wind-Ups until I complete th...   \n",
       "\n",
       "      score_com                                              title  \\\n",
       "135          18                      IAmArnold... Ask me anything.   \n",
       "136           4                      IAmArnold... Ask me anything.   \n",
       "137           2                      IAmArnold... Ask me anything.   \n",
       "138           2                      IAmArnold... Ask me anything.   \n",
       "139           2                      IAmArnold... Ask me anything.   \n",
       "...         ...                                                ...   \n",
       "10453         3  This subreddit is way too dead. Let's talk and...   \n",
       "13388         1                            Help set up my workshop   \n",
       "13339         1  Writers of Reddit. I have a few questions for ...   \n",
       "6611          2  What are the most visually appealing Xbox 360 ...   \n",
       "11506         1       What are you playing until March and beyond?   \n",
       "\n",
       "                                                selftext score  freq  \n",
       "135    Former Mr. Olympia, Conan, Terminator, and Gov...  5651   760  \n",
       "136    Former Mr. Olympia, Conan, Terminator, and Gov...  5651   760  \n",
       "137    Former Mr. Olympia, Conan, Terminator, and Gov...  5651   760  \n",
       "138    Former Mr. Olympia, Conan, Terminator, and Gov...  5651   760  \n",
       "139    Former Mr. Olympia, Conan, Terminator, and Gov...  5651   760  \n",
       "...                                                  ...   ...   ...  \n",
       "10453  Hey. This subreddit doesn't get enough love. H...    13     1  \n",
       "13388  I recently moved across country and bought a n...     2     1  \n",
       "13339  Hello everyone! I'm an architecture student an...     2     1  \n",
       "6611   What are some of the best visually, artwork, e...    77     1  \n",
       "11506  I know that right now I'm finishing Noble Knig...     8     1  \n",
       "\n",
       "[5417 rows x 8 columns]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "merged4 = merged4.assign(freq=merged4.apply(lambda x: merged4.subreddit.value_counts().to_dict()[x.subreddit], axis=1)) \\\n",
    "        .sort_values(by=['freq', 'subreddit'], ascending=[False, True])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('IAmA', 760),\n",
       " ('AskReddit', 511),\n",
       " ('Random_Acts_Of_Amazon', 104),\n",
       " ('loseit', 90),\n",
       " ('todayilearned', 86),\n",
       " ('atheism', 68),\n",
       " ('Christianity', 60),\n",
       " ('keto', 47),\n",
       " ('writing', 47),\n",
       " ('NoFap', 46),\n",
       " ('gamedev', 45),\n",
       " ('Guitar', 44),\n",
       " ('malefashionadvice', 38),\n",
       " ('depression', 36),\n",
       " ('leagueoflegends', 35),\n",
       " ('trees', 35),\n",
       " ('books', 34),\n",
       " ('nfl', 34),\n",
       " ('gaming', 32),\n",
       " ('AskHistorians', 31),\n",
       " ('rpg', 30),\n",
       " ('MLPLounge', 29),\n",
       " ('Games', 28),\n",
       " ('Metal', 27),\n",
       " ('WeAreTheMusicMakers', 27),\n",
       " ('exmormon', 27),\n",
       " ('booksuggestions', 26),\n",
       " ('AskMen', 24),\n",
       " ('GetMotivated', 24),\n",
       " ('RandomActsOfPolish', 24),\n",
       " ('DebateReligion', 23),\n",
       " ('DnD', 23),\n",
       " ('progresspics', 23),\n",
       " ('AskWomen', 22),\n",
       " ('SketchDaily', 22),\n",
       " ('TwoXChromosomes', 22),\n",
       " ('femalefashionadvice', 22),\n",
       " ('nba', 22),\n",
       " ('MakeupAddiction', 21),\n",
       " ('WTF', 21),\n",
       " ('comicbooks', 21),\n",
       " ('Fitness', 20),\n",
       " ('mylittlepony', 20),\n",
       " ('politics', 20),\n",
       " ('RedditLaqueristas', 19),\n",
       " ('running', 19),\n",
       " ('DIY', 18),\n",
       " ('Fantasy', 18),\n",
       " ('photography', 18),\n",
       " ('stopdrinking', 18),\n",
       " ('teenagers', 18),\n",
       " ('TrueFilm', 17),\n",
       " ('classicalmusic', 17),\n",
       " ('gaybros', 16),\n",
       " ('CFB', 15),\n",
       " ('edmproduction', 15),\n",
       " ('Catholicism', 14),\n",
       " ('asoiaf', 14),\n",
       " ('Drugs', 13),\n",
       " ('Homebrewing', 13),\n",
       " ('MensRights', 13),\n",
       " ('soccer', 13),\n",
       " ('tattoos', 13),\n",
       " ('Guildwars2', 12),\n",
       " ('Psychonaut', 12),\n",
       " ('katawashoujo', 12),\n",
       " ('sex', 12),\n",
       " ('truegaming', 12),\n",
       " ('worldbuilding', 12),\n",
       " ('BabyBumps', 11),\n",
       " ('OkCupid', 11),\n",
       " ('TrueAtheism', 11),\n",
       " ('hiphopheads', 11),\n",
       " ('india', 11),\n",
       " ('makinghiphop', 11),\n",
       " ('offmychest', 11),\n",
       " ('travel', 11),\n",
       " ('worldnews', 11),\n",
       " ('Documentaries', 10),\n",
       " ('IWantToLearn', 10),\n",
       " ('Meditation', 10),\n",
       " ('Minecraft', 10),\n",
       " ('SquaredCircle', 10),\n",
       " ('boardgames', 10),\n",
       " ('cars', 10),\n",
       " ('funny', 10),\n",
       " ('harrypotter', 10),\n",
       " ('knitting', 10),\n",
       " ('motorcycles', 10),\n",
       " ('stopsmoking', 10),\n",
       " ('yoga', 10),\n",
       " ('UpliftingNews', 9),\n",
       " ('askscience', 9),\n",
       " ('community', 9),\n",
       " ('fantasywriters', 9),\n",
       " ('gamegrumps', 9),\n",
       " ('pokemon', 9),\n",
       " ('xxfitness', 9),\n",
       " ('ADHD', 8),\n",
       " ('AMA', 8),\n",
       " ('Cooking', 8),\n",
       " ('DotA2', 8),\n",
       " ('NetflixBestOf', 8),\n",
       " ('Pathfinder_RPG', 8),\n",
       " ('Poetry', 8),\n",
       " ('anime', 8),\n",
       " ('aww', 8),\n",
       " ('graphic_design', 8),\n",
       " ('lotr', 8),\n",
       " ('magicTCG', 8),\n",
       " ('masseffect', 8),\n",
       " ('starcraft', 8),\n",
       " ('television', 8),\n",
       " ('trance', 8),\n",
       " ('vegan', 8),\n",
       " ('Android', 7),\n",
       " ('Borderlands', 7),\n",
       " ('Filmmakers', 7),\n",
       " ('TheLastAirbender', 7),\n",
       " ('exjw', 7),\n",
       " ('explainlikeimfive', 7),\n",
       " ('frugalmalefashion', 7),\n",
       " ('howtonotgiveafuck', 7),\n",
       " ('philosophy', 7),\n",
       " ('piano', 7),\n",
       " ('space', 7),\n",
       " ('startrek', 7),\n",
       " ('Anarchism', 6),\n",
       " ('AskScienceFiction', 6),\n",
       " ('C25K', 6),\n",
       " ('DebateAChristian', 6),\n",
       " ('Design', 6),\n",
       " ('Libertarian', 6),\n",
       " ('Paleo', 6),\n",
       " ('RandomActsOfGaming', 6),\n",
       " ('britishproblems', 6),\n",
       " ('casualiama', 6),\n",
       " ('civ', 6),\n",
       " ('darksouls', 6),\n",
       " ('doctorwho', 6),\n",
       " ('electronicmusic', 6),\n",
       " ('halo', 6),\n",
       " ('latterdaysaints', 6),\n",
       " ('learnprogramming', 6),\n",
       " ('literature', 6),\n",
       " ('minimalism', 6),\n",
       " ('mormondebate', 6),\n",
       " ('musictheory', 6),\n",
       " ('postrock', 6),\n",
       " ('printSF', 6),\n",
       " ('skyrim', 6),\n",
       " ('socialskills', 6),\n",
       " ('vexillology', 6),\n",
       " ('web_design', 6),\n",
       " ('ENFP', 5),\n",
       " ('Entrepreneur', 5),\n",
       " ('Frugal', 5),\n",
       " ('ModerationLog', 5),\n",
       " ('Monstercat', 5),\n",
       " ('Naruto', 5),\n",
       " ('Parenting', 5),\n",
       " ('PoliticalDiscussion', 5),\n",
       " ('Random_Acts_Of_Pizza', 5),\n",
       " ('ShitRedditSays', 5),\n",
       " ('Standup', 5),\n",
       " ('SuicideWatch', 5),\n",
       " ('Teachers', 5),\n",
       " ('Warhammer', 5),\n",
       " ('adventuretime', 5),\n",
       " ('amiugly', 5),\n",
       " ('audioengineering', 5),\n",
       " ('beer', 5),\n",
       " ('blackops2', 5),\n",
       " ('bodybuilding', 5),\n",
       " ('circlejerk', 5),\n",
       " ('climbing', 5),\n",
       " ('conspiracy', 5),\n",
       " ('dayz', 5),\n",
       " ('design_critiques', 5),\n",
       " ('drums', 5),\n",
       " ('entwives', 5),\n",
       " ('gonewild', 5),\n",
       " ('hockey', 5),\n",
       " ('homestuck', 5),\n",
       " ('leaves', 5),\n",
       " ('lgbt', 5),\n",
       " ('nerdcubed', 5),\n",
       " ('science', 5),\n",
       " ('seduction', 5),\n",
       " ('technology', 5),\n",
       " ('tipofmytongue', 5),\n",
       " ('unitedkingdom', 5),\n",
       " ('wow', 5),\n",
       " ('Animesuggest', 4),\n",
       " ('Buddhism', 4),\n",
       " ('Civcraft', 4),\n",
       " ('Cypher', 4),\n",
       " ('DeadSpace', 4),\n",
       " ('DebateCommunism', 4),\n",
       " ('Diablo', 4),\n",
       " ('EDC', 4),\n",
       " ('Fallout', 4),\n",
       " ('FancyFollicles', 4),\n",
       " ('Futurology', 4),\n",
       " ('INTP', 4),\n",
       " ('LongDistance', 4),\n",
       " ('NovelThisMonth', 4),\n",
       " ('Screenwriting', 4),\n",
       " ('Seireitei', 4),\n",
       " ('TeenMFA', 4),\n",
       " ('TrueReddit', 4),\n",
       " ('apple', 4),\n",
       " ('asktransgender', 4),\n",
       " ('backpacking', 4),\n",
       " ('batman', 4),\n",
       " ('bestof', 4),\n",
       " ('bicycling', 4),\n",
       " ('comics', 4),\n",
       " ('crochet', 4),\n",
       " ('feedthebeast', 4),\n",
       " ('food', 4),\n",
       " ('gallifrey', 4),\n",
       " ('gamingsuggestions', 4),\n",
       " ('guns', 4),\n",
       " ('horror', 4),\n",
       " ('improv', 4),\n",
       " ('infp', 4),\n",
       " ('islam', 4),\n",
       " ('lego', 4),\n",
       " ('letsplay', 4),\n",
       " ('malefashion', 4),\n",
       " ('news', 4),\n",
       " ('nexus4', 4),\n",
       " ('raspberry_pi', 4),\n",
       " ('rawdenim', 4),\n",
       " ('scifi', 4),\n",
       " ('shortscarystories', 4),\n",
       " ('silenthill', 4),\n",
       " ('singing', 4),\n",
       " ('startups', 4),\n",
       " ('tarot', 4),\n",
       " ('vinyl', 4),\n",
       " ('wedding', 4),\n",
       " ('wicked_edge', 4),\n",
       " ('windowsphone', 4),\n",
       " ('xxketo', 4),\n",
       " ('AmericanHorrorStory', 3),\n",
       " ('Anarcho_Capitalism', 3),\n",
       " ('AndroidQuestions', 3),\n",
       " ('Art', 3),\n",
       " ('AskSocialScience', 3),\n",
       " ('Bass', 3),\n",
       " ('CollegeBasketball', 3),\n",
       " ('CrazyIdeas', 3),\n",
       " ('DJs', 3),\n",
       " ('DeadBedrooms', 3),\n",
       " ('DebateAnAtheist', 3),\n",
       " ('Dexter', 3),\n",
       " ('DnB', 3),\n",
       " ('EDM', 3),\n",
       " ('EatCheapAndHealthy', 3),\n",
       " ('ExploreReligion', 3),\n",
       " ('ForeverAlone', 3),\n",
       " ('Frisson', 3),\n",
       " ('GameDeals', 3),\n",
       " ('GiftofGames', 3),\n",
       " ('Israel', 3),\n",
       " ('JRPG', 3),\n",
       " ('KarmaCourt', 3),\n",
       " ('KerbalSpaceProgram', 3),\n",
       " ('LGBTeens', 3),\n",
       " ('LifeProTips', 3),\n",
       " ('LucidDreaming', 3),\n",
       " ('MMFB', 3),\n",
       " ('Maplestory', 3),\n",
       " ('Military', 3),\n",
       " ('MorbidReality', 3),\n",
       " ('MvC3', 3),\n",
       " ('NoSleepOOC', 3),\n",
       " ('OceanSkys', 3),\n",
       " ('OnePiece', 3),\n",
       " ('POLITIC', 3),\n",
       " ('Pathfinder', 3),\n",
       " ('Random_Acts_of_Etsy', 3),\n",
       " ('Songwriters', 3),\n",
       " ('Stoicism', 3),\n",
       " ('TheGirlSurvivalGuide', 3),\n",
       " ('Warhammer40k', 3),\n",
       " ('Watches', 3),\n",
       " ('WoT', 3),\n",
       " ('WorldofTanks', 3),\n",
       " ('actuallesbians', 3),\n",
       " ('airsoft', 3),\n",
       " ('baseball', 3),\n",
       " ('bigboobproblems', 3),\n",
       " ('bipolar', 3),\n",
       " ('bodyweightfitness', 3),\n",
       " ('changemyview', 3),\n",
       " ('communism101', 3),\n",
       " ('d3hardcore', 3),\n",
       " ('education', 3),\n",
       " ('exchristian', 3),\n",
       " ('guitarlessons', 3),\n",
       " ('homestead', 3),\n",
       " ('intj', 3),\n",
       " ('kpop', 3),\n",
       " ('lost', 3),\n",
       " ('magicskyfairy', 3),\n",
       " ('marketing', 3),\n",
       " ('mbti', 3),\n",
       " ('mindcrack', 3),\n",
       " ('pathofexile', 3),\n",
       " ('pcgaming', 3),\n",
       " ('personalfinance', 3),\n",
       " ('pinkfloyd', 3),\n",
       " ('r4r', 3),\n",
       " ('radiohead', 3),\n",
       " ('randomactsofkindness', 3),\n",
       " ('ravens', 3),\n",
       " ('rct', 3),\n",
       " ('redditgetsdrawn', 3),\n",
       " ('rush', 3),\n",
       " ('secretsanta', 3),\n",
       " ('self', 3),\n",
       " ('sewing', 3),\n",
       " ('snowboarding', 3),\n",
       " ('socialanxiety', 3),\n",
       " ('socialism', 3),\n",
       " ('stonerrock', 3),\n",
       " ('tf2', 3),\n",
       " ('thesims', 3),\n",
       " ('thewalkingdead', 3),\n",
       " ('tolkienfans', 3),\n",
       " ('totalwar', 3),\n",
       " ('transgender', 3),\n",
       " ('typography', 3),\n",
       " ('washingtondc', 3),\n",
       " ('webdev', 3),\n",
       " ('weightlifting', 3),\n",
       " ('whatstheword', 3),\n",
       " ('2X_INTJ', 2),\n",
       " ('49ers', 2),\n",
       " ('AGOTBoardGame', 2),\n",
       " ('AdviceAnimals', 2),\n",
       " ('AirForce', 2),\n",
       " ('AllThingsTerran', 2),\n",
       " ('AmISexy', 2),\n",
       " ('AnimeSketch', 2),\n",
       " ('Anthropology', 2),\n",
       " ('Anxiety', 2),\n",
       " ('Aquariums', 2),\n",
       " ('AwesomeArtwork', 2),\n",
       " ('BDSMcommunity', 2),\n",
       " ('Baking', 2),\n",
       " ('Bitcoin', 2),\n",
       " ('BodyAcceptance', 2),\n",
       " ('Breadit', 2),\n",
       " ('BreakUps', 2),\n",
       " ('Calgary', 2),\n",
       " ('CanadaPolitics', 2),\n",
       " ('CannabisExtracts', 2),\n",
       " ('CompanyOfHeroes', 2),\n",
       " ('DCcomics', 2),\n",
       " ('DarkKnightDiscussion', 2),\n",
       " ('DharmaStudy', 2),\n",
       " ('Diablo3Strategy', 2),\n",
       " ('DoesAnybodyElse', 2),\n",
       " ('Dota2Trade', 2),\n",
       " ('Eve', 2),\n",
       " ('Firearms', 2),\n",
       " ('FoodPorn', 2),\n",
       " ('GirlGamers', 2),\n",
       " ('Gunners', 2),\n",
       " ('HistoryPorn', 2),\n",
       " ('Hungergames', 2),\n",
       " ('Illustration', 2),\n",
       " ('InsightfulQuestions', 2),\n",
       " ('JamesBond', 2),\n",
       " ('Juicing', 2),\n",
       " ('Kendama', 2),\n",
       " ('KitchenConfidential', 2),\n",
       " ('LawSchool', 2),\n",
       " ('LetsTalkMusic', 2),\n",
       " ('LiverpoolFC', 2),\n",
       " ('LizzieBennet', 2),\n",
       " ('Lolita', 2),\n",
       " ('LosAngeles', 2),\n",
       " ('MLPtunes', 2),\n",
       " ('MMA', 2),\n",
       " ('MMORPG', 2),\n",
       " ('MovieSuggestions', 2),\n",
       " ('MusicCritique', 2),\n",
       " ('MyLittleSupportGroup', 2),\n",
       " ('OneGameAMonth', 2),\n",
       " ('Patriots', 2),\n",
       " ('PennStateUniversity', 2),\n",
       " ('Permaculture', 2),\n",
       " ('Physics', 2),\n",
       " ('Planetside', 2),\n",
       " ('PointsPlus', 2),\n",
       " ('Portland', 2),\n",
       " ('Pottery', 2),\n",
       " ('RandomActsofMakeup', 2),\n",
       " ('RedHotChiliPeppers', 2),\n",
       " ('SRSDiscussion', 2),\n",
       " ('ShittyLifeProTips', 2),\n",
       " ('SocialEngineering', 2),\n",
       " ('StLouis', 2),\n",
       " ('StarWars', 2),\n",
       " ('TheDarkTower', 2),\n",
       " ('Theatre', 2),\n",
       " ('Tribes', 2),\n",
       " ('UUreddit', 2),\n",
       " ('Vocaloid', 2),\n",
       " ('WingChun', 2),\n",
       " ('WritingPrompts', 2),\n",
       " ('analog', 2),\n",
       " ('animation', 2),\n",
       " ('answers', 2),\n",
       " ('architecture', 2),\n",
       " ('askphilosophy', 2),\n",
       " ('askseddit', 2),\n",
       " ('assassinscreed', 2),\n",
       " ('bahai', 2),\n",
       " ('bdsm', 2),\n",
       " ('beatles', 2),\n",
       " ('bicycletouring', 2),\n",
       " ('blackladies', 2),\n",
       " ('bleach', 2),\n",
       " ('breakingbad', 2),\n",
       " ('business', 2),\n",
       " ('canada', 2),\n",
       " ('cats', 2),\n",
       " ('coheedandcambria', 2),\n",
       " ('college', 2),\n",
       " ('confession', 2),\n",
       " ('crafts', 2),\n",
       " ('creepy', 2),\n",
       " ('dogs', 2),\n",
       " ('doublebass', 2),\n",
       " ('dwarffortress', 2),\n",
       " ('electronic_cigarette', 2),\n",
       " ('europe', 2),\n",
       " ('fashion', 2),\n",
       " ('fireemblem', 2),\n",
       " ('fnv', 2),\n",
       " ('fountainpens', 2),\n",
       " ('futurebeatproducers', 2),\n",
       " ('gainit', 2),\n",
       " ('gamecollecting', 2),\n",
       " ('gamemusic', 2),\n",
       " ('gameofbands', 2),\n",
       " ('gameofthrones', 2),\n",
       " ('gatech', 2),\n",
       " ('gaymers', 2),\n",
       " ('getdisciplined', 2),\n",
       " ('golf', 2),\n",
       " ('gonewildaudio', 2),\n",
       " ('grandrapids', 2),\n",
       " ('guitarpedals', 2),\n",
       " ('happy', 2),\n",
       " ('hardstyle', 2),\n",
       " ('headphones', 2),\n",
       " ('hiking', 2),\n",
       " ('history', 2),\n",
       " ('homeschool', 2),\n",
       " ('ifyoulikeblank', 2),\n",
       " ('iosgaming', 2),\n",
       " ('ireland', 2),\n",
       " ('javascript', 2),\n",
       " ('jobs', 2),\n",
       " ('ketorecipes', 2),\n",
       " ('languagelearning', 2),\n",
       " ('ldscirclejerk', 2),\n",
       " ('learnart', 2),\n",
       " ('linguistics', 2),\n",
       " ('linux', 2),\n",
       " ('longboarding', 2),\n",
       " ('martialarts', 2),\n",
       " ('mcpublic', 2),\n",
       " ('medicine', 2),\n",
       " ('metalgearsolid', 2),\n",
       " ('networking', 2),\n",
       " ('nursing', 2),\n",
       " ('occult', 2),\n",
       " ('ouya', 2),\n",
       " ('photocritique', 2),\n",
       " ('polyamory', 2),\n",
       " ('pornfree', 2),\n",
       " ('premed', 2),\n",
       " ('productivity', 2),\n",
       " ('psychology', 2),\n",
       " ('punk', 2),\n",
       " ('quotes', 2),\n",
       " ('recipes', 2),\n",
       " ('relationship_advice', 2),\n",
       " ('rugbyunion', 2),\n",
       " ('rupaulsdragrace', 2),\n",
       " ('selfhelp', 2),\n",
       " ('selfpublish', 2),\n",
       " ('shittyadvice', 2),\n",
       " ('skateboarding', 2),\n",
       " ('smashbros', 2),\n",
       " ('speedrun', 2),\n",
       " ('standupshots', 2),\n",
       " ('steampunk', 2),\n",
       " ('survivor', 2),\n",
       " ('swoleacceptance', 2),\n",
       " ('swtor', 2),\n",
       " ('talesfromtechsupport', 2),\n",
       " ('tf2trade', 2),\n",
       " ('themountaingoats', 2),\n",
       " ('toronto', 2),\n",
       " ('tryptonaut', 2),\n",
       " ('violinist', 2),\n",
       " ('vita', 2),\n",
       " ('zelda', 2),\n",
       " ('zen', 2),\n",
       " ('100movies365days', 1),\n",
       " ('100sets', 1),\n",
       " ('24hoursupport', 1),\n",
       " ('3DS', 1),\n",
       " ('4chan', 1),\n",
       " ('52WeeksOfCocktails', 1),\n",
       " ('52book', 1),\n",
       " ('52weeksofcooking', 1),\n",
       " ('Adelaide', 1),\n",
       " ('AdventureRacing', 1),\n",
       " ('Advice', 1),\n",
       " ('AndroidGaming', 1),\n",
       " ('AndroidUsers', 1),\n",
       " ('Animals', 1),\n",
       " ('AnnArbor', 1),\n",
       " ('AntiJokes', 1),\n",
       " ('Archaeology', 1),\n",
       " ('ArcherFX', 1),\n",
       " ('Archery', 1),\n",
       " ('ArtBattle', 1),\n",
       " ('ArtEd', 1),\n",
       " ('AskAcademia', 1),\n",
       " ('AskGames', 1),\n",
       " ('AskPhotography', 1),\n",
       " ('AskRedditAfterDark', 1),\n",
       " ('AskScienceDiscussion', 1),\n",
       " ('AssassinOrder', 1),\n",
       " ('Astronomy', 1),\n",
       " ('AtheistTwelveSteppers', 1),\n",
       " ('Atlanta', 1),\n",
       " ('AustralianShepherd', 1),\n",
       " ('Automate', 1),\n",
       " ('Avengers', 1),\n",
       " ('BALLET', 1),\n",
       " ('BOLIVIA', 1),\n",
       " ('Bad_Cop_No_Donut', 1),\n",
       " ('BanjoKazooie', 1),\n",
       " ('BeardPorn', 1),\n",
       " ('Bioshock', 1),\n",
       " ('BipolarReddit', 1),\n",
       " ('Bluegrass', 1),\n",
       " ('Bonsai', 1),\n",
       " ('BostonBruins', 1),\n",
       " ('Bowyer', 1),\n",
       " ('Bravenewbies', 1),\n",
       " ('BritishTV', 1),\n",
       " ('BronyWeapons', 1),\n",
       " ('BuyItForLife', 1),\n",
       " ('CHIBears', 1),\n",
       " ('CODZombies', 1),\n",
       " ('CSUS', 1),\n",
       " ('CalPoly', 1),\n",
       " ('CallOfDuty', 1),\n",
       " ('Calligraphy', 1),\n",
       " ('CamelotUnchained', 1),\n",
       " ('CampingandHiking', 1),\n",
       " ('Charity', 1),\n",
       " ('CherokeeXJ', 1),\n",
       " ('China', 1),\n",
       " ('ChivalryGame', 1),\n",
       " ('ChristianMusic', 1),\n",
       " ('ChronicPain', 1),\n",
       " ('CircLoLjerk', 1),\n",
       " ('CircleMusic', 1),\n",
       " ('Clarinet', 1),\n",
       " ('Climbingvids', 1),\n",
       " ('Coffee', 1),\n",
       " ('Colts', 1),\n",
       " ('Connecticut', 1),\n",
       " ('Cornwall', 1),\n",
       " ('CourseraFilm', 1),\n",
       " ('Creativity', 1),\n",
       " ('Cricket', 1),\n",
       " ('CrunchBang', 1),\n",
       " ('CubeWorld', 1),\n",
       " ('Cubers', 1),\n",
       " ('DaftPunk', 1),\n",
       " ('Dallas', 1),\n",
       " ('Dance', 1),\n",
       " ('DavidBowie', 1),\n",
       " ('DebateFascism', 1),\n",
       " ('DebateaCommunist', 1),\n",
       " ('DecidingToBeBetter', 1),\n",
       " ('Dentistry', 1),\n",
       " ('Destiny', 1),\n",
       " ('DistroHopping', 1),\n",
       " ('DobermanPinscher', 1),\n",
       " ('Drag', 1),\n",
       " ('Drifting', 1),\n",
       " ('DrugNerds', 1),\n",
       " ('DunderMifflin', 1),\n",
       " ('DungeonsAndDragons', 1),\n",
       " ('ELINT', 1),\n",
       " ('Economics', 1),\n",
       " ('EnoughPaulSpam', 1),\n",
       " ('Eragon', 1),\n",
       " ('Etsy', 1),\n",
       " ('ExplainLikeImCalvin', 1),\n",
       " ('FCJbookclub', 1),\n",
       " ('FL_Studio', 1),\n",
       " ('FanTheories', 1),\n",
       " ('Fantasy_Bookclub', 1),\n",
       " ('FrenchHouse', 1),\n",
       " ('FreshStart', 1),\n",
       " ('FutureWhatIf', 1),\n",
       " ('GTA', 1),\n",
       " ('GameSociety', 1),\n",
       " ('Gamingmemories', 1),\n",
       " ('Glitch_in_the_Matrix', 1),\n",
       " ('GlobalOffensive', 1),\n",
       " ('GraphicDesign', 1),\n",
       " ('GreenBayPackers', 1),\n",
       " ('HIMYM', 1),\n",
       " ('HaloStory', 1),\n",
       " ('Heavymind', 1),\n",
       " ('HermitCraft', 1),\n",
       " ('HistoryofIdeas', 1),\n",
       " ('HomeImprovement', 1),\n",
       " ('HomeworkHelp', 1),\n",
       " ('HouseOfCards', 1),\n",
       " ('HumanPorn', 1),\n",
       " ('HuntsvilleAlabama', 1),\n",
       " ('ICanDrawThat', 1),\n",
       " ('IRstudies', 1),\n",
       " ('ITunes', 1),\n",
       " ('Iceland', 1),\n",
       " ('IndieGaming', 1),\n",
       " ('InfrastructurePorn', 1),\n",
       " ('Ingress', 1),\n",
       " ('InternetAMA', 1),\n",
       " ('Internet_Box', 1),\n",
       " ('Islaam', 1),\n",
       " ('Jazz', 1),\n",
       " ('JoeRogan', 1),\n",
       " ('Judaism', 1),\n",
       " ('JusticePorn', 1),\n",
       " ('Justrolledintotheshop', 1),\n",
       " ('KeepWriting', 1),\n",
       " ('Korean', 1),\n",
       " ('LV426', 1),\n",
       " ('LadyBoners', 1),\n",
       " ('LeagueofLegendsMeta', 1),\n",
       " ('Lettering', 1),\n",
       " ('LifeRPG', 1),\n",
       " ('LoLFanArt', 1),\n",
       " ('Louisville', 1),\n",
       " ('Lovecraft', 1),\n",
       " ('Luthier', 1),\n",
       " ('MCPE', 1),\n",
       " ('MLS', 1),\n",
       " ('Magicdeckbuilding', 1),\n",
       " ('MakeNewFriendsHere', 1),\n",
       " ('Marvel', 1),\n",
       " ('MechanicalKeyboards', 1),\n",
       " ('Metroid', 1),\n",
       " ('MineZ', 1),\n",
       " ('Minecraft360', 1),\n",
       " ('MinecraftInventions', 1),\n",
       " ('MonsterHunter', 1),\n",
       " ('MortalEngines', 1),\n",
       " ('Muse', 1),\n",
       " ('NASCAR', 1),\n",
       " ('NCSS', 1),\n",
       " ('NPBR', 1),\n",
       " ('Negareddit', 1),\n",
       " ('NewOrleans', 1),\n",
       " ('Newbwriters', 1),\n",
       " ('Nexus10', 1),\n",
       " ('Nexus7', 1),\n",
       " ('Nirvana', 1),\n",
       " ('PHP', 1),\n",
       " ('PS3', 1),\n",
       " ('PickAnAndroidForMe', 1),\n",
       " ('PipeTobacco', 1),\n",
       " ('Pizza', 1),\n",
       " ('Playdate', 1),\n",
       " ('PloungeMafia', 1),\n",
       " ('PlusSize', 1),\n",
       " ('PoetrySlam', 1),\n",
       " ('Portlandia', 1),\n",
       " ('PowerMetal', 1),\n",
       " ('Project52', 1),\n",
       " ('PropagandaPosters', 1),\n",
       " ('Protomen', 1),\n",
       " ('Purdue', 1),\n",
       " ('Python', 1),\n",
       " ('Quakers', 1),\n",
       " ('REDDITORSINRECOVERY', 1),\n",
       " ('RWF', 1),\n",
       " ('RaceTrackDesigns', 1),\n",
       " ('RandomActsOfChristmas', 1),\n",
       " ('RandomActsOfGifting', 1),\n",
       " ('RandomActsofCrafting', 1),\n",
       " ('Rateme', 1),\n",
       " ('RealJournalism', 1),\n",
       " ('Recordings', 1),\n",
       " ('RedditThroughHistory', 1),\n",
       " ('Redditropolis', 1),\n",
       " ('Romania', 1),\n",
       " ('RoomPorn', 1),\n",
       " ('SCP', 1),\n",
       " ('SF4', 1),\n",
       " ('SFGiants', 1),\n",
       " ('SFSU', 1),\n",
       " ('SRSAnime', 1),\n",
       " ('SRSWomen', 1),\n",
       " ('SavageGarden', 1),\n",
       " ('Scotch', 1),\n",
       " ('Scotland', 1),\n",
       " ('Shambhala', 1),\n",
       " ('Sherlock_Fanfiction', 1),\n",
       " ('ShittyPoetry', 1),\n",
       " ('Shortfilms', 1),\n",
       " ('ShouldIbuythisgame', 1),\n",
       " ('Sidehugs', 1),\n",
       " ('Silverbugs', 1),\n",
       " ('Sissies', 1),\n",
       " ('SkyPorn', 1),\n",
       " ('Smyrna', 1),\n",
       " ('SoccerBetting', 1),\n",
       " ('Songwriting', 1),\n",
       " ('StopSelfHarm', 1),\n",
       " ('Swimming', 1),\n",
       " ('SwingDancing', 1),\n",
       " ('Synesthesia', 1),\n",
       " ('TMNT', 1),\n",
       " ('TattooDesigns', 1),\n",
       " ('Team_Arabic', 1),\n",
       " ('TeraOnline', 1),\n",
       " ('TheAldmeriDominion', 1),\n",
       " ('TheHobbit', 1),\n",
       " ('TheRedPill', 1),\n",
       " ('TheseAreOurAlbums', 1),\n",
       " ('TopGear', 1),\n",
       " ('Transhuman', 1),\n",
       " ('TribeTwelve', 1),\n",
       " ('TrueAskReddit', 1),\n",
       " ('TruePokemon', 1),\n",
       " ('TrueTrueReddit', 1),\n",
       " ('Tulpas', 1),\n",
       " ('TumblrInAction', 1),\n",
       " ('TwinCities', 1),\n",
       " ('UBC', 1),\n",
       " ('UCSD', 1),\n",
       " ('USCivilWar', 1),\n",
       " ('USHistory', 1),\n",
       " ('USMilitarySO', 1),\n",
       " ('Unity3D', 1),\n",
       " ('VideoEditing', 1),\n",
       " ('Warmachine', 1),\n",
       " ('Wicca', 1),\n",
       " ('XFiles', 1),\n",
       " ('Xcom', 1),\n",
       " ('Yogscast', 1),\n",
       " ('YouShouldKnow', 1),\n",
       " ('ZenHabits', 1),\n",
       " ('abletonlive', 1),\n",
       " ('acting', 1),\n",
       " ('adventism', 1),\n",
       " ('adventuretrees', 1),\n",
       " ('advertising', 1),\n",
       " ('ainbow', 1),\n",
       " ('algorithms', 1),\n",
       " ('allcapsmotivation', 1),\n",
       " ('allthingsprotoss', 1),\n",
       " ('ambien', 1),\n",
       " ('ambient', 1),\n",
       " ('americanidol', 1),\n",
       " ('ancientgreece', 1),\n",
       " ('anonymous', 1),\n",
       " ('antichamber', 1),\n",
       " ('aoe2', 1),\n",
       " ('argentina', 1),\n",
       " ('artificial', 1),\n",
       " ('artstore', 1),\n",
       " ('asianamerican', 1),\n",
       " ('asmr', 1),\n",
       " ('astrology', 1),\n",
       " ('atheistparents', 1),\n",
       " ('australia', 1),\n",
       " ('aves', 1),\n",
       " ('avesNYC', 1),\n",
       " ('aviation', 1),\n",
       " ('babylon5', 1),\n",
       " ('badreligion', 1),\n",
       " ('baltimore', 1),\n",
       " ('barista', 1),\n",
       " ('battlestations', 1),\n",
       " ('bayarea', 1),\n",
       " ('baylor', 1),\n",
       " ('beards', 1),\n",
       " ('beertrade', 1),\n",
       " ('bestofnetflix', 1),\n",
       " ('beyondthebump', 1),\n",
       " ('bindingofisaac', 1),\n",
       " ('biology', 1),\n",
       " ('birdsofprey', 1),\n",
       " ('bjj', 1),\n",
       " ('blender', 1),\n",
       " ('blues', 1),\n",
       " ('boardsofcanada', 1),\n",
       " ('boston', 1),\n",
       " ('bostonceltics', 1),\n",
       " ('botany', 1),\n",
       " ('bourbon', 1),\n",
       " ('buffy', 1),\n",
       " ('buildaband', 1),\n",
       " ('buildapc', 1),\n",
       " ('bullying', 1),\n",
       " ('burlington', 1),\n",
       " ('californication', 1),\n",
       " ('castles', 1),\n",
       " ('chicago', 1),\n",
       " ('chicagoEDM', 1),\n",
       " ('chicagomusicscene', 1),\n",
       " ('chicagotrees', 1),\n",
       " ('childfree', 1),\n",
       " ('chomsky', 1),\n",
       " ('christcore', 1),\n",
       " ('christiananarchism', 1),\n",
       " ('classicalguitar', 1),\n",
       " ('cognitivelinguistics', 1),\n",
       " ('columbiamo', 1),\n",
       " ('comedy', 1),\n",
       " ('commandline', 1),\n",
       " ('communism', 1),\n",
       " ('computertechs', 1),\n",
       " ('conspiratard', 1),\n",
       " ('copywriting', 1),\n",
       " ('cordcutters', 1),\n",
       " ('counterfactuals', 1),\n",
       " ('coys', 1),\n",
       " ('craftit', 1),\n",
       " ('creepygaming', 1),\n",
       " ('creepypasta', 1),\n",
       " ('cringepics', 1),\n",
       " ('cripplingalcoholism', 1),\n",
       " ('crossdressing', 1),\n",
       " ('crossfit', 1),\n",
       " ('daddit', 1),\n",
       " ('dancehall', 1),\n",
       " ('dating_advice', 1),\n",
       " ('ddo', 1),\n",
       " ('deadmau5', 1),\n",
       " ('deadpool', 1),\n",
       " ('deaf', 1),\n",
       " ('declutter', 1),\n",
       " ('diabetes', 1),\n",
       " ('digimon', 1),\n",
       " ('discgolf', 1),\n",
       " ('docproduction', 1),\n",
       " ('drunk', 1),\n",
       " ('dub', 1),\n",
       " ('dustforce', 1),\n",
       " ('dvdcollection', 1),\n",
       " ('ecology', 1),\n",
       " ('ehlersdanlos', 1),\n",
       " ('elderscrollsonline', 1),\n",
       " ('eldertrees', 1),\n",
       " ('electricdaisycarnival', 1),\n",
       " ('energy', 1),\n",
       " ('engineering', 1),\n",
       " ('entertainment', 1),\n",
       " ('entj', 1),\n",
       " ('environment', 1),\n",
       " ('etymology', 1),\n",
       " ('evergreen', 1),\n",
       " ('evolution', 1),\n",
       " ('evolutionReddit', 1),\n",
       " ('excatholic', 1),\n",
       " ('exmuslim', 1),\n",
       " ('explainlikeIAmA', 1),\n",
       " ('faceoff', 1),\n",
       " ('fandomnatural', 1),\n",
       " ('fantasybaseball', 1),\n",
       " ('farcry', 1),\n",
       " ('fastfood', 1),\n",
       " ('feminineboys', 1),\n",
       " ('feminisms', 1),\n",
       " ('fibro', 1),\n",
       " ('fifthworldproblems', 1),\n",
       " ('findareddit', 1),\n",
       " ('findfashion', 1),\n",
       " ('firefly', 1),\n",
       " ('fitmeals', 1),\n",
       " ('fittings', 1),\n",
       " ('folkmetal', 1),\n",
       " ('footballmanagergames', 1),\n",
       " ('formula1', 1),\n",
       " ('france', 1),\n",
       " ('frugalbeauty', 1),\n",
       " ('furry', 1),\n",
       " ('futurebeats', 1),\n",
       " ('gadgets', 1),\n",
       " ('gamemaker', 1),\n",
       " ('gamernews', 1),\n",
       " ('gardening', 1),\n",
       " ('gay', 1),\n",
       " ('ghibli', 1),\n",
       " ('gifs', 1),\n",
       " ('glutenfree', 1),\n",
       " ('goth', 1),\n",
       " ('gradadmissions', 1),\n",
       " ('graffhelp', 1),\n",
       " ('greenday', 1),\n",
       " ('hawks', 1),\n",
       " ('highschool', 1),\n",
       " ('hinduism', 1),\n",
       " ('hiphop101', 1),\n",
       " ('hockeygoalies', 1),\n",
       " ('hometheater', 1),\n",
       " ('hookah', 1),\n",
       " ('horn', 1),\n",
       " ('hypotheticalsituation', 1),\n",
       " ('indianapolis', 1),\n",
       " ('intentionalcommunity', 1),\n",
       " ('introvert', 1),\n",
       " ('investing', 1),\n",
       " ('iran', 1),\n",
       " ('jailbreak', 1),\n",
       " ('jambands', 1),\n",
       " ('jewelers', 1),\n",
       " ('karaoke', 1),\n",
       " ('kickstarter', 1),\n",
       " ('knives', 1),\n",
       " ('labrats', 1),\n",
       " ('largeformat', 1),\n",
       " ('law', 1),\n",
       " ('legal', 1),\n",
       " ('lifehacks', 1),\n",
       " ('listentothis', 1),\n",
       " ('listentous', 1),\n",
       " ('livesound', 1),\n",
       " ('longrange', 1),\n",
       " ('love', 1),\n",
       " ('mac', 1),\n",
       " ('madmen', 1),\n",
       " ('malelivingspace', 1),\n",
       " ('math', 1),\n",
       " ('mcgill', 1),\n",
       " ('mead', 1),\n",
       " ('medicinehat', 1),\n",
       " ('metaphotography', 1),\n",
       " ('mflb', 1),\n",
       " ('minerapocalypse', 1),\n",
       " ('modelmakers', 1),\n",
       " ('montreal', 1),\n",
       " ('mtgcube', 1),\n",
       " ('musicals', 1),\n",
       " ('musicmakers', 1),\n",
       " ('mylittledota', 1),\n",
       " ('mylittlefanfic', 1),\n",
       " ('mylittlelistentothis', 1),\n",
       " ('nasa', 1),\n",
       " ('neilgaiman', 1),\n",
       " ('nerdfighters', 1),\n",
       " ('netflix', 1),\n",
       " ('newhampshire', 1),\n",
       " ('newzealand', 1),\n",
       " ('nin', 1),\n",
       " ('normalnudes', 1),\n",
       " ('northernireland', 1),\n",
       " ('nottheonion', 1),\n",
       " ('notzen', 1),\n",
       " ('nova', 1),\n",
       " ('ns2', 1),\n",
       " ('nutrition', 1),\n",
       " ('nuzlocke', 1),\n",
       " ('nyc', 1),\n",
       " ('oculus', 1),\n",
       " ('offbeat', 1),\n",
       " ('olympics', 1),\n",
       " ('opensource', 1),\n",
       " ('opera', 1),\n",
       " ('opiates', 1),\n",
       " ('orangecounty', 1),\n",
       " ('organ', 1),\n",
       " ('origami', 1),\n",
       " ('paganmusic', 1),\n",
       " ('painting', 1),\n",
       " ('patientgamers', 1),\n",
       " ('pbp', 1),\n",
       " ...]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(merged4[\"subreddit\"]).most_common()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get all comments for submission id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "https://api.pushshift.io/reddit/comment/search/?link_id=17ok2p,16lito,189nuu,17byde&subreddit=AskReddit&limit=1000\n"
     ]
    }
   ],
   "source": [
    "list_submission_ids = [\"16wirc\", \"162b0z\" ,\"15vob8\", \"184vmk\", \"17byde\", \"181c81\", \"181zkt\", \"16lito\", \"18slc9\", \"17ok2p\"]\n",
    "inspir_ids=[\"17ok2p\", \"16lito\", \"189nuu\", \"17byde\"]\n",
    "column_values = ['body','parent_id','subreddit_id', 'score', 'subreddit']\n",
    "\n",
    "print(len(list_submission_ids))\n",
    "\n",
    "str_list_ids = \",\".join(inspir_ids)\n",
    "\n",
    "url = f\"https://api.pushshift.io/reddit/comment/search/?link_id=\" + str_list_ids + \"&subreddit=AskReddit\"\n",
    "print(url)\n",
    "df_all=pd.DataFrame(columns=column_values)\n",
    "    \n",
    "    r = requests.get(url,  timeout=30)\n",
    "    # Check the status code, if successful, process the data\n",
    "    if r.status_code == 200:\n",
    "        response = json.loads(r.text)\n",
    "        data = response['data']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetchObjects(**kwargs):\n",
    "    # Default paramaters for API query\n",
    "    params = {\n",
    "        \"sort_type\":\"created_utc\",\n",
    "        \"sort\":\"asc\",\n",
    "        \"size\":10000\n",
    "        }\n",
    "\n",
    "    # Add additional paramters based on function arguments\n",
    "    for key,value in kwargs.items():\n",
    "        params[key] = value\n",
    "    \n",
    "    list_submission_ids = [\"16wirc\", \"162b0z\" ,\"15vob8\", \"184vmk\", \"17byde\", \"181c81\", \"181zkt\", \"16lito\", \"18slc9\", \"17ok2p\", \"189nuu\"]\n",
    "    inspir_ids=[\"17ok2p\", \"16lito\", \"189nuu\", \"17byde\"]\n",
    "    str_list_ids = \",\".join(list_submission_ids)\n",
    "        \n",
    "    # Perform an API request\n",
    "    r = requests.get(\"https://api.pushshift.io/reddit/comment/search/?link_id=\" + str_list_ids, params=params, timeout=30)\n",
    "\n",
    "    # Check the status code, if successful, process the data\n",
    "    if r.status_code == 200:\n",
    "        response = json.loads(r.text)\n",
    "        data = response['data']\n",
    "        sorted_data_by_id = sorted(data, key=lambda x: int(x['id'],36))\n",
    "        return sorted_data_by_id\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_reddit_data(**kwargs):\n",
    "    # Speficify the start timestamp\n",
    "    max_created_utc = 1356998400  # 01/01/2013 @ 12:00am (UTC)\n",
    "    max_id = 0\n",
    "    \n",
    "    for key,value in kwargs.items():\n",
    "        if key == \"columns\":\n",
    "            column_values = value\n",
    "    \n",
    "    df_all=pd.DataFrame(columns=column_values)\n",
    "    # While loop for recursive function\n",
    "    while 1:\n",
    "        nothing_processed = True\n",
    "        # Call the recursive function\n",
    "        objects = fetchObjects(**kwargs,after=max_created_utc)\n",
    "        print(len(objects))\n",
    "        df_new = pd.DataFrame(objects, columns = column_values)\n",
    "        df_all = pd.concat([df_all, df_new])\n",
    "\n",
    "        # Loop the returned data, ordered by date\n",
    "        for object in objects:\n",
    "            id = int(object['id'],36)\n",
    "            if id > max_id:\n",
    "                nothing_processed = False\n",
    "                created_utc = object['created_utc']\n",
    "                max_id = id\n",
    "                if created_utc > max_created_utc: max_created_utc = created_utc\n",
    "\n",
    "        # Exit if nothing happened\n",
    "        if nothing_processed: return df_all.reset_index(drop=True).sort_values(by=['score'], ascending=False)\n",
    "        max_created_utc -= 1\n",
    "\n",
    "        # Sleep a little before the next recursive function call\n",
    "        time.sleep(.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "66\n",
      "1\n",
      "(9267, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Otters hold hands when sleeping so they dont d...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3830</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>\\n\\n&amp;gt;Reddit, what is the happiest fact you ...</td>\n",
       "      <td>t1_c800pji</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3633</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>On 17 August 1999, a major earthquake struck T...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3168</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>I'm checking google for a picture now.\\nEDIT: ...</td>\n",
       "      <td>t1_c8006c2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3017</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>[The West Wing](http://www.youtube.com/watch?f...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>2980</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Otters rape baby seals.\\n\\nEdit: C'mon, since ...</td>\n",
       "      <td>t1_c8006c2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-17</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>Damn straight! Have an upvote.</td>\n",
       "      <td>t1_c800nmz</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-19</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5008</th>\n",
       "      <td>WOW it's the first time I've ever seen this me...</td>\n",
       "      <td>t1_c804925</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-24</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>I'm one day closer to death.</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-24</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7532</th>\n",
       "      <td>You're over-idealizing. You probably don't eve...</td>\n",
       "      <td>t1_c8056x5</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>-38</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9267 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   body   parent_id  \\\n",
       "54    Otters hold hands when sleeping so they dont d...   t3_16wirc   \n",
       "281   \\n\\n&gt;Reddit, what is the happiest fact you ...  t1_c800pji   \n",
       "88    On 17 August 1999, a major earthquake struck T...   t3_16wirc   \n",
       "96    I'm checking google for a picture now.\\nEDIT: ...  t1_c8006c2   \n",
       "157   [The West Wing](http://www.youtube.com/watch?f...   t3_16wirc   \n",
       "...                                                 ...         ...   \n",
       "101   Otters rape baby seals.\\n\\nEdit: C'mon, since ...  t1_c8006c2   \n",
       "410                      Damn straight! Have an upvote.  t1_c800nmz   \n",
       "5008  WOW it's the first time I've ever seen this me...  t1_c804925   \n",
       "49                         I'm one day closer to death.   t3_16wirc   \n",
       "7532  You're over-idealizing. You probably don't eve...  t1_c8056x5   \n",
       "\n",
       "     subreddit_id score  subreddit  \n",
       "54       t5_2qh1i  3830  AskReddit  \n",
       "281      t5_2qh1i  3633  AskReddit  \n",
       "88       t5_2qh1i  3168  AskReddit  \n",
       "96       t5_2qh1i  3017  AskReddit  \n",
       "157      t5_2qh1i  2980  AskReddit  \n",
       "...           ...   ...        ...  \n",
       "101      t5_2qh1i   -17  AskReddit  \n",
       "410      t5_2qh1i   -19  AskReddit  \n",
       "5008     t5_2qh1i   -24  AskReddit  \n",
       "49       t5_2qh1i   -24  AskReddit  \n",
       "7532     t5_2qh1i   -38  AskReddit  \n",
       "\n",
       "[9267 rows x 5 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_submission = extract_reddit_data(subreddit=\"AskReddit\", columns = ['body','parent_id','subreddit_id', 'score', 'subreddit'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Otters hold hands when sleeping so they dont d...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3830</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>\\n\\n&amp;gt;Reddit, what is the happiest fact you ...</td>\n",
       "      <td>t1_c800pji</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3633</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>On 17 August 1999, a major earthquake struck T...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3168</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>I'm checking google for a picture now.\\nEDIT: ...</td>\n",
       "      <td>t1_c8006c2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3017</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>[The West Wing](http://www.youtube.com/watch?f...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>2980</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5587</th>\n",
       "      <td>Theres always money in the banana stand.</td>\n",
       "      <td>t1_c803v4g</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5234</th>\n",
       "      <td>As well as Mac and Sweet Dee. Dude gets paid t...</td>\n",
       "      <td>t1_c80283l</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2380</th>\n",
       "      <td>I wonder if comedy clubs hire people to laugh ...</td>\n",
       "      <td>t1_c800vgt</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5220</th>\n",
       "      <td>A better word: spumiferous - adj. describing s...</td>\n",
       "      <td>t1_c801f29</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2364</th>\n",
       "      <td>That's what space is for</td>\n",
       "      <td>t1_c801cd2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8444 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   body   parent_id  \\\n",
       "54    Otters hold hands when sleeping so they dont d...   t3_16wirc   \n",
       "281   \\n\\n&gt;Reddit, what is the happiest fact you ...  t1_c800pji   \n",
       "88    On 17 August 1999, a major earthquake struck T...   t3_16wirc   \n",
       "96    I'm checking google for a picture now.\\nEDIT: ...  t1_c8006c2   \n",
       "157   [The West Wing](http://www.youtube.com/watch?f...   t3_16wirc   \n",
       "...                                                 ...         ...   \n",
       "5587           Theres always money in the banana stand.  t1_c803v4g   \n",
       "5234  As well as Mac and Sweet Dee. Dude gets paid t...  t1_c80283l   \n",
       "2380  I wonder if comedy clubs hire people to laugh ...  t1_c800vgt   \n",
       "5220  A better word: spumiferous - adj. describing s...  t1_c801f29   \n",
       "2364                           That's what space is for  t1_c801cd2   \n",
       "\n",
       "     subreddit_id score  subreddit  \n",
       "54       t5_2qh1i  3830  AskReddit  \n",
       "281      t5_2qh1i  3633  AskReddit  \n",
       "88       t5_2qh1i  3168  AskReddit  \n",
       "96       t5_2qh1i  3017  AskReddit  \n",
       "157      t5_2qh1i  2980  AskReddit  \n",
       "...           ...   ...        ...  \n",
       "5587     t5_2qh1i     1  AskReddit  \n",
       "5234     t5_2qh1i     1  AskReddit  \n",
       "2380     t5_2qh1i     1  AskReddit  \n",
       "5220     t5_2qh1i     1  AskReddit  \n",
       "2364     t5_2qh1i     1  AskReddit  \n",
       "\n",
       "[8444 rows x 5 columns]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_submission = df_submission[~(df_submission['score'] <= 0)]\n",
    "df_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Otters hold hands when sleeping so they dont d...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3830</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>&amp;gt;Reddit, what is the happiest fact you know...</td>\n",
       "      <td>t1_c800pji</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3633</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>On 17 August 1999, a major earthquake struck T...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3168</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>I'm checking google for a picture now.EDIT: So...</td>\n",
       "      <td>t1_c8006c2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3017</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>[The West Wing](A few years ago, Voyager 1 cro...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>2980</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5587</th>\n",
       "      <td>Theres always money in the banana stand.</td>\n",
       "      <td>t1_c803v4g</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5234</th>\n",
       "      <td>As well as Mac and Sweet Dee. Dude gets paid t...</td>\n",
       "      <td>t1_c80283l</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2380</th>\n",
       "      <td>I wonder if comedy clubs hire people to laugh ...</td>\n",
       "      <td>t1_c800vgt</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5220</th>\n",
       "      <td>A better word: spumiferous - adj. describing s...</td>\n",
       "      <td>t1_c801f29</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2364</th>\n",
       "      <td>That's what space is for</td>\n",
       "      <td>t1_c801cd2</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8444 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   body   parent_id  \\\n",
       "54    Otters hold hands when sleeping so they dont d...   t3_16wirc   \n",
       "281   &gt;Reddit, what is the happiest fact you know...  t1_c800pji   \n",
       "88    On 17 August 1999, a major earthquake struck T...   t3_16wirc   \n",
       "96    I'm checking google for a picture now.EDIT: So...  t1_c8006c2   \n",
       "157   [The West Wing](A few years ago, Voyager 1 cro...   t3_16wirc   \n",
       "...                                                 ...         ...   \n",
       "5587           Theres always money in the banana stand.  t1_c803v4g   \n",
       "5234  As well as Mac and Sweet Dee. Dude gets paid t...  t1_c80283l   \n",
       "2380  I wonder if comedy clubs hire people to laugh ...  t1_c800vgt   \n",
       "5220  A better word: spumiferous - adj. describing s...  t1_c801f29   \n",
       "2364                           That's what space is for  t1_c801cd2   \n",
       "\n",
       "     subreddit_id score  subreddit  \n",
       "54       t5_2qh1i  3830  AskReddit  \n",
       "281      t5_2qh1i  3633  AskReddit  \n",
       "88       t5_2qh1i  3168  AskReddit  \n",
       "96       t5_2qh1i  3017  AskReddit  \n",
       "157      t5_2qh1i  2980  AskReddit  \n",
       "...           ...   ...        ...  \n",
       "5587     t5_2qh1i     1  AskReddit  \n",
       "5234     t5_2qh1i     1  AskReddit  \n",
       "2380     t5_2qh1i     1  AskReddit  \n",
       "5220     t5_2qh1i     1  AskReddit  \n",
       "2364     t5_2qh1i     1  AskReddit  \n",
       "\n",
       "[8444 rows x 5 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for index in df_submission.index:\n",
    "    selftext = df_submission.loc[index,'body']\n",
    "    selftext = re.sub(r'http\\S+', '', selftext)\n",
    "    selftext = selftext.replace(\"\\n\", \"\")\n",
    "    selftext = re.sub('\\s{2,}', ' ', selftext)  # To remove more than one space\n",
    "    df_submission.loc[index,'body'] = selftext.rstrip() #remove new line\n",
    "df_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-115-e4d38da0bc06>:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_submission2 = df_submission2[(df_submission['body']).str.split().str.len().lt(201)]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_body</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>score_com</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Otters hold hands when sleeping so they dont d...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3830</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>&amp;gt;Reddit, what is the happiest fact you know...</td>\n",
       "      <td>t1_c800pji</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3633</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>On 17 August 1999, a major earthquake struck T...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>3168</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>[The West Wing](A few years ago, Voyager 1 cro...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>2980</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>Despite high infant mortality rates and lower ...</td>\n",
       "      <td>t3_16wirc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>2679</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5235</th>\n",
       "      <td>I think you missed the happy part. unless your...</td>\n",
       "      <td>t1_c805abc</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5558</th>\n",
       "      <td>So A termite walks into the bar and asks \"Is t...</td>\n",
       "      <td>t1_c801s5v</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5234</th>\n",
       "      <td>As well as Mac and Sweet Dee. Dude gets paid t...</td>\n",
       "      <td>t1_c80283l</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2380</th>\n",
       "      <td>I wonder if comedy clubs hire people to laugh ...</td>\n",
       "      <td>t1_c800vgt</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5220</th>\n",
       "      <td>A better word: spumiferous - adj. describing s...</td>\n",
       "      <td>t1_c801f29</td>\n",
       "      <td>t5_2qh1i</td>\n",
       "      <td>1</td>\n",
       "      <td>AskReddit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3670 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           comment_body          id  \\\n",
       "54    Otters hold hands when sleeping so they dont d...   t3_16wirc   \n",
       "281   &gt;Reddit, what is the happiest fact you know...  t1_c800pji   \n",
       "88    On 17 August 1999, a major earthquake struck T...   t3_16wirc   \n",
       "157   [The West Wing](A few years ago, Voyager 1 cro...   t3_16wirc   \n",
       "164   Despite high infant mortality rates and lower ...   t3_16wirc   \n",
       "...                                                 ...         ...   \n",
       "5235  I think you missed the happy part. unless your...  t1_c805abc   \n",
       "5558  So A termite walks into the bar and asks \"Is t...  t1_c801s5v   \n",
       "5234  As well as Mac and Sweet Dee. Dude gets paid t...  t1_c80283l   \n",
       "2380  I wonder if comedy clubs hire people to laugh ...  t1_c800vgt   \n",
       "5220  A better word: spumiferous - adj. describing s...  t1_c801f29   \n",
       "\n",
       "     subreddit_id score_com  subreddit  \n",
       "54       t5_2qh1i      3830  AskReddit  \n",
       "281      t5_2qh1i      3633  AskReddit  \n",
       "88       t5_2qh1i      3168  AskReddit  \n",
       "157      t5_2qh1i      2980  AskReddit  \n",
       "164      t5_2qh1i      2679  AskReddit  \n",
       "...           ...       ...        ...  \n",
       "5235     t5_2qh1i         1  AskReddit  \n",
       "5558     t5_2qh1i         1  AskReddit  \n",
       "5234     t5_2qh1i         1  AskReddit  \n",
       "2380     t5_2qh1i         1  AskReddit  \n",
       "5220     t5_2qh1i         1  AskReddit  \n",
       "\n",
       "[3670 rows x 5 columns]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_submission2 = df_submission[(df_submission['body']).str.split().str.len().gt(10)]\n",
    "df_submission2 = df_submission2[(df_submission['body']).str.split().str.len().lt(201)]\n",
    "df_submission2 = df_submission2.rename(columns={'body': \"comment_body\",'parent_id': \"id\", \"score\":\"score_com\"})\n",
    "df_submission2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission2.transpose().to_json('data/comments_for_AskReddit_specific.json', index = 'true')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}