FnR News - Data Cleaning - December 2022.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 13062,
     "status": "ok",
     "timestamp": 1674349770371,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "zRFIJBOUiB_Z",
    "outputId": "9c52c13a-98b9-45c8-b78d-ce64fe61bca1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6DAKgogd4izc"
   },
   "source": [
    "# Initial Cleaning stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "tVnAHf1-kcsN"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "MONGOLOGIN = os.environ.get(\"MONGOLOGIN\")\n",
    "import pandas as pd\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "PrqQHQh2k_Va"
   },
   "outputs": [],
   "source": [
    "client = pymongo.MongoClient(MONGOLOGIN)\n",
    "db = client['Database']\n",
    "articlesMongo = db[\"articles\"]\n",
    "linksMongo = db[\"links\"]\n",
    "blockedLinksMongo = db['blocked_links']\n",
    "\n",
    "articles_df = pd.DataFrame(list(articlesMongo.find()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "nZ9OqVH4lNVH"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "      <th>Invalid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid_1</td>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "      <td>577</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>covid_2</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "      <td>731</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>covid_3</td>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "      <td>748</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>covid_4</td>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "      <td>1924</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>covid_5</td>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "      <td>641</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16309</th>\n",
       "      <td>donald trump_443</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "      <td>1397</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16310</th>\n",
       "      <td>donald trump_444</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "      <td>4612</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16311</th>\n",
       "      <td>donald trump_445</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "      <td>3279</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16312</th>\n",
       "      <td>donald trump_446</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "      <td>2365</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16313</th>\n",
       "      <td>donald trump_447</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "      <td>4734</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16314 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    uid                       _id                  pub_date  \\\n",
       "0               covid_1  6374ae8bc8a337344e61280f  2022-10-27T21:30:00.272Z   \n",
       "1               covid_2  6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "2               covid_3  6374ae8bc8a337344e612813      2022-11-07T20:50:03Z   \n",
       "3               covid_4  6374ae8bc8a337344e612815                       NaN   \n",
       "4               covid_5  6374ae8bc8a337344e612829              Oct 17, 2022   \n",
       "...                 ...                       ...                       ...   \n",
       "16309  donald trump_443  638dc20c6f39fded0f8bf127                       NaN   \n",
       "16310  donald trump_444  638dc2176f39fded0f8bf128                       NaN   \n",
       "16311  donald trump_445  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "16312  donald trump_446  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "16313  donald trump_447  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16309  https://www.wbal.com/article/594732/111/trumps...  donald trump   \n",
       "16310  https://www.wgbh.org/news/national-news/2022/1...  donald trump   \n",
       "16311  https://www.yahoo.com/news/janes-directors-don...  donald trump   \n",
       "16312  https://www.yahoo.com/video/donald-trumps-2024...  donald trump   \n",
       "16313  https://www.yahoo.com/video/donald-trumps-turb...  donald trump   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...        google   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...        google   \n",
       "2      Beijing to improve COVID prevention policy ope...        google   \n",
       "3      China doubles down on zero-Covid pledge as pub...        google   \n",
       "4      Governor Newsom to End the COVID-19 State of E...        google   \n",
       "...                                                  ...           ...   \n",
       "16309  Trump's call to suspend Constitution not a 202...    duckduckgo   \n",
       "16310  Former President Donald Trump announces he is ...    duckduckgo   \n",
       "16311  ‘The Janes’ Directors On How Donald Trump Spur...        google   \n",
       "16312  Donald Trump's 2024 Presidential Campaign Is B...    duckduckgo   \n",
       "16313            Donald Trump's turbulence-filled launch          bing   \n",
       "\n",
       "                                         cleaned_content  wordcount  Invalid  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...        577      NaN  \n",
       "1      add potential swell covid 19 infection winter ...        731      NaN  \n",
       "2      beijing nov 7 reuters beijing improve way covi...        748      NaN  \n",
       "3      china reiterated unswerving commitment longsta...       1924      NaN  \n",
       "4      published oct 17 2022california pandemic respo...        641      NaN  \n",
       "...                                                  ...        ...      ...  \n",
       "16309  abc news washington republican ohio rep dave j...       1397      NaN  \n",
       "16310  donald trump tried overthrow result 2020 presi...       4612      NaN  \n",
       "16311  shortly donald trump became president united s...       3279      NaN  \n",
       "16312  donald trump suddenly reminded 2016 political ...       2365      NaN  \n",
       "16313  donald trump team starting institute new syste...       4734      NaN  \n",
       "\n",
       "[16314 rows x 10 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned = pd.read_excel('cleaned.xlsx', index_col=0)\n",
    "cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>content</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6374ae8bc8a337344e612806</td>\n",
       "      <td>2022-11-07T18:36:24-0500</td>\n",
       "      <td>https://www.cbsnews.com/boston/news/boston-uni...</td>\n",
       "      <td>covid</td>\n",
       "      <td>What's going on with the BU biolab COVID exper...</td>\n",
       "      <td>By Paula EbbenNovember 7, 2022 / 6:36 PM      ...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6374ae8bc8a337344e612807</td>\n",
       "      <td>2022-10-17</td>\n",
       "      <td>https://www.nature.com/articles/s41562-022-014...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Life expectancy changes since COVID-19 | Natur...</td>\n",
       "      <td>70k Accesses2333 AltmetricMetrics detailsThe C...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6374ae8bc8a337344e612808</td>\n",
       "      <td>2022-11-07T05:22:28+0000</td>\n",
       "      <td>https://www.cnbc.com/2022/11/07/oil-markets-pr...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Oil falls in choppy trade on mixed China Covid...</td>\n",
       "      <td>Oil prices fell on Monday, paring gains after ...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>Add in a potential swell in COVID-19 infection...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6374ae8bc8a337344e61280a</td>\n",
       "      <td>2022-11-06T23:14:25+0000</td>\n",
       "      <td>https://www.cnbc.com/2022/11/06/apple-warns-co...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Apple warns Covid restrictions in China are hu...</td>\n",
       "      <td>In this articleApple said in a statement on Su...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20334</th>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>ABC News(WASHINGTON) -- Republican Ohio Rep. D...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20335</th>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>Donald Trump, who tried to overthrow the resul...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20336</th>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>Shortly after Donald Trump became president of...</td>\n",
       "      <td>google</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20337</th>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>Donald Trump is suddenly being reminded it's n...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20338</th>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>Donald Trump's team is starting to institute a...</td>\n",
       "      <td>bing</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20339 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            _id                  pub_date  \\\n",
       "0      6374ae8bc8a337344e612806  2022-11-07T18:36:24-0500   \n",
       "1      6374ae8bc8a337344e612807                2022-10-17   \n",
       "2      6374ae8bc8a337344e612808  2022-11-07T05:22:28+0000   \n",
       "3      6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "4      6374ae8bc8a337344e61280a  2022-11-06T23:14:25+0000   \n",
       "...                         ...                       ...   \n",
       "20334  638dc20c6f39fded0f8bf127                             \n",
       "20335  638dc2176f39fded0f8bf128                             \n",
       "20336  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "20337  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "20338  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbsnews.com/boston/news/boston-uni...         covid   \n",
       "1      https://www.nature.com/articles/s41562-022-014...         covid   \n",
       "2      https://www.cnbc.com/2022/11/07/oil-markets-pr...         covid   \n",
       "3      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "4      https://www.cnbc.com/2022/11/06/apple-warns-co...         covid   \n",
       "...                                                  ...           ...   \n",
       "20334  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "20335  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "20336  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "20337  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "20338  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline  \\\n",
       "0      What's going on with the BU biolab COVID exper...   \n",
       "1      Life expectancy changes since COVID-19 | Natur...   \n",
       "2      Oil falls in choppy trade on mixed China Covid...   \n",
       "3      Bay Area could face COVID, flu and RSV in the ...   \n",
       "4      Apple warns Covid restrictions in China are hu...   \n",
       "...                                                  ...   \n",
       "20334  Trump's call to suspend Constitution not a 202...   \n",
       "20335  Former President Donald Trump announces he is ...   \n",
       "20336  ‘The Janes’ Directors On How Donald Trump Spur...   \n",
       "20337  Donald Trump's 2024 Presidential Campaign Is B...   \n",
       "20338            Donald Trump's turbulence-filled launch   \n",
       "\n",
       "                                                 content search_engine uid  \n",
       "0      By Paula EbbenNovember 7, 2022 / 6:36 PM      ...        google      \n",
       "1      70k Accesses2333 AltmetricMetrics detailsThe C...        google      \n",
       "2      Oil prices fell on Monday, paring gains after ...        google      \n",
       "3      Add in a potential swell in COVID-19 infection...        google      \n",
       "4      In this articleApple said in a statement on Su...        google      \n",
       "...                                                  ...           ...  ..  \n",
       "20334  ABC News(WASHINGTON) -- Republican Ohio Rep. D...    duckduckgo      \n",
       "20335  Donald Trump, who tried to overthrow the resul...    duckduckgo      \n",
       "20336  Shortly after Donald Trump became president of...        google      \n",
       "20337  Donald Trump is suddenly being reminded it's n...    duckduckgo      \n",
       "20338  Donald Trump's team is starting to institute a...          bing      \n",
       "\n",
       "[20339 rows x 8 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df = pd.merge(articles_df, cleaned[['url', 'uid']], on='url', how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid_y</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>content</th>\n",
       "      <th>search_engine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid_7</td>\n",
       "      <td>6374ae8bc8a337344e612806</td>\n",
       "      <td>2022-11-07T18:36:24-0500</td>\n",
       "      <td>https://www.cbsnews.com/boston/news/boston-uni...</td>\n",
       "      <td>covid</td>\n",
       "      <td>What's going on with the BU biolab COVID exper...</td>\n",
       "      <td>By Paula EbbenNovember 7, 2022 / 6:36 PM      ...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>covid_8</td>\n",
       "      <td>6374ae8bc8a337344e612807</td>\n",
       "      <td>2022-10-17</td>\n",
       "      <td>https://www.nature.com/articles/s41562-022-014...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Life expectancy changes since COVID-19 | Natur...</td>\n",
       "      <td>70k Accesses2333 AltmetricMetrics detailsThe C...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>covid_30</td>\n",
       "      <td>6374ae8bc8a337344e612808</td>\n",
       "      <td>2022-11-07T05:22:28+0000</td>\n",
       "      <td>https://www.cnbc.com/2022/11/07/oil-markets-pr...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Oil falls in choppy trade on mixed China Covid...</td>\n",
       "      <td>Oil prices fell on Monday, paring gains after ...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>covid_2</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>Add in a potential swell in COVID-19 infection...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>covid_18</td>\n",
       "      <td>6374ae8bc8a337344e61280a</td>\n",
       "      <td>2022-11-06T23:14:25+0000</td>\n",
       "      <td>https://www.cnbc.com/2022/11/06/apple-warns-co...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Apple warns Covid restrictions in China are hu...</td>\n",
       "      <td>In this articleApple said in a statement on Su...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20334</th>\n",
       "      <td>donald trump_443</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>ABC News(WASHINGTON) -- Republican Ohio Rep. D...</td>\n",
       "      <td>duckduckgo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20335</th>\n",
       "      <td>donald trump_444</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>Donald Trump, who tried to overthrow the resul...</td>\n",
       "      <td>duckduckgo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20336</th>\n",
       "      <td>donald trump_445</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>Shortly after Donald Trump became president of...</td>\n",
       "      <td>google</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20337</th>\n",
       "      <td>donald trump_446</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>Donald Trump is suddenly being reminded it's n...</td>\n",
       "      <td>duckduckgo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20338</th>\n",
       "      <td>donald trump_447</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>Donald Trump's team is starting to institute a...</td>\n",
       "      <td>bing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20339 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  uid_y                       _id                  pub_date  \\\n",
       "0               covid_7  6374ae8bc8a337344e612806  2022-11-07T18:36:24-0500   \n",
       "1               covid_8  6374ae8bc8a337344e612807                2022-10-17   \n",
       "2              covid_30  6374ae8bc8a337344e612808  2022-11-07T05:22:28+0000   \n",
       "3               covid_2  6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "4              covid_18  6374ae8bc8a337344e61280a  2022-11-06T23:14:25+0000   \n",
       "...                 ...                       ...                       ...   \n",
       "20334  donald trump_443  638dc20c6f39fded0f8bf127                             \n",
       "20335  donald trump_444  638dc2176f39fded0f8bf128                             \n",
       "20336  donald trump_445  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "20337  donald trump_446  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "20338  donald trump_447  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbsnews.com/boston/news/boston-uni...         covid   \n",
       "1      https://www.nature.com/articles/s41562-022-014...         covid   \n",
       "2      https://www.cnbc.com/2022/11/07/oil-markets-pr...         covid   \n",
       "3      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "4      https://www.cnbc.com/2022/11/06/apple-warns-co...         covid   \n",
       "...                                                  ...           ...   \n",
       "20334  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "20335  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "20336  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "20337  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "20338  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline  \\\n",
       "0      What's going on with the BU biolab COVID exper...   \n",
       "1      Life expectancy changes since COVID-19 | Natur...   \n",
       "2      Oil falls in choppy trade on mixed China Covid...   \n",
       "3      Bay Area could face COVID, flu and RSV in the ...   \n",
       "4      Apple warns Covid restrictions in China are hu...   \n",
       "...                                                  ...   \n",
       "20334  Trump's call to suspend Constitution not a 202...   \n",
       "20335  Former President Donald Trump announces he is ...   \n",
       "20336  ‘The Janes’ Directors On How Donald Trump Spur...   \n",
       "20337  Donald Trump's 2024 Presidential Campaign Is B...   \n",
       "20338            Donald Trump's turbulence-filled launch   \n",
       "\n",
       "                                                 content search_engine  \n",
       "0      By Paula EbbenNovember 7, 2022 / 6:36 PM      ...        google  \n",
       "1      70k Accesses2333 AltmetricMetrics detailsThe C...        google  \n",
       "2      Oil prices fell on Monday, paring gains after ...        google  \n",
       "3      Add in a potential swell in COVID-19 infection...        google  \n",
       "4      In this articleApple said in a statement on Su...        google  \n",
       "...                                                  ...           ...  \n",
       "20334  ABC News(WASHINGTON) -- Republican Ohio Rep. D...    duckduckgo  \n",
       "20335  Donald Trump, who tried to overthrow the resul...    duckduckgo  \n",
       "20336  Shortly after Donald Trump became president of...        google  \n",
       "20337  Donald Trump is suddenly being reminded it's n...    duckduckgo  \n",
       "20338  Donald Trump's team is starting to institute a...          bing  \n",
       "\n",
       "[20339 rows x 8 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = list(out_df)\n",
    "# move the column to head of list using index, pop and insert\n",
    "cols.insert(0, cols.pop(cols.index('uid_y')))\n",
    "out_df = out_df.loc[:, cols]\n",
    "out_df.pop('uid_x')\n",
    "out_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_df.to_excel('raw.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1672781141126,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "ntGjBLVilblR",
    "outputId": "bc94a04a-85fe-47b2-dd69-8fd68b49a0d9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid_1</td>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>covid_2</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>covid_3</td>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>covid_4</td>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>covid_5</td>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16309</th>\n",
       "      <td>donald trump_443</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16310</th>\n",
       "      <td>donald trump_444</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16311</th>\n",
       "      <td>donald trump_445</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16312</th>\n",
       "      <td>donald trump_446</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16313</th>\n",
       "      <td>donald trump_447</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16314 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    uid                       _id\n",
       "0               covid_1  6374ae8bc8a337344e61280f\n",
       "1               covid_2  6374ae8bc8a337344e612809\n",
       "2               covid_3  6374ae8bc8a337344e612813\n",
       "3               covid_4  6374ae8bc8a337344e612815\n",
       "4               covid_5  6374ae8bc8a337344e612829\n",
       "...                 ...                       ...\n",
       "16309  donald trump_443  638dc20c6f39fded0f8bf127\n",
       "16310  donald trump_444  638dc2176f39fded0f8bf128\n",
       "16311  donald trump_445  638dc22d6f39fded0f8bf129\n",
       "16312  donald trump_446  638dc23d6f39fded0f8bf12a\n",
       "16313  donald trump_447  638dc2446f39fded0f8bf12b\n",
       "\n",
       "[16314 rows x 2 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned[['uid', '_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iIp_bMfOllaE"
   },
   "outputs": [],
   "source": [
    "df = df.loc[~df.duplicated(['content'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 754701,
     "status": "ok",
     "timestamp": 1671159950788,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "mXGcsB_UWx3i",
    "outputId": "d0e9477d-524d-4af3-d366-01245c3475d5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "  word_tokens = word_tokenize(row.content)\n",
    "  # converts the words in word_tokens to lower case and then checks whether\n",
    "  #they are present in stop_words or not\n",
    "  filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]\n",
    "  #with no lower case conversion\n",
    "  filtered_sentence = []\n",
    "\n",
    "  for w in word_tokens:\n",
    "    if w not in stop_words:\n",
    "      filtered_sentence.append(w)\n",
    "\n",
    "  # Remove all the special characters\n",
    "  document = re.sub(r'\\W', ' ', str(filtered_sentence))\n",
    "\n",
    "  # remove all single characters\n",
    "  document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
    "\n",
    "  # Remove single characters from the start\n",
    "  document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n",
    "\n",
    "  # Substituting multiple spaces with single space\n",
    "  document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
    "\n",
    "  # Converting to Lowercase\n",
    "  document = document.lower()\n",
    "\n",
    "  # Lemmatization\n",
    "  document = document.split()\n",
    "\n",
    "\n",
    "  stemmer = WordNetLemmatizer()\n",
    "  document = [stemmer.lemmatize(word) for word in document]\n",
    "  document = ' '.join(document)\n",
    "\n",
    "  df.iloc[index]['cleaned_content'] = document\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iM5N9Yjtc8sU"
   },
   "outputs": [],
   "source": [
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1671159182499,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "a2inq6BreZw9",
    "outputId": "06860b79-6de1-424f-fdd9-45ac1c5d8d88"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16960, 8)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 311,
     "status": "ok",
     "timestamp": 1671159154992,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "y2KH7phwqAIc",
    "outputId": "2d71950b-7e21-466c-a5cc-4a760a53f072"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-78-1d6729c317be>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['cleaned_content'] = \"\"\n"
     ]
    }
   ],
   "source": [
    "df['cleaned_content'] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 641
    },
    "executionInfo": {
     "elapsed": 330,
     "status": "ok",
     "timestamp": 1671162711561,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "hgnTzMhqypeB",
    "outputId": "50a5c927-b539-4d70-8bdb-189b5d3f4b58"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-7b665321-238e-4456-9841-f6cdb979e880\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>content</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>Since the onset of the COVID-19 pandemic in Ma...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>Add in a potential swell in COVID-19 infection...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>BEIJING, Nov 7 (Reuters) - Beijing will improv...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>China has reiterated its unswerving commitment...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>Published: Oct 17, 2022California's pandemic r...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16955</th>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>ABC News(WASHINGTON) -- Republican Ohio Rep. D...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16956</th>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>Donald Trump, who tried to overthrow the resul...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16957</th>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>Shortly after Donald Trump became president of...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16958</th>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>Donald Trump is suddenly being reminded it's n...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16959</th>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>Donald Trump's team is starting to institute a...</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16960 rows × 8 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7b665321-238e-4456-9841-f6cdb979e880')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-7b665321-238e-4456-9841-f6cdb979e880 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-7b665321-238e-4456-9841-f6cdb979e880');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                            _id                  pub_date  \\\n",
       "0      6374ae8bc8a337344e61280f  2022-10-27T21:30:00.272Z   \n",
       "1      6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "2      6374ae8bc8a337344e612813      2022-11-07T20:50:03Z   \n",
       "3      6374ae8bc8a337344e612815                       NaN   \n",
       "4      6374ae8bc8a337344e612829              Oct 17, 2022   \n",
       "...                         ...                       ...   \n",
       "16955  638dc20c6f39fded0f8bf127                             \n",
       "16956  638dc2176f39fded0f8bf128                             \n",
       "16957  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "16958  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "16959  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16955  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "16956  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "16957  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "16958  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "16959  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...   \n",
       "2      Beijing to improve COVID prevention policy ope...   \n",
       "3      China doubles down on zero-Covid pledge as pub...   \n",
       "4      Governor Newsom to End the COVID-19 State of E...   \n",
       "...                                                  ...   \n",
       "16955  Trump's call to suspend Constitution not a 202...   \n",
       "16956  Former President Donald Trump announces he is ...   \n",
       "16957  ‘The Janes’ Directors On How Donald Trump Spur...   \n",
       "16958  Donald Trump's 2024 Presidential Campaign Is B...   \n",
       "16959            Donald Trump's turbulence-filled launch   \n",
       "\n",
       "                                                 content search_engine  \\\n",
       "0      Since the onset of the COVID-19 pandemic in Ma...        google   \n",
       "1      Add in a potential swell in COVID-19 infection...        google   \n",
       "2      BEIJING, Nov 7 (Reuters) - Beijing will improv...        google   \n",
       "3      China has reiterated its unswerving commitment...        google   \n",
       "4      Published: Oct 17, 2022California's pandemic r...        google   \n",
       "...                                                  ...           ...   \n",
       "16955  ABC News(WASHINGTON) -- Republican Ohio Rep. D...    duckduckgo   \n",
       "16956  Donald Trump, who tried to overthrow the resul...    duckduckgo   \n",
       "16957  Shortly after Donald Trump became president of...        google   \n",
       "16958  Donald Trump is suddenly being reminded it's n...    duckduckgo   \n",
       "16959  Donald Trump's team is starting to institute a...          bing   \n",
       "\n",
       "                                         cleaned_content  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...  \n",
       "1      add potential swell covid 19 infection winter ...  \n",
       "2      beijing nov 7 reuters beijing improve way covi...  \n",
       "3      china reiterated unswerving commitment longsta...  \n",
       "4      published oct 17 2022california pandemic respo...  \n",
       "...                                                  ...  \n",
       "16955  abc news washington republican ohio rep dave j...  \n",
       "16956  donald trump tried overthrow result 2020 presi...  \n",
       "16957  shortly donald trump became president united s...  \n",
       "16958  donald trump suddenly reminded 2016 political ...  \n",
       "16959  donald trump team starting institute new syste...  \n",
       "\n",
       "[16960 rows x 8 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 467
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1671162971864,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "9FqFWOC-EVpC",
    "outputId": "82008216-d7f1-4edb-972b-fa1e45222f4e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-33ad950a-8477-41dc-99c7-4bb3bac7e049\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16955</th>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16956</th>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td></td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16957</th>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16958</th>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16959</th>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16960 rows × 7 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-33ad950a-8477-41dc-99c7-4bb3bac7e049')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-33ad950a-8477-41dc-99c7-4bb3bac7e049 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-33ad950a-8477-41dc-99c7-4bb3bac7e049');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                            _id                  pub_date  \\\n",
       "0      6374ae8bc8a337344e61280f  2022-10-27T21:30:00.272Z   \n",
       "1      6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "2      6374ae8bc8a337344e612813      2022-11-07T20:50:03Z   \n",
       "3      6374ae8bc8a337344e612815                       NaN   \n",
       "4      6374ae8bc8a337344e612829              Oct 17, 2022   \n",
       "...                         ...                       ...   \n",
       "16955  638dc20c6f39fded0f8bf127                             \n",
       "16956  638dc2176f39fded0f8bf128                             \n",
       "16957  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "16958  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "16959  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16955  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "16956  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "16957  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "16958  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "16959  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...        google   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...        google   \n",
       "2      Beijing to improve COVID prevention policy ope...        google   \n",
       "3      China doubles down on zero-Covid pledge as pub...        google   \n",
       "4      Governor Newsom to End the COVID-19 State of E...        google   \n",
       "...                                                  ...           ...   \n",
       "16955  Trump's call to suspend Constitution not a 202...    duckduckgo   \n",
       "16956  Former President Donald Trump announces he is ...    duckduckgo   \n",
       "16957  ‘The Janes’ Directors On How Donald Trump Spur...        google   \n",
       "16958  Donald Trump's 2024 Presidential Campaign Is B...    duckduckgo   \n",
       "16959            Donald Trump's turbulence-filled launch          bing   \n",
       "\n",
       "                                         cleaned_content  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...  \n",
       "1      add potential swell covid 19 infection winter ...  \n",
       "2      beijing nov 7 reuters beijing improve way covi...  \n",
       "3      china reiterated unswerving commitment longsta...  \n",
       "4      published oct 17 2022california pandemic respo...  \n",
       "...                                                  ...  \n",
       "16955  abc news washington republican ohio rep dave j...  \n",
       "16956  donald trump tried overthrow result 2020 presi...  \n",
       "16957  shortly donald trump became president united s...  \n",
       "16958  donald trump suddenly reminded 2016 political ...  \n",
       "16959  donald trump team starting institute new syste...  \n",
       "\n",
       "[16960 rows x 7 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf = df.drop(['content'], axis=1)\n",
    "tdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qnmKfanay6tZ"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "tdf.to_excel('/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kLqAbLwKE8vL"
   },
   "outputs": [],
   "source": [
    "df = pd.read_excel('/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NnmOsESFiP_T"
   },
   "outputs": [],
   "source": [
    "df['wordcount'] = df['cleaned_content'].str.split().str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 1080,
     "status": "ok",
     "timestamp": 1673219868025,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "5By9AXadixKC",
    "outputId": "6f4da8bc-9785-4d74-bafe-cdb8c885b49e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-bf492196-19a2-4b06-a7bb-7ade9ba6234f\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "      <td>577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "      <td>731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "      <td>748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "      <td>1924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16309</th>\n",
       "      <td>16955</td>\n",
       "      <td>16955</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "      <td>1397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16310</th>\n",
       "      <td>16956</td>\n",
       "      <td>16956</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "      <td>4612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16311</th>\n",
       "      <td>16957</td>\n",
       "      <td>16957</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "      <td>3279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16312</th>\n",
       "      <td>16958</td>\n",
       "      <td>16958</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "      <td>2365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16313</th>\n",
       "      <td>16959</td>\n",
       "      <td>16959</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "      <td>4734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16314 rows × 10 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bf492196-19a2-4b06-a7bb-7ade9ba6234f')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-bf492196-19a2-4b06-a7bb-7ade9ba6234f button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-bf492196-19a2-4b06-a7bb-7ade9ba6234f');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "       Unnamed: 0  Unnamed: 0.1                       _id  \\\n",
       "0               0             0  6374ae8bc8a337344e61280f   \n",
       "1               1             1  6374ae8bc8a337344e612809   \n",
       "2               2             2  6374ae8bc8a337344e612813   \n",
       "3               3             3  6374ae8bc8a337344e612815   \n",
       "4               4             4  6374ae8bc8a337344e612829   \n",
       "...           ...           ...                       ...   \n",
       "16309       16955         16955  638dc20c6f39fded0f8bf127   \n",
       "16310       16956         16956  638dc2176f39fded0f8bf128   \n",
       "16311       16957         16957  638dc22d6f39fded0f8bf129   \n",
       "16312       16958         16958  638dc23d6f39fded0f8bf12a   \n",
       "16313       16959         16959  638dc2446f39fded0f8bf12b   \n",
       "\n",
       "                       pub_date  \\\n",
       "0      2022-10-27T21:30:00.272Z   \n",
       "1           2022-11-06 12:00:56   \n",
       "2          2022-11-07T20:50:03Z   \n",
       "3                           NaN   \n",
       "4                  Oct 17, 2022   \n",
       "...                         ...   \n",
       "16309                       NaN   \n",
       "16310                       NaN   \n",
       "16311  2022-12-04T18:00:00.000Z   \n",
       "16312  2022-11-30T21:00:11.000Z   \n",
       "16313  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16309  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "16310  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "16311  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "16312  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "16313  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...        google   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...        google   \n",
       "2      Beijing to improve COVID prevention policy ope...        google   \n",
       "3      China doubles down on zero-Covid pledge as pub...        google   \n",
       "4      Governor Newsom to End the COVID-19 State of E...        google   \n",
       "...                                                  ...           ...   \n",
       "16309  Trump's call to suspend Constitution not a 202...    duckduckgo   \n",
       "16310  Former President Donald Trump announces he is ...    duckduckgo   \n",
       "16311  ‘The Janes’ Directors On How Donald Trump Spur...        google   \n",
       "16312  Donald Trump's 2024 Presidential Campaign Is B...    duckduckgo   \n",
       "16313            Donald Trump's turbulence-filled launch          bing   \n",
       "\n",
       "                                         cleaned_content  wordcount  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...        577  \n",
       "1      add potential swell covid 19 infection winter ...        731  \n",
       "2      beijing nov 7 reuters beijing improve way covi...        748  \n",
       "3      china reiterated unswerving commitment longsta...       1924  \n",
       "4      published oct 17 2022california pandemic respo...        641  \n",
       "...                                                  ...        ...  \n",
       "16309  abc news washington republican ohio rep dave j...       1397  \n",
       "16310  donald trump tried overthrow result 2020 presi...       4612  \n",
       "16311  shortly donald trump became president united s...       3279  \n",
       "16312  donald trump suddenly reminded 2016 political ...       2365  \n",
       "16313  donald trump team starting institute new syste...       4734  \n",
       "\n",
       "[16314 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xuC_AcBDixyD"
   },
   "outputs": [],
   "source": [
    "df = df.loc[df.wordcount > 50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1672781174900,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "19fzTyTRjRUy",
    "outputId": "08c2226b-572c-409e-bb23-955c0a2a9b2e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-a7c97210-5f2e-4c32-8a2d-f7a65c1a8061\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "      <td>577.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "      <td>731.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "      <td>748.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "      <td>1924.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "      <td>641.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16955</th>\n",
       "      <td>16955</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "      <td>1397.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16956</th>\n",
       "      <td>16956</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "      <td>4612.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16957</th>\n",
       "      <td>16957</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "      <td>3279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16958</th>\n",
       "      <td>16958</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "      <td>2365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16959</th>\n",
       "      <td>16959</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald+trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "      <td>4734.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16314 rows × 9 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a7c97210-5f2e-4c32-8a2d-f7a65c1a8061')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-a7c97210-5f2e-4c32-8a2d-f7a65c1a8061 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-a7c97210-5f2e-4c32-8a2d-f7a65c1a8061');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "       Unnamed: 0                       _id                  pub_date  \\\n",
       "0               0  6374ae8bc8a337344e61280f  2022-10-27T21:30:00.272Z   \n",
       "1               1  6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "2               2  6374ae8bc8a337344e612813      2022-11-07T20:50:03Z   \n",
       "3               3  6374ae8bc8a337344e612815                       NaN   \n",
       "4               4  6374ae8bc8a337344e612829              Oct 17, 2022   \n",
       "...           ...                       ...                       ...   \n",
       "16955       16955  638dc20c6f39fded0f8bf127                       NaN   \n",
       "16956       16956  638dc2176f39fded0f8bf128                       NaN   \n",
       "16957       16957  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "16958       16958  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "16959       16959  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16955  https://www.wbal.com/article/594732/111/trumps...  donald+trump   \n",
       "16956  https://www.wgbh.org/news/national-news/2022/1...  donald+trump   \n",
       "16957  https://www.yahoo.com/news/janes-directors-don...  donald+trump   \n",
       "16958  https://www.yahoo.com/video/donald-trumps-2024...  donald+trump   \n",
       "16959  https://www.yahoo.com/video/donald-trumps-turb...  donald+trump   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...        google   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...        google   \n",
       "2      Beijing to improve COVID prevention policy ope...        google   \n",
       "3      China doubles down on zero-Covid pledge as pub...        google   \n",
       "4      Governor Newsom to End the COVID-19 State of E...        google   \n",
       "...                                                  ...           ...   \n",
       "16955  Trump's call to suspend Constitution not a 202...    duckduckgo   \n",
       "16956  Former President Donald Trump announces he is ...    duckduckgo   \n",
       "16957  ‘The Janes’ Directors On How Donald Trump Spur...        google   \n",
       "16958  Donald Trump's 2024 Presidential Campaign Is B...    duckduckgo   \n",
       "16959            Donald Trump's turbulence-filled launch          bing   \n",
       "\n",
       "                                         cleaned_content  wordcount  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...      577.0  \n",
       "1      add potential swell covid 19 infection winter ...      731.0  \n",
       "2      beijing nov 7 reuters beijing improve way covi...      748.0  \n",
       "3      china reiterated unswerving commitment longsta...     1924.0  \n",
       "4      published oct 17 2022california pandemic respo...      641.0  \n",
       "...                                                  ...        ...  \n",
       "16955  abc news washington republican ohio rep dave j...     1397.0  \n",
       "16956  donald trump tried overthrow result 2020 presi...     4612.0  \n",
       "16957  shortly donald trump became president united s...     3279.0  \n",
       "16958  donald trump suddenly reminded 2016 political ...     2365.0  \n",
       "16959  donald trump team starting institute new syste...     4734.0  \n",
       "\n",
       "[16314 rows x 9 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 15907,
     "status": "ok",
     "timestamp": 1674349827743,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "TAVc_T-Yh1Hy"
   },
   "outputs": [],
   "source": [
    "df.to_excel('/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Zl3Ly02Aijfh"
   },
   "source": [
    "# Post-cleaned start-point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 16602,
     "status": "ok",
     "timestamp": 1674349786970,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "y2lAlKjfnat-"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned_v2.xlsx'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_excel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned_v2.xlsx\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex_col\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\util\\_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    209\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    210\u001b[0m         kwargs[new_arg_name] \u001b[38;5;241m=\u001b[39m new_arg_value\n\u001b[1;32m--> 211\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\util\\_decorators.py:331\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m num_allow_args:\n\u001b[0;32m    326\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m    327\u001b[0m         msg\u001b[38;5;241m.\u001b[39mformat(arguments\u001b[38;5;241m=\u001b[39m_format_argument_list(allow_args)),\n\u001b[0;32m    328\u001b[0m         \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[0;32m    329\u001b[0m         stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[0;32m    330\u001b[0m     )\n\u001b[1;32m--> 331\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\io\\excel\\_base.py:482\u001b[0m, in \u001b[0;36mread_excel\u001b[1;34m(io, sheet_name, header, names, index_col, usecols, squeeze, dtype, engine, converters, true_values, false_values, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, parse_dates, date_parser, thousands, decimal, comment, skipfooter, convert_float, mangle_dupe_cols, storage_options)\u001b[0m\n\u001b[0;32m    480\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(io, ExcelFile):\n\u001b[0;32m    481\u001b[0m     should_close \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m--> 482\u001b[0m     io \u001b[38;5;241m=\u001b[39m \u001b[43mExcelFile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mio\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    483\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m engine \u001b[38;5;129;01mand\u001b[39;00m engine \u001b[38;5;241m!=\u001b[39m io\u001b[38;5;241m.\u001b[39mengine:\n\u001b[0;32m    484\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    485\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mEngine should not be specified when passing \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    486\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124man ExcelFile - ExcelFile already has the engine set\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    487\u001b[0m     )\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\io\\excel\\_base.py:1652\u001b[0m, in \u001b[0;36mExcelFile.__init__\u001b[1;34m(self, path_or_buffer, engine, storage_options)\u001b[0m\n\u001b[0;32m   1650\u001b[0m     ext \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxls\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1651\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1652\u001b[0m     ext \u001b[38;5;241m=\u001b[39m \u001b[43minspect_excel_format\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1653\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontent_or_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\n\u001b[0;32m   1654\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1655\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m ext \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m   1656\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m   1657\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExcel file format cannot be determined, you must specify \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1658\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124man engine manually.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1659\u001b[0m         )\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\io\\excel\\_base.py:1525\u001b[0m, in \u001b[0;36minspect_excel_format\u001b[1;34m(content_or_path, storage_options)\u001b[0m\n\u001b[0;32m   1522\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(content_or_path, \u001b[38;5;28mbytes\u001b[39m):\n\u001b[0;32m   1523\u001b[0m     content_or_path \u001b[38;5;241m=\u001b[39m BytesIO(content_or_path)\n\u001b[1;32m-> 1525\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1526\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcontent_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mis_text\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[0;32m   1527\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m handle:\n\u001b[0;32m   1528\u001b[0m     stream \u001b[38;5;241m=\u001b[39m handle\u001b[38;5;241m.\u001b[39mhandle\n\u001b[0;32m   1529\u001b[0m     stream\u001b[38;5;241m.\u001b[39mseek(\u001b[38;5;241m0\u001b[39m)\n",
      "File \u001b[1;32m~\\.conda\\envs\\DataScience\\lib\\site-packages\\pandas\\io\\common.py:865\u001b[0m, in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m    856\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mopen\u001b[39m(\n\u001b[0;32m    857\u001b[0m             handle,\n\u001b[0;32m    858\u001b[0m             ioargs\u001b[38;5;241m.\u001b[39mmode,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    861\u001b[0m             newline\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    862\u001b[0m         )\n\u001b[0;32m    863\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    864\u001b[0m         \u001b[38;5;66;03m# Binary mode\u001b[39;00m\n\u001b[1;32m--> 865\u001b[0m         handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mioargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    866\u001b[0m     handles\u001b[38;5;241m.\u001b[39mappend(handle)\n\u001b[0;32m    868\u001b[0m \u001b[38;5;66;03m# Convert BytesIO or file objects passed with an encoding\u001b[39;00m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned_v2.xlsx'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pymongo\n",
    "df = pd.read_excel('/content/drive/MyDrive/Colab Notebooks/NLP project/cleaned.xlsx', index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sv6ssc194s0M"
   },
   "source": [
    "## Func declaration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2101,
     "status": "ok",
     "timestamp": 1673995210437,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "j64pVMG3qncL",
    "outputId": "8b861fe5-27be-4ecc-dd72-89f1cdaf42cb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def clean_corpus(X):\n",
    "  documents = []\n",
    "  for sen in range(0, len(X)):\n",
    "    \n",
    "      # Remove all the special characters\n",
    "      document = re.sub(r'\\W', ' ', str(X[sen]))\n",
    "\n",
    "      document = re.sub(r'\\w*\\d\\w*', '', document)\n",
    "\n",
    "      # remove all single characters\n",
    "      document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
    "\n",
    "      # Remove single characters from the start\n",
    "      document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n",
    "\n",
    "      # Substituting multiple spaces with single space\n",
    "      document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
    "\n",
    "      # Converting to Lowercase\n",
    "      document = document.lower()\n",
    "\n",
    "      # Lemmatization\n",
    "      document = document.split()\n",
    "\n",
    "      # stop word removal\n",
    "      #document = remove_stop_words(document)\n",
    "\n",
    "      from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "      stemmer = WordNetLemmatizer()\n",
    "      document = [stemmer.lemmatize(word) for word in document]\n",
    "      document = ' '.join(document)\n",
    "\n",
    "      documents.append(document)\n",
    "\n",
    "  return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cFj9z0XUsAOr"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def create_top_words_file(df):\n",
    "  # Loop all topics and generate csv with top 100 words\n",
    "  for topic in df['topic'].unique():\n",
    "    sub_df = df['cleaned_content'].loc[df['topic'] == topic]\n",
    "    sub_df = sub_df.reset_index(drop=True)\n",
    "    cleaned = clean_corpus(sub_df)\n",
    "    vectorizer = CountVectorizer(stop_words=stopwords.words('english'))\n",
    "    X_cv = vectorizer.fit_transform(cleaned).toarray()\n",
    "    counts = np.zeros(len(X_cv[0]))\n",
    "\n",
    "    for count, x in enumerate(X_cv.transpose()):\n",
    "      counts[count] = x.sum()\n",
    "    out = {}\n",
    "    top_100_idx = np.argsort(counts)[-100:]\n",
    "    top_100_values = [counts[i] for i in top_100_idx]\n",
    "\n",
    "    for i in range(len(top_100_idx)):\n",
    "      out[vectorizer.get_feature_names_out()[np.flip(top_100_idx)[i]]] = np.flip(top_100_values)[i]\n",
    "\n",
    "    top_words_df = pd.DataFrame.from_dict(out, orient='index')\n",
    "\n",
    "    top_words_df.to_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/_top_words_{topic}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lDL2tZ-QxEP6"
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from nltk import ngrams\n",
    "\n",
    "def create_ngram_file(df, n):\n",
    "  for topic in df['topic'].unique():\n",
    "    sub_df = df['cleaned_content'].loc[df['topic'] == topic]\n",
    "    sub_df = sub_df.reset_index(drop=True)\n",
    "    cleaned = clean_corpus(sub_df)\n",
    "    ngram_counts = Counter()\n",
    "    for row in cleaned:\n",
    "      ngram_counts_temp = Counter(ngrams(row.split(), n))\n",
    "      for x in ngram_counts_temp.keys():\n",
    "        ngram_counts[x] = ngram_counts[x] + ngram_counts_temp[x]\n",
    "    pd.DataFrame.from_records(ngram_counts.most_common(100),columns=[f'{n}-gram', 'count']).to_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_{n}_gram_{topic}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1239078,
     "status": "ok",
     "timestamp": 1674007374805,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "J7l4-EpxCNRe",
    "outputId": "2379e770-7005-4554-fd2f-bc0189857813"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /root/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "\n",
    "def create_verb_file(df):\n",
    "  for topic in df['topic'].unique():\n",
    "    c = Counter()\n",
    "    subset = df['cleaned_content'].loc[df['topic'] == topic]\n",
    "    for row in subset:\n",
    "      pos = nltk.pos_tag(row.split())\n",
    "      for t in pos:\n",
    "        if 'JJ' in t[1]:\n",
    "          c[f'{t[0]}'] += 1\n",
    "    pd.DataFrame.from_records(c.most_common(100),columns=[f'adjective', 'count']).to_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_adjective_{topic}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "w4Lw7nBH41MI"
   },
   "source": [
    "## --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 520,
     "status": "ok",
     "timestamp": 1674349799414,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "Rj9qgQHY54Un"
   },
   "outputs": [],
   "source": [
    "# Unique Labeling\n",
    "df['uid'] = df['topic']\n",
    "df['uid'] = df['uid'].astype(str) + '_' + df.groupby('topic').cumcount().add(1).astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1674349800845,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "0nRzXlf6BfRp",
    "outputId": "150ff3b1-8949-4991-c084-fea8cb0f9a4a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-b0de3971-1288-4c63-83e9-fa0b052f7c1d\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid_1</td>\n",
       "      <td>6374ae8bc8a337344e61280f</td>\n",
       "      <td>2022-10-27T21:30:00.272Z</td>\n",
       "      <td>https://www.cbc.ca/news/canada/newfoundland-la...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Q&amp;A: Dr. Janice Fitzgerald on new variants, va...</td>\n",
       "      <td>google</td>\n",
       "      <td>since onset covid 19 pandemic march 2020 newfo...</td>\n",
       "      <td>577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>covid_2</td>\n",
       "      <td>6374ae8bc8a337344e612809</td>\n",
       "      <td>2022-11-06 12:00:56</td>\n",
       "      <td>https://www.sfchronicle.com/health/article/vir...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Bay Area could face COVID, flu and RSV in the ...</td>\n",
       "      <td>google</td>\n",
       "      <td>add potential swell covid 19 infection winter ...</td>\n",
       "      <td>731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>covid_3</td>\n",
       "      <td>6374ae8bc8a337344e612813</td>\n",
       "      <td>2022-11-07T20:50:03Z</td>\n",
       "      <td>https://www.reuters.com/world/china/beijing-im...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Beijing to improve COVID prevention policy ope...</td>\n",
       "      <td>google</td>\n",
       "      <td>beijing nov 7 reuters beijing improve way covi...</td>\n",
       "      <td>748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>covid_4</td>\n",
       "      <td>6374ae8bc8a337344e612815</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.cnn.com/2022/11/07/china/china-zer...</td>\n",
       "      <td>covid</td>\n",
       "      <td>China doubles down on zero-Covid pledge as pub...</td>\n",
       "      <td>google</td>\n",
       "      <td>china reiterated unswerving commitment longsta...</td>\n",
       "      <td>1924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>covid_5</td>\n",
       "      <td>6374ae8bc8a337344e612829</td>\n",
       "      <td>Oct 17, 2022</td>\n",
       "      <td>https://www.gov.ca.gov/2022/10/17/governor-new...</td>\n",
       "      <td>covid</td>\n",
       "      <td>Governor Newsom to End the COVID-19 State of E...</td>\n",
       "      <td>google</td>\n",
       "      <td>published oct 17 2022california pandemic respo...</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16309</th>\n",
       "      <td>donald trump_443</td>\n",
       "      <td>638dc20c6f39fded0f8bf127</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wbal.com/article/594732/111/trumps...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Trump's call to suspend Constitution not a 202...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>abc news washington republican ohio rep dave j...</td>\n",
       "      <td>1397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16310</th>\n",
       "      <td>donald trump_444</td>\n",
       "      <td>638dc2176f39fded0f8bf128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.wgbh.org/news/national-news/2022/1...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Former President Donald Trump announces he is ...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump tried overthrow result 2020 presi...</td>\n",
       "      <td>4612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16311</th>\n",
       "      <td>donald trump_445</td>\n",
       "      <td>638dc22d6f39fded0f8bf129</td>\n",
       "      <td>2022-12-04T18:00:00.000Z</td>\n",
       "      <td>https://www.yahoo.com/news/janes-directors-don...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>‘The Janes’ Directors On How Donald Trump Spur...</td>\n",
       "      <td>google</td>\n",
       "      <td>shortly donald trump became president united s...</td>\n",
       "      <td>3279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16312</th>\n",
       "      <td>donald trump_446</td>\n",
       "      <td>638dc23d6f39fded0f8bf12a</td>\n",
       "      <td>2022-11-30T21:00:11.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-2024...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Donald Trump's 2024 Presidential Campaign Is B...</td>\n",
       "      <td>duckduckgo</td>\n",
       "      <td>donald trump suddenly reminded 2016 political ...</td>\n",
       "      <td>2365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16313</th>\n",
       "      <td>donald trump_447</td>\n",
       "      <td>638dc2446f39fded0f8bf12b</td>\n",
       "      <td>2022-11-29T21:47:20.000Z</td>\n",
       "      <td>https://www.yahoo.com/video/donald-trumps-turb...</td>\n",
       "      <td>donald trump</td>\n",
       "      <td>Donald Trump's turbulence-filled launch</td>\n",
       "      <td>bing</td>\n",
       "      <td>donald trump team starting institute new syste...</td>\n",
       "      <td>4734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16314 rows × 9 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b0de3971-1288-4c63-83e9-fa0b052f7c1d')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-b0de3971-1288-4c63-83e9-fa0b052f7c1d button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-b0de3971-1288-4c63-83e9-fa0b052f7c1d');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                    uid                       _id                  pub_date  \\\n",
       "0               covid_1  6374ae8bc8a337344e61280f  2022-10-27T21:30:00.272Z   \n",
       "1               covid_2  6374ae8bc8a337344e612809       2022-11-06 12:00:56   \n",
       "2               covid_3  6374ae8bc8a337344e612813      2022-11-07T20:50:03Z   \n",
       "3               covid_4  6374ae8bc8a337344e612815                       NaN   \n",
       "4               covid_5  6374ae8bc8a337344e612829              Oct 17, 2022   \n",
       "...                 ...                       ...                       ...   \n",
       "16309  donald trump_443  638dc20c6f39fded0f8bf127                       NaN   \n",
       "16310  donald trump_444  638dc2176f39fded0f8bf128                       NaN   \n",
       "16311  donald trump_445  638dc22d6f39fded0f8bf129  2022-12-04T18:00:00.000Z   \n",
       "16312  donald trump_446  638dc23d6f39fded0f8bf12a  2022-11-30T21:00:11.000Z   \n",
       "16313  donald trump_447  638dc2446f39fded0f8bf12b  2022-11-29T21:47:20.000Z   \n",
       "\n",
       "                                                     url         topic  \\\n",
       "0      https://www.cbc.ca/news/canada/newfoundland-la...         covid   \n",
       "1      https://www.sfchronicle.com/health/article/vir...         covid   \n",
       "2      https://www.reuters.com/world/china/beijing-im...         covid   \n",
       "3      https://www.cnn.com/2022/11/07/china/china-zer...         covid   \n",
       "4      https://www.gov.ca.gov/2022/10/17/governor-new...         covid   \n",
       "...                                                  ...           ...   \n",
       "16309  https://www.wbal.com/article/594732/111/trumps...  donald trump   \n",
       "16310  https://www.wgbh.org/news/national-news/2022/1...  donald trump   \n",
       "16311  https://www.yahoo.com/news/janes-directors-don...  donald trump   \n",
       "16312  https://www.yahoo.com/video/donald-trumps-2024...  donald trump   \n",
       "16313  https://www.yahoo.com/video/donald-trumps-turb...  donald trump   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "0      Q&A: Dr. Janice Fitzgerald on new variants, va...        google   \n",
       "1      Bay Area could face COVID, flu and RSV in the ...        google   \n",
       "2      Beijing to improve COVID prevention policy ope...        google   \n",
       "3      China doubles down on zero-Covid pledge as pub...        google   \n",
       "4      Governor Newsom to End the COVID-19 State of E...        google   \n",
       "...                                                  ...           ...   \n",
       "16309  Trump's call to suspend Constitution not a 202...    duckduckgo   \n",
       "16310  Former President Donald Trump announces he is ...    duckduckgo   \n",
       "16311  ‘The Janes’ Directors On How Donald Trump Spur...        google   \n",
       "16312  Donald Trump's 2024 Presidential Campaign Is B...    duckduckgo   \n",
       "16313            Donald Trump's turbulence-filled launch          bing   \n",
       "\n",
       "                                         cleaned_content  wordcount  \n",
       "0      since onset covid 19 pandemic march 2020 newfo...        577  \n",
       "1      add potential swell covid 19 infection winter ...        731  \n",
       "2      beijing nov 7 reuters beijing improve way covi...        748  \n",
       "3      china reiterated unswerving commitment longsta...       1924  \n",
       "4      published oct 17 2022california pandemic respo...        641  \n",
       "...                                                  ...        ...  \n",
       "16309  abc news washington republican ohio rep dave j...       1397  \n",
       "16310  donald trump tried overthrow result 2020 presi...       4612  \n",
       "16311  shortly donald trump became president united s...       3279  \n",
       "16312  donald trump suddenly reminded 2016 political ...       2365  \n",
       "16313  donald trump team starting institute new syste...       4734  \n",
       "\n",
       "[16314 rows x 9 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1674008626771,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "22gwtvSNQmbK",
    "outputId": "cf20d4f3-3dff-4503-da14-cc84ef3f77f5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-d753dfc0-57b9-4313-9d16-fee36cf2fa24\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>adjective</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid</td>\n",
       "      <td>24881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>new</td>\n",
       "      <td>22061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pandemic</td>\n",
       "      <td>10214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>public</td>\n",
       "      <td>10106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>last</td>\n",
       "      <td>9205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>biggest</td>\n",
       "      <td>1796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>potential</td>\n",
       "      <td>1788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>senior</td>\n",
       "      <td>1783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>serious</td>\n",
       "      <td>1747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>thursday</td>\n",
       "      <td>1740</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d753dfc0-57b9-4313-9d16-fee36cf2fa24')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-d753dfc0-57b9-4313-9d16-fee36cf2fa24 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-d753dfc0-57b9-4313-9d16-fee36cf2fa24');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "    adjective  count\n",
       "0       covid  24881\n",
       "1         new  22061\n",
       "2    pandemic  10214\n",
       "3      public  10106\n",
       "4        last   9205\n",
       "..        ...    ...\n",
       "95    biggest   1796\n",
       "96  potential   1788\n",
       "97     senior   1783\n",
       "98    serious   1747\n",
       "99   thursday   1740\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_adjective_covid.csv', index_col=0)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "executionInfo": {
     "elapsed": 306,
     "status": "ok",
     "timestamp": 1674350643514,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "U9Y7dEAhsCeo"
   },
   "outputs": [],
   "source": [
    "t2 = pd.DataFrame(columns=['rank', 'top word', 'adj', 'verb', 'noun', '2gram', '3gram', '4gram', '5gram'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1674350655436,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "QO27zeDDB9X8",
    "outputId": "52bd1753-d64f-4709-97ee-619341c443fd"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-8160ce11-b798-45e9-9ddc-103813474c06\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank</th>\n",
       "      <th>top word</th>\n",
       "      <th>adj</th>\n",
       "      <th>verb</th>\n",
       "      <th>noun</th>\n",
       "      <th>2gram</th>\n",
       "      <th>3gram</th>\n",
       "      <th>4gram</th>\n",
       "      <th>5gram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>96</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>97</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>98</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>100</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 9 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8160ce11-b798-45e9-9ddc-103813474c06')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-8160ce11-b798-45e9-9ddc-103813474c06 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-8160ce11-b798-45e9-9ddc-103813474c06');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "    rank top word  adj verb noun 2gram 3gram 4gram 5gram\n",
       "0      1      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "1      2      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "2      3      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "3      4      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "4      5      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "..   ...      ...  ...  ...  ...   ...   ...   ...   ...\n",
       "95    96      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "96    97      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "97    98      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "98    99      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "99   100      NaN  NaN  NaN  NaN   NaN   NaN   NaN   NaN\n",
       "\n",
       "[100 rows x 9 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "executionInfo": {
     "elapsed": 337,
     "status": "ok",
     "timestamp": 1674350649523,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "9n7fU8kMCCEN"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.arange(100) + 1\n",
    "\n",
    "t2['rank'] = np.arange(100) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "executionInfo": {
     "elapsed": 50983,
     "status": "ok",
     "timestamp": 1674351767546,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "hky3AwexsOQ1"
   },
   "outputs": [],
   "source": [
    "#for topic in ['covid']:\n",
    "for topic in df['topic'].unique():\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_top_words_{topic}.csv', index_col=0)\n",
    "  t = t.reset_index()\n",
    "  t['0'] = t['0'].astype(int)\n",
    "  t['top word tuple'] = t[['index',\"0\"]].apply(tuple, axis=1)\n",
    "  t2['top word'] = t['top word tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_adjective_{topic}.csv', index_col=0)\n",
    "  t['adj tuple'] = t[[\"adjective\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['adj'] = t['adj tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_verbs_{topic}.csv', index_col=0)\n",
    "  t['verb tuple'] = t[[\"verb\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['verb'] = t['verb tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_nouns_{topic}.csv', index_col=0)\n",
    "  t['noun tuple'] = t[[\"noun\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['noun'] = t['noun tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_2_gram_{topic}.csv', index_col=0)\n",
    "  t['2gram tuple'] = t[[\"2-gram\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['2gram'] = t['2gram tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_3_gram_{topic}.csv', index_col=0)\n",
    "  t['3gram tuple'] = t[[\"3-gram\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['3gram'] = t['3gram tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_4_gram_{topic}.csv', index_col=0)\n",
    "  t['4gram tuple'] = t[[\"4-gram\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['4gram'] = t['4gram tuple']\n",
    "\n",
    "  t = pd.read_csv(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/_5_gram_{topic}.csv', index_col=0)\n",
    "  t['5gram tuple'] = t[[\"5-gram\",\"count\"]].apply(tuple, axis=1)\n",
    "  t2['5gram'] = t['5gram tuple']\n",
    "\n",
    "  t2.to_excel(f'/content/drive/MyDrive/Colab Notebooks/NLP project/processed/compiled_{topic}.xlsx')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1674350991913,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "Us2yWV6FsXdV",
    "outputId": "97ce8eb9-97cb-4f29-c4c6-225905d9e2d1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-4cb7ef09-d9e6-4a5c-8b0e-aceeb3e08ba2\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rank</th>\n",
       "      <th>top word</th>\n",
       "      <th>adj</th>\n",
       "      <th>verb</th>\n",
       "      <th>noun</th>\n",
       "      <th>2gram</th>\n",
       "      <th>3gram</th>\n",
       "      <th>4gram</th>\n",
       "      <th>5gram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>(covid, 89265)</td>\n",
       "      <td>(covid, 24881)</td>\n",
       "      <td>(said, 37972)</td>\n",
       "      <td>(covid, 47855)</td>\n",
       "      <td>(('covid', 'vaccine'), 7735)</td>\n",
       "      <td>(('may', 'earn', 'affiliate'), 3735)</td>\n",
       "      <td>(('may', 'earn', 'affiliate', 'commission'), 3...</td>\n",
       "      <td>(('may', 'earn', 'affiliate', 'commission', 'p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>(said, 37972)</td>\n",
       "      <td>(new, 22061)</td>\n",
       "      <td>(covid, 15442)</td>\n",
       "      <td>(health, 31184)</td>\n",
       "      <td>(('long', 'covid'), 5445)</td>\n",
       "      <td>(('earn', 'affiliate', 'commission'), 3732)</td>\n",
       "      <td>(('earn', 'affiliate', 'commission', 'purchase...</td>\n",
       "      <td>(('earn', 'affiliate', 'commission', 'purchase...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>(health, 31184)</td>\n",
       "      <td>(pandemic, 10214)</td>\n",
       "      <td>(say, 9981)</td>\n",
       "      <td>(people, 28697)</td>\n",
       "      <td>(('public', 'health'), 4561)</td>\n",
       "      <td>(('affiliate', 'commission', 'purchase'), 3725)</td>\n",
       "      <td>(('affiliate', 'commission', 'purchase', 'some...</td>\n",
       "      <td>(('affiliate', 'commission', 'purchase', 'some...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>(people, 28697)</td>\n",
       "      <td>(public, 10106)</td>\n",
       "      <td>(reported, 9352)</td>\n",
       "      <td>(vaccine, 24892)</td>\n",
       "      <td>(('covid', 'case'), 4415)</td>\n",
       "      <td>(('commission', 'purchase', 'something'), 3725)</td>\n",
       "      <td>(('commission', 'purchase', 'something', 'reco...</td>\n",
       "      <td>(('commission', 'purchase', 'something', 'reco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>(vaccine, 28499)</td>\n",
       "      <td>(last, 9205)</td>\n",
       "      <td>(including, 8032)</td>\n",
       "      <td>(case, 23113)</td>\n",
       "      <td>(('zero', 'covid'), 4063)</td>\n",
       "      <td>(('purchase', 'something', 'recommended'), 3725)</td>\n",
       "      <td>(('purchase', 'something', 'recommended', 'lin...</td>\n",
       "      <td>(('purchase', 'something', 'recommended', 'lin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>96</td>\n",
       "      <td>(research, 5648)</td>\n",
       "      <td>(biggest, 1796)</td>\n",
       "      <td>(lost, 1365)</td>\n",
       "      <td>(team, 3727)</td>\n",
       "      <td>(('covid', 'the'), 906)</td>\n",
       "      <td>(('since', 'start', 'pandemic'), 282)</td>\n",
       "      <td>(('tribune', 'content', 'agency', 'llc'), 180)</td>\n",
       "      <td>(('serve', 'people', 'south', 'carolina', 'bey...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>97</td>\n",
       "      <td>(global, 5584)</td>\n",
       "      <td>(potential, 1788)</td>\n",
       "      <td>(hospitalized, 1363)</td>\n",
       "      <td>(county, 3687)</td>\n",
       "      <td>(('first', 'time'), 898)</td>\n",
       "      <td>(('the', 'white', 'house'), 281)</td>\n",
       "      <td>(('updated', 'covid', 'booster', 'shot'), 179)</td>\n",
       "      <td>(('year', 'news', 'world', 'report', 'named'),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>98</td>\n",
       "      <td>(resident, 5561)</td>\n",
       "      <td>(senior, 1783)</td>\n",
       "      <td>(started, 1344)</td>\n",
       "      <td>(point, 3678)</td>\n",
       "      <td>(('said', 'dr'), 874)</td>\n",
       "      <td>(('received', 'least', 'one'), 277)</td>\n",
       "      <td>(('content', 'agency', 'llc', 'microsoft'), 178)</td>\n",
       "      <td>(('news', 'world', 'report', 'named', 'musc'),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>99</td>\n",
       "      <td>(cdc, 5543)</td>\n",
       "      <td>(serious, 1747)</td>\n",
       "      <td>(show, 1343)</td>\n",
       "      <td>(support, 3675)</td>\n",
       "      <td>(('nursing', 'home'), 870)</td>\n",
       "      <td>(('article', 'originally', 'appeared'), 275)</td>\n",
       "      <td>(('agency', 'llc', 'microsoft', 'may'), 178)</td>\n",
       "      <td>(('world', 'report', 'named', 'musc', 'health'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>100</td>\n",
       "      <td>(measure, 5529)</td>\n",
       "      <td>(thursday, 1740)</td>\n",
       "      <td>(seeing, 1324)</td>\n",
       "      <td>(population, 3646)</td>\n",
       "      <td>(('last', 'month'), 865)</td>\n",
       "      <td>(('standard', 'the', 'thomson'), 274)</td>\n",
       "      <td>(('llc', 'microsoft', 'may', 'earn'), 178)</td>\n",
       "      <td>(('report', 'named', 'musc', 'health', 'no'), ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 9 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4cb7ef09-d9e6-4a5c-8b0e-aceeb3e08ba2')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-4cb7ef09-d9e6-4a5c-8b0e-aceeb3e08ba2 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-4cb7ef09-d9e6-4a5c-8b0e-aceeb3e08ba2');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "    rank          top word                adj                  verb  \\\n",
       "0      1    (covid, 89265)     (covid, 24881)         (said, 37972)   \n",
       "1      2     (said, 37972)       (new, 22061)        (covid, 15442)   \n",
       "2      3   (health, 31184)  (pandemic, 10214)           (say, 9981)   \n",
       "3      4   (people, 28697)    (public, 10106)      (reported, 9352)   \n",
       "4      5  (vaccine, 28499)       (last, 9205)     (including, 8032)   \n",
       "..   ...               ...                ...                   ...   \n",
       "95    96  (research, 5648)    (biggest, 1796)          (lost, 1365)   \n",
       "96    97    (global, 5584)  (potential, 1788)  (hospitalized, 1363)   \n",
       "97    98  (resident, 5561)     (senior, 1783)       (started, 1344)   \n",
       "98    99       (cdc, 5543)    (serious, 1747)          (show, 1343)   \n",
       "99   100   (measure, 5529)   (thursday, 1740)        (seeing, 1324)   \n",
       "\n",
       "                  noun                         2gram  \\\n",
       "0       (covid, 47855)  (('covid', 'vaccine'), 7735)   \n",
       "1      (health, 31184)     (('long', 'covid'), 5445)   \n",
       "2      (people, 28697)  (('public', 'health'), 4561)   \n",
       "3     (vaccine, 24892)     (('covid', 'case'), 4415)   \n",
       "4        (case, 23113)     (('zero', 'covid'), 4063)   \n",
       "..                 ...                           ...   \n",
       "95        (team, 3727)       (('covid', 'the'), 906)   \n",
       "96      (county, 3687)      (('first', 'time'), 898)   \n",
       "97       (point, 3678)         (('said', 'dr'), 874)   \n",
       "98     (support, 3675)    (('nursing', 'home'), 870)   \n",
       "99  (population, 3646)      (('last', 'month'), 865)   \n",
       "\n",
       "                                               3gram  \\\n",
       "0               (('may', 'earn', 'affiliate'), 3735)   \n",
       "1        (('earn', 'affiliate', 'commission'), 3732)   \n",
       "2    (('affiliate', 'commission', 'purchase'), 3725)   \n",
       "3    (('commission', 'purchase', 'something'), 3725)   \n",
       "4   (('purchase', 'something', 'recommended'), 3725)   \n",
       "..                                               ...   \n",
       "95             (('since', 'start', 'pandemic'), 282)   \n",
       "96                  (('the', 'white', 'house'), 281)   \n",
       "97               (('received', 'least', 'one'), 277)   \n",
       "98      (('article', 'originally', 'appeared'), 275)   \n",
       "99             (('standard', 'the', 'thomson'), 274)   \n",
       "\n",
       "                                                4gram  \\\n",
       "0   (('may', 'earn', 'affiliate', 'commission'), 3...   \n",
       "1   (('earn', 'affiliate', 'commission', 'purchase...   \n",
       "2   (('affiliate', 'commission', 'purchase', 'some...   \n",
       "3   (('commission', 'purchase', 'something', 'reco...   \n",
       "4   (('purchase', 'something', 'recommended', 'lin...   \n",
       "..                                                ...   \n",
       "95     (('tribune', 'content', 'agency', 'llc'), 180)   \n",
       "96     (('updated', 'covid', 'booster', 'shot'), 179)   \n",
       "97   (('content', 'agency', 'llc', 'microsoft'), 178)   \n",
       "98       (('agency', 'llc', 'microsoft', 'may'), 178)   \n",
       "99         (('llc', 'microsoft', 'may', 'earn'), 178)   \n",
       "\n",
       "                                                5gram  \n",
       "0   (('may', 'earn', 'affiliate', 'commission', 'p...  \n",
       "1   (('earn', 'affiliate', 'commission', 'purchase...  \n",
       "2   (('affiliate', 'commission', 'purchase', 'some...  \n",
       "3   (('commission', 'purchase', 'something', 'reco...  \n",
       "4   (('purchase', 'something', 'recommended', 'lin...  \n",
       "..                                                ...  \n",
       "95  (('serve', 'people', 'south', 'carolina', 'bey...  \n",
       "96  (('year', 'news', 'world', 'report', 'named'),...  \n",
       "97  (('news', 'world', 'report', 'named', 'musc'),...  \n",
       "98  (('world', 'report', 'named', 'musc', 'health'...  \n",
       "99  (('report', 'named', 'musc', 'health', 'no'), ...  \n",
       "\n",
       "[100 rows x 9 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 347,
     "status": "ok",
     "timestamp": 1674350811377,
     "user": {
      "displayName": "James S",
      "userId": "08343857009872710683"
     },
     "user_tz": -780
    },
    "id": "oxT4gd8ssYsq",
    "outputId": "38e4ad78-84df-4fd8-a0e8-b6e9d3d57f70"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-69df7e02-2b2a-459c-b4ae-1110296ff44d\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>covid</td>\n",
       "      <td>89265.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>said</td>\n",
       "      <td>37972.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>health</td>\n",
       "      <td>31184.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>people</td>\n",
       "      <td>28697.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vaccine</td>\n",
       "      <td>28499.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>research</td>\n",
       "      <td>5648.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>global</td>\n",
       "      <td>5584.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>resident</td>\n",
       "      <td>5561.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>cdc</td>\n",
       "      <td>5543.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>measure</td>\n",
       "      <td>5529.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-69df7e02-2b2a-459c-b4ae-1110296ff44d')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-69df7e02-2b2a-459c-b4ae-1110296ff44d button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-69df7e02-2b2a-459c-b4ae-1110296ff44d');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "       index        0\n",
       "0      covid  89265.0\n",
       "1       said  37972.0\n",
       "2     health  31184.0\n",
       "3     people  28697.0\n",
       "4    vaccine  28499.0\n",
       "..       ...      ...\n",
       "95  research   5648.0\n",
       "96    global   5584.0\n",
       "97  resident   5561.0\n",
       "98       cdc   5543.0\n",
       "99   measure   5529.0\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Post-cleaned stage, touch ups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_excel(r\"G:\\My Drive\\Colab Notebooks\\NLP project\\cleaned_v2.xlsx\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "      <th>Invalid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9676</th>\n",
       "      <td>monkeypox_810</td>\n",
       "      <td>637e09294f434d9cf37c5a26</td>\n",
       "      <td>2022-11-17</td>\n",
       "      <td>https://www.qmul.ac.uk/media/news/2022/smd/fir...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>- Queen Mary University of LondonFacebookTwitt...</td>\n",
       "      <td>bing</td>\n",
       "      <td>an international collaboration clinician estab...</td>\n",
       "      <td>607</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640</th>\n",
       "      <td>Donald Trump sent his own plane to transport 2...</td>\n",
       "      <td>6374ae8bc8a337344e612f16</td>\n",
       "      <td>2016-01-24T18:48:13+00:00</td>\n",
       "      <td>https://www.washingtonpost.com/blogs/liveblog-...</td>\n",
       "      <td>Donald Trump sent his own plane to transport 2...</td>\n",
       "      <td>- The Washington Post</td>\n",
       "      <td>google</td>\n",
       "      <td>we ve three day melting still lot snow road ma...</td>\n",
       "      <td>3447</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>722</th>\n",
       "      <td>china taiwan conflict_91</td>\n",
       "      <td>6374ae8bc8a337344e612aed</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.fmprc.gov.cn/eng/zxxx_662805/20220...</td>\n",
       "      <td>china taiwan conflict</td>\n",
       "      <td>​Statement by the Ministry of Foreign Affairs ...</td>\n",
       "      <td>google</td>\n",
       "      <td>on  august disregard china strong opposition s...</td>\n",
       "      <td>1074</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10351</th>\n",
       "      <td>russia_114</td>\n",
       "      <td>637ee27a982aaca4f90b1288</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://en.defence-ua.com/news/turning_washing...</td>\n",
       "      <td>russia</td>\n",
       "      <td>​Turning Washing Machines into Tornado-S and T...</td>\n",
       "      <td>href</td>\n",
       "      <td>the armed force russian federation received ba...</td>\n",
       "      <td>235</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12227</th>\n",
       "      <td>russia_181</td>\n",
       "      <td>63807d659737d819e4c50c15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://en.defence-ua.com/news/ukraines_air_de...</td>\n",
       "      <td>russia</td>\n",
       "      <td>​Ukraine’s Air Defense Forces Shot Down 51 of ...</td>\n",
       "      <td>href</td>\n",
       "      <td>that according statement air force command arm...</td>\n",
       "      <td>100</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7956</th>\n",
       "      <td>monkeypox_763</td>\n",
       "      <td>637b6dbab8d6a467e0608d9a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.who.int/zh/news/item/23-07-2022-se...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>关于多国猴痘疫情的《国际卫生条例(2005)》突发事件委员会世界卫生组织世界卫生组织世界卫生...</td>\n",
       "      <td>href</td>\n",
       "      <td>推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...</td>\n",
       "      <td>661</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14109</th>\n",
       "      <td>pizzagate_382</td>\n",
       "      <td>6385f824faaec7e24ebf8a1a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://pansci.asia/archives/321172</td>\n",
       "      <td>pizzagate</td>\n",
       "      <td>最成功也最危險！美國陰謀論之王：QAnon - PanSci 泛科學</td>\n",
       "      <td>google</td>\n",
       "      <td>編按 還記得今年年初 川普的支持者闖入美國國會大廈嗎 現場各處出現了 的旗幟 這些 代表的是...</td>\n",
       "      <td>1372</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953</th>\n",
       "      <td>monkeypox_761</td>\n",
       "      <td>637b6da9b8d6a467e0608d8b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.who.int/zh/news-room/fact-sheets/d...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>猴痘世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界...</td>\n",
       "      <td>href</td>\n",
       "      <td>推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...</td>\n",
       "      <td>290</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14114</th>\n",
       "      <td>pizzagate_387</td>\n",
       "      <td>6385f8bdfaaec7e24ebf8a22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://theinitium.com/article/20221129-opinio...</td>\n",
       "      <td>pizzagate</td>\n",
       "      <td>華思睿：被馬斯克玩壞的Twitter，還能活多久？｜評論｜國際｜科技｜深度｜端傳媒 Init...</td>\n",
       "      <td>google</td>\n",
       "      <td>他一人就可以決定這個在全球擁有數億用戶的社交媒體平台的最終命運 沒有人可以阻擋他 他一人就可...</td>\n",
       "      <td>273</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1624</th>\n",
       "      <td>Donald Trump sent his own plane to transport 2...</td>\n",
       "      <td>6374ae8bc8a337344e612f28</td>\n",
       "      <td>2021-08-31 02:14:15 AM</td>\n",
       "      <td>https://www.foxnews.com/live-news/live-updates...</td>\n",
       "      <td>Donald Trump sent his own plane to transport 2...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>google</td>\n",
       "      <td>suicide bombing outside abbey gate kabul airpo...</td>\n",
       "      <td>3435</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15445 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     uid  \\\n",
       "9676                                       monkeypox_810   \n",
       "1640   Donald Trump sent his own plane to transport 2...   \n",
       "722                             china taiwan conflict_91   \n",
       "10351                                         russia_114   \n",
       "12227                                         russia_181   \n",
       "...                                                  ...   \n",
       "7956                                       monkeypox_763   \n",
       "14109                                      pizzagate_382   \n",
       "7953                                       monkeypox_761   \n",
       "14114                                      pizzagate_387   \n",
       "1624   Donald Trump sent his own plane to transport 2...   \n",
       "\n",
       "                            _id                   pub_date  \\\n",
       "9676   637e09294f434d9cf37c5a26                 2022-11-17   \n",
       "1640   6374ae8bc8a337344e612f16  2016-01-24T18:48:13+00:00   \n",
       "722    6374ae8bc8a337344e612aed                        NaN   \n",
       "10351  637ee27a982aaca4f90b1288                        NaN   \n",
       "12227  63807d659737d819e4c50c15                        NaN   \n",
       "...                         ...                        ...   \n",
       "7956   637b6dbab8d6a467e0608d9a                        NaN   \n",
       "14109  6385f824faaec7e24ebf8a1a                        NaN   \n",
       "7953   637b6da9b8d6a467e0608d8b                        NaN   \n",
       "14114  6385f8bdfaaec7e24ebf8a22                        NaN   \n",
       "1624   6374ae8bc8a337344e612f28     2021-08-31 02:14:15 AM   \n",
       "\n",
       "                                                     url  \\\n",
       "9676   https://www.qmul.ac.uk/media/news/2022/smd/fir...   \n",
       "1640   https://www.washingtonpost.com/blogs/liveblog-...   \n",
       "722    https://www.fmprc.gov.cn/eng/zxxx_662805/20220...   \n",
       "10351  https://en.defence-ua.com/news/turning_washing...   \n",
       "12227  https://en.defence-ua.com/news/ukraines_air_de...   \n",
       "...                                                  ...   \n",
       "7956   https://www.who.int/zh/news/item/23-07-2022-se...   \n",
       "14109                https://pansci.asia/archives/321172   \n",
       "7953   https://www.who.int/zh/news-room/fact-sheets/d...   \n",
       "14114  https://theinitium.com/article/20221129-opinio...   \n",
       "1624   https://www.foxnews.com/live-news/live-updates...   \n",
       "\n",
       "                                                   topic  \\\n",
       "9676                                           monkeypox   \n",
       "1640   Donald Trump sent his own plane to transport 2...   \n",
       "722                                china taiwan conflict   \n",
       "10351                                             russia   \n",
       "12227                                             russia   \n",
       "...                                                  ...   \n",
       "7956                                           monkeypox   \n",
       "14109                                          pizzagate   \n",
       "7953                                           monkeypox   \n",
       "14114                                          pizzagate   \n",
       "1624   Donald Trump sent his own plane to transport 2...   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "9676   - Queen Mary University of LondonFacebookTwitt...          bing   \n",
       "1640                               - The Washington Post        google   \n",
       "722    ​Statement by the Ministry of Foreign Affairs ...        google   \n",
       "10351  ​Turning Washing Machines into Tornado-S and T...          href   \n",
       "12227  ​Ukraine’s Air Defense Forces Shot Down 51 of ...          href   \n",
       "...                                                  ...           ...   \n",
       "7956   关于多国猴痘疫情的《国际卫生条例(2005)》突发事件委员会世界卫生组织世界卫生组织世界卫生...          href   \n",
       "14109                 最成功也最危險！美國陰謀論之王：QAnon - PanSci 泛科學        google   \n",
       "7953   猴痘世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界...          href   \n",
       "14114  華思睿：被馬斯克玩壞的Twitter，還能活多久？｜評論｜國際｜科技｜深度｜端傳媒 Init...        google   \n",
       "1624                                                 NaN        google   \n",
       "\n",
       "                                         cleaned_content  wordcount  Invalid  \n",
       "9676   an international collaboration clinician estab...        607      NaN  \n",
       "1640   we ve three day melting still lot snow road ma...       3447      NaN  \n",
       "722    on  august disregard china strong opposition s...       1074      NaN  \n",
       "10351  the armed force russian federation received ba...        235      NaN  \n",
       "12227  that according statement air force command arm...        100      NaN  \n",
       "...                                                  ...        ...      ...  \n",
       "7956   推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...        661      NaN  \n",
       "14109  編按 還記得今年年初 川普的支持者闖入美國國會大廈嗎 現場各處出現了 的旗幟 這些 代表的是...       1372      NaN  \n",
       "7953   推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...        290      NaN  \n",
       "14114  他一人就可以決定這個在全球擁有數億用戶的社交媒體平台的最終命運 沒有人可以阻擋他 他一人就可...        273      NaN  \n",
       "1624   suicide bombing outside abbey gate kabul airpo...       3435      NaN  \n",
       "\n",
       "[15445 rows x 10 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.loc[~(df['Invalid'] == 1)]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.to_excel(r\"G:\\My Drive\\Colab Notebooks\\NLP project\\cleaned_v2.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "      <th>Invalid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10351</th>\n",
       "      <td>russia_114</td>\n",
       "      <td>637ee27a982aaca4f90b1288</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://en.defence-ua.com/news/turning_washing...</td>\n",
       "      <td>russia</td>\n",
       "      <td>​Turning Washing Machines into Tornado-S and T...</td>\n",
       "      <td>href</td>\n",
       "      <td>the armed force russian federation received ba...</td>\n",
       "      <td>235</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12227</th>\n",
       "      <td>russia_181</td>\n",
       "      <td>63807d659737d819e4c50c15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://en.defence-ua.com/news/ukraines_air_de...</td>\n",
       "      <td>russia</td>\n",
       "      <td>​Ukraine’s Air Defense Forces Shot Down 51 of ...</td>\n",
       "      <td>href</td>\n",
       "      <td>that according statement air force command arm...</td>\n",
       "      <td>100</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9948</th>\n",
       "      <td>russia_85</td>\n",
       "      <td>637e31eea48aa2486e7c5bf6</td>\n",
       "      <td>2022-11-22T15:46:01.000Z</td>\n",
       "      <td>https://news.yahoo.com/man-mothers-wives-russi...</td>\n",
       "      <td>russia</td>\n",
       "      <td>\"Are you a man or what? \": mothers and wives o...</td>\n",
       "      <td>href</td>\n",
       "      <td>the civil movement family member russian mobil...</td>\n",
       "      <td>2700</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6797</th>\n",
       "      <td>brexit_552</td>\n",
       "      <td>637ae9e4c61d4e39b098a895</td>\n",
       "      <td>2021-06-23</td>\n",
       "      <td>https://es.euronews.com/2021/06/23/muchos-de-l...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>\"Muchos de los problemas del Reino Unido sigue...</td>\n",
       "      <td>href</td>\n",
       "      <td>comparte esta noticiael líder del conservador ...</td>\n",
       "      <td>5585</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6820</th>\n",
       "      <td>brexit_561</td>\n",
       "      <td>637aea72c61d4e39b098a943</td>\n",
       "      <td>2022-06-15</td>\n",
       "      <td>https://fr.euronews.com/my-europe/2022/06/15/n...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>\"Nous serons très fermes, calmes, mais dans le...</td>\n",
       "      <td>href</td>\n",
       "      <td>partager cet articlel ukraine serum au cœur de...</td>\n",
       "      <td>4667</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6984</th>\n",
       "      <td>brexit_620</td>\n",
       "      <td>637af2cdc61d4e39b098af07</td>\n",
       "      <td>2021-06-23</td>\n",
       "      <td>https://per.euronews.com/2021/06/23/brexit-cha...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>نگاهی به چالش‌ها و پیامدهای برکسیت در پنجمین س...</td>\n",
       "      <td>href</td>\n",
       "      <td>هم ucرسانی این مطلب از زمان تصمیم بهت ucآور بر...</td>\n",
       "      <td>5332</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7896</th>\n",
       "      <td>brexit_1063</td>\n",
       "      <td>637b6baad5a586abbe608791</td>\n",
       "      <td>2021-01-26</td>\n",
       "      <td>https://arabic.euronews.com/my-europe/2021/01/...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>هذه هي الأسس التنظيمية لتحديد حصص الصيادين الأ...</td>\n",
       "      <td>href</td>\n",
       "      <td>شارك في هذا المقالتعتمد بلجيكا على قدر واسع في...</td>\n",
       "      <td>2470</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6279</th>\n",
       "      <td>monkeypox_505</td>\n",
       "      <td>637abdf07e9bad29dbc2507d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.who.int/zh/news/item/01-11-2022-th...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>关于多国猴痘疫情的《国际卫生条例（2005）》突发事件委员会\\n第三次会议的报告\\n国际关注...</td>\n",
       "      <td>href</td>\n",
       "      <td>推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...</td>\n",
       "      <td>1256</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7956</th>\n",
       "      <td>monkeypox_763</td>\n",
       "      <td>637b6dbab8d6a467e0608d9a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.who.int/zh/news/item/23-07-2022-se...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>关于多国猴痘疫情的《国际卫生条例(2005)》突发事件委员会世界卫生组织世界卫生组织世界卫生...</td>\n",
       "      <td>href</td>\n",
       "      <td>推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...</td>\n",
       "      <td>661</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7953</th>\n",
       "      <td>monkeypox_761</td>\n",
       "      <td>637b6da9b8d6a467e0608d8b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.who.int/zh/news-room/fact-sheets/d...</td>\n",
       "      <td>monkeypox</td>\n",
       "      <td>猴痘世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界...</td>\n",
       "      <td>href</td>\n",
       "      <td>推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...</td>\n",
       "      <td>290</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2719 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 uid                       _id                  pub_date  \\\n",
       "10351     russia_114  637ee27a982aaca4f90b1288                       NaN   \n",
       "12227     russia_181  63807d659737d819e4c50c15                       NaN   \n",
       "9948       russia_85  637e31eea48aa2486e7c5bf6  2022-11-22T15:46:01.000Z   \n",
       "6797      brexit_552  637ae9e4c61d4e39b098a895                2021-06-23   \n",
       "6820      brexit_561  637aea72c61d4e39b098a943                2022-06-15   \n",
       "...              ...                       ...                       ...   \n",
       "6984      brexit_620  637af2cdc61d4e39b098af07                2021-06-23   \n",
       "7896     brexit_1063  637b6baad5a586abbe608791                2021-01-26   \n",
       "6279   monkeypox_505  637abdf07e9bad29dbc2507d                       NaN   \n",
       "7956   monkeypox_763  637b6dbab8d6a467e0608d9a                       NaN   \n",
       "7953   monkeypox_761  637b6da9b8d6a467e0608d8b                       NaN   \n",
       "\n",
       "                                                     url      topic  \\\n",
       "10351  https://en.defence-ua.com/news/turning_washing...     russia   \n",
       "12227  https://en.defence-ua.com/news/ukraines_air_de...     russia   \n",
       "9948   https://news.yahoo.com/man-mothers-wives-russi...     russia   \n",
       "6797   https://es.euronews.com/2021/06/23/muchos-de-l...     brexit   \n",
       "6820   https://fr.euronews.com/my-europe/2022/06/15/n...     brexit   \n",
       "...                                                  ...        ...   \n",
       "6984   https://per.euronews.com/2021/06/23/brexit-cha...     brexit   \n",
       "7896   https://arabic.euronews.com/my-europe/2021/01/...     brexit   \n",
       "6279   https://www.who.int/zh/news/item/01-11-2022-th...  monkeypox   \n",
       "7956   https://www.who.int/zh/news/item/23-07-2022-se...  monkeypox   \n",
       "7953   https://www.who.int/zh/news-room/fact-sheets/d...  monkeypox   \n",
       "\n",
       "                                                headline search_engine  \\\n",
       "10351  ​Turning Washing Machines into Tornado-S and T...          href   \n",
       "12227  ​Ukraine’s Air Defense Forces Shot Down 51 of ...          href   \n",
       "9948   \"Are you a man or what? \": mothers and wives o...          href   \n",
       "6797   \"Muchos de los problemas del Reino Unido sigue...          href   \n",
       "6820   \"Nous serons très fermes, calmes, mais dans le...          href   \n",
       "...                                                  ...           ...   \n",
       "6984   نگاهی به چالش‌ها و پیامدهای برکسیت در پنجمین س...          href   \n",
       "7896   هذه هي الأسس التنظيمية لتحديد حصص الصيادين الأ...          href   \n",
       "6279   关于多国猴痘疫情的《国际卫生条例（2005）》突发事件委员会\\n第三次会议的报告\\n国际关注...          href   \n",
       "7956   关于多国猴痘疫情的《国际卫生条例(2005)》突发事件委员会世界卫生组织世界卫生组织世界卫生...          href   \n",
       "7953   猴痘世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界卫生组织世界...          href   \n",
       "\n",
       "                                         cleaned_content  wordcount  Invalid  \n",
       "10351  the armed force russian federation received ba...        235      NaN  \n",
       "12227  that according statement air force command arm...        100      NaN  \n",
       "9948   the civil movement family member russian mobil...       2700      NaN  \n",
       "6797   comparte esta noticiael líder del conservador ...       5585      NaN  \n",
       "6820   partager cet articlel ukraine serum au cœur de...       4667      NaN  \n",
       "...                                                  ...        ...      ...  \n",
       "6984   هم ucرسانی این مطلب از زمان تصمیم بهت ucآور بر...       5332      NaN  \n",
       "7896   شارك في هذا المقالتعتمد بلجيكا على قدر واسع في...       2470      NaN  \n",
       "6279   推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...       1256      NaN  \n",
       "7956   推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...        661      NaN  \n",
       "7953   推出鼓励儿童开展身体活动和锻炼身体的数字应用程序世卫组织强调口腔卫生被忽视问题影响到全球近半...        290      NaN  \n",
       "\n",
       "[2719 rows x 10 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['search_engine'] == 'href']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.loc[~(df['url'].str.contains('://fr.euro'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.loc[~(df['url'].str.contains('://es.euro'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = df.loc[~(df['url'].str.contains('ru.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('hu.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('it.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('pt.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('de.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('gr.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('per.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('fa.euronews'))]\n",
    "df = df.loc[~(df['url'].str.contains('arabic.euronews'))]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>url</th>\n",
       "      <th>topic</th>\n",
       "      <th>headline</th>\n",
       "      <th>search_engine</th>\n",
       "      <th>cleaned_content</th>\n",
       "      <th>wordcount</th>\n",
       "      <th>Invalid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7119</th>\n",
       "      <td>brexit_700</td>\n",
       "      <td>637afbddc61d4e39b098bab7</td>\n",
       "      <td>2020-01-23</td>\n",
       "      <td>https://www.euronews.com/2020/01/23/brexit-fin...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>‘Brexit finish line’ crossed as British parlia...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlerights group welcomed inspiration...</td>\n",
       "      <td>4226</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8310</th>\n",
       "      <td>brexit_1429</td>\n",
       "      <td>637b8c11d5a586abbe60b222</td>\n",
       "      <td>2019-10-11</td>\n",
       "      <td>https://www.euronews.com/2019/10/11/watch-live...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>‘Much work needed’ to secure Brexit deal ahead...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlea man stabbed three people shoppi...</td>\n",
       "      <td>4603</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8273</th>\n",
       "      <td>brexit_1392</td>\n",
       "      <td>637b8960d5a586abbe60ac02</td>\n",
       "      <td>2019-09-03</td>\n",
       "      <td>https://www.euronews.com/2019/09/03/no-to-no-d...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>‘No to no-deal Brexit’: the 21 Tory rebels who...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlesen joe manchin w va wo t seek an...</td>\n",
       "      <td>4090</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5993</th>\n",
       "      <td>Emission trading scheme_419</td>\n",
       "      <td>637aaf8d325bba68bbc24c7a</td>\n",
       "      <td>2022-11-15</td>\n",
       "      <td>https://www.euronews.com/green/2022/11/15/not-...</td>\n",
       "      <td>Emission trading scheme</td>\n",
       "      <td>‘Not backtracking’: EU climate chief announces...</td>\n",
       "      <td>bing</td>\n",
       "      <td>olivier de schutter un special rapporteur extr...</td>\n",
       "      <td>4691</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8351</th>\n",
       "      <td>brexit_1470</td>\n",
       "      <td>637b8fb2d5a586abbe60ba79</td>\n",
       "      <td>2020-02-19</td>\n",
       "      <td>https://www.euronews.com/2020/02/19/speak-engl...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>‘Speak English’ in order to work in UK, Britai...</td>\n",
       "      <td>href</td>\n",
       "      <td>share article passenger begin leaving quaranti...</td>\n",
       "      <td>4620</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8298</th>\n",
       "      <td>brexit_1417</td>\n",
       "      <td>637b8b2fd5a586abbe60b009</td>\n",
       "      <td>2019-10-03</td>\n",
       "      <td>https://www.euronews.com/2019/10/03/why-is-bor...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>Why is Boris Johnson’s latest Brexit plan a he...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlepoland janusz wojciechowski need ...</td>\n",
       "      <td>4631</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8237</th>\n",
       "      <td>brexit_1356</td>\n",
       "      <td>637b861bd5a586abbe60a45b</td>\n",
       "      <td>2019-05-17</td>\n",
       "      <td>https://www.euronews.com//2019/05/17/why-is-fa...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>Why is Farage's Brexit Party leading the polls...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articletaipei taiwan taiwan legislature ...</td>\n",
       "      <td>4768</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8426</th>\n",
       "      <td>brexit_1545</td>\n",
       "      <td>637b972cd5a586abbe60cc01</td>\n",
       "      <td>2021-01-28</td>\n",
       "      <td>https://www.euronews.com/my-europe/2021/01/28/...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>Will the Netherlands fill the English-speaking...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articleasylum attack european border un ...</td>\n",
       "      <td>4572</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8394</th>\n",
       "      <td>brexit_1513</td>\n",
       "      <td>637b9301d5a586abbe60c1a7</td>\n",
       "      <td>2020-12-29</td>\n",
       "      <td>https://www.euronews.com/2020/12/29/will-there...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>Will there be UK food import delays after fina...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlein village near salzburg austria ...</td>\n",
       "      <td>4223</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7123</th>\n",
       "      <td>brexit_704</td>\n",
       "      <td>637afc36c61d4e39b098bbb5</td>\n",
       "      <td>2020-01-29</td>\n",
       "      <td>https://www.euronews.com/2020/01/29/brexit-mep...</td>\n",
       "      <td>brexit</td>\n",
       "      <td>'You'll be missed': Brexit deal approved by ME...</td>\n",
       "      <td>href</td>\n",
       "      <td>share articlebrexit architect nigel farage tol...</td>\n",
       "      <td>4748</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>395 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              uid                       _id    pub_date  \\\n",
       "7119                   brexit_700  637afbddc61d4e39b098bab7  2020-01-23   \n",
       "8310                  brexit_1429  637b8c11d5a586abbe60b222  2019-10-11   \n",
       "8273                  brexit_1392  637b8960d5a586abbe60ac02  2019-09-03   \n",
       "5993  Emission trading scheme_419  637aaf8d325bba68bbc24c7a  2022-11-15   \n",
       "8351                  brexit_1470  637b8fb2d5a586abbe60ba79  2020-02-19   \n",
       "...                           ...                       ...         ...   \n",
       "8298                  brexit_1417  637b8b2fd5a586abbe60b009  2019-10-03   \n",
       "8237                  brexit_1356  637b861bd5a586abbe60a45b  2019-05-17   \n",
       "8426                  brexit_1545  637b972cd5a586abbe60cc01  2021-01-28   \n",
       "8394                  brexit_1513  637b9301d5a586abbe60c1a7  2020-12-29   \n",
       "7123                   brexit_704  637afc36c61d4e39b098bbb5  2020-01-29   \n",
       "\n",
       "                                                    url  \\\n",
       "7119  https://www.euronews.com/2020/01/23/brexit-fin...   \n",
       "8310  https://www.euronews.com/2019/10/11/watch-live...   \n",
       "8273  https://www.euronews.com/2019/09/03/no-to-no-d...   \n",
       "5993  https://www.euronews.com/green/2022/11/15/not-...   \n",
       "8351  https://www.euronews.com/2020/02/19/speak-engl...   \n",
       "...                                                 ...   \n",
       "8298  https://www.euronews.com/2019/10/03/why-is-bor...   \n",
       "8237  https://www.euronews.com//2019/05/17/why-is-fa...   \n",
       "8426  https://www.euronews.com/my-europe/2021/01/28/...   \n",
       "8394  https://www.euronews.com/2020/12/29/will-there...   \n",
       "7123  https://www.euronews.com/2020/01/29/brexit-mep...   \n",
       "\n",
       "                        topic  \\\n",
       "7119                   brexit   \n",
       "8310                   brexit   \n",
       "8273                   brexit   \n",
       "5993  Emission trading scheme   \n",
       "8351                   brexit   \n",
       "...                       ...   \n",
       "8298                   brexit   \n",
       "8237                   brexit   \n",
       "8426                   brexit   \n",
       "8394                   brexit   \n",
       "7123                   brexit   \n",
       "\n",
       "                                               headline search_engine  \\\n",
       "7119  ‘Brexit finish line’ crossed as British parlia...          href   \n",
       "8310  ‘Much work needed’ to secure Brexit deal ahead...          href   \n",
       "8273  ‘No to no-deal Brexit’: the 21 Tory rebels who...          href   \n",
       "5993  ‘Not backtracking’: EU climate chief announces...          bing   \n",
       "8351  ‘Speak English’ in order to work in UK, Britai...          href   \n",
       "...                                                 ...           ...   \n",
       "8298  Why is Boris Johnson’s latest Brexit plan a he...          href   \n",
       "8237  Why is Farage's Brexit Party leading the polls...          href   \n",
       "8426  Will the Netherlands fill the English-speaking...          href   \n",
       "8394  Will there be UK food import delays after fina...          href   \n",
       "7123  'You'll be missed': Brexit deal approved by ME...          href   \n",
       "\n",
       "                                        cleaned_content  wordcount  Invalid  \n",
       "7119  share articlerights group welcomed inspiration...       4226      NaN  \n",
       "8310  share articlea man stabbed three people shoppi...       4603      NaN  \n",
       "8273  share articlesen joe manchin w va wo t seek an...       4090      NaN  \n",
       "5993  olivier de schutter un special rapporteur extr...       4691      NaN  \n",
       "8351  share article passenger begin leaving quaranti...       4620      NaN  \n",
       "...                                                 ...        ...      ...  \n",
       "8298  share articlepoland janusz wojciechowski need ...       4631      NaN  \n",
       "8237  share articletaipei taiwan taiwan legislature ...       4768      NaN  \n",
       "8426  share articleasylum attack european border un ...       4572      NaN  \n",
       "8394  share articlein village near salzburg austria ...       4223      NaN  \n",
       "7123  share articlebrexit architect nigel farage tol...       4748      NaN  \n",
       "\n",
       "[395 rows x 10 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[(df['url'].str.contains('euronews'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNottOSzHMpXDkV/zlMkB96",
   "mount_file_id": "1NhMhZ-8k3JcC5ikf77fV6Ud8bdqD6yyf",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "690fcc030dbde6f3f652c519b06a0d95da528e66105ed58c88e506e5818277cb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}