text_process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experimenting with altered text processing techniques...\n",
    "\n",
    "Saved to dataframe as shown:\n",
    "\n",
    "ENTER  DATAFRAME NAMES HERE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Python Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python38\\lib\\site-packages\\scipy\\sparse\\sparsetools.py:21: DeprecationWarning: `scipy.sparse.sparsetools` is deprecated!\n",
      "scipy.sparse.sparsetools is a private module for scipy.sparse, and should not be used.\n",
      "  _deprecated()\n"
     ]
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "import json\n",
    "import spacy\n",
    "\n",
    "# Required Libraries\n",
    "\n",
    "#Base and Cleaning \n",
    "import json\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import emoji\n",
    "import regex\n",
    "import re\n",
    "import string\n",
    "from collections import Counter\n",
    "\n",
    "#Visualizations\n",
    "import plotly.express as px\n",
    "import matplotlib.pyplot as plt \n",
    "import pyLDAvis.gensim\n",
    "import chart_studio\n",
    "import chart_studio.plotly as py \n",
    "import chart_studio.tools as tls\n",
    "\n",
    "#Natural Language Processing (NLP)\n",
    "import spacy\n",
    "import gensim\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.models.ldamulticore import LdaMulticore\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from gensim.parsing.preprocessing import STOPWORDS as SW\n",
    "from sklearn.decomposition import LatentDirichletAllocation, TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from pprint import pprint\n",
    "from wordcloud import STOPWORDS\n",
    "stopwords = set(STOPWORDS)\n",
    "\n",
    "import pyLDAvis.gensim\n",
    "import chart_studio\n",
    "import chart_studio.plotly as py \n",
    "import chart_studio.tools as tls\n",
    "\n",
    "#Natural Language Processing (NLP)\n",
    "import spacy\n",
    "from spacy.tokenizer import Tokenizer\n",
    "import gensim\n",
    "from spacy.tokenizer import Tokenizer\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.models.ldamulticore import LdaMulticore\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from gensim.parsing.preprocessing import STOPWORDS as SW\n",
    "from sklearn.decomposition import LatentDirichletAllocation, TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from pprint import pprint\n",
    "\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "\n",
    "from nltk import FreqDist, word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\shive\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\shive\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\shive\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# Example 1: import nltk (natural language tool kit), a popular python package for text mining\n",
    "import nltk\n",
    "# stopwords, FreqDist, word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import FreqDist, word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')    # Lemmatizer Software\n",
    "# Punkt Sentence Tokenizer. This tokenizer divides a text into a list of sentences,\n",
    "nltk.download('punkt')\n",
    "# StopWords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "\n",
    "# from nltk.corpus import stopwords\n",
    "#regular expression\n",
    "import re\n",
    "#csv\n",
    "import csv\n",
    "# Excel-like format\n",
    "import pandas as pd\n",
    "#data visualization\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MongoDB collection and retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient('mongodb://kmAdmin:kmadmin@129.130.10.108:7017/knowledge-map')\n",
    "db = client['knowledge-map']\n",
    "theses = db['theses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for document in db.theses.find():\n",
    "  data.append(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in data:\n",
    "  document['_id'] = str(document['_id'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "theses_data = [data for data in db.theses.find()]\n",
    "df = pd.DataFrame(theses_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>title</th>\n",
       "      <th>authors</th>\n",
       "      <th>text</th>\n",
       "      <th>text_processed</th>\n",
       "      <th>title_manual</th>\n",
       "      <th>authors_manual</th>\n",
       "      <th>year_manual</th>\n",
       "      <th>text_processed_optimized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6046389562f2776f1f5c11b6</td>\n",
       "      <td>Evolutionary Tree Genetic Programming</td>\n",
       "      <td>jan.antolik@matfyz.cz</td>\n",
       "      <td>Evolutionary Tree Genetic Programming\\r\\n\\r\\nj...</td>\n",
       "      <td>[evolutionary, tree, genetic, programming, jan...</td>\n",
       "      <td>Evolutionary Tree Genetic Programming</td>\n",
       "      <td>Jan Antolik</td>\n",
       "      <td>2004</td>\n",
       "      <td>[evolutionary, tree, genetic_programme, abstra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6046389762f2776f1f5c11b7</td>\n",
       "      <td>PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...</td>\n",
       "      <td>by ASHISH KUMAR ASHOK</td>\n",
       "      <td>PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...</td>\n",
       "      <td>[predictive, data, mining, collaborative, edit...</td>\n",
       "      <td>PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...</td>\n",
       "      <td>ASHISH KUMAR ASHOK</td>\n",
       "      <td>2011</td>\n",
       "      <td>[predictive, datum, mining, collaborative, edi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6046389862f2776f1f5c11b8</td>\n",
       "      <td>LocoLit: A Real-Time View of Local Attractions...</td>\n",
       "      <td>B.E., Muffakham Jah College of Engineering, 20...</td>\n",
       "      <td>LocoLit: A Real-Time View of Local Attractions...</td>\n",
       "      <td>[locolit, real, time, view, local, attraction,...</td>\n",
       "      <td>LocoLit: A Real-Time View of Local Attractions</td>\n",
       "      <td>Atef Khan</td>\n",
       "      <td>2018</td>\n",
       "      <td>[locolit, real, time, view, local, attraction,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6046389962f2776f1f5c11b9</td>\n",
       "      <td>Evolving locomotion for virtual quadrupeds by ...</td>\n",
       "      <td>B.S., Kansas State University, 2018 A REPORT</td>\n",
       "      <td>Evolving locomotion for virtual quadrupeds</td>\n",
       "      <td>[]</td>\n",
       "      <td>Evolving locomotion for virtual quadrupeds</td>\n",
       "      <td>Caleb Compton</td>\n",
       "      <td>2020</td>\n",
       "      <td>[evolve, locomotion, virtual, quadruped]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6046389b62f2776f1f5c11ba</td>\n",
       "      <td>Hierarchical and Partitioning-Based Hybridized...</td>\n",
       "      <td>CHANDRAVYAS ANNAKULA</td>\n",
       "      <td>Hierarchical and Partitioning-Based Hybridized...</td>\n",
       "      <td>[hierarchical, partitioning, base, hybridize, ...</td>\n",
       "      <td>Hierarchical and Partitioning-Based Hybridized...</td>\n",
       "      <td>CHANDRAVYAS ANNAKULA</td>\n",
       "      <td>2017</td>\n",
       "      <td>[hierarchical, partitioning, base, hybridize, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6046389f62f2776f1f5c11bb</td>\n",
       "      <td>ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...</td>\n",
       "      <td>by CHRISTOPHER HENRY MEYER B.S., Kansas State ...</td>\n",
       "      <td>ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...</td>\n",
       "      <td>[improve, natural, language, processing, phras...</td>\n",
       "      <td>ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...</td>\n",
       "      <td>CHRISTOPHER HENRY MEYER</td>\n",
       "      <td>2008</td>\n",
       "      <td>[improve, processing, phrase, base, syntactic,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>604638aa62f2776f1f5c11bc</td>\n",
       "      <td>Building a scalable distributed data platform ...</td>\n",
       "      <td>A REPORT submitted in partial fulfillment of the</td>\n",
       "      <td>Building a scalable distributed data platform ...</td>\n",
       "      <td>[build, scalable, distribute, data, platform, ...</td>\n",
       "      <td>Building a scalable distributed data platform ...</td>\n",
       "      <td>DHANANJAY MEHTA</td>\n",
       "      <td>2017</td>\n",
       "      <td>[build, scalable, distribute, data, platform, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>604638b062f2776f1f5c11bd</td>\n",
       "      <td>COMPARATIVE TEXT SUMMARIZATION OF PRODUCT REVI...</td>\n",
       "      <td>DINESH REDDY SINGI REDDY</td>\n",
       "      <td>COMPARATIVE  TEXT SUMMARIZATION OF PRODUCT REV...</td>\n",
       "      <td>[comparative, text, summarization, product, re...</td>\n",
       "      <td>COMPARATIVE TEXT SUMMARIZATION OF PRODUCT REVIEWS</td>\n",
       "      <td>DINESH REDDY SINGI REDDY</td>\n",
       "      <td>2010</td>\n",
       "      <td>[comparative, text, summarization, product, re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>604638b262f2776f1f5c11be</td>\n",
       "      <td>AUTOMATED GENRE CLASSIFICATION IN LITERATURE b...</td>\n",
       "      <td>A THESIS submitted in partial fulfillment of the</td>\n",
       "      <td>AUTOMATED GENRE CLASSIFICATION IN LITERATURE b...</td>\n",
       "      <td>[automated, genre, classification, literature,...</td>\n",
       "      <td>AUTOMATED GENRE CLASSIFICATION IN LITERATURE</td>\n",
       "      <td>EMILY JORDAN</td>\n",
       "      <td>2014</td>\n",
       "      <td>[classification, literature, emily, thesis, su...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>604638b362f2776f1f5c11bf</td>\n",
       "      <td>AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...</td>\n",
       "      <td>by ABHIJIT ERANDE B. E., University of Pune, 2005</td>\n",
       "      <td>AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...</td>\n",
       "      <td>[automatic, detection, significant, feature, e...</td>\n",
       "      <td>AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...</td>\n",
       "      <td>ABHIJIT ERANDE</td>\n",
       "      <td>2009</td>\n",
       "      <td>[automatic, detection, significant, feature, e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>604638bd62f2776f1f5c11c0</td>\n",
       "      <td>ALGORITHM SELECTION FOR SORTING AND PROBABILIS...</td>\n",
       "      <td>byHAIPENG GUO M.S., Beijing University of Aero...</td>\n",
       "      <td>ALGORITHM SELECTION FOR SORTING AND PROBABILIS...</td>\n",
       "      <td>[algorithm, selection, sort, probabilistic, in...</td>\n",
       "      <td>ALGORITHM SELECTION FOR SORTING AND PROBABILIS...</td>\n",
       "      <td>HAIPENG GUO</td>\n",
       "      <td>2003</td>\n",
       "      <td>[sort, probabilistic, inference, machine, base...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>604638bd62f2776f1f5c11c1</td>\n",
       "      <td>Genetic Programming and Multi-Agent Layered Le...</td>\n",
       "      <td>develop a solution. One important observation ...</td>\n",
       "      <td>Genetic Programming and Multi-Agent Layered Le...</td>\n",
       "      <td>[genetic, programming, multi, agent, layer, le...</td>\n",
       "      <td>Genetic Programming and Multi-Agent Layered Le...</td>\n",
       "      <td>Steven M. Gustafson</td>\n",
       "      <td>2000</td>\n",
       "      <td>[genetic_programme, multi, agent, layer, learn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>604638bf62f2776f1f5c11c2</td>\n",
       "      <td>USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...</td>\n",
       "      <td>by HAOLEI WANG B.S.,Dongbei University of Fina...</td>\n",
       "      <td>USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...</td>\n",
       "      <td>[density, base, clustering, improve, skeleton,...</td>\n",
       "      <td>USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...</td>\n",
       "      <td>HAOLEI WANG</td>\n",
       "      <td>2012</td>\n",
       "      <td>[use, density, base, cluster, improve, embed, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>604638ca62f2776f1f5c11c3</td>\n",
       "      <td>M.S., Emporia State University, 2008 M.S., Kan...</td>\n",
       "      <td>requirements for the degree DOCTOR OF PHILOSOP...</td>\n",
       "      <td>M.S., Emporia State University, 2008 M.S., Kan...</td>\n",
       "      <td>[ms, emporia, state, university, 2008, ms, kan...</td>\n",
       "      <td>Affective Intelligence in Built Environments</td>\n",
       "      <td>Heath Landon Yates</td>\n",
       "      <td>2018</td>\n",
       "      <td>[requirement, abstract, abstract, dissertation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>604638cc62f2776f1f5c11c4</td>\n",
       "      <td>SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...</td>\n",
       "      <td>A THESIS</td>\n",
       "      <td>SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...</td>\n",
       "      <td>[security, risk, prioritization, logical, atta...</td>\n",
       "      <td>SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...</td>\n",
       "      <td>HUSSAIN ALMOHRI</td>\n",
       "      <td>2008</td>\n",
       "      <td>[security, risk, prioritization, logical, thes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>604638d062f2776f1f5c11c5</td>\n",
       "      <td>BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...</td>\n",
       "      <td>B.E., Shanghai Jiaotong Univeristy, China, 200...</td>\n",
       "      <td>BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...</td>\n",
       "      <td>[bioinformatics, analyse, alternative, splicin...</td>\n",
       "      <td>BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...</td>\n",
       "      <td>JING XIA</td>\n",
       "      <td>2008</td>\n",
       "      <td>[alternative_splice, base, machine, base, abst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>604638d862f2776f1f5c11c6</td>\n",
       "      <td>submitted in partial fulfillment of the requir...</td>\n",
       "      <td>DOCTOR OF PHILOSOPHY</td>\n",
       "      <td>submitted in partial fulfillment of the requir...</td>\n",
       "      <td>[submit, partial, fulfillment, requirement, de...</td>\n",
       "      <td>Bringing Computational Thinking to K-12 and Hi...</td>\n",
       "      <td>Joshua Levi Weese</td>\n",
       "      <td>2017</td>\n",
       "      <td>[submit, partial_fulfillment, requirement, phi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>604638da62f2776f1f5c11c7</td>\n",
       "      <td>A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...</td>\n",
       "      <td>by JOSHUA L. WEESE B.S., Kansas State Universi...</td>\n",
       "      <td>A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...</td>\n",
       "      <td>[convolutive, model, polyphonic, instrument, i...</td>\n",
       "      <td>A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...</td>\n",
       "      <td>JOSHUA L. WEESE</td>\n",
       "      <td>2013</td>\n",
       "      <td>[pitch, detection, use, combined, classificati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>604638de62f2776f1f5c11c8</td>\n",
       "      <td>APPROXIMATE INFERENCE OF BAYESIAN NETWORKS THR...</td>\n",
       "      <td>JULIE ANN THORNTON B.S., Kansas State Universi...</td>\n",
       "      <td>APPROXIMATE INFERENCE OF BAYESIAN NETWORKS THR...</td>\n",
       "      <td>[approximate, inference, bayesian, network, ed...</td>\n",
       "      <td>APPROXIMATE INFERENCE OF BAYESIAN NETWORKS  TH...</td>\n",
       "      <td>JULIE ANN THORNTON</td>\n",
       "      <td>2005</td>\n",
       "      <td>[abstract, table, content, chapter, introducti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>604638e062f2776f1f5c11c9</td>\n",
       "      <td>Density-based and partitioning-based clusterin...</td>\n",
       "      <td>ARUNA SAI KANNAMAREDDY</td>\n",
       "      <td>Density-based and partitioning-based clusterin...</td>\n",
       "      <td>[density, base, partitioning, base, clustering...</td>\n",
       "      <td>Density-based and partitioning-based clusterin...</td>\n",
       "      <td>ARUNA SAI KANNAMAREDDY</td>\n",
       "      <td>2017</td>\n",
       "      <td>[density, base, partition, base, cluster, larg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>604638e262f2776f1f5c11ca</td>\n",
       "      <td>IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...</td>\n",
       "      <td>by KEERTHI KORIVI B.E. , Osmania University, I...</td>\n",
       "      <td>IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...</td>\n",
       "      <td>[identify, poverty, drive, need, augment, cens...</td>\n",
       "      <td>IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...</td>\n",
       "      <td>KEERTHI KORIVI</td>\n",
       "      <td>2016</td>\n",
       "      <td>[identify, poverty, drive, need, augment, cens...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>604638e962f2776f1f5c11cb</td>\n",
       "      <td>A semi-supervised clustering method for payloa...</td>\n",
       "      <td>Kyu Seok Lee B.S., Korea University, 2008</td>\n",
       "      <td>A semi-supervised clustering method for payloa...</td>\n",
       "      <td>[semi, supervised, clustering, method, payload...</td>\n",
       "      <td>A semi-supervised clustering method for payloa...</td>\n",
       "      <td>Kyu Seok Lee</td>\n",
       "      <td>2020</td>\n",
       "      <td>[semi_supervise, clustering, method, abstract,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>604638ed62f2776f1f5c11cc</td>\n",
       "      <td>Data augmentation using generative adversarial...</td>\n",
       "      <td>byLei Luo M.A., Kansas State University, 2017</td>\n",
       "      <td>Data augmentation using generative adversarial...</td>\n",
       "      <td>[datum, augmentation, generative, adversarial,...</td>\n",
       "      <td>Data augmentation using generative adversarial...</td>\n",
       "      <td>Lei Luo</td>\n",
       "      <td>2020</td>\n",
       "      <td>[datum, augmentation, use, generative_adversar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>604638f362f2776f1f5c11cd</td>\n",
       "      <td>Graph mining for role extraction in predictive...</td>\n",
       "      <td>by Luis Enrique Bobadilla Dias</td>\n",
       "      <td>Graph mining for role extraction in predictive...</td>\n",
       "      <td>[graph, mining, role, extraction, predictive, ...</td>\n",
       "      <td>Graph mining for role extraction in predictive...</td>\n",
       "      <td>Luis Enrique Bobadilla Dias</td>\n",
       "      <td>2020</td>\n",
       "      <td>[mining, high, performance, compute, system, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>604638f662f2776f1f5c11ce</td>\n",
       "      <td>Data Analytics on a Yelp Data Set by Maitreyi ...</td>\n",
       "      <td>B.Tech., Gitam University, India, 2015 A REPORT</td>\n",
       "      <td>Data Analytics on a Yelp Data Set by Maitreyi ...</td>\n",
       "      <td>[data, analytics, yelp, data, set, maitreyi, t...</td>\n",
       "      <td>Data Analytics on a Yelp Data Set</td>\n",
       "      <td>Maitreyi Tata</td>\n",
       "      <td>2017</td>\n",
       "      <td>[yelp, datum, set, maitreyi, report, abstract,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>604638fe62f2776f1f5c11cf</td>\n",
       "      <td>AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...</td>\n",
       "      <td>by MAJED ALSADHAN B.S., Kansas State Universit...</td>\n",
       "      <td>AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...</td>\n",
       "      <td>[application, topic, model, algorithm, text, a...</td>\n",
       "      <td>AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...</td>\n",
       "      <td>MAJED ALSADHAN</td>\n",
       "      <td>2014</td>\n",
       "      <td>[application, topic, text, analytic, intellige...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>6046390362f2776f1f5c11d0</td>\n",
       "      <td>TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...</td>\n",
       "      <td>MANIKANTA BABU KARANAM</td>\n",
       "      <td>TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...</td>\n",
       "      <td>[tackle, problem, diversity, recommender, syst...</td>\n",
       "      <td>TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...</td>\n",
       "      <td>MANIKANTA BABU KARANAM</td>\n",
       "      <td>2010</td>\n",
       "      <td>[tackle, problem, diversity, abstract, system,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>6046390562f2776f1f5c11d1</td>\n",
       "      <td>GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...</td>\n",
       "      <td>SACCHAROMYCES CEREVISIAE</td>\n",
       "      <td>GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...</td>\n",
       "      <td>[graph, base, protein, protein, interaction, p...</td>\n",
       "      <td>GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...</td>\n",
       "      <td>MARTIN SAMUEL RAO PARADESI</td>\n",
       "      <td>2008</td>\n",
       "      <td>[base, abstract, list, table, chapter, introdu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>6046390a62f2776f1f5c11d2</td>\n",
       "      <td>College of Engineering</td>\n",
       "      <td>Manhattan, Kansas 2016</td>\n",
       "      <td>College of Engineering\\r\\n\\r\\nManhattan, Kansa...</td>\n",
       "      <td>[college, engineering, manhattan, kansas, 2016...</td>\n",
       "      <td>HIERARCHICAL BAYESIAN TOPIC MODELING WITH SENT...</td>\n",
       "      <td>MING YANG</td>\n",
       "      <td>2016</td>\n",
       "      <td>[abstract, abstract, dissertation, submit, par...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>6046391062f2776f1f5c11d3</td>\n",
       "      <td>Generative versus Sampling-Based Approaches to...</td>\n",
       "      <td>by Nasik Muhammad Nafi</td>\n",
       "      <td>Generative versus Sampling-Based Approaches to...</td>\n",
       "      <td>[generative, versus, sampling, base, approach,...</td>\n",
       "      <td>Generative versus Sampling-Based Approaches to...</td>\n",
       "      <td>Nasik Muhammad Naf</td>\n",
       "      <td>2019</td>\n",
       "      <td>[sampling, base, detection, abstract, table, c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         _id  \\\n",
       "0   6046389562f2776f1f5c11b6   \n",
       "1   6046389762f2776f1f5c11b7   \n",
       "2   6046389862f2776f1f5c11b8   \n",
       "3   6046389962f2776f1f5c11b9   \n",
       "4   6046389b62f2776f1f5c11ba   \n",
       "5   6046389f62f2776f1f5c11bb   \n",
       "6   604638aa62f2776f1f5c11bc   \n",
       "7   604638b062f2776f1f5c11bd   \n",
       "8   604638b262f2776f1f5c11be   \n",
       "9   604638b362f2776f1f5c11bf   \n",
       "10  604638bd62f2776f1f5c11c0   \n",
       "11  604638bd62f2776f1f5c11c1   \n",
       "12  604638bf62f2776f1f5c11c2   \n",
       "13  604638ca62f2776f1f5c11c3   \n",
       "14  604638cc62f2776f1f5c11c4   \n",
       "15  604638d062f2776f1f5c11c5   \n",
       "16  604638d862f2776f1f5c11c6   \n",
       "17  604638da62f2776f1f5c11c7   \n",
       "18  604638de62f2776f1f5c11c8   \n",
       "19  604638e062f2776f1f5c11c9   \n",
       "20  604638e262f2776f1f5c11ca   \n",
       "21  604638e962f2776f1f5c11cb   \n",
       "22  604638ed62f2776f1f5c11cc   \n",
       "23  604638f362f2776f1f5c11cd   \n",
       "24  604638f662f2776f1f5c11ce   \n",
       "25  604638fe62f2776f1f5c11cf   \n",
       "26  6046390362f2776f1f5c11d0   \n",
       "27  6046390562f2776f1f5c11d1   \n",
       "28  6046390a62f2776f1f5c11d2   \n",
       "29  6046391062f2776f1f5c11d3   \n",
       "\n",
       "                                                title  \\\n",
       "0               Evolutionary Tree Genetic Programming   \n",
       "1   PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...   \n",
       "2   LocoLit: A Real-Time View of Local Attractions...   \n",
       "3   Evolving locomotion for virtual quadrupeds by ...   \n",
       "4   Hierarchical and Partitioning-Based Hybridized...   \n",
       "5   ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...   \n",
       "6   Building a scalable distributed data platform ...   \n",
       "7   COMPARATIVE TEXT SUMMARIZATION OF PRODUCT REVI...   \n",
       "8   AUTOMATED GENRE CLASSIFICATION IN LITERATURE b...   \n",
       "9   AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...   \n",
       "10  ALGORITHM SELECTION FOR SORTING AND PROBABILIS...   \n",
       "11  Genetic Programming and Multi-Agent Layered Le...   \n",
       "12  USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...   \n",
       "13  M.S., Emporia State University, 2008 M.S., Kan...   \n",
       "14  SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...   \n",
       "15  BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...   \n",
       "16  submitted in partial fulfillment of the requir...   \n",
       "17  A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...   \n",
       "18  APPROXIMATE INFERENCE OF BAYESIAN NETWORKS THR...   \n",
       "19  Density-based and partitioning-based clusterin...   \n",
       "20  IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...   \n",
       "21  A semi-supervised clustering method for payloa...   \n",
       "22  Data augmentation using generative adversarial...   \n",
       "23  Graph mining for role extraction in predictive...   \n",
       "24  Data Analytics on a Yelp Data Set by Maitreyi ...   \n",
       "25  AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...   \n",
       "26  TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...   \n",
       "27  GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...   \n",
       "28                             College of Engineering   \n",
       "29  Generative versus Sampling-Based Approaches to...   \n",
       "\n",
       "                                              authors  \\\n",
       "0                               jan.antolik@matfyz.cz   \n",
       "1                               by ASHISH KUMAR ASHOK   \n",
       "2   B.E., Muffakham Jah College of Engineering, 20...   \n",
       "3        B.S., Kansas State University, 2018 A REPORT   \n",
       "4                                CHANDRAVYAS ANNAKULA   \n",
       "5   by CHRISTOPHER HENRY MEYER B.S., Kansas State ...   \n",
       "6    A REPORT submitted in partial fulfillment of the   \n",
       "7                            DINESH REDDY SINGI REDDY   \n",
       "8    A THESIS submitted in partial fulfillment of the   \n",
       "9   by ABHIJIT ERANDE B. E., University of Pune, 2005   \n",
       "10  byHAIPENG GUO M.S., Beijing University of Aero...   \n",
       "11  develop a solution. One important observation ...   \n",
       "12  by HAOLEI WANG B.S.,Dongbei University of Fina...   \n",
       "13  requirements for the degree DOCTOR OF PHILOSOP...   \n",
       "14                                           A THESIS   \n",
       "15  B.E., Shanghai Jiaotong Univeristy, China, 200...   \n",
       "16                               DOCTOR OF PHILOSOPHY   \n",
       "17  by JOSHUA L. WEESE B.S., Kansas State Universi...   \n",
       "18  JULIE ANN THORNTON B.S., Kansas State Universi...   \n",
       "19                             ARUNA SAI KANNAMAREDDY   \n",
       "20  by KEERTHI KORIVI B.E. , Osmania University, I...   \n",
       "21          Kyu Seok Lee B.S., Korea University, 2008   \n",
       "22      byLei Luo M.A., Kansas State University, 2017   \n",
       "23                     by Luis Enrique Bobadilla Dias   \n",
       "24    B.Tech., Gitam University, India, 2015 A REPORT   \n",
       "25  by MAJED ALSADHAN B.S., Kansas State Universit...   \n",
       "26                             MANIKANTA BABU KARANAM   \n",
       "27                           SACCHAROMYCES CEREVISIAE   \n",
       "28                             Manhattan, Kansas 2016   \n",
       "29                             by Nasik Muhammad Nafi   \n",
       "\n",
       "                                                 text  \\\n",
       "0   Evolutionary Tree Genetic Programming\\r\\n\\r\\nj...   \n",
       "1   PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...   \n",
       "2   LocoLit: A Real-Time View of Local Attractions...   \n",
       "3          Evolving locomotion for virtual quadrupeds   \n",
       "4   Hierarchical and Partitioning-Based Hybridized...   \n",
       "5   ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...   \n",
       "6   Building a scalable distributed data platform ...   \n",
       "7   COMPARATIVE  TEXT SUMMARIZATION OF PRODUCT REV...   \n",
       "8   AUTOMATED GENRE CLASSIFICATION IN LITERATURE b...   \n",
       "9   AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...   \n",
       "10  ALGORITHM SELECTION FOR SORTING AND PROBABILIS...   \n",
       "11  Genetic Programming and Multi-Agent Layered Le...   \n",
       "12  USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...   \n",
       "13  M.S., Emporia State University, 2008 M.S., Kan...   \n",
       "14  SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...   \n",
       "15  BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...   \n",
       "16  submitted in partial fulfillment of the requir...   \n",
       "17  A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...   \n",
       "18  APPROXIMATE INFERENCE OF BAYESIAN NETWORKS THR...   \n",
       "19  Density-based and partitioning-based clusterin...   \n",
       "20  IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...   \n",
       "21  A semi-supervised clustering method for payloa...   \n",
       "22  Data augmentation using generative adversarial...   \n",
       "23  Graph mining for role extraction in predictive...   \n",
       "24  Data Analytics on a Yelp Data Set by Maitreyi ...   \n",
       "25  AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...   \n",
       "26  TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...   \n",
       "27  GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...   \n",
       "28  College of Engineering\\r\\n\\r\\nManhattan, Kansa...   \n",
       "29  Generative versus Sampling-Based Approaches to...   \n",
       "\n",
       "                                       text_processed  \\\n",
       "0   [evolutionary, tree, genetic, programming, jan...   \n",
       "1   [predictive, data, mining, collaborative, edit...   \n",
       "2   [locolit, real, time, view, local, attraction,...   \n",
       "3                                                  []   \n",
       "4   [hierarchical, partitioning, base, hybridize, ...   \n",
       "5   [improve, natural, language, processing, phras...   \n",
       "6   [build, scalable, distribute, data, platform, ...   \n",
       "7   [comparative, text, summarization, product, re...   \n",
       "8   [automated, genre, classification, literature,...   \n",
       "9   [automatic, detection, significant, feature, e...   \n",
       "10  [algorithm, selection, sort, probabilistic, in...   \n",
       "11  [genetic, programming, multi, agent, layer, le...   \n",
       "12  [density, base, clustering, improve, skeleton,...   \n",
       "13  [ms, emporia, state, university, 2008, ms, kan...   \n",
       "14  [security, risk, prioritization, logical, atta...   \n",
       "15  [bioinformatics, analyse, alternative, splicin...   \n",
       "16  [submit, partial, fulfillment, requirement, de...   \n",
       "17  [convolutive, model, polyphonic, instrument, i...   \n",
       "18  [approximate, inference, bayesian, network, ed...   \n",
       "19  [density, base, partitioning, base, clustering...   \n",
       "20  [identify, poverty, drive, need, augment, cens...   \n",
       "21  [semi, supervised, clustering, method, payload...   \n",
       "22  [datum, augmentation, generative, adversarial,...   \n",
       "23  [graph, mining, role, extraction, predictive, ...   \n",
       "24  [data, analytics, yelp, data, set, maitreyi, t...   \n",
       "25  [application, topic, model, algorithm, text, a...   \n",
       "26  [tackle, problem, diversity, recommender, syst...   \n",
       "27  [graph, base, protein, protein, interaction, p...   \n",
       "28  [college, engineering, manhattan, kansas, 2016...   \n",
       "29  [generative, versus, sampling, base, approach,...   \n",
       "\n",
       "                                         title_manual  \\\n",
       "0               Evolutionary Tree Genetic Programming   \n",
       "1   PREDICTIVE DATA MINING IN A COLLABORATIVE EDIT...   \n",
       "2      LocoLit: A Real-Time View of Local Attractions   \n",
       "3          Evolving locomotion for virtual quadrupeds   \n",
       "4   Hierarchical and Partitioning-Based Hybridized...   \n",
       "5   ON IMPROVING NATURAL LANGUAGE PROCESSING THROU...   \n",
       "6   Building a scalable distributed data platform ...   \n",
       "7   COMPARATIVE TEXT SUMMARIZATION OF PRODUCT REVIEWS   \n",
       "8        AUTOMATED GENRE CLASSIFICATION IN LITERATURE   \n",
       "9   AUTOMATIC DETECTION OF SIGNIFICANT FEATURES AN...   \n",
       "10  ALGORITHM SELECTION FOR SORTING AND PROBABILIS...   \n",
       "11  Genetic Programming and Multi-Agent Layered Le...   \n",
       "12  USING DENSITY-BASED CLUSTERING TO IMPROVE SKEL...   \n",
       "13       Affective Intelligence in Built Environments   \n",
       "14  SECURITY RISK PRIORITIZATION FOR LOGICAL ATTAC...   \n",
       "15  BIOINFORMATICS ANALYSES OF ALTERNATIVE SPLICIN...   \n",
       "16  Bringing Computational Thinking to K-12 and Hi...   \n",
       "17  A CONVOLUTIVE MODEL FOR POLYPHONIC INSTRUMENT ...   \n",
       "18  APPROXIMATE INFERENCE OF BAYESIAN NETWORKS  TH...   \n",
       "19  Density-based and partitioning-based clusterin...   \n",
       "20  IDENTIFYING POVERTY-DRIVEN NEED BY AUGMENTING ...   \n",
       "21  A semi-supervised clustering method for payloa...   \n",
       "22  Data augmentation using generative adversarial...   \n",
       "23  Graph mining for role extraction in predictive...   \n",
       "24                  Data Analytics on a Yelp Data Set   \n",
       "25  AN APPLICATION OF TOPIC MODELING ALGORITHMS TO...   \n",
       "26  TACKLING THE PROBLEMS OF DIVERSITY IN RECOMMEN...   \n",
       "27  GRAPH-BASED PROTEIN-PROTEIN INTERACTION PREDIC...   \n",
       "28  HIERARCHICAL BAYESIAN TOPIC MODELING WITH SENT...   \n",
       "29  Generative versus Sampling-Based Approaches to...   \n",
       "\n",
       "                 authors_manual  year_manual  \\\n",
       "0                   Jan Antolik         2004   \n",
       "1            ASHISH KUMAR ASHOK         2011   \n",
       "2                     Atef Khan         2018   \n",
       "3                 Caleb Compton         2020   \n",
       "4          CHANDRAVYAS ANNAKULA         2017   \n",
       "5       CHRISTOPHER HENRY MEYER         2008   \n",
       "6               DHANANJAY MEHTA         2017   \n",
       "7      DINESH REDDY SINGI REDDY         2010   \n",
       "8                  EMILY JORDAN         2014   \n",
       "9                ABHIJIT ERANDE         2009   \n",
       "10                  HAIPENG GUO         2003   \n",
       "11          Steven M. Gustafson         2000   \n",
       "12                  HAOLEI WANG         2012   \n",
       "13           Heath Landon Yates         2018   \n",
       "14              HUSSAIN ALMOHRI         2008   \n",
       "15                     JING XIA         2008   \n",
       "16            Joshua Levi Weese         2017   \n",
       "17              JOSHUA L. WEESE         2013   \n",
       "18           JULIE ANN THORNTON         2005   \n",
       "19       ARUNA SAI KANNAMAREDDY         2017   \n",
       "20               KEERTHI KORIVI         2016   \n",
       "21                 Kyu Seok Lee         2020   \n",
       "22                      Lei Luo         2020   \n",
       "23  Luis Enrique Bobadilla Dias         2020   \n",
       "24                Maitreyi Tata         2017   \n",
       "25               MAJED ALSADHAN         2014   \n",
       "26       MANIKANTA BABU KARANAM         2010   \n",
       "27   MARTIN SAMUEL RAO PARADESI         2008   \n",
       "28                    MING YANG         2016   \n",
       "29           Nasik Muhammad Naf         2019   \n",
       "\n",
       "                             text_processed_optimized  \n",
       "0   [evolutionary, tree, genetic_programme, abstra...  \n",
       "1   [predictive, datum, mining, collaborative, edi...  \n",
       "2   [locolit, real, time, view, local, attraction,...  \n",
       "3            [evolve, locomotion, virtual, quadruped]  \n",
       "4   [hierarchical, partitioning, base, hybridize, ...  \n",
       "5   [improve, processing, phrase, base, syntactic,...  \n",
       "6   [build, scalable, distribute, data, platform, ...  \n",
       "7   [comparative, text, summarization, product, re...  \n",
       "8   [classification, literature, emily, thesis, su...  \n",
       "9   [automatic, detection, significant, feature, e...  \n",
       "10  [sort, probabilistic, inference, machine, base...  \n",
       "11  [genetic_programme, multi, agent, layer, learn...  \n",
       "12  [use, density, base, cluster, improve, embed, ...  \n",
       "13  [requirement, abstract, abstract, dissertation...  \n",
       "14  [security, risk, prioritization, logical, thes...  \n",
       "15  [alternative_splice, base, machine, base, abst...  \n",
       "16  [submit, partial_fulfillment, requirement, phi...  \n",
       "17  [pitch, detection, use, combined, classificati...  \n",
       "18  [abstract, table, content, chapter, introducti...  \n",
       "19  [density, base, partition, base, cluster, larg...  \n",
       "20  [identify, poverty, drive, need, augment, cens...  \n",
       "21  [semi_supervise, clustering, method, abstract,...  \n",
       "22  [datum, augmentation, use, generative_adversar...  \n",
       "23  [mining, high, performance, compute, system, a...  \n",
       "24  [yelp, datum, set, maitreyi, report, abstract,...  \n",
       "25  [application, topic, text, analytic, intellige...  \n",
       "26  [tackle, problem, diversity, abstract, system,...  \n",
       "27  [base, abstract, list, table, chapter, introdu...  \n",
       "28  [abstract, abstract, dissertation, submit, par...  \n",
       "29  [sampling, base, detection, abstract, table, c...  "
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define preprocessing (text_processed) function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(tokens): #tokens should be 1 string containing the entire document\n",
    "    # Lowercase\n",
    "    tokens = tokens.lower()\n",
    "\n",
    "    # Remove useless characters\n",
    "    tokens = re.sub(\"[^a-zA-Z0-9]\", \" \", tokens)\n",
    "    tokens\n",
    "\n",
    "    # tokenization or word split\n",
    "    tokens = word_tokenize(tokens)\n",
    "\n",
    "    # remove stopwords\n",
    "    tokens = [ i for i in tokens if i not in stopwords.words('english')]\n",
    "\n",
    "    # save all words more than 3 characters\n",
    "    tokens = [ i for i in tokens if len(i) >= 3 ]\n",
    "\n",
    "    # Init the Wordnet Lemmatizer\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "    tokens = (lemmatizer.lemmatize(word) for word in tokens)\n",
    "    \n",
    "    return list(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "tknes = preprocess(df.iloc[18]['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data exploration, unigrams and bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tkens' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-229-8c1a18ffc1c6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0munigrams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtkens\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mfdist\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFreqDist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munigrams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mfdist\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'tkens' is not defined"
     ]
    }
   ],
   "source": [
    "unigrams = tkens.copy()\n",
    "\n",
    "fdist = nltk.FreqDist(unigrams)\n",
    "fdist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigrams = nltk.bigrams(tkens.copy())\n",
    "\n",
    "fdist_bi = nltk.FreqDist(bigrams)\n",
    "fdist_bi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tkens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apply function to dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can take a long time to run.\n",
    "\n",
    "df['text_processed'] = df['text'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(r\"C:\\Users\\shive\\Desktop\\llnl-ksu-recipes\\tkm-repo\\text_processed.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEXT_PROCESS2 MORE PREPROCESSING\n",
    "\n",
    "source: https://towardsdatascience.com/6-tips-to-optimize-an-nlp-topic-model-for-interpretability-20742f3047e2\n",
    "\n",
    "https://nicharuc.github.io/topic_modeling/\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function definitions..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Source: https://github.com/nicharuc/topic_modeling/blob/master/clean_text.py\n",
    "def clean_all(df, col_name):\n",
    "    \"\"\"\n",
    "    Combines all functions used to clean and lemmatize the comments.\n",
    "    :param df: data frame with comments\n",
    "    :param col_name: column name in data frame containing comments\n",
    "    :return: data frame with comments column lemmatized\n",
    "    \"\"\"\n",
    "\n",
    "    # encode for only ascii characters\n",
    "    df[col_name] = df[col_name].map(ascii_rm)\n",
    "    \n",
    "    # lowercase texts\n",
    "    df[col_name] = df[col_name].map(lambda x: x.lower())\n",
    "\n",
    "\n",
    "    # lemmatize words\n",
    "    df[col_name] = df[col_name].astype(str).map(lemma)\n",
    "\n",
    "    # remove punctuation\n",
    "    df[col_name] = df[col_name].map(punc_n)\n",
    "\n",
    "\n",
    "    # filter only english comments/non blank comments\n",
    "    df['language'] = df[col_name].map(get_language)\n",
    "    df = df.loc[df['language'] == 'english']\n",
    "    df = df.drop('language', axis=1)\n",
    "    df = df[df[col_name] != \"\"]\n",
    "    \n",
    "    return df\n",
    "\n",
    "def ascii_rm(comment):\n",
    "    \"\"\"\n",
    "    Parses comments and keep only ascii characters\n",
    "    :param comment: a comment\n",
    "    :return: comment with only ascii characters\n",
    "    \"\"\"\n",
    "    #comment = comment.encode('ascii', errors = 'ignore')\n",
    "    return comment\n",
    "\n",
    "def get_language(text):\n",
    "    \"\"\"\n",
    "    Determines what language the comment is written in and filters only English comments.\n",
    "    :param text: comment\n",
    "    :return: language of comment\n",
    "    \"\"\"\n",
    "    words = set(nltk.wordpunct_tokenize(text.lower()))\n",
    "    return max(((lang, len(words & stopwords)) for lang, stopwords in STOPWORDS_DICT.items()), key = lambda x: x[1])[0]\n",
    "\n",
    "def punc_n(comment):\n",
    "    \"\"\"\n",
    "    Removes punctuations from comments.\n",
    "    :param comment: a comment\n",
    "    :return: comment without punctuations\n",
    "    \"\"\"\n",
    "    regex = re.compile('[' + re.escape('!\"#%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~')+'0-9\\\\r\\\\t\\\\n]')\n",
    "    nopunct = regex.sub(\" \", comment)\n",
    "    nopunct_words = nopunct.split(' ')\n",
    "    filter_words = [word.strip() for word in nopunct_words if word != '']\n",
    "    words = ' '.join(filter_words)\n",
    "    return words\n",
    "\n",
    "def lemma(comment):\n",
    "    \"\"\"\n",
    "    Lemmatize comments using spacy lemmatizer.\n",
    "    :param comment: a comment\n",
    "    :return: lemmatized comment\n",
    "    \"\"\"\n",
    "    lemmatized = nlp(comment)\n",
    "    lemmatized_final = ' '.join([word.lemma_ for word in lemmatized if word.lemma_ != '\\'s'])\n",
    "    return lemmatized_final\n",
    "\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "stop_word_list = ['a', 'about', 'above', 'across', 'after', 'afterwards', 'again', 'against', 'all', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'amoungst', 'amount', 'an', 'and', 'another', 'any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere', 'are', 'around', 'as', 'at', 'back', 'be', 'became', 'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'below', 'beside', 'besides', 'between', 'beyond', 'bill', 'both', 'bottom', 'but', 'by', 'call', 'can', 'cannot', 'cant', 'co', 'computer', 'con', 'could', 'couldnt', 'cry', 'de', 'describe', 'detail', 'did', 'didn', 'do', 'does', 'doesn', 'doing', 'don', 'done', 'down', 'due', 'during', 'each', 'eg', 'eight', 'either', 'eleven', 'else', 'elsewhere', 'empty', 'enough', 'etc', 'even', 'ever', 'every', 'everyone', 'everything', 'everywhere', 'except', 'few', 'fifteen', 'fify', 'fill', 'find', 'fire', 'first', 'five', 'for', 'former', 'formerly', 'forty', 'found', 'four', 'from', 'front', 'full', 'further', 'get', 'give', 'go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her', 'here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'however', 'hundred', 'i', 'ie', 'if', 'in', 'inc', 'indeed', 'interest', 'into', 'is', 'it', 'its', 'itself', 'just', 'keep', 'kg', 'km', 'last', 'latter', 'latterly', 'least', 'less', 'ltd', 'made', 'make', 'many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine', 'more', 'moreover', 'most', 'mostly', 'move', 'much', 'must', 'my', 'myself', 'name', 'namely', 'neither', 'never', 'nevertheless', 'next', 'nine', 'no', 'nobody', 'none', 'noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of', 'off', 'often', 'on', 'once', 'one', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'part', 'per', 'perhaps', 'please', 'put', 'quite', 'rather', 'rather', 're', 'really', 'regarding', 'same', 'say', 'see', 'seem', 'seemed', 'seeming', 'seems', 'serious', 'several', 'she', 'should', 'show', 'side', 'since', 'sincere', 'six', 'sixty', 'so', 'some', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhere', 'still', 'such', 'system', 'take', 'ten', 'than', 'that', 'the', 'their', 'them', 'themselves', 'then', 'thence', 'there', 'thereafter', 'thereby', 'therefore', 'therein', 'thereupon', 'these', 'they', 'thick', 'thin', 'third', 'this', 'those', 'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together', 'too', 'top', 'toward', 'towards', 'twelve', 'twenty', 'two', 'un', 'under', 'unless', 'until', 'up', 'upon', 'us', 'use', 'used', 'using', 'various', 'very', 'very', 'via', 'was', 'we', 'well', 'were', 'what', 'whatever', 'when', 'whence', 'whenever', 'where', 'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', 'whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with', 'within', 'without', 'would', 'yet', 'you', 'your', 'yours', 'yourself', 'yourselves']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient('mongodb://kmAdmin:kmadmin@129.130.10.108:7017/knowledge-map')\n",
    "db = client['knowledge-map']\n",
    "theses = db['etdr']\n",
    "\n",
    "data = []\n",
    "for document in db.etdr.find():\n",
    "  data.append(document)\n",
    "\n",
    "for document in data:\n",
    "  document['_id'] = str(document['_id'])\n",
    "\n",
    "theses_data = [data for data in db.etdr.find()]\n",
    "df = pd.DataFrame(theses_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>dc.contributor.author</th>\n",
       "      <th>dc.identifier.uri</th>\n",
       "      <th>dc.description.abstract</th>\n",
       "      <th>dc.language.iso</th>\n",
       "      <th>dc.subject</th>\n",
       "      <th>dc.title</th>\n",
       "      <th>dc.type</th>\n",
       "      <th>dc.description.degree</th>\n",
       "      <th>dc.description.level</th>\n",
       "      <th>...</th>\n",
       "      <th>dc.source</th>\n",
       "      <th>dc.publisher</th>\n",
       "      <th>dc.subject.umi</th>\n",
       "      <th>dc.citation.issue</th>\n",
       "      <th>dc.citation.atitle</th>\n",
       "      <th>dc.subject.LCNAF</th>\n",
       "      <th>dc.language</th>\n",
       "      <th>dc.format.extent</th>\n",
       "      <th>dc.format.mimetype</th>\n",
       "      <th>dc.contributor.advisor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>61673cafc5b630858c0d87c2</td>\n",
       "      <td>Mitchell, Tyler</td>\n",
       "      <td>https://hdl.handle.net/2097/40992</td>\n",
       "      <td>A study of the Higgs boson coupling to gauge b...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Higgs boson, High energy physics, Coupling, C...</td>\n",
       "      <td>Search for anomalous couplings of Higgs bosons...</td>\n",
       "      <td>Dissertation</td>\n",
       "      <td>Doctor of Philosophy</td>\n",
       "      <td>Doctoral</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>61673cafc5b630858c0d87c3</td>\n",
       "      <td>Carter, Elizabeth</td>\n",
       "      <td>https://hdl.handle.net/2097/40991</td>\n",
       "      <td>In 1985, The Kansas Farm Winery Act was passed...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Grape, Wine, Transaction Costs, Supply Chain]</td>\n",
       "      <td>Analysis of drivers and constraints in the Kan...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61673cb0c5b630858c0d87c4</td>\n",
       "      <td>Delfelder, Christopher James</td>\n",
       "      <td>https://hdl.handle.net/2097/40994</td>\n",
       "      <td>As more upland gamebird habitat is cleared for...</td>\n",
       "      <td>en</td>\n",
       "      <td>[ring-necked pheasant, reduced protein, feathe...</td>\n",
       "      <td>Effects of nutrient levels on pen-raised ring-...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61673cb0c5b630858c0d87c5</td>\n",
       "      <td>Beer, Lynn</td>\n",
       "      <td>https://hdl.handle.net/2097/40995</td>\n",
       "      <td>In the fast-changing world of business today, ...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Job Crafting, Relational Demography, Engagement]</td>\n",
       "      <td>How job crafting can link demographic similari...</td>\n",
       "      <td>Dissertation</td>\n",
       "      <td>Doctor of Philosophy</td>\n",
       "      <td>Doctoral</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>61673cb1c5b630858c0d87c6</td>\n",
       "      <td>Wikander, Yvonne</td>\n",
       "      <td>https://hdl.handle.net/2097/40983</td>\n",
       "      <td>Cytauxzoon felis is a tick-transmitted, obliga...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Cytauxzoon felis, Domestic cat, Cytauxzoonosi...</td>\n",
       "      <td>Investigations into Cytauxzoon felis among the...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        _id         dc.contributor.author  \\\n",
       "0  61673cafc5b630858c0d87c2               Mitchell, Tyler   \n",
       "1  61673cafc5b630858c0d87c3             Carter, Elizabeth   \n",
       "2  61673cb0c5b630858c0d87c4  Delfelder, Christopher James   \n",
       "3  61673cb0c5b630858c0d87c5                    Beer, Lynn   \n",
       "4  61673cb1c5b630858c0d87c6              Wikander, Yvonne   \n",
       "\n",
       "                   dc.identifier.uri  \\\n",
       "0  https://hdl.handle.net/2097/40992   \n",
       "1  https://hdl.handle.net/2097/40991   \n",
       "2  https://hdl.handle.net/2097/40994   \n",
       "3  https://hdl.handle.net/2097/40995   \n",
       "4  https://hdl.handle.net/2097/40983   \n",
       "\n",
       "                             dc.description.abstract dc.language.iso  \\\n",
       "0  A study of the Higgs boson coupling to gauge b...           en_US   \n",
       "1  In 1985, The Kansas Farm Winery Act was passed...           en_US   \n",
       "2  As more upland gamebird habitat is cleared for...              en   \n",
       "3  In the fast-changing world of business today, ...           en_US   \n",
       "4  Cytauxzoon felis is a tick-transmitted, obliga...           en_US   \n",
       "\n",
       "                                          dc.subject  \\\n",
       "0  [Higgs boson, High energy physics, Coupling, C...   \n",
       "1     [Grape, Wine, Transaction Costs, Supply Chain]   \n",
       "2  [ring-necked pheasant, reduced protein, feathe...   \n",
       "3  [Job Crafting, Relational Demography, Engagement]   \n",
       "4  [Cytauxzoon felis, Domestic cat, Cytauxzoonosi...   \n",
       "\n",
       "                                            dc.title       dc.type  \\\n",
       "0  Search for anomalous couplings of Higgs bosons...  Dissertation   \n",
       "1  Analysis of drivers and constraints in the Kan...        Thesis   \n",
       "2  Effects of nutrient levels on pen-raised ring-...        Thesis   \n",
       "3  How job crafting can link demographic similari...  Dissertation   \n",
       "4  Investigations into Cytauxzoon felis among the...        Thesis   \n",
       "\n",
       "  dc.description.degree dc.description.level  ... dc.source dc.publisher  \\\n",
       "0  Doctor of Philosophy             Doctoral  ...       NaN          NaN   \n",
       "1     Master of Science              Masters  ...       NaN          NaN   \n",
       "2     Master of Science              Masters  ...       NaN          NaN   \n",
       "3  Doctor of Philosophy             Doctoral  ...       NaN          NaN   \n",
       "4     Master of Science              Masters  ...       NaN          NaN   \n",
       "\n",
       "  dc.subject.umi dc.citation.issue dc.citation.atitle dc.subject.LCNAF  \\\n",
       "0            NaN               NaN                NaN              NaN   \n",
       "1            NaN               NaN                NaN              NaN   \n",
       "2            NaN               NaN                NaN              NaN   \n",
       "3            NaN               NaN                NaN              NaN   \n",
       "4            NaN               NaN                NaN              NaN   \n",
       "\n",
       "  dc.language dc.format.extent dc.format.mimetype dc.contributor.advisor  \n",
       "0         NaN              NaN                NaN                    NaN  \n",
       "1         NaN              NaN                NaN                    NaN  \n",
       "2         NaN              NaN                NaN                    NaN  \n",
       "3         NaN              NaN                NaN                    NaN  \n",
       "4         NaN              NaN                NaN                    NaN  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_linebreak(line):\n",
    "    line = line.replace('\\n',' ')\n",
    "    line = line.replace('\\r',' ')\n",
    "    line = line.replace('\\t', ' ')\n",
    "    line = line.replace('- ', ' ')\n",
    "    return line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_processed2'] = df['dc.description.abstract'].apply(remove_linebreak)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lowercase\n",
    "df['text_processed2'] = df['text_processed2'].map(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation\n",
    "df['text_processed2'] = df['text_processed2'].apply(punc_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmatize words\n",
    "df['text_processed2'] = df['text_processed2'].astype(str).map(lemma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'in the era of big datum variable selection with high dimensional datum have draw increase attention with a large number of predictor there rise a big challenge for model fitting and prediction in this dissertation -PRON- propose three different yet interconnect methodology which include theory computation and real application for various scenario of regression analysis the primary goal in this dissertation be to develop powerful bayesian solution to high dimensional datum challenge use a new variable selection strategy call hybrid search to effectively reduce computation cost in high dimensional datum analysis -PRON- propose novel computational strategy that can quickly evaluate a large number of marginal likelihood simultaneously within a single computation in chapter -PRON- discuss background and current challenge in high dimensional variable selection the motivation of -PRON- study be also justify in chapter -PRON- introduce a new bayesian method of good subset selection in the context of linear regression the propose method rapidly find the good subset via a hybrid search algorithm that combine deterministic local search and stochastic global search in chapter on the basis of the approach in chapter -PRON- extend -PRON- to a framework of multivariate linear regression model which analyze the relationship between multiple response variable and a common set of predictor in chapter -PRON- propose a general bayesian method to perform high dimensional variable selection for various datum type such as binary count continuous and time to event survival datum use bayesian approximation technique -PRON- develop a general computing strategy that enable -PRON- to assess the marginal likelihood of many candidate model within a single computation in addition to accelerate the convergence -PRON- employ a hybrid search algorithm that can quickly explore the model space and accurately obtain the global maximum of marginal posterior probability'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = df['text_processed2'].tolist()\n",
    "d[18]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>dc.contributor.author</th>\n",
       "      <th>dc.identifier.uri</th>\n",
       "      <th>dc.description.abstract</th>\n",
       "      <th>dc.language.iso</th>\n",
       "      <th>dc.subject</th>\n",
       "      <th>dc.title</th>\n",
       "      <th>dc.type</th>\n",
       "      <th>dc.description.degree</th>\n",
       "      <th>dc.description.level</th>\n",
       "      <th>...</th>\n",
       "      <th>dc.publisher</th>\n",
       "      <th>dc.subject.umi</th>\n",
       "      <th>dc.citation.issue</th>\n",
       "      <th>dc.citation.atitle</th>\n",
       "      <th>dc.subject.LCNAF</th>\n",
       "      <th>dc.language</th>\n",
       "      <th>dc.format.extent</th>\n",
       "      <th>dc.format.mimetype</th>\n",
       "      <th>dc.contributor.advisor</th>\n",
       "      <th>text_processed2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>61673cafc5b630858c0d87c2</td>\n",
       "      <td>Mitchell, Tyler</td>\n",
       "      <td>https://hdl.handle.net/2097/40992</td>\n",
       "      <td>A study of the Higgs boson coupling to gauge b...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Higgs boson, High energy physics, Coupling, C...</td>\n",
       "      <td>Search for anomalous couplings of Higgs bosons...</td>\n",
       "      <td>Dissertation</td>\n",
       "      <td>Doctor of Philosophy</td>\n",
       "      <td>Doctoral</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>a study of the higgs boson couple to gauge bos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>61673cafc5b630858c0d87c3</td>\n",
       "      <td>Carter, Elizabeth</td>\n",
       "      <td>https://hdl.handle.net/2097/40991</td>\n",
       "      <td>In 1985, The Kansas Farm Winery Act was passed...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Grape, Wine, Transaction Costs, Supply Chain]</td>\n",
       "      <td>Analysis of drivers and constraints in the Kan...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>in the kansas farm winery act be pass allow fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>61673cb0c5b630858c0d87c4</td>\n",
       "      <td>Delfelder, Christopher James</td>\n",
       "      <td>https://hdl.handle.net/2097/40994</td>\n",
       "      <td>As more upland gamebird habitat is cleared for...</td>\n",
       "      <td>en</td>\n",
       "      <td>[ring-necked pheasant, reduced protein, feathe...</td>\n",
       "      <td>Effects of nutrient levels on pen-raised ring-...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>as more upland gamebird habitat be clear for r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61673cb0c5b630858c0d87c5</td>\n",
       "      <td>Beer, Lynn</td>\n",
       "      <td>https://hdl.handle.net/2097/40995</td>\n",
       "      <td>In the fast-changing world of business today, ...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Job Crafting, Relational Demography, Engagement]</td>\n",
       "      <td>How job crafting can link demographic similari...</td>\n",
       "      <td>Dissertation</td>\n",
       "      <td>Doctor of Philosophy</td>\n",
       "      <td>Doctoral</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>in the fast change world of business today emp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>61673cb1c5b630858c0d87c6</td>\n",
       "      <td>Wikander, Yvonne</td>\n",
       "      <td>https://hdl.handle.net/2097/40983</td>\n",
       "      <td>Cytauxzoon felis is a tick-transmitted, obliga...</td>\n",
       "      <td>en_US</td>\n",
       "      <td>[Cytauxzoon felis, Domestic cat, Cytauxzoonosi...</td>\n",
       "      <td>Investigations into Cytauxzoon felis among the...</td>\n",
       "      <td>Thesis</td>\n",
       "      <td>Master of Science</td>\n",
       "      <td>Masters</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cytauxzoon felis be a tick transmit obligate h...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        _id         dc.contributor.author  \\\n",
       "0  61673cafc5b630858c0d87c2               Mitchell, Tyler   \n",
       "1  61673cafc5b630858c0d87c3             Carter, Elizabeth   \n",
       "2  61673cb0c5b630858c0d87c4  Delfelder, Christopher James   \n",
       "3  61673cb0c5b630858c0d87c5                    Beer, Lynn   \n",
       "4  61673cb1c5b630858c0d87c6              Wikander, Yvonne   \n",
       "\n",
       "                   dc.identifier.uri  \\\n",
       "0  https://hdl.handle.net/2097/40992   \n",
       "1  https://hdl.handle.net/2097/40991   \n",
       "2  https://hdl.handle.net/2097/40994   \n",
       "3  https://hdl.handle.net/2097/40995   \n",
       "4  https://hdl.handle.net/2097/40983   \n",
       "\n",
       "                             dc.description.abstract dc.language.iso  \\\n",
       "0  A study of the Higgs boson coupling to gauge b...           en_US   \n",
       "1  In 1985, The Kansas Farm Winery Act was passed...           en_US   \n",
       "2  As more upland gamebird habitat is cleared for...              en   \n",
       "3  In the fast-changing world of business today, ...           en_US   \n",
       "4  Cytauxzoon felis is a tick-transmitted, obliga...           en_US   \n",
       "\n",
       "                                          dc.subject  \\\n",
       "0  [Higgs boson, High energy physics, Coupling, C...   \n",
       "1     [Grape, Wine, Transaction Costs, Supply Chain]   \n",
       "2  [ring-necked pheasant, reduced protein, feathe...   \n",
       "3  [Job Crafting, Relational Demography, Engagement]   \n",
       "4  [Cytauxzoon felis, Domestic cat, Cytauxzoonosi...   \n",
       "\n",
       "                                            dc.title       dc.type  \\\n",
       "0  Search for anomalous couplings of Higgs bosons...  Dissertation   \n",
       "1  Analysis of drivers and constraints in the Kan...        Thesis   \n",
       "2  Effects of nutrient levels on pen-raised ring-...        Thesis   \n",
       "3  How job crafting can link demographic similari...  Dissertation   \n",
       "4  Investigations into Cytauxzoon felis among the...        Thesis   \n",
       "\n",
       "  dc.description.degree dc.description.level  ... dc.publisher dc.subject.umi  \\\n",
       "0  Doctor of Philosophy             Doctoral  ...          NaN            NaN   \n",
       "1     Master of Science              Masters  ...          NaN            NaN   \n",
       "2     Master of Science              Masters  ...          NaN            NaN   \n",
       "3  Doctor of Philosophy             Doctoral  ...          NaN            NaN   \n",
       "4     Master of Science              Masters  ...          NaN            NaN   \n",
       "\n",
       "  dc.citation.issue dc.citation.atitle dc.subject.LCNAF dc.language  \\\n",
       "0               NaN                NaN              NaN         NaN   \n",
       "1               NaN                NaN              NaN         NaN   \n",
       "2               NaN                NaN              NaN         NaN   \n",
       "3               NaN                NaN              NaN         NaN   \n",
       "4               NaN                NaN              NaN         NaN   \n",
       "\n",
       "  dc.format.extent dc.format.mimetype dc.contributor.advisor  \\\n",
       "0              NaN                NaN                    NaN   \n",
       "1              NaN                NaN                    NaN   \n",
       "2              NaN                NaN                    NaN   \n",
       "3              NaN                NaN                    NaN   \n",
       "4              NaN                NaN                    NaN   \n",
       "\n",
       "                                     text_processed2  \n",
       "0  a study of the higgs boson couple to gauge bos...  \n",
       "1  in the kansas farm winery act be pass allow fo...  \n",
       "2  as more upland gamebird habitat be clear for r...  \n",
       "3  in the fast change world of business today emp...  \n",
       "4  cytauxzoon felis be a tick transmit obligate h...  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Form bigrams and trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "finder = nltk.collocations.BigramCollocationFinder.from_documents([comment.split() for comment in df.text_processed2])\n",
    "# Filter only those that occur at least 50 times\n",
    "finder.apply_freq_filter(50)\n",
    "bigram_scores = finder.score_ngrams(bigram_measures.pmi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram_measures = nltk.collocations.TrigramAssocMeasures()\n",
    "finder = nltk.collocations.TrigramCollocationFinder.from_documents([comment.split() for comment in df.text_processed2])\n",
    "# Filter only those that occur at least 50 times\n",
    "finder.apply_freq_filter(50)\n",
    "trigram_scores = finder.score_ngrams(trigram_measures.pmi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_pmi = pd.DataFrame(bigram_scores)\n",
    "bigram_pmi.columns = ['bigram', 'pmi']\n",
    "bigram_pmi.sort_values(by='pmi', axis = 0, ascending = False, inplace = True)\n",
    "\n",
    "trigram_pmi = pd.DataFrame(trigram_scores)\n",
    "trigram_pmi.columns = ['trigram', 'pmi']\n",
    "trigram_pmi.sort_values(by='pmi', axis = 0, ascending = False, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for bigrams with only noun-type structures\n",
    "def bigram_filter(bigram):\n",
    "    tag = nltk.pos_tag(bigram)\n",
    "    if tag[0][1] not in ['JJ', 'NN'] and tag[1][1] not in ['NN']:\n",
    "        return False\n",
    "    if bigram[0] in stop_word_list or bigram[1] in stop_word_list:\n",
    "        return False\n",
    "    if 'n' in bigram or 't' in bigram:\n",
    "        return False\n",
    "    if 'PRON' in bigram:\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for trigrams with only noun-type structures\n",
    "def trigram_filter(trigram):\n",
    "    tag = nltk.pos_tag(trigram)\n",
    "    if tag[0][1] not in ['JJ', 'NN'] and tag[1][1] not in ['JJ','NN']:\n",
    "        return False\n",
    "    if trigram[0] in stop_word_list or trigram[-1] in stop_word_list or trigram[1] in stop_word_list:\n",
    "        return False\n",
    "    if 'n' in trigram or 't' in trigram:\n",
    "         return False\n",
    "    if 'PRON' in trigram:\n",
    "        return False\n",
    "    return True "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can set pmi threshold to whatever makes sense - eyeball through and select threshold where n-grams stop making sense\n",
    "# choose top 500 ngrams in this case ranked by PMI that have noun like structures\n",
    "filtered_bigram = bigram_pmi[bigram_pmi.apply(lambda bigram:\\\n",
    "                                              bigram_filter(bigram['bigram'])\\\n",
    "                                              and bigram.pmi > 5, axis = 1)][:500]\n",
    "\n",
    "filtered_trigram = trigram_pmi[trigram_pmi.apply(lambda trigram: \\\n",
    "                                                 trigram_filter(trigram['trigram'])\\\n",
    "                                                 and trigram.pmi > 5, axis = 1)][:500]\n",
    "\n",
    "\n",
    "bigrams = [' '.join(x) for x in filtered_bigram.bigram.values if len(x[0]) > 2 or len(x[1]) > 2]\n",
    "trigrams = [' '.join(x) for x in filtered_trigram.trigram.values if len(x[0]) > 2 or len(x[1]) > 2 and len(x[2]) > 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['monte carlo',\n",
       " 'triticum aestivum',\n",
       " 'saudi arabia',\n",
       " 'flint hill',\n",
       " 'fort riley',\n",
       " 'palmer amaranth',\n",
       " 'liver abscess',\n",
       " 'remote sensing',\n",
       " 'hessian fly',\n",
       " 'ray diffraction',\n",
       " 'mass spectrometry',\n",
       " 'escherichia coli',\n",
       " 'semi structured',\n",
       " 'skeletal muscle',\n",
       " 'halogen bond',\n",
       " 'randomly assign',\n",
       " 'log cfu',\n",
       " 'wind turbine',\n",
       " 'cross sectional',\n",
       " 'tallgrass prairie']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examples of bigrams\n",
    "bigrams[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['wheat triticum aestivum',\n",
       " 'x ray diffraction',\n",
       " 'semi structured interview',\n",
       " 'sorghum sorghum bicolor',\n",
       " 'log cfu g',\n",
       " 'world war ii',\n",
       " 'kansas city missouri',\n",
       " 'kansas state university',\n",
       " 'statistically significant difference',\n",
       " 'net present value']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examples of trigrams\n",
    "trigrams[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenate n-grams\n",
    "def replace_ngram(x):\n",
    "    for gram in trigrams:\n",
    "        x = x.replace(gram, '_'.join(gram.split()))\n",
    "    for gram in bigrams:\n",
    "        x = x.replace(gram, '_'.join(gram.split()))\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_processed = pd.DataFrame(df.text_processed2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize reviews + remove stop words + remove names + remove words with less than 2 characters\n",
    "reviews_w_ngrams = reviews_w_ngrams.text.map(lambda x: [word for word in x.split()\\\n",
    "                                                 if word not in stop_word_list\\\n",
    "                                                              and len(word) > 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [evolutionary, tree, genetic, programming, nja...\n",
       "1    [predictive, datum_mining, collaborative, edit...\n",
       "2    [locolit, real_time, view, local, attraction, ...\n",
       "4    [hierarchical, partition, base, hybridize, blo...\n",
       "5    [improve, natural_language_processing, phrase,...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_w_ngrams.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_w_ngrams = text_processed.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_w_ngrams.text_processed2 = reviews_w_ngrams.text_processed2.map(lambda x: replace_ngram(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize reviews + remove stop words + text_processed2 names + remove words with less than 2 characters\n",
    "reviews_w_ngrams = reviews_w_ngrams.reviewText.map(lambda x: [word for word in x.split()\\\n",
    "                                                 if word not in stop_word_list\\\n",
    "                                                              and len(word) > 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [study, higgs, boson, couple, gauge, boson, pr...\n",
       "1    [kansas, farm, winery, act, pass, allow, sale,...\n",
       "2    [upland, gamebird, habitat, clear, residential...\n",
       "3    [fast, change, world, business, today, employe...\n",
       "4    [cytauxzoon, felis, tick, transmit, obligate, ...\n",
       "Name: text_processed2, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_w_ngrams.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Filter for only nouns\n",
    "def noun_only(x):\n",
    "    pos_comment = nltk.pos_tag(x)\n",
    "    filtered = [word[0] for word in pos_comment if word[1] in ['NN']]\n",
    "    # to filter both noun and verbs\n",
    "    #filtered = [word[0] for word in pos_comment if word[1] in ['NN','VB', 'VBD', 'VBG', 'VBN', 'VBZ']]\n",
    "    return filtered\n",
    "\n",
    "    \n",
    "f = reviews_w_ngrams.map(noun_only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_pron(x):\n",
    "    fixed = [word for word in x if word.lower() != 'pron']\n",
    "    return fixed\n",
    "    \n",
    "final_reviews = f.map(remove_pron)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       [study, gauge, boson, subscript, subscript, su...\n",
       "1       [farm, winery, act, pass, sale, manufacture, s...\n",
       "2       [gamebird, habitat, housing, agriculture, oper...\n",
       "3       [change, world, business, today, employee, rec...\n",
       "4       [cytauxzoon, transmit, obligate, hemoprotozoal...\n",
       "                              ...                        \n",
       "6939    [transformation, trigger, absorption, importan...\n",
       "6940    [holobiont, concept, plant, treat, plant, cons...\n",
       "6941    [study, effect, supply, information, fat, cont...\n",
       "6942    [intelligence, organization, factor, raison, d...\n",
       "6943    [population, change, growth, environment, crop...\n",
       "Name: text_processed2, Length: 6944, dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_processed2'] = final_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6944"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_reviews)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA Model grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import nltk\n",
    "import re\n",
    "import string\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gensim\n",
    "from gensim import corpora\n",
    "import pyLDAvis.gensim\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = corpora.Dictionary(df['text_processed2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_term_matrix = [dictionary.doc2bow(doc) for doc in df['text_processed2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 5\n",
      "Round: 6\n",
      "Round: 7\n",
      "Round: 8\n",
      "Round: 9\n",
      "Round: 10\n",
      "Round: 11\n",
      "Round: 12\n",
      "Round: 13\n",
      "Round: 14\n",
      "Round: 15\n",
      "Round: 16\n",
      "Round: 17\n",
      "Round: 18\n",
      "Round: 19\n",
      "Round: 20\n",
      "Round: 21\n",
      "Round: 22\n",
      "Round: 23\n",
      "Round: 24\n"
     ]
    }
   ],
   "source": [
    "coherence = []\n",
    "for k in range(5,25):\n",
    "    print('Round: '+str(k))\n",
    "    Lda = gensim.models.ldamodel.LdaModel\n",
    "    ldamodel = Lda(doc_term_matrix, num_topics=k, id2word = dictionary, passes=40,\\\n",
    "                   iterations=200, chunksize = 10000, eval_every = None)\n",
    "    \n",
    "    cm = gensim.models.coherencemodel.CoherenceModel(model=ldamodel, texts=final_reviews,\\\n",
    "                                                     dictionary=dictionary, coherence='c_v')\n",
    "    coherence.append((k,cm.get_coherence()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val = [x[0] for x in coherence]\n",
    "y_val = [x[1] for x in coherence]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABHsklEQVR4nO3deXxU1fn48c+TPUAg7EtYwq4syhIUxX0pLq2gba2KVq1LtWrtt5YWq19/LdZqS7X9tnWpdatVa61FxKpFa0VlEUgACVsgAQIJa3YI2fP8/rg3OAyTyUwyk8nyvF+veTFz7j3nnplc5pl7zj3niKpijDHGBCoq0hUwxhjTvljgMMYYExQLHMYYY4JigcMYY0xQLHAYY4wJigUOY4wxQbHAYVqdiLwkIr+I0LFFRF4UkWIRWd0KxxsqIkdEJDrcx4oUETlPRPIiXQ/TeixwGERkl4gcFJGuHmm3isjSCFYrXM4CLgYGq+ppnhtE5Kful/wREakUkTqP15uaczBV3a2q3VS1LhSVDycRuU5E0t33u09E3heRsyJdL9P2WOAwDaKBeyNdiWA145f8MGCXqpZ7b1DVX7pf8t2AO4CVDa9VdXwo6ttWicgPgd8BvwT6A0OBp4BZrVyPmNY8nmkeCxymwQLgRyKS7L1BRFJFRD3/U4vIUhG51X1+k4gsF5HfikiJiOwQkTPd9D3u1cyNXsX2EZEPReSwiHwiIsM8yj7J3VYkIlkicrXHtpdE5GkReU9EyoHzfdR3kIgsdvNni8htbvotwHPAGe6v6p8H+uG472eNiJS6/57p9Vk8KiKrRaRMRN4WkV6+PjsR6eU2le11m8sWuel9RORf7udXJCKficgJ/z/d9/4br7S33S9+ROQnIpLvfq5ZInJhAO+tBzAfuEtVF6pquarWqOo7qjrX3SdeRH7n1nuv+zzeq5z73L/1PhG52SM9XkR+IyK7ReSAiDwjIonutvNEJM+t937gRRGJEpF5IpIjIoUi8oaPz/NGt7wCEXnA41jR7pVjjvsZZIjIEHdbo+eVCZKq2qOTP4BdwEXAQuAXbtqtwFL3eSqgQIxHnqXAre7zm4Ba4GacK5dfALuBJ4F44CvAYaCbu/9L7utz3O3/Byxzt3UF9rhlxQCTgQJgnEfeUmAGzg+fBB/v51OcX8sJwCTgEHCBR12XBfCZ3ORRp15AMXCDW6dr3de9PT6LfGCCW/9/Aq/4+uyAd4G/Az2BWOBcN/1R4Bk3LRY4GxAf9TrH/XzEfd0TqAAGAWPdbYM8jj0ygPd6ifv3i/Gzz3zgc6Af0BdYATzsbjvPzT/frftlwFGgp7v9t8Bi93NMAt4BHvXK+yv3XEjEufL9HBjspv0J+JvX5/lnd99TgSrgZHf7XCDT/SzE3d6bJs4rewT5nRHpCtgj8g++DBwTcL6U+xJ84NjusW2iu39/j7RCYJL7/CXgdY9t3YA6YAjwLeAzr/r9Cfh/Hnlf9vNehrhlJXmkPQq85FHXYAPHDcBqr+0rgZs8PovHPLaNA6pxguixzw4YCNQ3fKF6lTcfeBsY1US9BCcon+O+vg34r/t8FHDQ/VvGBvH3nwPsb2KfHOAyj9czcZr8wPnyr/A6Pw4C0936luMRwIAzgJ0eeavx+AEAbAEu9Hg9EKhxP8OGz3Owx/bVwDXu8yxglo/6+z2v7BHcw5qqzDGquhH4FzCvGdkPeDyvcMvzTuvm8XqPx3GPAEU4v5qHAae7TTYlIlKC88U2wFdeHwYBRap62CMtF0gJ/K34LDPXK827zD1e22KBPl55hrh1K/ZxjAVANvCB29Tn82+gzjfe6zhXPQDXAa+627KBHwA/Aw6KyOsiMsjvO3MU4jQd+utf8P4Mct20Y2Woaq3H66M4f+++QBcgw+Pv+W83vcEhVa30eD0MeMtj/y04Pwb6e+yz38exwPmMc3zUP5DzygTIAofx9v9wfsV6fik2dCR38Uhr6X+4IQ1PRKQbTjPGXpwv4E9UNdnj0U1V7/TI629K571ALxFJ8kgbitOU1Fx7cb54PHmXOcRrWw1OU4inPW7dkr0PoKqHVfU+VR0BXAH80E//xN+Ab7j9QqfjNI01lPOaqp7l1ldxmoCashKnuWe2n328P4OhblpTCnB+NIz3+Hv2UOcGhGPV9sqzB7jU6xxIUNVA/oZ7gJGNpDd1XpkAWeAwx3F/tf4d+L5H2iGcL8nr3c7H7+D7P2cwLhORs0QkDngY+FxV9+Bc8YwRkRtEJNZ9TBORkwOs/x6c9vdHRSRBRE4BbgFeaUFd33PrdJ2IxIjIt3Cao/7lsc/1IjJORLrgNDu9qV634KrqPuB94CkR6em+t3MAROSrIjJKRASnubAOp1nL13tch/OF/BywRFVL3DLGisgFbqd1Jc4Xts8yvMorBR4CnhSR2SLSxa3bpSLya3e3vwEPikhfEenj7t/kZ6qq9Tj9Eb8VkX5uPVNEZKafbM8Aj7iBEfeYgd7d9RzwsIiMFscpItKbFp5X5ngWOIwv83E6Ez3dhtPxWAiMx/lybonXcK5uioCpwPXg/PLG6Uy/BucX7X6+7DgN1LU4beF7gbdw2rH/09yKqmoh8FXgPpz3/2Pgq6rqeUXxV5z+l/04nfLfx7cbcK5GtuL0A/zATR8N/Ac4gnMF8JSqfuynWq/h9GW85pEWDzyGE1T243Rk3w8gInPEz1gUVX0c+CHwIM7NBHuAu4FF7i6/ANKBDTidz2vdtED8BKcZ7nMRKXPf51g/+/8fTmf6ByJyGKej/PQAj/UE8AbwAVAGPA8khui8Mq6GOzOMMc0kzkDJV1T1uUjXxZjWYFccxhhjgmKBwxhjTFCsqcoYY0xQ7IrDGGNMUDrFhGJ9+vTR1NTUSFfDGGPalYyMjAJV7eud3ikCR2pqKunp6ZGuhjHGtCsi4j1jAmBNVcYYY4JkgcMYY0xQLHAYY4wJigUOY4wxQbHAYYwxJiid4q4qY4zpTBaty2fBkiz2llQwKDmRuTPHMntyS5akOZ4FDmOM6UAWrcvn/oWZVNQ4s/rnl1Rw/8JMgJAFj7A2VYnIJe6i8NmNrWjm7vd1dwH6NPd1qohUiMh69/GMx75TRSTTLfP37voFxhhjgAVLso4FjQYVNXUsWJIVsmOE7YpDRKKBJ4GLgTxgjYgsVtXNXvsl4SxOv8qriBxVneSj6Kdx1oZYhbPAziU4i+MYY0ynl19S4TN9byPpzRHOK47TgGxV3aGq1TjrJPtaxethnAVVKn1sO46IDAS6q+rn7trLL+N/uUtjjOk0MvNKiY323QgzKDkxZMcJZ+BIwVlFrEEex69jjYhMAYao6rs+8g8XkXUi8omInO1RZp6/Mj3Kvl1E0kUk/dChQ81+E8YY09aVV9Uy/53NzHpyGYmx0cRFH//VnhgbzdyZ/hZdDE7EOsdFJApnmcebfGzeBwxV1UIRmQosEpHxwZSvqs8CzwKkpaXZ3PHGmA7poy0HeOjtTeSXVDDn9KH8+JKT+HjrwXZ7V1U+MMTj9WA3rUESMAFY6vZvDwAWi8gVqpoOVAGoaoaI5ABj3PyD/ZRpjDGdwsGySn72zibey9zP6H7dePOOM0hL7QU4d0+FMlB4C2fgWAOMFpHhOF/u1wDXNWxU1VKgT8Nrd93mH6lquoj0BYpUtU5ERgCjgR2qWiQiZSIyHadz/NvAH8L4Howxpk2pr1deXb2bX7+/laq6en70lTHcfs5I4mJabzx32AKHqtaKyN3AEiAaeEFVN4nIfCBdVRf7yX4OMF9EaoB64A5VLXK3fQ94CUjEuZvK7qgyxnQKWfsPc//CDazdXcKZI3vzyJUTGd6na6vXo1MsHZuWlqa2Hocxpr3wHvl974WjyS0q50+f7CApIYYHLx/HVVNSCPcwNhHJUNU073QbOW6MMW2Ir5HfP/nnBhS4akoKD14+jl5d4yJaRwscxhjThvga+a1A765xPHH1pIjUyZvNjmuMMW1IYyO8i8qrW7kmjbPAYYwxbUhjI7xDOfK7pSxwGGNMGzJ35tgTpg0J9cjvlrLAYYwxbcisSYMY0D2BmCgneKQkJ/LoVRPDOqAvWNY5bowxbcja3cXsKa7gF7MncP30YZGujk92xWGMMW3IC8t3kZQQw1VT2s4VhjcLHMYY00bsK63g3xv3c820IXSJa7sNQhY4jDGmjXjl81xUlW+fkRrpqvhlgcMYY9qAypo6Xlu1m4tO7s+QXl0iXR2/LHAYY0wbsHj9XoqP1nDTjNRIV6VJFjiMMSbCVJUXV+zipAFJnDGid6Sr0yQLHMYYE2GrdxaxZV8ZN52ZGvYZb0PBAocxxkTYi8t3kdwlllmT2u4tuJ4scBhjTATlFR/lg837ufa0oSTGRUe6OgGxwGGMMRH0189zEZE2O0rcFwscxhgTIUera3l99R5mju9PShua/bYpFjiMMSZCFq3bS2lFDTfPGB7pqgTFAocxxkSAqvLSip2MH9SdtGE9I12doIQ1cIjIJSKSJSLZIjLPz35fFxEVkTT39cUikiEime6/F3jsu9Qtc7376BfO92CMMeGwIqeQbQeOtJtbcD2FbRYtEYkGngQuBvKANSKyWFU3e+2XBNwLrPJILgC+pqp7RWQCsATwvE9tjqqmh6vuxhgTbi8u30XvrnF87dRBka5K0MJ5xXEakK2qO1S1GngdmOVjv4eBXwGVDQmquk5V97ovNwGJIhIfxroaY0yr2V14lI+2HuC604eSENs+bsH1FM7AkQLs8Xidx/FXDYjIFGCIqr7rp5yvA2tVtcoj7UW3mep/pZFrPBG5XUTSRST90KFDzXwLxhgTen9ZuYvodnYLrqeIdY6LSBTwBHCfn33G41yNfNcjeY6qTgTOdh83+Mqrqs+qapqqpvXt2zd0FTfGmBYor6rljTV7uGziQPp3T4h0dZolnIEjHxji8Xqwm9YgCZgALBWRXcB0YLFHB/lg4C3g26qa05BJVfPdfw8Dr+E0iRljTLuwcG0eh6tq28UsuI0JZ+BYA4wWkeEiEgdcAyxu2KiqparaR1VTVTUV+By4QlXTRSQZeBeYp6rLG/KISIyI9HGfxwJfBTaG8T0YY0zI1Nc7s+CeOrgHk4ckR7o6zRa2wKGqtcDdOHdEbQHeUNVNIjJfRK5oIvvdwCjgIa/bbuOBJSKyAViPcwXz53C9B2OMCaXPsgvYcaicm2cMb3e34HoSVY10HcIuLS1N09Pt7l1jTGTd/OJqNu4tY/lPLiAupu2PvxaRDFVN805v+zU3xpgOYMehI3ycdYg5pw9tF0HDn/Zde2OMaSdeXplLbLQw5/T2eQuup7CNHDfGNN+idfksWJLF3pIKBiUnMnfmWGZPbh+L/JgTHa6s4R/pe/jaKYPom9T+xzJb4DCmjVm0Lp/7F2ZSUVMHQH5JBfcvzASw4NFO/SM9j/LqunZ9C64na6oypo1ZsCTrWNBoUFFTx4IlWRGqkWmJ+nrlLyt3MXVYT04ZnBzp6oSEBQ5j2pi9JRVBpZu2bem2g+QWHuWmM1MjXZWQscBhTBszqJGV4Ab2aJ/TU3R2Ly7fxYDuCVwyYUCkqxIyFjiMaWPmzhxLgo/bNU8e2J3OMO6qI8k+eJjPthdwwxnDiI3uOF+31jluwsLuCmq+2ZNT2Lq/jGc+2QHAoB4JDO/TlY+2HuSxf29l3iUntetRx60h0udfw/Hz3ebFHgmxrXbs1mCBw4Sc3RXUcvEx0UQJbPjZTLrFx1Bfrzy0eCN/coOJBY/GRfr88z4+wCPvbaFbQkyHOf87zrWTaTPsrqCWW7u7mJMGdKdbvPPbLipKmH/FBK6fPpQ/fbKDx97fas1WjYj0+Rfp47cGu+IwIWd3BbVMXb2ybncJV3r9Oo2KEh6eNQGAP33qXnlcalce3iJ9/kX6+K3BAocJuUHJicfadr3TTdOy9h/mSFUtU4f1PGGbyPHBQ4H7LXgcp2eXOIqOVp+Q3lrnX++ucRSUR+74rcGaqkzIzZ05lnivu4LiY6KYO3NshGrUvmTkFgH4DBzwZfC4Yfownv10B49as9Uxb6TvobiimigfcfSc0X3Cfvw9RUepqK3D+/CJsdEd6vy3wGFCbvbkFOacPvS4tElDkjtMx2C4ZeQW0y8pnsE9G/+FKiLMnzX+WPD45XtbOn3w+MuKXfz4zQ2cNaoPj141kZTkRATnrrQx/bvx5to8Vu8sCtvxj1bXctvL6USL8NPLTj52/JTkRB69amKHOv+tqcqERWJcNNFRwqafz+Sx97fyyue55BUfZXDPLq1y/EjfjtkSGbuLSUvt2WTzU0PwEIE/f7YTgJ9ednKnbLZ6amk2v/53FheP688fr5tMfEw035r25Y+X0qM1XPn0cr7713TevusshvYO7XmoqvzoH1+w7cBhXrz5NM4d05fbzhkR0mO0JXbFYcJiQ14pY/onkRAbze3njECEY7eShlvD7ZD5JRUoX96OuWhdfpN5I+1gWSV7iiqYMtR3M5U3EeHnV4zn22cM48+f7eSRdzvXlYeq8pslWfz631nMmjSIp+ZMIT4m+oT9enSJ5fkbp1GvcMtf1nC4siak9fjjf7N5L3M/9196MueO6RvSstsiCxwm5FSVzPxSTknpATidgt+YOoS/p+/hQFll2I/fnm+HzMgtBhrv3/ClIXjceMYwnlvWeYJHfb3y83c288ePs7lm2hCeuHqS39HZw/t05enrp7CzoJx7/raO2rr6kNTjg037efzDbVw1OYVbzx4ekjLbOgscJuTyiisoOVrDxME9jqXdee5I6uqVZz8N/1VHe74dMiO3mPiYKMYP6tH0zh5EhJ95BI+bXlzDmY9+xPB57zLjsf+2i6utYNTVK/MWbuClFbu45azhPHrVRKJ99Yh7OXNkH+bPmsDSrEP88r2tLa7HtgOH+Z+/r+fUwT345VUTO00zYVgDh4hcIiJZIpItIvP87Pd1EVERSfNIu9/NlyUiM4Mt00TOhrxSAE7xCBxDe3dh1qRBvLoql8IjVWE9fmO3PbaH2yHTc4s5dXBys5YWbQgeZ4/uwyfbDrG3tLLdNdUFoqaunntfX8cb6Xl8/8LRPHh5cP06150+lO/MGM4Ly3fy2qrdza5HydFqbns5nS7xMfzphjQSYk9sIuuowhY4RCQaeBK4FBgHXCsi43zslwTcC6zySBsHXAOMBy4BnhKR6EDLNJG1Ib+E2Ghh7ICk49K/d94oqmrreX7ZzrAe31dzQWy0tPnbIStr6ti0t5QpQTRTeRMRcg4eOSG9vTTVNaWypo47X8ngXxv2cf+lJ/HDi8c061f+Ty87ifPG9uWhtzeyIqcg6Py1dfXc/do69pVU8sz1UxnQyWYuDucVx2lAtqruUNVq4HVglo/9HgZ+BXg2fs8CXlfVKlXdCWS75QVapomgzLxSThrQ/YROylH9unH5xIG8vDKXEh8DtEJlZU4hsdFCf3eJzrjoKGKjozinjXdaZuaXUlOnpLUgcADsK/Xdj9Qemur8OVpdyy1/WcN/thzk4Vnj+e65I5tdVkx0FL+/djLD+3TlzlfWsrOgPKj8v3xvK8uyC/jFlROC6o/qKMIZOFKAPR6v89y0Y0RkCjBEVd8NMG+TZXqUfbuIpItI+qFDh5r3DkzQGjrGPfs3PN11/iiOVNXy0opdYTn+h5sP8MHmA/zw4rGseuAidj12Oe/ccxbVtfX86v2Wt2mHU/oup2O8JVcc0L6b6hpTWlHDDc+vZmVOIY9/81RuOCO1xWV2T3DutIoS506r0orA7rT6R/oeXli+k5tnpHJ12pAW16M9iljnuIhEAU8A94WjfFV9VlXTVDWtb9+2/UuzI8ktPMrhytpjd1R5O3lgdy4e158Xl+8K+S2R5VW1/L+3NzKmf7fjmqvGDkjilrOG8/f0PaTvCt8AsJbKyC1mRJ+u9Ooa16Jy5s4cS6JXe3tCOxu5v2hdPjMe+y/D573L9F9+xOX/9xkb8kp48ropfH3q4JAdZ2jvLjxz/VT2FB3l7tfWNnmn1drdxTzw1kZmjOrNA5edHLJ6tDfhDBz5gGc4HuymNUgCJgBLRWQXMB1Y7HaQN5a3qTJNhG3IdzrGG7viALjnglGUVtTwyufN75j05f8+2s7e0kp+eeXEE27L/P6FoxnUI4EHF22kJkS3YYaSqrJ2d3FImj1mT045NnK6wYzRfdrNAEjvcTj7yyrJK6ng5hnDuXTiwJAf7/QRvXlk9kQ+217A/H9tbnS/A2WV3PHXDAb0SOCP104hpgMtzBSscL7zNcBoERkuInE4nd2LGzaqaqmq9lHVVFVNBT4HrlDVdHe/a0QkXkSGA6OB1U2VaSIvM6+EuJgoxvRPanSfUwYnc+6Yvjz32Q6OVteG5Lib95bx/LKdXDNtCGmpvU7Y3jU+hoe+Np6t+w/zlzA1k7XEzoJyisqrQ9ZePntyCsvnXcCuxy7n0gkDWLOziPKq0HzW4eZrHA7Auxv2he2YV08bwu3njODllbm8vHLXCdsra+q4/a8ZlFfV8udvp9GzhVeF7V3YAoeq1gJ3A0uALcAbqrpJROaLyBVN5N0EvAFsBv4N3KWqdY2VGa73YIK3Ia+UcQO7N7lM5j0XjKKwvJq/rd7jd79A1NcrDyzKJDkxlnmXntTofjPH9+eCk/rx2w+3sa+0bXUUNwz8S0sNfUfrrWePoKyyljcz8kJedjhEahzOTy45iQtP6sfP39nMZ9u/7BdVVX66MJMv9pTwxLcmnXC3YGcU1rmqVPU94D2vtIca2fc8r9ePAI8EUqZpG+rrlY35pQG1Qael9mL6iF48+2kOc04f2qJ74F9bvZt1u0t44upTSe7S+C/BhhHWFz3xCfPf2czT109t9jFDLSO3mB6JsYzo0y3kZU8d1pPJQ5N5ftlOrp8+LKCBcpEUqWn5o6OE/7t2Mt94egW3vZxO94RYDh2uIikhhrLKWv7nojHMHD8grHVoLzpvI50JuR0F5ZRX1zGxkY5xb9+/YDQHyqpa9Ev44OFKfvXvrZw5svcJCx/5MqRXF+65YBTvb9zPx1kHm33cUMvILWbK0GSiwvSlftvZI9hddJQPNx8IS/mhdPG4/iektda05N3iY7g6bQiVNfUcPFyFAmWVtUQJDOnVfu9KCzULHCZkMvNLAKcPIxBnjOzNlKHJPL00p9kd1r/41xaqaup5ePaEgAeC3XbOCEb07cr/e3sTlT7a0ltb6dEath884rNvJlRmjh/AkF6JPPdZ60w02Vy7Csp5MyOPYb27MKhHQkSmJfc1QLVe4fEPtrXK8dsDCxwmZDbklZIQG8XIvl0D2l9EuOeC0eSXVPBWM6bD+HTbIRZ/sZc7zxvJyL6BN/HEx0Tzi1kT2F10lKeW5gR93FBbu9sdvxHgjLjNER0l3HzmcNJzi1nnHq+tqayp43uvriUmWnjttumsuP9Cdj52OcvnXdCqd4S157nOWosFDhMymXmljB/UI6jbFM8b25cJKd156uNs6uoDn9G1sqaO/317ozPy97zgRxCfOaoPsyYN4pmlOew4dOIUHa0pI7eY6Cjh1CHBTWwYrKunDSEpIYbnwjzlS3P9/J3NbN5Xxm+vnnTcrcStrSMOoAw1CxwmJGrr6tm0tyzg/o0GIsLd549mV+FR/rVhb8D5nvw4m9zCozwye0KzO9YfuPxk4mOjeOjtTRGdhjwjt5jxg7rTJS6866p1i4/hutOH8n7mPvYUHQ3rsYK1aF0+f1u9mzvPG8n5J/WLaF18DaDsaEu/tpQFDhMSOYfKqaipO25G3EB9ZVx/xvTvxpMfZ1MfwFVH9sHDPPNJDldOTuHMUc1fR7pfUgJzZ45lWXYB/wrjGAF/aurqWb+nJKzNVJ5uOjOVKBFeXL6rVY4XiOyDh/npW5mcltqL+y4eE+nqHDeAsqMu/dpSFjhMSGzIKwFoVuCIihLuOn8U2w4c4YPN+/3uq6o88NZGusTF8MDlLZ/yYc7pw5iY0oOH/7U55FOgBGLrvsNU1NS12kR5A3sk8tVTBvL3Nbspi8D79Xa0upbvvbqWxNho/nDd5DYzGrthAGUk+ljag7bxVzLtXmZ+KV3johnezHEIXz1lEMP7dOUP/83222z0ZkYeq3YWMe/Sk+jTLb651T0mOkp45MoJHDpSFZG7ZjJynbmzwjHwrzG3nj2C8uo6Xl8d2ilfgqWqPLhoI9sPHuF310yif/fONTV5e2aBw4TEhrxSxqf0aPbgsugo4XvnjWTT3jKWZvmezbiovJpfvreFtGE9+VYIZyU9ZXAy158+jJdX7mKjO9dWa0nPLWZQjwQG9mi9jtcJKT2YPqIXLy7fFdF5u/6RnsfCtfl8/4LRnD3aJiJtTyxwmBarqatn876yRmfEDdTsySmkJCfy+/9u93nV8eh7WzhcWcsjV04M+UC5H80cS6+ucTy4aGNA/Syhsja3uMXTqDfHbWePYF9pJe9lRqZvZ8u+Mv73bWeW2e9fODoidTDNZ4HDtNi2A4eprq33OyNuIGKjo7jzvJGs213CipzC47at2lHIPzLyuPXsEWGZK6hHYiwPXH4y6/eU8Lc1rdOEs7ekgr2llS1euKk5zh/bjxF9u/LcZztb/Y6yI1W13PXqWronxvK7b01u81OgmBMFFDhEpL+IPC8i77uvx4nILeGtmmkvGpp3Ah0x7s83pg6mf/d4/vDf7cfSqmvreWDRRgb3TOTeMP46nT0phekjevHrf2dREOZ10eHLiQ2nDgvfiPHGREUJt541gsz8UlbtbL01SlSV+xdmsquwnD9cO5m+SS3vpzKtL9ArjpdwZqQd5L7eBvwgDPUx7dCGvFKSEmIY1qtLi8tKiI3mu+eM5PMdRaxxF1169tMcsg8e4eFZE0iMa/5kiE0REX4xewJHq2t59L3wrxaYkVtMYmw0Jw+MzGyrV01JoVfXuFadhuSVVbt554u93PeVsUwf0bvVjmtCK9DA0UdV3wDq4diU6ZGf5Me0CZn5pUxM6RGyfodrTxtKt/gYrn9uFanz3uU3H2zj1ME9WmVg2Kh+Sdx29gj+uTaPVTsKm87QAhm5xUwakhyxW1ATYqO5fvow/rPlYKuMns/MK+XhdzZz3ti+3NmC9cJN5AV6xpaLSG9AAURkOtC6t5+YNqmqto4t+8pa3L/hacmm/VTW1FFV++UdP1n7D7OoGfNZNcc9F4ymZ5dY5riBa8Zj/w35sY9W17J5X1mrjd9ozA3ThxEXE+VzYr9QKq2o4a7X1tK7WxxPXD0pbLMAm9YRaOD4Ic5KeyNFZDnwMnBP2Gpl2o1t+49QU6eckpIcsjIXLMmi1uvOpsraehYsyQrZMfxZsmk/5VV1x+qQX1LB/QszQxo8vthTSl29MrUVx2/40jcpnisnpfBmRh5F5dVhOYaq8uM3v2BvSQV/vG5yi9dUN5EX0OQ4qrpWRM4FxgICZKlq5IedmojbcGwq9dBdcUR6dtIFS7Ko9hrfUFFTx4IlWSEbQdww8G/KkMgGDoBbzx7O39P38OrnudwTopsPFq3LZ8GSLPaWVNA9IZbSyhoevPzkiNwIYEIv0Luq7gK6qeomVd0IdBOR74W3aqY9yMwrJblLLIN7hm4AW6RnJ22NwJWRW8zoft3o0SU2ZGU21+j+SZw3ti9/WZkbkvVJFq3L5/6FmeSXVKBAaWUNUQK97Uqjwwi0qeo2VS1peKGqxcBtYamRaVc25Dkd44EuohSISM9OGu7AVV+vrN1d0qrTjDTl1rNGUHCkisXrA5+huDELlmRR4RWA6hV+YwshdRiBBo5o8fhmEJFooMmfDyJyiYhkiUi2iMzzsf0OEckUkfUiskxExrnpc9y0hke9iExyty11y2zYFtk5mDuxypo6th04HPRU6k2J9OykvgJXdJSELHDlHDpCaUVNq82IG4gZo3pz0oAknlu2o8UDAn2tFw62EFJHEugCAP8G/i4if3Jff9dNa5QbXJ4ELgbygDUislhVN3vs9pqqPuPufwXwBHCJqr4KvOqmTwQWqep6j3xzVDU9wLqbMNmyr4zaeg1p/0aD2ZNTIjYjacNxG9rou8RFU1lbH7I7oL4c+Nd2AoeIcNvZI7jvH1/w6fYCzh3TvLmjNuaXEhst1NSdGHxsIaSOI9Arjp8AHwN3uo+PgB83kec0IFtVd6hqNfA6MMtzB1Ut83jZFfd2Xy/XunlNG5PpjhifGIIR422N57TaH913HtFRwu8/2t50xgBk5BbTq2scw/sEtsRua/naqYPolxTfrAGBpUdr+N9FG/naH5cRHxNNbPTxTZe2EFLHElDgUNV6VX1aVb/hPv6kqk31oqUAezxe57lpxxGRu0QkB/g18H0f5XwL+JtX2otuM9X/ejahmda1Ia+U3l3jGNSjY0+HPaBHAjdMH8Y/1+aFZKBcRm4xU4b2DGm/UCjExURx45mpfLa9gK37y5rOgNNf88aaPZz/+FJeXZXLjWeksnzeBSz4xqm2EFIHFuhdVTNE5EMR2SYiO0Rkp4iEZJ4CVX1SVUfiXNU86HXc04Gj7p1cDeao6kTgbPdxQyN1vl1E0kUk/dAh39N0m5bJzCtl4uDQdoy3VXeeN5L4mGh+95+WXXUUlVezo6C8TXWMe5pz+lASY6N57rOmBwRuzC/lqqdX8ON/bmBEn668c89Z/OyK8fRIjLWFkDq4QJuqnsfpfzgLmAakuf/6kw94Lpow2E1rzOvAbK+0a/C62lDVfPffw8BrOE1iJ1DVZ1U1TVXT+va1uf5D7Wh1LdsPHm7xVOrtRZ9u8dw8I5V3NuwN+Ne4L2vbYP+Gp+QucXwzbTBvr8/nYFmlz31Kjlbz4KJMvvbHZeQVH+Xxb57KP+44g/GDOse5YAIPHKWq+r6qHlTVwoZHE3nWAKNFZLiIxOEEgcWeO4iI52ijy4HtHtuigKvx6N8QkRgR6eM+jwW+CnhejZhWsnlvGfXaMfs3GvPdc0bSLT6GJ1pwW2l6bjGx0RLyO9FC6TszhlNbr7y8Mve49Pp65fXVu7ng8U94bdVubjwjlY/uO4+vTx3cKa46zZcCvavqYxFZACwEjs03raprG8ugqrUicjfOrLrRwAuquklE5gPpqroYuFtELgJqgGLgRo8izgH2qKpnk1g8sMQNGtHAf4A/B/geTAhtyGuYSr3tfgGGWo8usdx29gie+HAbG/JKmjWN/NrcYiak9CAhNnyz/LZUap+uTBzUgyeXZvPHj7NJSU7kmmlD+GjrQdbvKWFaak9+fsXpjBvUPdJVNRESaOA43f03zSNNgQv8ZVLV94D3vNIe8nh+r5+8S4HpXmnlwNSAamzCKjO/lH5J8Z1uneibZ6Ty4vKdPP7BNv7yHZ+tpI2qrq3ni7wSbpg+LEy1C41F6/LZuv8wDcM58ksqePzDbSQlxPDE1ady5eQUu8Lo5AKdq+r8cFfEtC+Z+aWd6mqjQVJCLHecO5JH39/Kml1FTEsNfO6lTXtLqQrheJBw8TVXF0C3+BiumjI4AjUybY2tAGiCdqSqlpxDR5gYwhlx25Nvn5FKn27x/GZJVlCjrNviwD9fGhvhvb/Ud2e56XxsBUATtE35pah2rv4NT4lx0dx9/khW7Sw6YW10fzJyixnSK5F+bbx5L9KTTJq2z1YANEFrGDE+oQ3fGRRu154+lEE9EvjNB4Fddagq6bnFTG1D81M1JtKTTJq2z1YANEHbkFfKoB4J9E2Kj3RVIiY+Jpp7LhzNut0lfJx1sMn984orOHS4iqlB9IlESqQnmTRtX6B3VXmvANgX+EbYamXatMz80pAuFdtefWPqYJ75JIffLNnGeWP6+V0O9Vj/Rju44oDITjJp2r4mrzjcWW7PdR9n4syMO15VN4S5bqYNKq2oYWdBebPGMHQ0sdFR/OCi0WzeV8a/N+33u29GbjHd4mMYOyCplWpnTPg0GTjcyQyvVdXahhUAbdnYzmtTw4y4nbh/w9MVp6Ywql83nvhwG3X1jfd1ZOQWM3loMtF+rkqMaS8C7eNYLiJ/FJGzRWRKwyOsNTNt0gYLHMeJjhJ+ePEYsg8eYfEXvqdiO1JVy9b9ZW1q4SZjWiLQPo5J7r/zPdKaHDluOp7MvFIG90ykp60ffcwl4wcwbmB3fvef7Xz1lEHERh//e2z97hLqlTY7I64xwQp0PY7zfTwsaHRCG/JLOu34jcZERQn3fWUMuYVH+WdG3gnbM3KLEYFJQ5Jbv3LGhIGNHDcBKy6vZk9RRacdMe7PBSf1Y9KQZH7/0Xaqao8f4pSeW8TY/kkkJcRGqHbGhJaNHDcBaxj4Z1ccJxIRfvSVsewtreT11V8ufFlXr6zfXWLNVKZDsZHjHdSidfnMeOy/DJ/3LjMe+y+L1vlbQyswx0aM24I9Ps0Y1ZvpI3rxx4+zqah2/ntsO3CYw1W1bX5+KmOCYSPHO6BF6/K5f2Em+SUVKM602PcvzGxx8NiQV0Jq7y706GJNLr6ICPd9ZSyHDlfx8spdgOfAv7Y/YtyYQAUaOLxHjr8M3BO2WpkWWbAki4qa4y8IK2rqWLAkq0XlOmuMJ7eojI5uWmovzh3Tl2c+yeFwZQ1rc4vpmxTPkF42QaDpOAK9q2otNnK83WhsWuzG0gNRcKSKvaWVnWaN8Za47ytjKD5aw4vLdx2b2NAWPjIdSaDjOABOA1LdPFNEBFV9OSy1Mi0yKDmRfB9BoiXTYjf0b9gcVU07ZXAyE1N68NsPt6FAydFqFq3Lt7mfTIcR6O24fwV+A5wFTHMfaX4zmYiZO3MscdEn/mm/NW1Is8vMzCtFBMbbOtNNWrQun20HDtMwAUlZZW1I+piMaSsCveJIA8ZpMMudmYiZPTmFdzfs48MtBwAY0D2Bqto6Xl65i1mTBjGsd9egy9yQV8qIPl1tLEIAFizJoqr2+KVXG/qY7KrDdASBdo5vBAYEW7iIXCIiWSKSLSLzfGy/Q0QyRWS9iCwTkXFueqqIVLjp60XkGY88U9082SLye7HGY58KyquYOqwnux67nM9/eiFv3nkmdfXKt19YTcGRqqDLy8wvsRlxAxSOPiZj2hK/gUNE3hGRxUAfYLOILBGRxQ2PJvJGA08ClwLjgGsbAoOH11R1oqpOAn4NPOGxLUdVJ7mPOzzSnwZuA0a7j0uafpudy+HKGjbklXLmyN7H0kb27cbzN03jQFklN7+4hvKq2oDLO1BWyYGyKpvYMEC29Krp6JpqqvpNC8o+DchW1R0AIvI6MAvY3LCDqpZ57N8V8NsUJiIDge6q+rn7+mVgNvB+C+rZ4azZVURdvXKGR+AAmDK0J09eN4Xb/5rBHa9k8PyN04iLafqiMzPPRowHY+7Msdy/MPO4W6Jt6VXTkfj91lDVTxoewFYgyX1scdP8SQH2eLzOc9OOIyJ3iUgOzhXH9z02DReRdSLyiYic7VGm5yxyPst0y71dRNJFJP3QoUNNVLVjWZ5dSFxMlM9pvC88uT+PXjmRz7YXMO+fGwJaL3tDfilRAuOsYzwgtvSq6egC6hwXkauBBcBSQIA/iMhcVX2zpRVQ1SeBJ0XkOuBB4EZgHzBUVQtFZCqwSETGB1nus8CzAGlpaZ2qU39FTiFpw3qSEBvtc/vV04ZwoKySxz/cRr/uCcy79CS/5WXmlTCqXze6xAVz93bnZkuvmo4s0G+CB4BpqnoQQET6Av8B/AWOfMDz/s/BblpjXsfpv0BVq4Aq93mGe0Uyxs0/OIgyO52i8mq27CvjR18Z43e/uy8YxYHDlTzzSQ79kuL5zlnDfe6nqmTml3LumH7hqK4xph0K9K6qqIag4SoMIO8aYLSIDBeROOAanGlLjhGR0R4vLwe2u+l93c51RGQETif4DlXdB5SJyHT3bqpvA28H+B46hc93FAJw5qg+fvcTEX5+xQQuGT+Ah9/dzDtf7PW5377SSgqOVFv/hjHmmECvOP4tIkuAv7mvvwW85y+DqtaKyN0407FHAy+o6iYRmQ+kq+pi4G4RuQioAYpxmqkAzgHmi0gNzoy8d6hqkbvtezjTvCfidIpbx7iHFTkFdIuPCWhqkOgo4XfXTOLbz6/mvje+oHfXuBMCzoY8GzFujDme38AhIqOA/qo6V0Suwhk5DrASeLWpwlX1PbwCjKo+5PH83kby/RP4ZyPb0oEJTR27s1qRU8hpw3sR42PkuC8JsdH8+dtpfPNPK7j9rxm88d0zjusEz8wvITpKGDfQOsaNMY6mvl1+B5QBqOpCVf2hqv4QeMvdZtqQ/aWV7DhUftz4jUD06BLLSzefRlJCDDe9uJo9RUePbduQV8qY/kmNdrQbYzqfpgJHf1XN9E5001LDUiPTbCt3FACcMH4jEIOSE/nLd06jsqaOG19cTVF59bGOcZsR1xjjqanAkexnmw2DbWOWZxeS3CWWkwc0r1lpTP8knr9pGvnFFVz55HJO/+VHlBytYcmm/TZBnzHmmKYCR7qI3OadKCK3AhnhqZJpDlVlZU4hZ4zoTVRU86fvmpbaizmnDyW36CgHDztzWpVU1NjsrsaYY5q6q+oHwFsiMocvA0UaEAdcGcZ6mSDtLjpKfkkFd5w7osVlLdl04IQ0m93VGNPAb+BQ1QPAmSJyPl/eyfSuqv437DUzQVmR44zfOGOk//EbgbDZXY0x/gQ0jkNVPwY+DnNdTAusyCmkX1I8I/sGv9aGt3CsIGiM6TgCHTlu2jCnf6OAGaP6hGRt67kzx5Lodfutze5qjGlgs9Z1ANsPHqHgSHWzbsP1paEfY8GSLPaWVDAoOZG5M8da/4YxBrDA0SGsyHbGbwQ78M8fm93VGNMYa6rqAFbkFDK0VxcG9+wS6aoYYzoBCxztXF298vmOwpBebRhjjD/WVBUmi9blt0ofwaa9pZRV1oasf8MYY5pigSMMFq3LP27N6fySCu5f6Ez5Ferg8eX4DQscxpjWYU1VYbBgSdaxoNGgYeR1qK3IKWR0v270S0oIednGGOOLBY4waK2R19W19azZWWT9G8aYVmWBIwwaG2Ed6pHXX+SVUFFTF5JpRowxJlAWOMJg7syxxHmtwJcQExXykdcrsgsRgTNG2BWHMab1WOAIg9mTUzhvbN/j0q47fWgYOsYLmDCoBz26xIa0XGOM8SesgUNELhGRLBHJFpF5PrbfISKZIrJeRJaJyDg3/WIRyXC3ZYjIBR55lrplrncf/cL5Hprr0JEqpgxNJvuRSxnWuwsZucWoasjKr6iuY93uEuvfMMa0urAFDhGJBp4ELgXGAdc2BAYPr6nqRFWdBPwaeMJNLwC+pqoTgRuBv3rlm6Oqk9zHwXC9h+Yqq6zhiz0lnDWqDzHRUXz3nJF8kVd67NbZUEjPLaK6rt5uwzXGtLpwXnGcBmSr6g5VrQZeB2Z57qCqZR4vuwLqpq9T1b1u+iYgUUTiw1jXkPo8p5B6hRmjnE7rr09NoV9SPE9+nB2yY6zIKSQmSpiW2itkZRpjTCDCGThSgD0er/PctOOIyF0ikoNzxfF9H+V8HVirqlUeaS+6zVT/K43MIy4it4tIuoikHzp0qPnvohmWZxeQGBvN5KE9AYiPiea2s0ewIqeQdbuLQ3KMFTmFTBqSTNd4G8NpjGldEe8cV9UnVXUk8BPgQc9tIjIe+BXwXY/kOW4T1tnu44ZGyn1WVdNUNa1v376+dgmbZdkFnDa8F3ExX368154+lB6JsTy1NKfF5ZdV1pCZZ/0bxpjICGfgyAeGeLwe7KY15nVgdsMLERkMvAV8W1WPfduqar7772HgNZwmsTZjX2kFOYfKOWvU8WMrusXHcNOZqXy4+QDbDhxu0TFW7yiiXkOzTKwxxgQrnIFjDTBaRIaLSBxwDbDYcwcRGe3x8nJgu5ueDLwLzFPV5R77x4hIH/d5LPBVYGMY30PQlmc7HeAzRp34pX7Tmal0iYvm6RZedazIKSQ+JorJQ5NbVI4xxjRH2AKHqtYCdwNLgC3AG6q6SUTmi8gV7m53i8gmEVkP/BDnDircfKOAh7xuu40HlojIBmA9zhXMn8P1HppjRXYBvbvGcdKApBO29ewax3WnDWXxF3vZU3S0+cfIKSAttScJXsu7GmNMawhrz6qqvge855X2kMfzexvJ9wvgF40UOzVkFQwxVWVZdgFnjupDVJTvtb9vPXsEf1m5iz99msMvZk8M+hiFR6rYuv+wrf9tjImYiHeOdyTZB49w8HAVZ41qvNN6QI8EvjF1MG+k53HwcGXQx1i5w2kKs45xY0ykWOAIoWXu2t+++jc8ffeckdTW1fPCsl1BH2NFTiHd4mOYmNKjOVU0xpgWs8ARQsuzCxjWu+m1v1P7dOWyiQN55fNcSitqgjrGypxCTh/ei5ho+9MZYyLDvn1CpKauns93FDV5tdHge+eN4khVLX9duSvgY+wtqWBnQblNM2KMiSgLHCGyIa+EI1W1J4zfaMy4Qd05f2xfXli+i4rquqYz4FxtAJxp4zeMMRFkgSNElm0Pfm2Mu84fRVF5Na+v2R3Q/ityCunZJdbnrb7GGNNaLHCEyHJ3bYyeXeMCzpOW2ovTUnvx7Kc7qK6t97uvqrIyp4AzRvZu9FZfY4xpDRY4QqC8qpZ1u4sD7t/w9L3zR7KvtJJF6/3NxgK5hUfZW1pp04wYYyLOAkcIrN5VRE2dBty/4encMX0ZP6g7zyzNoa6+8YWeVuTY+A1jTNtggSMElm8vIC4mirTUnkHnFRG+d94odhSUs2TT/saPkVPAgO4JjOjTtSVVNcaYFrPAEQLLsgtIG9b8uaMumTCAEX268tTSbJ/Ly9bXK5/nFHLmyN40svyIMca0GgscLXTosDN3VHP6NxpERwnfPXcEG/PL+HR7wQnbtx08TGF5tY3fMMa0CRY4WmhFjvNF35z+DU9XTh7MwB4JPOVjedkV7lTtFjiMMW2BBY4WWpFdSPeEGCa0cO6ouJgobj17BKt2FpGRW3T8MXIKA5rKxBhjWoMFjhY4No36yD5Eh2BsxbWnDaFnl1ie+vjLhZ5q6+pZtaPQ7qYyxrQZFjhaILfwKPklFcwYHZqxFV3iYrh5xnA+2nqQLfvKANi0t4zDVbU2fsMY02ZY4GiBhmnUW9q/4enGM1Lp6rG8bMP4jWCmMjHGmHCywNECy7MLSElOJLV36PoeenSJ5frpw/jXhr3sKihnRU4BY/p3o29SfMiOYYwxLWGBo5nq6pUVOYXMGBX6sRW3nDWcKBEu+/1nfLa9gPziChat8z8liTHGtBYLHM20aW8ppRU1LRq/0ZgVOYUocNSdbr28uo77F2Za8DDGtAlhDRwicomIZIlItojM87H9DhHJFJH1IrJMRMZ5bLvfzZclIjMDLbO1NPRvhGNtjAVLsk6Yt6qipo4FS7JCfixjjAlW2AKHiEQDTwKXAuOAaz0Dg+s1VZ2oqpOAXwNPuHnHAdcA44FLgKdEJDrAMlvFiuxCThqQFJa+h70lFUGlG2NMawrnFcdpQLaq7lDVauB1YJbnDqpa5vGyK9DwM3sW8LqqVqnqTiDbLa/JMltDZU0dq3cFvkxssAYlJwaVbowxrSmcgSMF2OPxOs9NO46I3CUiOThXHN9vIm9AZbrl3i4i6SKSfujQoWa/CV8ycouprq0P6W24nubOHEui14SJibHRzJ05NizHM8aYYES8c1xVn1TVkcBPgAdDWO6zqpqmqml9+/YNVbGA078REyWcNrxXSMttMHtyCo9eNZGU5EQESElO5NGrJjJ7ss8YaYwxrSomjGXnA0M8Xg920xrzOvB0AHmDKTMslmcXMGVoT7rGh+/jmz05xQKFMaZNCucVxxpgtIgMF5E4nM7uxZ47iMhoj5eXA9vd54uBa0QkXkSGA6OB1YGUGW4lR6vJzC8NW/+GMca0dWH7yayqtSJyN7AEiAZeUNVNIjIfSFfVxcDdInIRUAMUAze6eTeJyBvAZqAWuEtV6wB8lRmu9+DLypxCVGHGKJsCxBjTOYWzqQpVfQ94zyvtIY/n9/rJ+wjwSCBltqZl2QV0jYvm1CHJkaqCMcZEVMQ7x9ubFTmFTB/Rm9ho++iMMZ2TffsFIa/4KDsLyq1/wxjTqVngCELDEq5nhWj9DWOMaY8scARhWXYBfZPiGd2vW6SrYowxEWOBI0D19cry7ALOGtUn5NOoG2NMe2KBI0BZBw5TWF5t/RvGmE7PAkeAlrvTqNv4DWNMZ2eBI0DLsgsY0bcrA3vYDLXGmM7NAkcAqmvrWb2zKGyz4RpjTHtigSMA6/eUcLS6zvo3jDEGCxwBWZZdQJTA9BHWv2GMMRY4ArA8u4BTBifTIzE20lUxxpiIs8DRhMOVNazfU2L9G8YY47LA0YRVO4qoq1fr3zDGGJcFjiYsyy4gITaKKcOSI10VY4xpEyxwNGF5dgHTUnsRHxMd6aoYY0ybYIGjEYvW5TP9lx+x/eARvthTwqJ1rb60uTHGtElhXQGwvVq0Lp/7F2ZSUVMHQFllLfcvzARg9uSUSFbNGGMizq44fFiwJOtY0GhQUVPHgiVZEaqRMca0HWENHCJyiYhkiUi2iMzzsf2HIrJZRDaIyEciMsxNP19E1ns8KkVktrvtJRHZ6bFtUqjrvbekIqh0Y4zpTMIWOEQkGngSuBQYB1wrIuO8dlsHpKnqKcCbwK8BVPVjVZ2kqpOAC4CjwAce+eY2bFfV9aGu+6Bk3xMZNpZujDGdSTivOE4DslV1h6pWA68Dszx3cAPEUffl58BgH+V8A3jfY7+wmztzLImxx99FlRgbzdyZY1urCsYY02aFM3CkAHs8Xue5aY25BXjfR/o1wN+80h5xm7d+KyLxvgoTkdtFJF1E0g8dOhRMvZk9OYVHr5pISnIiAqQkJ/LoVROtY9wYY2gjd1WJyPVAGnCuV/pAYCKwxCP5fmA/EAc8C/wEmO9dpqo+624nLS1Ng63T7MkpFiiMMcaHcF5x5ANDPF4PdtOOIyIXAQ8AV6hqldfmq4G3VLWmIUFV96mjCngRp0nMGGNMKwln4FgDjBaR4SISh9PktNhzBxGZDPwJJ2gc9FHGtXg1U7lXIYiIALOBjaGvujHGmMaEralKVWtF5G6cZqZo4AVV3SQi84F0VV0MLAC6Af9w4gC7VfUKABFJxbli+cSr6FdFpC8gwHrgjnC9B2OMMScS1aCb/9udtLQ0TU9Pj3Q1jDGmXRGRDFVN8063kePGGGOC0imuOETkEJDbzOx9gIIWHN7yW37Lb/nba/5hqtr3hFRVtYefB05/jOW3/Jbf8ne6/I09rKnKGGNMUCxwGGOMCYoFjqY9a/ktv+W3/J00v0+donPcGGNM6NgVhzHGmKBY4DDGGBMUCxx+iMguEcl0VxoMeui5iCSLyJsislVEtojIGUHkHeu1CmKZiPwgyOP/j4hsEpGNIvI3EUkIMv+9bt5NgRxbRF4QkYMistEjrZeIfCgi291/ewaZ/5vu8etF5IQRrAHkX+B+/htE5C0RSQ4y/8Nu3vUi8oGIDAomv8e2+0RERaRPkMf/mYjke5wHlwV7fBG5x/0MNonIr4M8/t89jr1LRNYHmX+SiHze8H9IRBqdlLSR/KeKyEr3/+E7ItLdT/4hIvKxOKuKbhKRe930gM5BP/kDOgf95A/oHPSTP6BzsLH8HtubPAcDFo57fDvKA9gF9GlB/r8At7rP44DkZpYTjTOV/LAg8qQAO4FE9/UbwE1B5J+AM4FkF5w5zf4DjGoizznAFGCjR9qvgXnu83nAr4LMfzIwFliKs1pksMf/ChDjPv9VM47f3eP594Fngsnvpg/BmbMt19/51Mjxfwb8KMC/ma/857t/u3j3db9g6++x/XHgoSCP/wFwqfv8MmBpkPnXAOe6z78DPOwn/0Bgivs8CdiGs/poQOegn/wBnYN+8gd0DvrJH9A52Fj+YM7BQB92xREmItID5z/C8wCqWq2qJc0s7kIgR1WDHf0eAySKSAxOANgbRN6TgVWqelRVa3Emm7zKXwZV/RQo8kqehRNAcf+dHUx+Vd2iqlmBVLiR/B+49YfGV5n0l7/M42VXoNG7SRp5/wC/BX7sL28T+QPSSP47gcfUXbJAfc9C3eTxRURwljnwXlStqfwKNFwl9MDPOdhI/jHAp+7zD4Gv+8m/T1XXus8PA1twfkAFdA42lj/Qc9BP/oDOQT/5AzoH/bx/CPAcDJQFDv8U+EBEMkTk9iDzDgcOAS+KyDoReU5EujazHr5WQfRLVfOB3wC7gX1Aqap+4D/XcTYCZ4tIbxHpgvNrcUgTeXzpr6r73Of7gf7NKCNUvoPvVSb9EpFHRGQPMAd4KMi8s4B8Vf0i2ON6uNttqnihsWYWP8bg/B1XicgnIjKtmXU4GzigqtuDzPcDYIH7+f0GZyG2YGziyyWnv0mA56A4s2tPBlbRjHPQK3/Q/OQP6Bz0zh/sOeiZP0Tn4HEscPh3lqpOAS4F7hKRc4LIG4Nz2f20qk4GynEuk4MizlomVwD/CDJfT5z/cMOBQUBXcVZaDIiqbsG5rP4A+DfOFPZ1wdTBR5lKiH7xBEtEHgBqgVeDzauqD6jqEDfv3UEcswvwU4IMNl6eBkYCk3B+ADweZP4YoBcwHZgLvOFePQTrhLVxAnQn8D/u5/c/uFfgQfgO8D0RycBpfqluKoOIdAP+CfzA69d6QOegv/yBaCx/oOegr/zBnIOe+d3jtfQcPIEFDj/cX+0Nl/dvEdxqg3lAnqo2/OJ4EyeQBOtSYK2qHggy30XATlU9pM4KiguBM4MpQFWfV9WpqnoOUIzTZhqsA/Ll4lsDgUabSsJFRG4CvgrMcb84mutV/DSV+DASJ3B/ISK7cJoo1orIgEALUNUDqlqnqvXAnwl+xcs8YKE6VgP1OBPfBcxt6rwK+HuQxwa4EefcA+fHT1D1V9WtqvoVVZ2KE7hymqhrLM6X5quq2nDcgM/BRvIHrLH8gZ6DARzf7znoI3+Lz0FfLHA0QkS6ikhSw3OcDq6AVxtU1f3AHhEZ6yZdCGxuRlWa+0tvNzBdRLq4vzAvxGnzDJiI9HP/HYrzxfFaM+qxGOfLA/fft5tRRrOJyCU4bbtXqOrRZuQf7fFyFrA10Lyqmqmq/VQ1VVVTcb7Ep7jnRqDHH+jx8kqCX/FyEU4HOSIyBucmjWBnS70I2KqqeUHmA6dP41z3+QVAUE1dHudgFPAg8IyffQXnimaLqj7hsSmgc9BP/kDr6jN/oOegn/wBnYO+8ofiHPSpub3qHf0BjAC+cB+bgAeaUcYkIB3YgPMfuGeQ+bsChUCPZr6Hn7sn2Ubgr7h31gSR/zOcYPcFcGEA+/8Npzmlxj1BbwF6Ax/hfGH8B+gVZP4r3edVwAFgSZD5s4E9OE1t6/F/V5Sv/P90P78NwDs4nZUB5/favgv/d1X5Ov5fgUz3+IuBgUHmjwNecd/DWuCCYOsPvATc0cy//1lAhnsOrQKmBpn/Xpwr3W3AY7izXTSS/yycZqgNHn/vywI9B/3kD+gc9JM/oHPQT/6AzsHG8gdzDgb6sClHjDHGBMWaqowxxgTFAocxxpigWOAwxhgTFAscxhhjgmKBwxhjTFAscJgOyZ0F9HGP1z8SkZ+FqOyXROQboSirieN8U5xZlT/2SJsoX85WWyQiO93n/wmy7PkiclHoa206g5hIV8CYMKkCrhKRR1U12AFvYSMiMfrlhHdNuQW4TVWXNSSoaibO+CBE5CXgX6r6ZrD1UNWQTkFhOhe74jAdVS3Oesv/473B+4pBRI64/57nTgT4tojsEJHHRGSOiKwWZz2IkR7FXCTO+hLbROSrbv5ocdZeWONOSvhdj3I/E5HF+Jg9QESudcvfKCK/ctMewhnQ9byILGjqzfoqo+G9ichvxVmf4SMR6ev9GYjINBFZISJfuO81SUTGu8/Xu+9ldGPHNp2PBQ7TkT0JzBFnivtAnQrcgTOt/A3AGFU9DXgOuMdjv1SceZcuB54RZ5GsW3BmIZ4GTANuE5Hh7v5TgHtVdYznwcRZlOdXONNxTAKmichsVZ2PM+vAHFWd66/CjZXhbu4KpKvqeJyp8f+fV944nDmo7lXVU3GmF6lwP4P/U9VJQBrOyGljAAscpgNTZ2bRl3EWvwnUGnXWNajCmVCvYSr6TJxg0eANVa1XZ5rxHcBJOPOZfVucVfJW4Ux10fBLfbWq7vRxvGk4ixsdcpuwXsVZxyUY/sqo58vJCV/BuYrxNBbYp6prwPnM3DJWAj8VkZ/gLCBWEWSdTAdmgcN0dL/DuRLwXAulFvfcdyfPi/PYVuXxvN7jdT3H9wl6z9WjgAD3qOok9zFcv1wDpbwlbyKEAppjSFVfw5nOvwJ4T0QuCGutTLtigcN0aKpahLNs7i0eybuAqe7zK4DYZhT9TRGJcvs9RgBZOEtz3ulObY2IjJGmF+9aDZwrIn1EJBpnNuRPgqyLvzKigIb+nOuAZV55s4CB4i7w5PZvxIjICGCHqv4eZzbZU4Ksk+nA7K4q0xk8zvGL3/wZeFtEvsBZpKo5VwO7cb6wu+PMHFspIs/hNGetdae4PoSfpXLBWe5TROYBH+NcsbyrqkFNPd9EGeXAaSLyIM46FN/yylstIt8C/iAiiThXGBfhLBN7g4jU4Kya98tg6mQ6Npsd15gOTESOqGq3SNfDdCzWVGWMMSYodsVhjDEmKHbFYYwxJigWOIwxxgTFAocxxpigWOAwxhgTFAscxhhjgvL/AZfE6+n2rgH5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x_val,y_val)\n",
    "plt.scatter(x_val,y_val)\n",
    "plt.title('Number of Topics vs. Coherence')\n",
    "plt.xlabel('Number of Topics')\n",
    "plt.ylabel('Coherence')\n",
    "plt.xticks(x_val)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 has a coherence score of: 0.2779166617490351\n",
      "6 has a coherence score of: 0.3311007761527292\n",
      "7 has a coherence score of: 0.3013784239925935\n",
      "8 has a coherence score of: 0.35598332982751946\n",
      "9 has a coherence score of: 0.34027534940004506\n",
      "10 has a coherence score of: 0.4097773007643951\n",
      "11 has a coherence score of: 0.3807246796830988\n",
      "12 has a coherence score of: 0.4034330658044365\n",
      "13 has a coherence score of: 0.3803881822910133\n",
      "14 has a coherence score of: 0.35306678991921636\n",
      "15 has a coherence score of: 0.41492279574007895\n",
      "16 has a coherence score of: 0.40084959134756337\n",
      "17 has a coherence score of: 0.36936776364929547\n",
      "18 has a coherence score of: 0.37781590314549723\n",
      "19 has a coherence score of: 0.3964452619047776\n",
      "20 has a coherence score of: 0.40985524715594723\n",
      "21 has a coherence score of: 0.3930025546272143\n",
      "22 has a coherence score of: 0.41031905326427903\n",
      "23 has a coherence score of: 0.4430369479858099\n",
      "24 has a coherence score of: 0.4519169126814753\n"
     ]
    }
   ],
   "source": [
    "for x in coherence:\n",
    "    print(str(x[0]) + \" has a coherence score of: \" + str(x[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 26\n",
      "Round: 27\n",
      "Round: 28\n",
      "Round: 29\n",
      "Round: 30\n",
      "Round: 31\n",
      "Round: 32\n",
      "Round: 33\n",
      "Round: 34\n"
     ]
    }
   ],
   "source": [
    "coherence = []\n",
    "for k in range(26,35):\n",
    "    print('Round: '+str(k))\n",
    "    Lda = gensim.models.ldamodel.LdaModel\n",
    "    ldamodel = Lda(doc_term_matrix, num_topics=k, id2word = dictionary, passes=40,\\\n",
    "                   iterations=200, chunksize = 10000, eval_every = None)\n",
    "    \n",
    "    cm = gensim.models.coherencemodel.CoherenceModel(model=ldamodel, texts=final_reviews,\\\n",
    "                                                     dictionary=dictionary, coherence='c_v')\n",
    "    coherence.append((k,cm.get_coherence()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8DElEQVR4nO3deXxU5fX48c/JHkggQEKAEAgIBAIoIOKCuwIurVptq7hUbWutra2t1qptf7a1tlpp7Wrt5hdb19pWrVYruKC1uCCLSCYQZAlLBpKwZIPsOb8/7g0OccgCM3NnMuf9es2LmeduJ5MwZ57lPo+oKsYYY0xnCV4HYIwxJjpZgjDGGBOUJQhjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE5QlCBMWIvKwiNzt0bVFRBaKyF4RWRaB640SkXoRSQz3tbwiIqeLyHav4zCRZQkiTohImYhUikj/gLIvisjrHoYVLicDc4CRqjorcIOIfMf9MK8XkUYRaQt47Tuci6nqVlXNUNW2UAQfTiJyuYgsd3/eHSLyHxE52eu4THSyBBFfEoGbvA6itw7jm/looExV93XeoKo/cT/MM4AvA293vFbVyaGIN1qJyM3AL4GfALnAKOB3wIURjiMpktczh88SRHxZAHxLRLI6bxCRAhHRwP+8IvK6iHzRfX6NiCwVkV+ISLWIbBKRk9zybW7t5OpOp80WkZdFpE5E3hCR0QHnnuhu2yMipSLy2YBtD4vIgyLyoojsA84IEu8IEXnOPX6DiFznln8B+DNwovst+Yc9fXPcn+c9Ealx/z2p03txj4gsE5FaEfmXiAwO9t6JyGC3icvvNnM965Zni8i/3fdvj4i8KSIf+z/o/uw/61T2L/cDHhG5TUTK3fe1VETO6sHPNhC4C/iqqj6tqvtUtUVVn1fVW919UkXkl27cfvd5aqfz3OL+rneIyLUB5aki8jMR2SoiFSLyexFJd7edLiLb3bh3AgtFJEFEbheRjSKyW0SeCvJ+Xu2eb5eIfDfgWoluTXCj+x6sEJF8d9sh/67MYVBVe8TBAygDzgaeBu52y74IvO4+LwAUSAo45nXgi+7za4BW4FqcmsjdwFbgASAVmAvUARnu/g+7r091t/8K+J+7rT+wzT1XEjAd2AUUBRxbA8zG+RKTFuTn+S/Ot980YBpQBZwZEOv/evCeXBMQ02BgL3CVG9N89/WQgPeiHJjixv9P4NFg7x3wAvA3YBCQDJzmlt8D/N4tSwZOASRIXKe674+4rwcBDcAIoNDdNiLg2kf14Gc9x/39JXWxz13AO8BQIAd4C/iRu+109/i73NjPA/YDg9ztvwCec9/HTOB54J5Ox/7U/VtIx6nJvgOMdMv+ADzR6f38k7vvMUATMMndfiuwxn0vxN0+hG7+ruxxGJ8bXgdgjwj9oj9KEFNwPnxz6H2C+DBg21R3/9yAst3ANPf5w8CTAdsygDYgH7gUeLNTfH8Avh9w7F+7+Fny3XNlBpTdAzwcEGtvE8RVwLJO298Grgl4L+4N2FYENOMkywPvHTAcaO/44Ox0vruAfwHjuolLcJLvqe7r64DX3OfjgEr3d5nci9//FcDObvbZCJwX8HoeTlMdOB/yDZ3+PiqBE9x49xGQqIATgc0BxzYTkOiBtcBZAa+HAy3ue9jxfo4M2L4MuMx9XgpcGCT+Lv+u7NH7hzUxxRlVLQb+Ddx+GIdXBDxvcM/XuSwj4PW2gOvWA3twvgWPBo53m1qqRaQa5wNsWLBjgxgB7FHVuoCyLUBez3+UoOfc0qms8zm3ddqWDGR3OibfjW1vkGssADYAi90muqC/A3U+2Z7EqcUAXA485m7bAHwD+AFQKSJPisiILn8yx26cJr+u2v87vwdb3LID51DV1oDX+3F+3zlAP2BFwO/zJbe8Q5WqNga8Hg08E7D/Wpyknxuwz84g1wLnPd4YJP6e/F2ZXrAEEZ++j/OtNPDDr6NDt19A2ZH+x8rveCIiGTjND36cD9o3VDUr4JGhqjcEHNvVNMN+YLCIZAaUjcJpAjpcfpwPmECdz5nfaVsLThNGoG1ubFmdL6Cqdap6i6qOBS4Abu6i/+AJ4NNuv83xOE1aHed5XFVPduNVnKab7ryN00xzURf7dH4PRrll3dmF8+VgcsDvc6A6AwEOhN3pmG3AuZ3+BtJUtSe/w23AUYco7+7vyvSCJYg45H4L/Rvw9YCyKpwPwyvdTsDPE/w/YW+cJyIni0gK8CPgHVXdhlODmSAiV4lIsvs4TkQm9TD+bTjt4/eISJqIHA18AXj0CGJ90Y3pchFJEpFLcZqR/h2wz5UiUiQi/XCai/6hnYa2quoO4D/A70RkkPuznQogIp8QkXEiIjjNfG04zVHBfsZVOB+8fwYWqWq1e45CETnT7TxuxPlgDnqOTuerAe4EHhCRi0SknxvbuSJyn7vbE8D3RCRHRLLd/bt9T1W1Hae/4BciMtSNM09E5nVx2O+BH7sJEPeaPR1N9WfgRyIyXhxHi8gQjvDvynycJYj4dRdOp16g63A6AHcDk3E+hI/E4zi1lT3AscCV4HyTxunUvgznG+pOPurA7Kn5OG3VfuAZnHbmVw43UFXdDXwCuAXn5/828AlVDawhPILTP7ITp3P86wR3FU7tYh1OO/033PLxwCtAPc43+t+p6pIuwnocp6/h8YCyVOBenOSxE6dD+Q4AEblCuriXQ1V/DtwMfA+nU38bcCPwrLvL3cBy4AOcTuCVbllP3IbTfPaOiNS6P2dhF/v/CqdTe7GI1OF0WB/fw2vdDzwFLAZqgYeA9BD9XZkAHaMkjDFdEOeGwkdV9c9ex2JMpFgNwhhjTFCWIIwxxgRlTUzGGGOCshqEMcaYoPrMpFnZ2dlaUFDgdRjGGBNTVqxYsUtVc4Jt6zMJoqCggOXLl3sdhjHGxBQR6TyDwAHWxGSMMSYoSxDGGGOCsgRhjDEmKEsQxhhjgrIEYYwxJqg+M4qpr3l2VTkLFpXir25gRFY6t84r5KLpR7LcgTHG9I4liCj07Kpy7nh6DQ0tzkzS5dUN3PH0GgBLEsaYiLEmpii0YFHpgeTQoaGljQWLSj2KyBgTjyxBRCF/dUOvyo0xJhwsQUShEVnpvSo3xphwsAQRhW6dV0iiyEFlKYkJ3DqvqwW6jDEmtCxBRKGLpucxfGAaqUnOryc5URCBySMGeByZMSaeWIKIQs2t7VTUNXLt7DGU3Xs+b377TDLTkrj+kRXUNrZ4HZ4xJk5YgohC6yvqaGnTAzWGYQPTeODyGWzZs59bnlpNe7st8mSMCT9LEFGoxF8LHNykdPzYIXz3vEm8XFLBg29s9Co0Y+LSs6vKmX3va4y5/QVm3/saz64q9zqkiLAb5aKQz19D/5RECob0P6j82tkFrN5ezc8WlzIlbyCnTQi6xocxJoTi+cZVq0FEoWJ/LUUjBpCQcPBIJhHhnounUpibydefWMW2Pfs9itCY+BHPN65agogybe3K2h21TB4xMOj2filJ/OGqY1FVrn9kBQ3NbUH3M8aERjzfuGoJIsqU7d7H/uY2iroY0jp6SH9+NX86a3fW8t1n1qBqndbGhEs837hqCSLK+IJ0UAdzRuFQvnn2BJ5eVc5f3z7kkrLGmCP0xVPGfKxMBL559ngPooksSxBRxldeQ0piAuOHZna7741njOPsSUP50b9LeK9sTwSiMyb+VNY1IQK5makIMLhfCqpOX2FfZwkiyvj8tUwYlkFKUve/moQE4f5Lp5E/uB9feWwlFbWNEYjQmPjR0NzGE8u2Mrcol3e/ezab7z2flXfO4Ysnj+Hht8p4frXf6xDDyhJEFFFVfP4aJg8P3kEdzIC0ZH5/5bHsa2rlK4+tpLm1PYwRGhNfnn2/nOr9LVw7++BmptvOncixowdx+z8/YENlvUfRhZ8liCjir2lk7/4WpuT1bs6lwmGZ3Pfpo1mxZS93v1ASpuiMiS+qysKlmykaPoDjxww+aFtyYgIPXD6DtOREvvLYCvY3t3oUZXhZgogivvIaAIoOMcS1K584egRfOnUsf317C/9csT3UoZnDFK934PYFSzfsZn1FPdfOLkA6za4MzhQ4v7psOh9W1vPdZ4r75GhCSxBRxOevRQQmDe++gzqYb88r5MSxQ/jOM2sodpON8U7HHbjl1Q0oH92Ba0kiNixcupkh/VP45DEjDrnPyeOz+cZZE3hmVTlPLNsWwegiwxJEFPH5axmb3Z9+KYc3A0pSYgK/vXw6Q/qn8OVHV7B3X3OIIzS9Ec934Ma6sl37eK20kiuOH0VacmKX+37tzHGcOiGHHzznY832vvXFzBJEFPH5a5iS1/vmpUBDMlJ58Mpjqaxt4utPrqLNZn71THkc34Eb6x5+q4ykBOHKE0Z3u29CgvDLS6cxJCOFrzy+gpr9fWdKfksQUWLPvmZ21DSGZFGgY/KzuOvCybz54S7uf9m+rXphsW/nIbfFwx24sayusYV/rNjO+VOHM3RAWo+OGdw/hQeumMGO6kZu+fv7fWZKfksQUcLnd6qmh5qDqbcumzWK+bPyeWDJRl4qPvSHlQm9xb6dfPXxlYwa3I+0TvezpCcn2tKxUe7vy7dT39T6saGt3ZkxahDfPX8Sr6yt5I9vbgpTdJFlCSJK9HSKjd74wQWTOSY/i2/9fXWfHqsdTTqSQ9GIgTz/tZO595KjyXNrDKlJCdxz8dQ+P0V0LGtrV/7ydhkzRmVxTH5Wr4+/5qQCzp86nAWLSnln0+7QBxhhliCiRHF5DXlZ6WT1SwnZOVOTEnnwihmkJiVw/SPLqW/qm2O1o0Vgcvjr52cxMD2Zi6bnsfT2M7nh9KNobVfOKBzqdZimC0vWVbJl9/5e1x46iAj3XjKV0YP78bUnVlFZF9uzG1iCiBIl/tqQ1h46jMhK5zeXT6ds936+9dTqPjlWOxq8XFLxseQQaG5RLm3typLSSo8iND2x8K3NDBuQxjlThh32OTLTkvndlTOoa2zh60+sorUtdmc3sAQRBeqbWtm8e1/I+h86O+mobO44dyIv+Xby+zf6RttoNHm5pIKvPLbikMkB4JiRWQzNTGVxifUHRavSnXUs3bCbq04cTXLikX00Thw2gB9fNJV3Nu3h/pfXhyjCyLMEEQXW7qhFlV5PsdEbXzh5DJ84ejgLFq3jzQ+rwnadeNOT5ADOUMizi3J5o7SKxhZb5CkaPfzWZlKTErh81qiQnO+SY0cyf1Y+v3t9I6+urQjJOSPNEkQU6JhiI1w1CHDaRu/79NGMH2rLlYbKgeQwfECXyaHD3KJc9jW38fbG2O+87Gv27mvm6ZXlXDwjj0H9Q9cP+P1PTmbyiAHc/NTqmPw/ZwkiCvj8tQzpn0LugNSwXqdfShK/v+pYWtuVGx5bYd9kj8BByeELx3ebHABOPGoIGalJ1swUhR5ftpWm1nauOenwOqcPJS05kd9dMYN2Vb76+EqaWmPr/5wliCjg89dSNGJA0AnBQm1Mdn9+eek0istr++wEY+F2OMkBnFFlpxXm8HJJZZ+5kaovaGlr55G3tzB73BAKhx3ePGhdGT2kPz//zDF8sL2Gu/+9NuTnDydLEB5ram1jfUXdEU+x0RtnTcrlprPG88+V23n03a0Ru25f8MphJocOc4ty2VXfxKpt1eEJ0PTaS8U72VnbyLUhrj0Emjt5GNefOpZH3tnCv96PnckaLUF47MOKelrbNSxDXLty01njOaMwh7ue97Fiiy1X2hOvlFRwwxEkB4DTC4eSlCC8XBKbnZZ90cKlmxk9pB9nTgzvPSrfmlfIrILB3PH0Gj6sqAvrtUIlrAlCRM4RkVIR2SAit3ex3yUioiIys1P5KBGpF5FvhTNOL4V6io2eciYYm86IrHRueHRlzN/QE26hSA4AA9OTOWHsEOuHiBKrt1Wzcms1V59YQEJCeJt4kxMT+M3l0+mXksgNj61kXwzcuBq2BCEiicADwLlAETBfRIqC7JcJ3AS8G+Q09wP/CVeM0cDnryUjNYnRg/tF/NoD+znLldY1tnLjY6toieEbesIpVMmhw9zJuWyq2mfTn0SBhUs3k5GaxGdmjozI9XIHpPHr+dPZVFXPHU+vifo+wHDWIGYBG1R1k6o2A08CFwbZ70fAT4GDvsKKyEXAZsAXxhg9V1xeQ9HwAWH/9nIok4YP4N5LprKsbA8/eTG2OtAiIdTJAeDsSbkA1szkscraRl5Ys4NPHzuSzLQj/7321ElHZXPL3EKeW+2P+j7AcCaIPCBwiaXtbtkBIjIDyFfVFzqVZwC3AT/s6gIi8iURWS4iy6uqYu/mr7Z2Ze2OOooi3P/Q2YXT8vjCyWNYuLTMVjsL0JEcJoUwOYAz/cnUvIG8bM1Mnnr0nS20tivXnFQQ8WvfcNpRnFGYw4+eL2F1FA9Y8KyTWkQScJqQbgmy+QfAL1S1yzq4qv5RVWeq6sycnJwwRBlem3fto6GlLeId1MHcfu5Ejh8zmNuf/oASd2bZeBaYHB4JYXLoMLcol1Xbqq3vxyONLW089u5WziwcSkF2/4hfPyFB+MWl08jJTOUrj62ken90rv4YzgRRDuQHvB7plnXIBKYAr4tIGXAC8JzbUX08cJ9b/g3gOyJyYxhj9URHB3Ukh7geSnJiAr+9fAZZ6Slc/+jyqP2DjYRX14Y3OQDMmZyLKry61ibv88Lzq/3s3td82LO2hkJWvxR+d8UMKusaufmp1VF5b0w4E8R7wHgRGSMiKcBlwHMdG1W1RlWzVbVAVQuAd4ALVHW5qp4SUP5L4Ceq+tswxuoJn7+WlKQExg3N8DoUAHIyU3nwyhlU1DRx05Pvx+Vypa+ureDLj4Y3OQAU5mYyanC/LleeM+GhqixcWsaE3AxmjxviaSzH5Gdx5yeKeG1dJQ++sdHTWIIJW4JQ1VbgRmARsBZ4SlV9InKXiFwQruvGEp+/hsLczCOeOTKUpo8axA8umMwb66v41SuxOwvl4YhUcgBnbqw5Rbks3bjb1umIsGWb91Cyo5ZrZ4+JyOwF3bnyhNFccMwIfr64lLc27vI6nIOE9ZNJVV9U1QmqepSq/tgtu1NVnwuy7+mqujxI+Q9U9WfhjNMLqoovTGtAHKn5s/L57MyR/Pq1DXEz0iaSyaHD3KJcmlvb+e/62BtgEcsWLi0jq18yF02LjpX9RIR7Lp7K2JwMvv7EKipqo6dfKnq+usaZ8uoGqve3MDkK+h86ExHuunAKR48cyM1/e59NVX17vL4XyQHg2NGDGNQv2ZqZImjbnv0sLtnJ/FmjSE9J9DqcA/qnJvHgFTPY19TG1x6PnkWGLEF4JBxrUIdSWnIiD155LMlJCVz/yIqYuOvzcHiVHACSEhM4a1Iur62rtJsUI+Svb5chIlx1wmivQ/mY8bmZ3HOxc0/SgsWlXocDWILwjM9fS4LApGHRmSAA8rLS+c386Wysqufb//gg6u/67K2DksPnI5scOswpyqW2sZVlm20+rHDb19TKk+9t45wpwxiRle51OEFdND2PK44fxR/e2BQVNUtLEB4p8dcwNicjqqq5wcwel81t50zkhTU7+NObfWe50lfXVnDDoys/Sg79Ip8cAE4dn0NackLc9PV46emV26lrbOXzswu8DqVL/+8TRUzNG8gtf1/N1t3eLjJkCcIjxeW1TInS5qXOvnTqWM6bOox7/7OO+15ax+x7X2PM7S8w+97XYvLO647kMHF4pqfJASA9JZGTx+Ww2Lezz9XQokl7u7LwrTKOHjmQGaMGeR1OlzoWGRLwfGEvSxAe2F3fxM7axojP4Hq4nOVKj2FoZhq/e30j5dUNKE5H+x1Pr4mpJBFNyaHD3Mm5+GsaD/RLmdD774dVbKrax7WzC6JiaGt38gf34/7PTsPnr+WHz5d4FoclCA9Eewd1MBmpSbQH+Ybb0NLGgkXR0aHWnWhMDgBnTRxKgsBia2YKm4VLy8jJTOX8qSO8DqXHzi7K5YbTj+KJZVv554rtnsRgCcIDxR6tAXGkquqagpaXVzfw1sZdUX3DV7QmB4AhGanMHD3Y+iHCZENlPW+sr+LK40eTkhRbH3m3zJnA8WMG891n17BuZ+RrmLH1bvURPn8tIwelR9WHVE90NfLj8j+9y9QfLGLeL/7Lbf/4gMff3UqJvzYqxnO/ts5JDoXDoi85dJg7OZe1O2rZtsfbTsm+6C9vlZGSmMDlx4/yOpReS3IXGcpMS+Yrj62M+JcwSxAeKInSO6i7c+u8QtKTDx51lZ6cyI8vmsLD1x7HTWeNZ3hWGotKdvKdZ9Zw3q/fZOoPFvPZP7zNPS+u5cU1O/BXN0S0M/a1dRV8+REnOTz6hehMDuAMdwVbIyLUahpa+OfK7XzymBHkZKZ6Hc5hGZqZxm/mT6ds1z5u+2dkh5snRexKBoC6xhY279rHp6ZHx23+vXGRG/OCRaX4qxsYkZXOrfMKD5SfXuis6auqbNm9n/e3VR94LFxaRrNbmxiamcq0/CymjcpiWn4WR4/MIiM19H+KsZIcAEYP6U9hbiaLS3by+ZO9m2G0r3nqvW3sb27j2igf2tqdE8YO4dZ5E/npS+uYVTCYqyO0hoUliAhbu8NZrHxKXuzVIMBJEhd1k9xEhILs/hRk9z+wb1NrG2t31PH+1r28v62a1dtrDnTKisD4oRlO0sgfxLT8LCbkZpB0BJMYxlJy6DCnKJcH39jI3n3NDOqf4nU4Ma+tXfnL22XMGjM4KqbUP1LXnzqWFVv2cPcLJRw9ciDTIzBc1xJEhPlitIP6SKUmJboJIOtAWfX+5oNqGS+XVPDUcme0RnpyIlPzBh6oZUzLz2L4wLQeDVGMxeQATj/Eb5ds4LV1lVxybGTWSO7LXi6pYPveBr53/iSvQwmJhATh55+Zxvm/eZOvPraSF75+Sti/SFiCiDCfv5bsjBSGxmh7aChl9Uvh9MKhBzVNbd3jNE2t2uokjYcP1TQ1Mouj8z9qmnp2VTkLFpVSXt0AQP6g9JhKDgBT8wYybEAai0t2WoIIgf9bupm8rHTmFA3zOpSQGdgvmQevOJZLHnyLb/ztfRZec1xY17O3BBFhPn8tRSMGxsTNOpEmIowe0p/RQ/pz4bSPmqbW7ag7qKbRuWlqUL8UVm7dS0vbR513VXVNLCmt7LY5LJp0rBHxjxXbaWxpIy05uqdhiWY+fw3LNu/hO+dNJDGMH6BemDpyIN+/oIjvPlPMA0s28LWzxoftWpYgIqiptY0PK+o4ozD21s/2SmpSIsfkZ3FMfhZXu2UdTVOrt9Xw/ra9vLG+is6L3zW2trNgUWlMJQhw+iEeeWcL//twF2e7I5tM7y1cWkZ6ciKXzoy9oa09cfmsUSwv28v9L6/n4bfK2LOv+WODRkLBhrlG0Pqd9bS2a9z1P4RaR9PUTWePZ+G1sz6WHDr43eamWHLC2CFkpibZcNcjsKu+iefe93PJsXkx1cTYGyLCCWMHA7B7X3PYpr6xBBFBH3VQx+YIpmiVd4gb+KJ1SueupCQlcMbEobyytiIu1wQPhcff3UpzWzvXnNS3hwv/+tUNdP4LCfXUN5YgIqjYX0NmahKjBvfzOpQ+5VA38N06r9CjiI7MnKJcdu9rZtXWvV6HEnOaW9t55J0tnDohh3FDM7wOJ6wOVUMOZc3ZEkQE+fy1TBoxIKyjDuLRRdPzuOfiqeRlpSM4NYp7Lp4ac/0PHU4vzCE5UWzyvsPw4podVNU1xfyNcT1xqBpyKGvO1kkdIW3tyroddVw2K9/rUPqkntzAFysy05I58ahsFvt2cse5E23EWw+pKguXbmZsdn9OG9/3B4LcOq+QO55eQ0PAehGhrjlbDSJCNu+qp6GlzTqoTY/MLcqlbPd+NlTWex1KzFi51blD/5rZBXFRS49EzdlqEBFSXO5M1RurU2yYyJpTlMv3ni1mcUkF43MzvQ4nJixcupnMtCQumRE/NxmGu+ZsNYgI8flrSElK4Kicvt1xZkIjd0Aax+RnWT9ED+2oaeA/xTu57Lh8+odh4sd4ZQkiQnz+WiYOyyT5CCagM/FlblEuq7dVU1Hb6HUoUe+Rt7egqnzuxAKvQ+lT7NMqAlQVn7/W+h9Mr8y1NSJ6pKG5jceXbWVOUS75NoQ8pCxBRMD2vQ3UNLTYDXKmV8YNzaBgSD9rZurGs++XU72/hWtn9+0b47xgCSICfH6ng9oShOkNEWHu5GG8vXEXdY0tXocTlTqGtk4aPoDjxwz2Opw+xxJEBJT4a0gQmDjMEoTpnTlFubS0Ka+XVnkdSlR6a+Nu1lfUc+3sArtfJAwsQURAsb+WcUMzSE+x6ZtN78wYNYgh/VOsH+IQFi7dzJD+KVxwzAivQ+mTLEFEgM9fYx3U5rAkJghnT8plybpKmlvbvQ4nqmzZvY9X11Vy+fGjbO2MMLEEEWa76puoqG2y/gdz2OYU5VLX1Mq7m3d7HUpUefitMhJFuPKE0V6H0mdZggizjg7qIksQ5jCdPD6b9OREFvusmalDXWMLf1++nfOPHk7ugDSvw+mzLEGEWXF5xxoQ1sRkDk9aciKnTsjm5ZIKVG2NCIB/rNhOfVOrDW0NM0sQYVbiryV/cDoD0/vmylYmMuYWDWNnbSNr3C8c8ay9XfnLW2XMGJXFtPwsr8Pp03qUIEQkV0QeEpH/uK+LROQL4Q2tb/D5a5g83GoP5sicOXEoiQlio5mAJaWVlO3eb7WHCOhpDeJhYBHQMZZsPfCNMMTTp9Q1tlC2e7/N4GqO2KD+KRxXMMj6IYCFS8sYNiCNc6YM8zqUPq+nCSJbVZ8C2gFUtRVo6/oQU3LgDmqrQZgjN6doGKUVdWzZvc/rUDyzvqKO/23YxVUnjraJLyOgp+/wPhEZAs4a2SJyAtBtY6iInCMipSKyQURu72K/S0RERWSm+3qOiKwQkTXuv2f2MM6oYlNsmFCyyfucG+NSkxK4fNYor0OJCz1NEDcDzwFHichS4K/A17o6QEQSgQeAc4EiYL6IFAXZLxO4CXg3oHgX8ElVnQpcDTzSwzijis9fS3ZGKkNtGJ4JgfzB/Zg4LDNum5n27mvm6ZXlfGp6HoP6p3gdTlzoUYJQ1ZXAacBJwPXAZFX9oJvDZgEbVHWTqjYDTwIXBtnvR8BPgQOT3qvqKlX1uy99QLqIpPYk1mji89dY/4MJqbmTh7F8yx521zd5HUrEPfHeVppa27lmdoHXocSNno5i+iqQoao+VS0GMkTkK90clgdsC3i93S0LPO8MIF9VX+jiPJcAK1U1pv5HNLa08WFlvTUvmZCaW5RLu8Kr6yq9DiWiWtraeeTtLZx01BCb9DKCetrEdJ2qVne8UNW9wHVHcmERSQDuB27pYp/JOLWL6w+x/UsislxElldVRddsl+sr6mhrV+ugNiE1ecQA8rLS464fYpFvJztqGm1oa4T1NEEkSsBcum7/QneNgOVAfsDrkW5Zh0xgCvC6iJQBJwDPBXRUjwSeAT6nqhuDXUBV/6iqM1V1Zk5OTg9/lMiwDmoTDiLCnKJc3vywiobm+BlIuHBpGaMG9+PMiUO9DiWu9DRBvAT8TUTOEpGzgCfcsq68B4wXkTEikgJchtPRDYCq1qhqtqoWqGoB8A5wgaouF5Es4AXgdlVd2rsfKToUl9eQmZbEKFsC0YTYnKJcGlvaefPD6Ko1h8sH26tZsWUvV59UQGKCrfkQST1NELcBS4Ab3MerwLe7OsC9V+JGnBvs1gJPqapPRO4SkQu6ud6NwDjgThF5333E1FcHn7+WouEDbBETE3KzxgxmQFpS3CxFunBpGf1TEvnMzJFehxJ3knqyk6q2Aw+6jx5T1ReBFzuV3XmIfU8PeH43cHdvrhVN2tqVdTtruXyWTUNsQi85MYEzJw7l1bUVtLa1k9SHbxirrG3k3x/4ueL40QxIs/nMIq2no5hmi8jLIrJeRDaJyGYR2RTu4GLVpqp6GlvabYirCZu5k4exd38LK7bs9TqUsHr03a20tivXnFTgdShxqUc1COAh4JvACmyKjW4V+22KbxNep07IISUxgcUlFRw/dojX4YRFU2sbj7+7hTMLh1KQ3d/rcOJST+umNar6H1WtVNXdHY+wRhbDfOW1pCYlcFSO/VGb8MhITWL2uCF9eo2I51fvYFd9sw1t9VBPE8QSEVkgIieKyIyOR1gji2E+fy0Th2X26bZh4705RcPYumc/pRV1XocScqrKwqWbmZCbwexxfbOGFAt6+gl2PDAT+Anwc/fxs3AFFctU1VkDIs+al0x4nV00FBF4uQ/OzbRs8x58/lquOWmMjQT0UE9HMZ0R7kD6iu17G6htbLUb5EzYDc1MY3p+FotLKvjaWeO9DiekFi4tI6tfMp+antf9ziZsbEW5EPNZB7WJoDlFw1hTXoO/usHrUELi2VXlHP/jV3jJt5PWNmWRb6fXIcU1W1EuxHz+WhIThInDMr0OxcSBuZOdNSJeWRv7zUzPrirnjqfXUFHnzMtZ39TKHU+v4dlV5d0cacLFVpQLseLyGsblZJCWnOh1KCYOHJWTwdic/n1i8r4Fi0ppaDn4Y6WhpY0Fi0o9isiEdUW5eOTz11r/g4mouUXDeHvjbmoaWrwO5YiUH6KZrK80n8WisK0oF4+q6pqorGuiyBKEiaA5Rbm0tiuvl8buGhENzW0kJwYfrTQiKz3C0ZgO3Y5icqf2Ps19FAIClKpqbH9dCYOODuopNsTVRND0/CyyM1JZXFLBhdNib9RPe7vyjb+torVNSUlMoLmt/cC29OREbp1X6GF08a3bGoSqtgHzVbW1Y0U5Sw7BdawBYTUIE0kJCcKcoqG8UVpFU2vsdQ3e+9I6Fvkq+N4nirjv00eTl5WOAHlZ6dxz8VQusqGununpXExLReS3wN+AfR2F7lrVxuXz1zBqcD+bddJE3NyiYTyxbBtvb9zN6YWxMzP+4+9u5Y//3cRVJ4zm87MLEBFLCFGkpwlimvvvXQFlCpwZ0mhinHVQG6+ceNQQ+qUksrikImYSxH/XV/H//lXM6YU5fP+TRXbHdBSyO6lDpLaxhS279/PZmfnd72xMiKUlJ3J6YQ6vlFRw94VTSIjylddKd9bx1cdWMn5oBr+ZP93mLYtSdid1iJRY/4Px2NyiYVTWNbF6e7XXoXSpsq6Rzz/8HukpifzfNceRaU2yUcvupA6Rjg5qa2IyXjmjcCiJCRLVN801NLdx3V+Ws2dfMw9dfZwNYY1ydid1iPj8NeRkpjI0M83rUEycGtgvmRPGDo7atarb25Wbn3qfD8pr+NVl05g60oaDRzu7kzpEfOW1TLHag/HYnEm5bKisZ1NVvdehfMxPF63jP8U7+e55k5g7eZjX4ZgesDupQ6CxpY0NVfU2g6vx3Bz3gzfampmeWLaVP7yxiStPGMUXTrYV4mJFjxKEe7/DacBJwPXAZFX9IJyBxZLSnXW0tav1PxjP5WWlM3nEgKhKEG9+WMX3ni3mtAk5/OCTk204awzpzdiyWcAxwAxgvoh8LjwhxZ6ODmqbYsNEg7lFw1ixdS9V7rTZXlpfUcdXHnWGs/72chvOGmt6Osz1EZwlRk8GjnMfM8MYV0wp9tcwIC2JkYNsRIbx3pyiXFThtXXe1iKq6pq4duF7pKUk8pANZ41JPb2TeiZQpKoazmBilc9fS9GIAVZ1NlFh0vBMRg5KZ7GvgkuPG+VJDA3NbXzxr8vZva+Jp64/kTwbzhqTelrfKwZs2EEQrW3trNtRax3UJmqICHOKcnlzwy72NbVG/PoHhrNur+ZXl03n6JFZEY/BhEaXCUJEnheR54BsoEREFonIcx2PyIQY3Tbt2kdTaztT8qyD2kSPuUXDaG5t580PqyJ+7fsWlfKf4p1859xJzLPhrDGtuyamn0UkihhWXO7cDmI1CBNNjisYRFa/ZBb7KjhnyvCIXffJZVv5/Rsbufz4UXzxFBvOGuu6TBCq+kbHcxHJxemcBlimqrG7fFUI+fy1pCYlMDa7v9ehGHNAUmICZ04cyqtrK2lta4/I6KH/fbiL7z1bzCnjs/nhBTactS/o6SimzwLLgM8AnwXeFZFPhzOwWOHz1zBx+AAbvmeiztyiYdQ0tLCsbE/Yr/VhRR03PLaCo3IyeOCKGSTb/4c+oae/xe8Cx6nq1ar6OZx7Iv5f+MKKDaqKz29TbJjodOqEbFKTEsJ+01xVXRPXPvweqUmJPHTNTFswqw/paYJI6NSktLsXx/ZZ2/Y0UNfYav0PJir1S0nilPHZLPZVEK4R6o0tbVz31+Xsqm/ioatnMnJQv7Bcx3ijpx/yL7kjmK4RkWuAF4AXwxdWbPD5OzqorQZhotOcolzKqxtYu6Mu5Odub1dueWo1q7dX88tLp3FMflbIr2G81d0w13EiMltVbwX+ABztPt4G/hiB+KKaz19LYoJQOCzT61CMCeqsSbmIwOKSnSE/94LFpbywZgd3nDsxoiOlTOR0V4P4JVALoKpPq+rNqnoz8Iy7La4V+2sYPzSDtOREr0MxJqjsjFSOHTUo5P0QT723jQdf38j8WaO47pSxIT23iR7dJYhcVV3TudAtKwhLRDGkY4oNY6LZ3Mm5+Py1bN+7PyTnW7phF995Zg2njM/mrgttOGtf1l2CyOpiW1xPrlJZ10hVXZN1UJuoN6fIuZv5lRDUIjZU1vHlR1cwNqe/DWeNA939dpeLyHWdC0Xki8CK8IQUGw5M8W01CBPlxmT3Z/zQjCNeinRXfRPXLHSHs159nA1njQPdTbXxDeAZEbmCjxLCTCAF+FQY44p6PneKDWtiMrFgTlEuf/jvJqr3N5PVL6XXxwcOZ33ySyeSP9iGs8aDLmsQqlqhqicBPwTK3McPVfVEVe12WISInCMipSKyQURu72K/S0RERWRmQNkd7nGlIjKvpz9QpPj8tYwe0s/muDcxYe7kYbS1K0tKez9DTnu7csvfV7NqazW/+Ow0ptlw1rjRo/UgVHUJsKQ3JxaRROABYA6wHXhPRJ5T1ZJO+2UCNwHvBpQVAZcBk4ERwCsiMkFV23oTQzj5/LVMtRXkTIw4Om8guQNSWeyr4FPTR/bq2J+/XMoLH+zg9nMncu5UG84aT8LZwzQL2KCqm1S1GXgSuDDIfj8Cfgo0BpRdCDypqk2quhnY4J4vKtQ0tLB1z35rXjIxIyFBOHtSLm+sr6Kxpeffs55avo0Hlmxk/qx8rj/VhrPGm3AmiDxgW8Dr7W7ZASIyA8hX1Rd6e6x7/JdEZLmILK+qity89yVuB7XdQW1iydzJw9jf3MZbG3f1aP+3NuziO093DGedYsNZ45BnY9REJAG4H7jlcM+hqn9U1ZmqOjMnJyd0wXXjoyk2rInJxI4Txg4mIzWpRzfNbais58uPrmBMtg1njWfh/K2XA/kBr0e6ZR0ygSnA6yJSBpwAPOd2VHd3rKdK/LXkDkglJzPV61CM6bHUpEROL8zh5ZJK2tsPPXnf7vomrn14GSlJCfzfNTacNZ6FM0G8B4wXkTEikoLT6XxgmVJVrVHVbFUtUNUC4B3gAlVd7u53mYikisgYYDzOehRRodhfY7UHE5PmFOWyq76JVduqg27vGM5aWdvEnz4304azxrmwJQhVbQVuBBYBa4GnVNUnIneJyAXdHOsDngJKgJeAr0bLCKbGljY2Vu2z/gcTk86YOJTkRAk6eV97u/Ktv69m5dZqfnHpNKaPGuRBhCaa9GiY6+FS1RfpNC24qt55iH1P7/T6x8CPwxbcYVq3s462drUEYWLSgLRkThg7hJdLKrjj3EkHbbv/5fX8+4Md3HbORM6z4awGW/Sn16yD2sS6uUW5bKrax4bK+gNlf1++jd8u2cClM/P58mk2nNU4LEH0UnF5LQPTkxk5KK7nKjQx7OyiXOCjNSLe2riLO55ew+xxQ7j7Uzac1XwkrE1MfVGJv4ai4QPsP5GJWcMHppM/KJ1fvLye+14qRQRyM9P43RXH2nBWcxD7a+iFlrZ21u6sY0qe9T+Y2PXsqnJ21DTS0uYMdVWFvfubWbKu9/M0mb7NEkQvbKyqp7m13fofTExbsKiU1k73QTS1trNgUalHEZloZQmiF3zlNsWGiX3+6oZelZv4ZQmiF3z+WtKSExibk+F1KMYcthFZwQdYHKrcxC9LEL3g89cwafgAEhOsg9rErlvnFZKenHhQWXpyIrfOK/QoIhOtLEH0UHu7UuKvteYlE/Mump7HPRdPJS8rHQHystK55+KpXDT9YxMmmzhnw1x7aNve/dQ1tVoHtekTLpqeZwnBdMtqED3kszUgjDFxxhJED/n8NSQlCBNyM70OxRhjIsISRA8Vl9cybmgGaZ0694wxpq+yBNFDPn+t9T8YY+KKJYgeqKxtZFd9k02xYYyJK5YgeqDYpvg2xsQhSxA90DHFxqTh1kFtjIkfliB6wOevpWBIPzJt8XZjTByxBNEDvh01TM6z5iVjTHyxBNGNmv0tbNvTYDfIGWPijiWIbvh2WAe1MSY+WYLoRolNsWGMiVOWILrh89cybEAa2RmpXodijDERZQmiG8XlNVZ7MMbEJUsQXWhobmNjVb0lCGNMXLIE0YV1O2tpV2yIqzEmLlmC6EKxdVAbY+KYJYgulPhrGJieTJ4t5m6MiUOWILrgc9egFhGvQzHGmIizBHEILW3trNtZxxTrfzDGxClLEIewobKe5tZ2638wxsQtSxCH4LMOamNMnLMEcQg+fw3pyYmMyc7wOhRjjPGEJYhD8PlrmTQ8k8QE66A2xsQnSxBBtLcrJf5am8HVGBPXLEEEsXXPfuqbWq3/wRgT1yxBBNHRQW1DXI0x8cwSRBA+fw1JCcL4XOugNsbEL0sQQRT7axmfm0lqUqLXoRhjjGfCmiBE5BwRKRWRDSJye5DtXxaRNSLyvoj8T0SK3PJkEfmLu22tiNwRzjgDqSolflsDwhhjwpYgRCQReAA4FygC5nckgACPq+pUVZ0G3Afc75Z/BkhV1anAscD1IlIQrlgDVdY1sau+mSmWIIwxcS6cNYhZwAZV3aSqzcCTwIWBO6hqbcDL/oB2bAL6i0gSkA40A4H7hk1xeQ1ga0AYY0w4E0QesC3g9Xa37CAi8lUR2YhTg/i6W/wPYB+wA9gK/ExV9wQ59ksislxElldVVYUkaJ+/FhGYNNxqEMaY+OZ5J7WqPqCqRwG3Ad9zi2cBbcAIYAxwi4iMDXLsH1V1pqrOzMnJCUk8Pn8NBUP6k5GaFJLzGWNMrApngigH8gNej3TLDuVJ4CL3+eXAS6raoqqVwFJgZjiC7KxjDQhjjIl34UwQ7wHjRWSMiKQAlwHPBe4gIuMDXp4PfOg+3wqc6e7THzgBWBfGWAGo3t/M9r0NNsWGMcYAYWtHUdVWEbkRWAQkAv+nqj4RuQtYrqrPATeKyNlAC7AXuNo9/AFgoYj4AAEWquoH4Yq1Q4lN8W2MMQeEtaFdVV8EXuxUdmfA85sOcVw9zlDXiLI1IIwx5iOed1JHE5+/huED0xiSkep1KMYY4zlLEAGKrYPaGGMOsAThamhuY1NVPUXWQW2MMYAliAPW7qylXbEpNowxxmUJwuWzKTaMMeYgliBcPn8tWf2SGTEwzetQjDEmKliCcPn8tUwZMRAR8ToUY4yJCpYggJa2dkp31tkIJmOMCWAJAviwop7mtnaKLEEYY8wBliBwbpADbA4mY4wJYAkCp/+hX0oiY7L7ex2KMcZEjbhPEM+uKufxd7eyv7mNU+9bwrOrupqR3Bhj4kdcJ4hnV5Vz+z8/oLmtHYDy6gbueHqNJQljjCHOE8SCRaU0trYfVNbQ0saCRaUeRWSMMdEjrhOEv7qhV+XGGBNP4jpBjMhK71W5McbEk7hOELfOKyQ9OfGgsvTkRG6dV+hRRMYYEz3CuqJctLtoeh7g9EX4qxsYkZXOrfMKD5QbY0w8i+sEAU6SsIRgjDEfF9dNTMYYYw7NEoQxxpigLEEYY4wJyhKEMcaYoCxBGGOMCUpU1esYQkJEqoAtR3CKbGBXiMIJJYurdyyu3rG4eqcvxjVaVXOCbegzCeJIichyVZ3pdRydWVy9Y3H1jsXVO/EWlzUxGWOMCcoShDHGmKAsQXzkj14HcAgWV+9YXL1jcfVOXMVlfRDGGGOCshqEMcaYoCxBGGOMCSruEoSI5IvIEhEpERGfiNwUsO1rIrLOLb8vGuISkb+JyPvuo0xE3o+SuKaJyDtuXMtFZFaUxHWMiLwtImtE5HkRGRDhuNJEZJmIrHbj+qFbPkZE3hWRDe7vNCVK4rrRjUlFJDuSMXUT12MiUioixSLyfyKSHCVxPeSWfSAi/xCRjGiIK2D7r0WkPmQXVNW4egDDgRnu80xgPVAEnAG8AqS624ZGQ1yd9vk5cGc0xAUsBs51y88DXo+SuN4DTnPLPw/8KMJxCZDhPk8G3gVOAJ4CLnPLfw/cECVxTQcKgDIgO5IxdRPXee42AZ6IovdrQMA+9wO3R0Nc7uuZwCNAfaiuF3c1CFXdoaor3ed1wFogD7gBuFdVm9xtlVESFwAiIsBncf6zRENcCnR8Ox8I+KMkrgnAf93dXgYuiXBcqqod3+CS3YcCZwL/cMv/AlwUDXGp6ipVLYtkLD2M60V3mwLLgJFRElctHPj/mI7zu/U8LhFJBBYA3w7l9eIuQQQSkQKcb1Dv4nywnOI2A7whIsdFSVwdTgEqVPVDT4LiY3F9A1ggItuAnwF3RElcPuBCd9NngHwP4kl0mwIrcZLURqBaVVvdXbYTkPy9iktV3+3mkIjoKi63aekq4KVoiUtEFgI7gYnAb6IkrhuB51R1RyivFbcJwm07/CfwDfdbQRIwGKcaeSvwlPstweu4OswnwrWHQEHiugH4pqrmA98EHoqSuD4PfEVEVuA0PTVHOiZVbVPVaTjfemfhfJB4rnNcIjLF45CAbuP6HfBfVX0zWuJS1WuBETi11kujIK5Tcb4MhTxZxWWCcL+V/BN4TFWfdou3A0+7VbhlQDvOBFhex4WIJAEXA3+LZDzdxHU10PH87zgfhJ7HparrVHWuqh6Lk1A3RjquDqpaDSwBTgSy3N8jOP+xy6MgrnO8iiGYznGJyPeBHOBmD8MK+n6pahvwJBFuwgwUENcZwDhgg4iUAf1EZEMorhF3CcKtFTwErFXV+wM2PYvzRiMiE4AUIjhrYxdxAZwNrFPV7ZGKpwdx+YHT3OdnAhFt+jpUXCIy1P03AfgeTodwJOPKEZEs93k6MAfnm+YS4NPublcD/4qCuNZFMoZgDhWXiHwRmAfMV9X2KImrVETGuWUCXECE38NDxLVCVYepaoGqFgD7VXVcSC4Yqt7uWHkAJ+N0LH0AvO8+zsNJCI8CxcBK4MxoiMvd9jDw5Sh7v04GVgCrcdr+j42SuG7CGdG0HrgXd7aACMZ1NLDKjasYd9QZMBans3UDTo0rNUri+jpO7bkVJ+n/OUriasWp/XX8biM9eu9jceF8oV4KrHHLHiNgVJOX71enfUI2ismm2jDGGBNU3DUxGWOM6RlLEMYYY4KyBGGMMSYoSxDGGGOCsgRhjDEmKEsQJqa5s5D+POD1t0TkByE698Mi8unu9zzi63xGRNaKyJKAsqny0Sy+e0Rks/v8lV6e+y4ROTv0UZt4kNT9LsZEtSbgYhG5R1UjdmNjd0QkST+ae6k7XwCuU9X/dRSo6hpgmnuuh4F/q+o/gh7dBVW9s7fHGNPBahAm1rXirMf7zc4bOtcAOubJF5HT3QkZ/yUim0TkXhG5wp1nf42IHBVwmrPFWe9ivYh8wj0+UUQWiMh77roA1wec900ReQ4oCRLPfPf8xSLyU7fsTpyb/h4SkQXd/bDBztHxs4nIL8RZI+BVEcnp/B6IyHEi8pY4awksE5FMEZnsPn/f/VnGd/+Wm3hhCcL0BQ8AV4jIwF4ccwzwZWASzmyhE1R1FvBn4GsB+xXgzDN1PvB7EUnD+cZfo6rHAccB14nIGHf/GcBNqjoh8GIiMgL4Kc60JNOA40TkIlW9C1gOXKGqt3YV8KHO4W7uDyxX1cnAG8D3Ox2bgjOX102qegzO9C0N7nvwK3Umf5uJc1e1MYAlCNMHqDOL619xpo3oqffUWVOiCWdKh8Vu+RqcpNDhKVVtV2ea9U04M7POBT4nzpTL7wJDgI5v3stUdXOQ6x2Hs6hSldv09Bhwai/i7e4c7Xw0meOjOLWSQIXADlV9D5z3zD3H28B3ROQ2YLSqNvQyJtOHWYIwfcUvcb7Z9w8oa8X9G3cn7wtc5rMp4Hl7wOt2Du6b6zwXjeKs6vU1VZ3mPsaoakeC2XckP0QI9WgOHVV9HGfSuQbgRRE5M6xRmZhiCcL0Caq6B2dZzy8EFJcBx7rPL8BZfau3PiMiCW6/xFigFFgE3OBON46ITBCR/l2dBGeivtNEJFuc1b/m4zQF9UZX50jgo9liLwf+1+nYUmC4uAthuf0PSSIyFtikqr/GmWH26F7GZPowG8Vk+pKf46ys1eFPwL9EZDXOimSH8+1+K84H8wCcGXUbReTPOM1QK91pn6voZglRVd0hIrfjTPstwAuq2qspv7s5xz6cxWO+h7PS2KWdjm0WkUuB37jTRDfg9EN8FrhKRFpwVkn7SW9iMn2bzeZqTB8gIvWqmuF1HKZvsSYmY4wxQVkNwhhjTFBWgzDGGBOUJQhjjDFBWYIwxhgTlCUIY4wxQVmCMMYYE9T/BzAvwAZBTZBVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_val = [x[0] for x in coherence]\n",
    "y_val = [x[1] for x in coherence]\n",
    "\n",
    "plt.plot(x_val,y_val)\n",
    "plt.scatter(x_val,y_val)\n",
    "plt.title('Number of Topics vs. Coherence')\n",
    "plt.xlabel('Number of Topics')\n",
    "plt.ylabel('Coherence')\n",
    "plt.xticks(x_val)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26 has a coherence score of: 0.3715152121029969\n",
      "27 has a coherence score of: 0.45459511292850296\n",
      "28 has a coherence score of: 0.4343243436999488\n",
      "29 has a coherence score of: 0.4308995416420955\n",
      "30 has a coherence score of: 0.4468994137258929\n",
      "31 has a coherence score of: 0.39543802082627294\n",
      "32 has a coherence score of: 0.41411048730500605\n",
      "33 has a coherence score of: 0.450962184824083\n",
      "34 has a coherence score of: 0.4300747253475661\n"
     ]
    }
   ],
   "source": [
    "for x in coherence:\n",
    "    print(str(x[0]) + \" has a coherence score of: \" + str(x[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA final model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "Lda = gensim.models.ldamodel.LdaModel\n",
    "ldamodel = Lda(doc_term_matrix, num_topics=15, id2word = dictionary, passes=40,\\\n",
    "               iterations=200,  chunksize = 10000, eval_every = None, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------ Topic 0 ------\n",
      "development sport transit transportation growth plan street corridor equity study\n",
      "\n",
      "------ Topic 1 ------\n",
      "cell gene disease protein study virus infection expression host pathogen\n",
      "\n",
      "------ Topic 2 ------\n",
      "study student research experience program relationship work education school group\n",
      "\n",
      "------ Topic 3 ------\n",
      "design community project application base approach development analysis research process\n",
      "\n",
      "------ Topic 4 ------\n",
      "study membrane cell reactor compound result fly reaction tumor activity\n",
      "\n",
      "------ Topic 5 ------\n",
      "population study wheat datum resistance selection crash habitat trait site\n",
      "\n",
      "------ Topic 6 ------\n",
      "product food consumer study result research problem preference choice time\n",
      "\n",
      "------ Topic 7 ------\n",
      "network cost industry business power market model company product datum\n",
      "\n",
      "------ Topic 8 ------\n",
      "study surface property material result method structure energy test base\n",
      "\n",
      "------ Topic 9 ------\n",
      "model test study datum result effect simulation exercise time function\n",
      "\n",
      "------ Topic 10 ------\n",
      "price model farm market producer increase change production risk result\n",
      "\n",
      "------ Topic 11 ------\n",
      "plant yield crop soil water study biomass seed increase treatment\n",
      "\n",
      "------ Topic 12 ------\n",
      "increase treatment effect oil study feed starch corn wheat grain\n",
      "\n",
      "------ Topic 13 ------\n",
      "treatment increase soil study effect concentration beef cow calf decrease\n",
      "\n",
      "------ Topic 14 ------\n",
      "model study datum area distribution base result location water field\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Filtering for words \n",
    "words = [re.findall(r'\"([^\"]*)\"',t[1]) for t in ldamodel.print_topics()]\n",
    "\n",
    "# Create Topics\n",
    "topics = [' '.join(t[0:10]) for t in words]\n",
    "\n",
    "# Getting the topics\n",
    "for id, t in enumerate(topics): \n",
    "    print(f\"------ Topic {id} ------\")\n",
    "    print(t, end=\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(r\"C:\\Users\\shive\\Desktop\\llnl-ksu-recipes\\tkm-repo\\etdr_processed.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "320px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}