Build_Different_Simserver_Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim import utils, corpora, models, similarities\n",
    "from simserver import SessionServer\n",
    "import simserver\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "import psycopg2\n",
    "from tqdm import tqdm\n",
    "import re\n",
    "import preprocess_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# connect to database\n",
    "dbname = 'podcast'\n",
    "username = 'lindsay'\n",
    "con = psycopg2.connect(database = dbname, user = username)\n",
    "cursor = con.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# download data from database\n",
    "query = \"SELECT id, summary, episode_descriptions, episode_names FROM podcast\"\n",
    "cursor.execute(query, con)\n",
    "query_results = cursor.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "# put into dataframe, concatenate text and remove non-alphanumeric characters for each podcast\n",
    "query_df = pd.DataFrame({'id' : [x[0] for x in query_results],\n",
    "                         'summary' : [x[1] for x in query_results],\n",
    "                         'episode_descriptions' : [x[2] for x in query_results],\n",
    "                         'episode_names' : [x[3] for x in query_results]})\n",
    "\n",
    "podcast_text = pd.DataFrame(columns=['id', 'text'])\n",
    "for ind, row in tqdm(query_df.iterrows(), total = query_df.shape[0]):\n",
    "    # concatenate\n",
    "    text = ' '.join([row['summary'], row['episode_descriptions'], row['episode_names']])\n",
    "    \n",
    "    # remove non-alphanumeric, non-space\n",
    "    text = re.sub(r'([^\\s\\w]|_)+', '', text)\n",
    "    \n",
    "    podcast_text = podcast_text.append(pd.DataFrame({'id' : [row['id']],\n",
    "                                                    'text' : [text]}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert text to pre-processed tokens\n",
    "podcast_text['tokens'] = [preprocess_text.preprocess_text(x) for x in podcast_text['text']]\n",
    "\n",
    "# save tokenized text\n",
    "podcast_text.to_pickle('gensim/tokenized_preprocessed_Feb_10_2016.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert to corpus\n",
    "corpus = [{'id' : int(x['id']),\n",
    "           'tokens' : x['tokens']}\n",
    "         for ind, x in podcast_text.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# setup servers\n",
    "server_lsi = SessionServer('/tmp/simserver_lsi/')\n",
    "server_lda = SessionServer('/tmp/simserver_lda/')\n",
    "server_logentropy = SessionServer('/tmp/simserver_logentropy/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# upload corpus to servers\n",
    "utils.upload_chunked(server_lsi, corpus, chunksize=1000)\n",
    "utils.upload_chunked(server_lda, corpus, chunksize=1000)\n",
    "utils.upload_chunked(server_logentropy, corpus, chunksize=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicta5f2d7\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicta5f2d7'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictdf3400\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictdf3400'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict7a1e3b\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict7a1e3b'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicte58ac4\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicte58ac4'\n",
      "WARNING:gensim.models.ldamodel:too few updates, training might not converge; consider increasing the number of passes or iterations to improve accuracy\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict118d5f\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict118d5f'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictcf1c5c\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictcf1c5c'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict2b6b48\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict2b6b48'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicte7c2b8\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldicte7c2b8'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict9383a8\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict9383a8'\n"
     ]
    }
   ],
   "source": [
    "# train models\n",
    "server_lsi.train(corpus, method='lsi')\n",
    "server_lda.train(corpus, method='lda')\n",
    "server_logentropy.train(corpus, method='logentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict6f029b\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict6f029b'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict295a51\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict295a51'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictcd2161\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictcd2161'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict26a536\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict26a536'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict7805cc\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict7805cc'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictfd8d5a\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictfd8d5a'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictbcb079\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictbcb079'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict47538d\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldict47538d'\n",
      "ERROR:sqlitedict:failed to delete /var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictfacfa2\n",
      "Traceback (most recent call last):\n",
      "  File \"//anaconda/lib/python2.7/site-packages/sqlitedict.py\", line 283, in terminate\n",
      "    os.remove(self.filename)\n",
      "OSError: [Errno 2] No such file or directory: '/var/folders/hg/kw83c_j57xqd055hr739mhvm0000gn/T/sqldictfacfa2'\n"
     ]
    }
   ],
   "source": [
    "# index documents\n",
    "server_lsi.index(corpus)\n",
    "server_lda.index(corpus)\n",
    "server_logentropy.index(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}