1_nltk_work.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%% libraries\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import io\n",
    "import itertools\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "%matplotlib inline\n",
    "\n",
    "# run for jupyter notebook\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "newspaper_name = 'The Himalayan Times'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/media/alal/LAL_DATA/Newspapers/The Himalayan Times'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "root = '/media/alal/LAL_DATA/Newspapers/' + newspaper_name + '/'\n",
    "os.chdir(root)\n",
    "%pwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pick file, remove punctuation and stopwords\n",
    "tmp = '/home/alal/tmp'\n",
    "inp = root + '/raw_txts'\n",
    "out = root + '/word_frequencies/'\n",
    "\n",
    "if not os.path.exists(out):\n",
    "    os.makedirs(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_word_freqs(inputfile,outdir):\n",
    "    filterout= set(stopwords.words('english')+\n",
    "               list(string.punctuation)+\n",
    "               ['\\'\\'','``','\\'s','’',\"“\",\"”\",\n",
    "                'the','said','nepal','world','kathmandu'])\n",
    "    cols = ['word','freq']\n",
    "\n",
    "    base = os.path.abspath(inputfile)\n",
    "    wdir, fname = outdir, os.path.split(base)[1]\n",
    "    writepath = wdir + '/wfreqs_' + fname.split('.')[0] + '.csv'\n",
    "\n",
    "    f = open(inputfile)\n",
    "    raw = f.read()\n",
    "    tokens = [token.lower() for token in nltk.word_tokenize(raw)]\n",
    "    cleaned = [token for token in tokens if token not in filterout]\n",
    "    \n",
    "    fdict = dict(nltk.FreqDist(cleaned))\n",
    "    df = pd.DataFrame(list(fdict.items()),columns=cols)\n",
    "    df = df.sort_values('freq',ascending=0)\n",
    "    \n",
    "    df.to_csv(writepath,columns=['word','freq'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**sentence splitter relies on nltk data**\n",
    "\n",
    "run \n",
    "```{python}\n",
    "nltk.download()\n",
    "```\n",
    "\n",
    "and select 'all packages' (3 GB download to `/home/<user>/` ) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.data.path.append('/media/alal/LAL_DATA/Newspapers/nltk_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_sentences(inputfile,outdir):\n",
    "    base = os.path.abspath(inputfile)\n",
    "    wdir, fname = outdir, os.path.split(base)[1]\n",
    "    writepath = wdir + '/sentences_' + fname.split('.')[0] + '.txt'\n",
    "\n",
    "    f = open(inputfile)\n",
    "    raw = f.read()\n",
    "    string = raw.replace('\\n',\" \")\n",
    "    sentences = [token.lower() for token in nltk.tokenize.sent_tokenize(string)]\n",
    "\n",
    "    outF = open(writepath, \"w\")\n",
    "    sentences = map(lambda x: x+\"\\n\", sentences)\n",
    "\n",
    "    outF.writelines(sentences)\n",
    "    outF.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parallelize execution of word-counter function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pick file, remove punctuation and stopwords\n",
    "tmp = '/home/alal/tmp'\n",
    "inp = root + 'raw_txts'\n",
    "out = root + '/word_frequencies/'\n",
    "\n",
    "if not os.path.exists(out):\n",
    "    os.makedirs(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "files = glob.glob(inp+'/THT_*.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "num_cores = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 747 ms, sys: 89.5 ms, total: 837 ms\n",
      "Wall time: 2min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results = Parallel(n_jobs=num_cores)(delayed(write_word_freqs)(i,out) \\\n",
    "                                     for i in files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parallelize execution of sentence splitter "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pick file, remove punctuation and stopwords\n",
    "tmp = '/home/alal/tmp'\n",
    "inp = root + 'raw_txts'\n",
    "out = root + '/sentences/'\n",
    "\n",
    "if not os.path.exists(out):\n",
    "    os.makedirs(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "files = glob.glob(inp+'/THT_*.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "num_cores = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 159 ms, sys: 57.2 ms, total: 217 ms\n",
      "Wall time: 39.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results = Parallel(n_jobs=num_cores)(delayed(write_sentences)(i,out) \\\n",
    "                                     for i in files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}