0_batch_converter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%% libraries\n",
    "import os\n",
    "import sys\n",
    "import glob\n",
    "import io\n",
    "import itertools\n",
    "import textract\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "%matplotlib inline\n",
    "\n",
    "# run for jupyter notebook\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Wrappers for textract functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%% reader functions\n",
    "def pdf_to_txt(inpath, outpath):\n",
    "    try:\n",
    "        text = textract.process(inpath, method='pdftotext')\n",
    "        base = os.path.abspath(inpath)\n",
    "        wdir, fname = outpath, os.path.split(base)[1]\n",
    "        writepath = wdir + '/' + fname.split('.')[0] + '.txt'\n",
    "\n",
    "        with open(writepath, 'wb') as f:\n",
    "            f.write(text)\n",
    "    except:\n",
    "        print(inpath, ' has incompatible characters. Run again')\n",
    "        pass\n",
    "    \n",
    "    \n",
    "def read_pdf(inpath):\n",
    "    text = textract.process(inpath, method='pdftotext')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def convert_size(size_bytes):\n",
    "   if size_bytes == 0:\n",
    "       return \"0B\"\n",
    "   size_name = (\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\", \"YB\")\n",
    "   i = int(math.floor(math.log(size_bytes, 1024)))\n",
    "   p = math.pow(1024, i)\n",
    "   s = round(size_bytes / p, 2)\n",
    "   return \"%s %s\" % (s, size_name[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set working directory, Preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "newspaper_name = 'The Himalayan Times'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = '/media/alal/LAL_DATA/Newspapers/' + newspaper_name\n",
    "os.chdir(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/media/alal/LAL_DATA/Newspapers/The Himalayan Times'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% directories\n",
    "input = root \n",
    "output = root + '/raw_txts/'\n",
    "\n",
    "if not os.path.exists(output):\n",
    "    os.makedirs(output)\n",
    "\n",
    "%pwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Construct list of files from all subdirectories, store file sizes in dictionary*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pdfs = []\n",
    "sizes = {}\n",
    "\n",
    "for root, dirs, files in os.walk(input):\n",
    "    for file in files:\n",
    "        if file.endswith(\".pdf\") and file[0] != '.':\n",
    "            ff = os.path.join(root, file)\n",
    "            pdfs.append(ff)\n",
    "            size = os.path.getsize(ff) # in bytes\n",
    "            sizes[file] = size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f93b39dbcc0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'461.36 KB'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'145.43 MB'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEQCAYAAACz0c/rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl0ZGeZ3/HvU7vWVrulXrw07TZgvICxI4OX4CEsgzFbhvHkmGUGcwwOMCGQZGaOIYQQTpJjyBnOQHJYeoyHZcAsxjgwxyxmgIEwtrFsbLDdeMEb7W631bu22p/8cavUUqskVUl1Vfd2/T7n6KhUdeve954q/fTorfe+r7k7IiJy/Et0ugEiIrI2FPgiIl1CgS8i0iUU+CIiXUKBLyLSJRT4IiJdInKBb2bXm9kzZnZfm/b3cTO738x2mtmnzMzasV8RkbiJXOADXwAubceOzOwi4GLgBcDZwPnAH7Rj3yIicRO5wHf3nwEH5t5nZqeZ2ffN7C4z+7mZPa/Z3QE5IANkgTSwt60NFhGJicgF/iJ2AO91938B/AXw6Wae5O63AT8B9tS+fuDuO0NrpYhIhKU63YDlmFk/cBHwzTnd79naY28EPtrgaU+5+6vM7NnAGcDJtftvNbNLav9FiIh0lcgHPsF/IYfc/YXHPuDuNwE3LfHcPwJud/dJADP7HnABoMAXka4T+S4ddz8CPGZmfwJggXOafPqTwB+YWcrM0gQf2KpLR0S6UuQC38xuAG4DTjezXWZ2FfAW4Cozuxe4H3hDk7u7Efgd8BvgXuBed/9uCM0WEYk80/TIIiLdIXIVvoiIhCNSH9oODw/7tm3bOt0MEZHYuOuuu/a5+0gz20Yq8Ldt28bY2FinmyEiEhtm9kSz26pLR0SkSyjwRUS6hAJfRKRLKPBFRLqEAl9EpEso8EVEuoQCX0SkSyjwZUm/eGQf9z11uNPNEJE2iNSFVxIt4xMF3nLdHQA88j9eTSqp+kAkzvQbLIu664mDs7cf2HOkgy0RkXZQ4MuiHt47MXv7ob2THWyJiLSDAl8W9dShGYZ60yQMnjww3enmiMgqqQ9fFvXUoRmedUIv+yaL/F6BLxJ7CnxZ1P7JIlvW5ejNpHhi/1SnmyMiq6QuHVnUoekiQ70Ztp7Qy66DM51ujoisUmiBb2anm9k9c76OmNn7wzqetN/B6RLre9OMDGTZP1WkUtVymCJxFlqXjrs/CLwQwMySwFPAt8M6nrRXvlRhplRhfV+G/myKStU5OF1kuD/b6aaJyAqtVZfOy4HfuXvTK7NIZx2cLgKwvjczG/L7JgudbJKIrNJaBf4VwA2NHjCzq81szMzGxsfH16g5spyDUyUA1vemGe7PAMGVtyISX6EHvpllgNcD32z0uLvvcPdRdx8dGWlqHV5ZA4fqFX5fhpEBVfgix4O1qPBfDdzt7nvX4FjSJgengwp/qDfNcD3wJ4qdbJKIrNJaBP6bWKQ7R6JrshAE/kAuzUA2RSaZYN+UKnyROAs18M2sF3glcFOYx5H2myxUAOjPpDAz1vWmOVyr+kUknkK90tbdp4ENYR5DwjFVKAPQm00CwYe3hxT4IrGmK22loalCmUwqQbo2B/5QT4ZDM+rDF4kzBb40NFUs0589+g/gOlX4IrGnwJeGpgoV+mrdOQBDPQp8kbhT4EtDk4UyfZmjFf76PnXpiMSdAl8ami6W6ZvbpdOTJl+qki9VOtgqEVkNBb40NFmozAv8od40AIdn1K0jElcKfGloqlCmf14ffjCfTn1SNRGJHwW+NDRVKNM7tw+/VuHrg1uR+FLgS0NBhT9/WCYo8EXiTIEvDc2UKvRk5nTp9AZdOofUpSMSWwp8WaBSdUoVJ5s6+vZY1xNU+EfyqvBF4kqBLwsUy1UAsqmjFX5fJknCYCJf7lSzRGSVFPiyQH2sfS599O1hZvRnUwp8kRhT4MsChQYVPgRz46tLRyS+FPiyQKMKH2AgpwpfJM4U+LLAYhX+YC7NhCp8kdhS4MsCqvBFjk9hL3E4ZGY3mtlvzWynmV0Y5vGkPRbvw1fgi8RZqEscAp8Evu/ul5tZBugN+XjSBotX+PrQViTOQgt8MxsELgGuBHD3IqDLNGNguQrf3TGzTjRNRFYhzC6d7cA48Hdm9iszu87M+kI8nrRJobx4hV+pOjOaE18klsIM/BRwHvAZdz8XmAKuOXYjM7vazMbMbGx8fDzE5kiz8qXFK3zQ1bYicRVm4O8Cdrn7HbWfbyT4AzCPu+9w91F3Hx0ZGQmxOdKseoWfPabCH6zNp6OhmSLxFFrgu/vTwO/N7PTaXS8HHgjreNI+9Qo/t0iFf0QVvkgshT1K573AV2ojdB4F3h7y8aQNFq3w1aUjEmuhBr673wOMhnkMab+jffgLP7QFdemIxJWutJUFCuUKmVRiwdBLfWgrEm8KfFmgUKqSSy18a6jCF4k3Bb4sUChXyKaTC+7XIigi8abAlwXypeqCi65Ai6CIxJ0CXxYolCsLLrqq03w6IvGlwJcFCqXqghE6dQO5FJOq8EViSYEvC+TLFXIN+vChFvgFBb5IHCnwZYGlKnz14YvElwJfFliqwu/PpVXhi8SUAl8WWK4PXxW+SDwp8GWBJfvwsykmCxqlIxJHCnxZYLk+/HypSqlSXeNWichqKfBlgXxpqT78YD6dKfXji8SOAl8WKJSXrvBB0yuIxJECX+Zx9yUDXzNmisSXAl/mKZRrc+Ev1qWTDWbM1NBMkfhR4Ms8s4G/TIWvkToi8aPAl3kKpWB5w+U+tFWXjkj8hLrEoZk9DkwAFaDs7lruMOKWrfCz9QpfgS8SN2EvYg7wr9x93xocR9og32SFrxkzReJHXToyz3IVfk9aq16JxFXYge/AD83sLjO7utEGZna1mY2Z2dj4+HjIzZHlLFfh11e9UpeOSPyEHfgXu/t5wKuBPzezS47dwN13uPuou4+OjIyE3BxZznIVPgSrXqnCF4mfUAPf3XfXvj8DfBt4UZjHk9UrlJeu8KG+CIqGZYrETWiBb2Z9ZjZQvw38IXBfWMeT9siX6hdeLf7WUJeOSDyFOUpnE/BtM6sf56vu/v0QjydtUK/wF1vEHIKROgenimvVJBFpk9AC390fBc4Ja/8SjnqFn1umwn/ywPRaNUlE2kTDMmWe+pW2S1X4WvVKJJ4U+DJPvtxcha8Lr0TiR4Ev8xTqH9ouWeGnmSlVKGvVK5FYUeDLPPlyhXTSSCZs0W3qi6BMFSpr1SwRaQMFvswTrGe7eHUPc2bM1Fh8kVhR4Ms8hXJlyf570IyZInGlwJd58q1U+PrgViRWFPgyT6FcWXIeHTjah6+ROiLxosCXefKl6qLr2dbNLmSuLh2RWFHgyzzNVPgDudpC5qrwRWJFgS/zFErVZT+0ne3S0SgdkVhR4Ms8QYW/dJdObyaJmSp8kbhR4Ms8+SYq/PqqV+rDF4kXBb7M00yFD8FYfA3LFImXpgLfzL5lZq8xM/2BOM41U+FDMBZfXToi8dJsgH8GeDPwsJlda2bPC7FN0kHNVvha9UokfpoKfHf/kbu/BTgPeBy41cz+2czebmbpMBsoa6tQri47LBNqC5kr8EVipekuGjPbAFwJvAP4FfBJgj8At4bSMllz7k6+VFlyAfO6oEtHwzJF4qSpJQ7N7CbgecCXgde5+57aQ183s7FlnpsExoCn3P21q2mshKtcdapOcxW+unREYqfZNW2vc/db5t5hZll3L7j76DLPfR+wExhcSQNl7eRryxs2VeFrlI5I7DTbpfPfG9x323JPMrOTgdcA17XSKOmMQm15w2yTo3SmixUqVQ+7WSLSJktW+Ga2GTgJ6DGzc4H6MkiDQG8T+/8b4K+AgSWOcTVwNcDWrVub2KWEZbbCb3KUDgRz4q/r0ef2InGwXJfOqwg+qD0Z+MSc+yeADy71RDN7LfCMu99lZi9dbDt33wHsABgdHVW52EGtVPiD9QnUFPgisbFk4Lv7F4Evmtkfu/u3Wtz3xcDrzewyIAcMmtnfu/tbV9hWCVm9wm9qHH5Oc+KLxM1yXTpvdfe/B7aZ2X889nF3/0SDp9Uf+wDwgdp+Xgr8hcI+2lrqw9eMmSKxs1yXTl/te3/YDZHOK5SCwG+qD1/LHIrEznJdOp+rff9vqzmIu/8U+Olq9iHhy5drXTpNVPj1hcwV+CLx0ezkaR83s0EzS5vZP5rZPjNT98xxpl7hN3Ph1Wwfvi6+EomNZsfh/6G7HwFeC+wCngv8ZWitko4olFu78Ar0oa1InDQb+PVxd5cBN7j7gZDaIx3USoXfl0lhpoXMReKk2akVvmtmvwVmgPeY2QiQD69Z0gn5Fir8RMLoz2hOfJE4aXZ65GuAC4FRdy8BU8AbwmyYrL1WKnyozZipYZkisdFshQ9wBsF4/LnP+VKb2yMd1MrkaaAJ1ETiptnpkb8MnAbcA1RqdzsK/ONKoVwlYZBK2PIbA+t60hyeUYUvEhfNVvijwJnurrlujmOFcrD4iVlzgT/Um+GpQzMht0pE2qXZUTr3AZvDbIh0Xr7U3PKGdUO9aQ5NF0NskYi0U7MV/jDwgJn9EijU73T314fSKumIZhcwr1vfm+bQtLp0ROKi2cD/SJiNkGjIl6rkmphWoW6oN8NMqdL0Orgi0llNBb67/5OZPQt4jrv/yMx6Af2GH2darfCHeoPr8Q5Nl9i8Tm8Hkahrdi6ddwI3Ap+r3XUScHNYjZLOaLXCX9+bAeDQjPrxReKg2d/uPydY0OQIgLs/DGwMq1HSGS1X+LWVrg5OqR9fJA6aDfyCu8+WcbWLrzRE8ziTL1Wbmhq5bqhe4WukjkgsNPvb/U9m9kGCxcxfCXwT+G54zZJOKJSrK+vD18VXIrHQbOBfA4wDvwH+LXAL8KGwGiWdEVx41Xof/kFV+CKx0OwonaqZ3Qzc7O7jIbdJOqRQaq3C78kkyaYSGosvEhNLlnMW+IiZ7QN+CzxoZuNm9uHldmxmOTP7pZnda2b3m9mqlkmU8BXKlZb68CGo8g9MqcIXiYPlfrvfTzA653x33+DuJwAvBi42s/+wzHMLwMvc/RzghcClZnbBqlssocmXqk0tYD7X8ECGfZOF5TcUkY5bLvD/DHiTuz9Wv8PdHwXeWntsUR6YrP2Yrn1pZE+EraTCH+7PKvBFYmK53+60u+879s5aP366wfbzmFnSzO4BngFudfc7GmxztZmNmdnY+Lg+HuiUStUpVbz1Cr8/y74JdemIxMFygb/Ub/Kyv+XuXnH3FwInAy8ys7MbbLPD3UfdfXRkZGS5XUpIji5+0lqFPzKQZf9UgWpV/7yJRN1yo3TOMbMjDe43INfsQdz9kJn9FLiUYKpliZiZFle7qhvuz1KqOIdnSqzvy4TRNBFpkyXLOXdPuvtgg68Bd1+yS8fMRsxsqHa7B3gFwUgfiaB6hd/TcuAHIa9+fJHoa+3/99ZsAX5iZr8G7iTow/+HEI8nq5CvL2DeapdOfxaAcQW+SOS1soh5S9z918C5Ye1f2qvVBczrRgZqgT+hwBeJujArfImRlQb+cK3C3zepkToiUafAF+Bol06rffjretKkEqY+fJEYUOALsPJhmYmEsaE/wz516YhEngJfgJUPy4SgH18f2opEnwJfgJUPywRNryASFwp8ASBfXtmwTICNA1n2HlHgi0SdAl8AKKyiS2fzYI59kwVKlWq7myUibaTAFwBmirXAb3HyNIBN63K4ayy+SNQp8AWAfLlCMmGkk9byczcPBtMq7T2Sb3ezRKSNFPgC1Bc/SWDWeuBvUuCLxIICX4BglM5K+u/haOA/fViBLxJlCnwBgnH4Kw38DX0Z0knjaY3UEYk0Bb4AUChVW77Kti6RMDYO5HhGXToikabAF2B1XToAmwazPK3AF4k0Bb4Aq+vSAdi8LqfAF4k4Bb4AQYW/kmkV6jYN5tirD21FIk2BL0BtWOYK+/AhCPypYoXJQrmNrRKRdlLgCxBceJVdTZeOhmaKRF5ogW9mp5jZT8xsp5ndb2bvC+tYsnr5YmVF0yrU6eIrkegLbU1boAz8J3e/28wGgLvM7FZ3fyDEY8oK5ctVejIr//u/eZ0qfJGoC63Cd/c97n537fYEsBM4KazjyepMF8v0Zlb+93/TYLC2rUbqiETXmvThm9k24FzgjgaPXW1mY2Y2Nj4+vhbNkWNUqk6+VKU3s/Iund5MioFcShdfiURY6IFvZv3At4D3u/uRYx939x3uPuruoyMjI2E3RxqYLgYja/pWUeFD8MGtKnyR6Ao18M0sTRD2X3H3m8I8lqxcfS78nlVU+FC/+Erz6YhEVZijdAz4PLDT3T8R1nFk9aZqgd+XXV3g6+IrkWgLs8K/GPhT4GVmdk/t67IQjycrNFW7WGo1H9pC0KUzPlmgUvV2NEtE2iy0YZnu/v+A1lfTkDU3U1vPdjUf2kIwUqdSdfZPFthYG5cvItGhK22lbRX+7EIo+uBWJJIU+MJ0sT0Vvi6+Eok2Bb7MBn47hmUC7J3QSB2RKFLgy+w4/N5VjtLZ0J8lmTCN1BGJKAW+tK1LJ5kwNg5o5SuRqFLgC9OFMmasarbMuo2DOc2YKRJRCnxhqhisdpVIrH4U7ebBrD60FYkoBb4wXaysekhm3ZZ1Pew5nMddF1+JRI0CX5gullc9rULdyet7mCyUOZLXUociUaPAF6YKq1vAfK4Th3oAeOrgTFv2JyLto8AXZkpl+rLt6dKpB/7uQwp8kahR4AtThcqqh2TWnVSv8BX4IpGjwJfa8obtCfwNfRkyqYQqfJEIUuALE/kyA7l0W/aVSBgnDfWwS4EvEjkKfKkFfvtmyj5xKKcKXySCFPhdrlJ1Jgvtq/ABTlzXo1E6IhGkwO9yk7W58AfbWOGftL6HZyYKFMqVtu1TRFZPgd/lJvIlgLZ26Zy8vhfQWHyRqAlzEfPrzewZM7svrGPI6k3k6xV++7p0Th0OAv/x/VNt26eIrF6YFf4XgEtD3L+0QT3w29mHv21DHwCP7Ztu2z5FZPVCC3x3/xlwIKz9S3scmWl/l84JfRkGcike36cKXyRKOt6Hb2ZXm9mYmY2Nj493ujldZ6LQ/sA3M04d7lOXjkjEdDzw3X2Hu4+6++jIyEinm9N1wujSgaBb5zFV+CKR0vHAl846PB1U+Ot62hv4pw738dShGWaKGpopEhUK/C53YLrIQDZFJtXet8IZWwZwh4f2TrR1vyKycmEOy7wBuA043cx2mdlVYR1LVu7gVJGhvvZW9wBnbBkEYOeeI23ft4isTPs+qTuGu78prH1L+xycLnFCb6bt+z1lfS99maQCXyRC1KXT5Q5OF1nf1/7ATySM520ZZOcedemIRIUCv8sdmCqGUuEDnHXiIPfvPky5Ug1l/yLSGgV+lzs0XWIopMA/f9sJTBUrPKBuHZFIUOB3sUK5wmShzPre9n9oC/Di7ScAcPuj+0PZv4i0RoHfxcYnCgBsHMyGsv+NAzm2j/Rx+6OaYUMkChT4XWzvkTwAmwZzoR3jkueM8ItH9s1OwywinaPA72J7DgeBv3ldeIH/unNOpFCu8oP794Z2DBFpjgK/iz1dD/wQK/zztg5xygk9fGPs96EdQ0Sao8DvYnuP5MmmEm2fR2cuM+PtF53KLx87oA9vRTpMgd/Fdh/Os3ldDjML9ThvfvFWNg5k+Z+37NSYfJEOUuB3scfGp2ZXpwpTLp3kw687k1/vOsyOnz8a+vFEpDEFfpeqVp3H9k1x2kj/mhzvNc/fwqvP3swnfvgQd6hrR6QjFPhd6ukjeWZKFbaPhF/hQ9CX/7HLX8DWDb28+yt388BuXX0rstZCmy1Toq0+T/1aVfgAg7k0n3/b+bz5b2/n8s/+M+94yXbO37aeqsOug9M8dXCG/ZNFpksV+rNJNg/28PyTB7notGFy6eSatVPkeKXA71J3P3GQZMJ4/snr1vS4pw73cdN7LuK/3Hw/n/rHh+c9lkoYG/oz9GZSTOTL7J8q4A5DvWne+ZLtvPMl29u+UItIN1Hgd6k7Hz/ImVsG6c+u/Vtgy7oernvbKM8cyfP4/mnM4KShHjYN5kgmjo4YmiqUueuJg3zptsf5Xz94kO/cs5uPX/4CzjllaM3bLHI8UOB3of2TBe58/ABX/ctTO9qOjYM5Ni5x0VdfNsUlzx3hkueO8KMH9vKhm+/jjz79C956wbO49OzNFMpVHnx6ggefnqAnk+TSszbzkucMhz7MVCSuFPhd6Gt3/p5y1XnjeSd3uilNe8WZm3jR9hO49nu/5at3PMmXbnti9rHNgzmmimW+eseTvOqsTVz7xheEsqiLSNyZu4e3c7NLgU8CSeA6d792qe1HR0d9bGwstPYI3PfUYf7N527jgu0buP7K8zvdnBU5MFXkgd1HyKQSPHdTP0O9GQrlCn/3i8f56x8+yFBvhr981em8+uzNZFIJxicK7D1SYO+RPNPFCr2ZJM/a0MtpI/36MFhiz8zucvfRprYNK/DNLAk8BLwS2AXcCbzJ3R9Y7DkK/HCUK1UeGZ/k5l/t5vpfPMZwX4Yb330RJw71dLppbXf/7sN84Kbf8Otdh5fdNmGw9YRenr2xn02DOTb0ZxnpzzDcn2V4IMtIf5bN63Kr+qPg7lSqTjJhketqqlQd9/a0rVJ1poplpgplposVUgkjl06SSyXJZRJkkonInf/xopXAD7NL50XAI+7+aK1RXwPeACwa+Cv12v/9c/KlKnP/eM37M+YL72u07dy/fT5n6/r9i/1trO9r/v4X7mv+fY22XXjM+e1rfH5H29f48UKpSrFSJWFw2fO38OHXncnGgfAmTOuks05cx83vuZg7HjvAr35/kErF2TiYZeNgjk0DOfqySSYLZR7bN8VDT0/wyPgkj45PcfeThzg4XWz4Gq/vTdOTTpJKJkglDRyq7lRr392DwJt7X6lcpVCpUiwHU0mYQToZBF86aWRSidmfEwnDvfYu8eC1m/2Z4PV1guPMfa0bPk59G5/z2NHt3aFUa1e56g3blk7W2pZKkDCo1s6vfo5zv1eqTrFSJV9aesoMM4LwTyfoSSfJpZPzPqDvdut7M3zjXReGfpwwA/8kYO4UibuAFx+7kZldDVwNsHXr1hUd6Nkj/ZQqtbf/nPfQ3LdTvbqYf9/CbedWIfPejlb/Nufxhs+fe9/Cba3RTuc+vmj7bNFjHtvuY5+fSSU4fdMAF502HOpUyFGRSBgXnraBC0/bsOg2Z524Dl4w/75ypcqBqSLjkwX2TxZ5ZqLAnkMz7J3IUyxXKVWCcDMgmTASZphBwoyEMVspJ+oBmkqQTSVJJYxypUqx4rNhW6oEf4SL5erRPzIWvKZmVvs+/31ps9vY7GM2531pVv/Zjnnu/Oekkwmy6QSZZBIz5rWt/lUsBz9Xqk4iYSSN2ncLzn3O7XTS6Mum6M+m6Mum6M0kqVSdmVKFfKlKvlSZ81Wt3V+hGmJ3ctwM5sKbwHCuMAO/0Z/vBa+wu+8AdkDQpbOSA/3NFeeu5Gki86SSiWVHDonEWZhXsewCTpnz88nA7hCPJyIiSwgz8O8EnmNmp5pZBrgC+E6IxxMRkSWE1qXj7mUz+3fADwiGZV7v7veHdTwREVlaqBdeufstwC1hHkNERJqjmahERLqEAl9EpEso8EVEuoQCX0SkS4Q6eVqrzGwceGLZDTtnGNjX6Ua0gc4jeo6Xc9F5rL1nuftIMxtGKvCjzszGmp2kKMp0HtFzvJyLziPa1KUjItIlFPgiIl1Cgd+aHZ1uQJvoPKLneDkXnUeEqQ9fRKRLqMIXEekSCnwRkS6hwF+CmZ1gZrea2cO17+sX2a5iZvfUviIzBbSZXWpmD5rZI2Z2TYPHs2b29drjd5jZtrVv5fKaOI8rzWx8zmvwjk60czlmdr2ZPWNm9y3yuJnZp2rn+WszO2+t29iMJs7jpWZ2eM7r8eG1buNyzOwUM/uJme00s/vN7H0NtonF69ESd9fXIl/Ax4FrarevAT62yHaTnW5rgzYlgd8B24EMcC9w5jHbvAf4bO32FcDXO93uFZ7HlcD/6XRbmziXS4DzgPsWefwy4HsEq8VdANzR6Tav8DxeCvxDp9u5zDlsAc6r3R4AHmrwvorF69HKlyr8pb0B+GLt9heBf93BtrRqdhF5dy8C9UXk55p7fjcCL7dGi+N2VjPnEQvu/jPgwBKbvAH4kgduB4bMbMvatK55TZxH5Ln7Hne/u3Z7AthJsA73XLF4PVqhwF/aJnffA8EbBNi4yHY5Mxszs9vNLCp/FBotIn/sG3p2G3cvA4eBxVf+7oxmzgPgj2v/dt9oZqc0eDwOmj3XOLjQzO41s++Z2VmdbsxSal2Z5wJ3HPPQ8fR6ACEvgBIHZvYjYHODh/5zC7vZ6u67zWw78GMz+427/649LVyxZhaRb2qh+Q5rpo3fBW5w94KZvYvgv5aXhd6y9ovD69GMuwnmd5k0s8uAm4HndLhNDZlZP/At4P3ufuTYhxs8JY6vx6yuD3x3f8Vij5nZXjPb4u57av/KPbPIPnbXvj9qZj8lqBY6HfjNLCJf32aXmaWAdUTvX/Vlz8Pd98/58W+Bj61Bu8LQzGsWeXOD091vMbNPm9mwu0dqMjIzSxOE/Vfc/aYGmxwXr8dc6tJZ2neAt9Vuvw34v8duYGbrzSxbuz0MXAw8sGYtXFwzi8jPPb/LgR977dOqCFn2PI7pV309QX9sHH0H+LPa6JALgMP1LsU4MbPN9c+CzOxFBDmzf+lnra1a+z4P7HT3Tyyy2XHxeszV9RX+Mq4FvmFmVwFPAn8CYGajwLvc/R3AGcDnzKxK8Ma+1t07Hvi+yCLyZvZRYMzdv0Pwhv+ymT1CUNlf0bkWN9bkefx7M3s9UCY4jys71uAlmNkNBCNYhs1sF/BfgTSAu3+WYP3ny4BHgGng7Z1p6dKaOI/LgXebWRmYAa6IYCFxMfCnwG/M7J7afR8EtkK8Xo9WaGoFEZEuoS4dEZEuocAXEekSCnwRkS6hwBcR6RIKfBGRDlluIrpjtt1am/DtV7Wryi9r9XgKfBGRzvkCcGmT234I+Ia7n0swhPrTrR5MgS8i0iGNJqIzs9PM7PtmdpeKFNZ9AAAA9ElEQVSZ/dzMnlffHBis3V7HCq761YVXIiLRsoPgws6HzezFBJX8y4CPAD80s/cCfcCi08IsRoEvIhIRtcncLgK+OWem8mzt+5uAL7j7X5vZhQRXyZ/t7tVm96/AFxGJjgRwyN1f2OCxq6j197v7bWaWA4ZZZFLHxXYuIiIRUJtp9DEzq8/bZWZ2Tu3hJ4GX1+4/A8gB463sX3PpiIh0yNyJ6IC9BBPR/Rj4DMEyjGnga+7+UTM7k2D6736CD3D/yt1/2NLxFPgiIt1BXToiIl1CgS8i0iUU+CIiXUKBLyLSJRT4IiJdQoEvItIlFPgiIl3i/wPkMHMqpNG19gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f93edb0b198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ser = pd.Series(sizes)\n",
    "ser.plot.density()\n",
    "convert_size(ser.min())\n",
    "convert_size(ser.max())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bulk convert all pdfs in root - only run once\n",
    "### Runtime ~= 30 mins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*bulk convert, experimenting with embarrasingly parallel execution*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "num_cores = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/alal/LAL_DATA/Newspapers/The Himalayan Times/2014/THT_2014_04_23.pdf  has incompatible characters. Run again\n",
      "/media/alal/LAL_DATA/Newspapers/The Himalayan Times/2014/THT_2014_05_01.pdf  has incompatible characters. Run again\n",
      "CPU times: user 1.98 s, sys: 404 ms, total: 2.39 s\n",
      "Wall time: 35min 58s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "results = Parallel(n_jobs=num_cores)(delayed(pdf_to_txt)(p,output) \\\n",
    "                                     for p in pdfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}