TextMining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import nltk.stem\n",
    "from nltk.corpus import stopwords\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "spanish_stemmer = nltk.stem.SnowballStemmer('spanish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "spanish_stopwords = stopwords.words('spanish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\echain\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "palabras = [\"Problemas\",\"Camiones\",\"Autos\",\"Corriendo\", \"Aprendia\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_word</th>\n",
       "      <th>lemmatized_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Problemas</td>\n",
       "      <td>Problemas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Camiones</td>\n",
       "      <td>Camiones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Autos</td>\n",
       "      <td>Autos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Corriendo</td>\n",
       "      <td>Corriendo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Aprendia</td>\n",
       "      <td>Aprendia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_word lemmatized_word\n",
       "0     Problemas       Problemas\n",
       "1      Camiones        Camiones\n",
       "2         Autos           Autos\n",
       "3     Corriendo       Corriendo\n",
       "4      Aprendia        Aprendia"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatized_words=[lemmatizer.lemmatize(word=word,pos='v') for word in palabras]\n",
    "lemmatizeddf= pd.DataFrame({'original_word': palabras,'lemmatized_word': lemmatized_words})\n",
    "lemmatizeddf=lemmatizeddf[['original_word','lemmatized_word']]\n",
    "lemmatizeddf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from es_lemmatizer import lemmatize\n",
    "import spacy\n",
    "from spacy.lang.es.examples import sentences\n",
    "nlp = spacy.load('es_core_news_sm')\n",
    "nlp.add_pipe(lemmatize, after=\"tagger\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.lang.es.examples import sentences "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('es')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lematizacion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Iba caminar por lo callar hasta que se caer uno casar desde el cielo'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"Iba caminando por la calle hasta que se cayó una casa desde el cielo\"\n",
    "\n",
    "# Parse the sentence using the loaded 'en' model object `nlp`\n",
    "doc = nlp(sentence)\n",
    "\n",
    "# Extract the lemma for each token and join\n",
    "\" \".join([token.lemma_ for token in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pattern\n",
    "from pattern.es import lemma, lexeme\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\" \".join([lemma(wd) for wd in sentence.split()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Separar texto por oraciones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "texto1 = 'A comienzos de los años ochenta surgieron los primeros esfuerzos de minería de textos que necesitaban una gran cantidad de esfuerzo humano, pero los avances tecnológicos han permitido que esta área progrese de manera rápida en la última década. La minería de textos es un área multidisciplinar basada en la recuperación de información, minería de datos, aprendizaje automático, estadísticas y la lingüística computacional.1​ Como la mayor parte de la información (más de un 80%) se encuentra actualmente almacenada como texto, se cree que la minería de textos tiene un gran valor comercial. Se le presta cada vez un mayor interés a la minería de textos multilingual: la habilidad de ganar información en otros idiomas.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "frases = sent_tokenize(texto1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['A comienzos de los años ochenta surgieron los primeros esfuerzos de minería de textos que necesitaban una gran cantidad de esfuerzo humano, pero los avances tecnológicos han permitido que esta área progrese de manera rápida en la última década.', 'La minería de textos es un área multidisciplinar basada en la recuperación de información, minería de datos, aprendizaje automático, estadísticas y la lingüística computacional.1\\u200b Como la mayor parte de la información (más de un 80%) se encuentra actualmente almacenada como texto, se cree que la minería de textos tiene un gran valor comercial.', 'Se le presta cada vez un mayor interés a la minería de textos multilingual: la habilidad de ganar información en otros idiomas.']\n"
     ]
    }
   ],
   "source": [
    "print(frases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 11: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 21: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 30: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 34: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 38: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 40: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 41: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 42: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 45: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 46: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 47: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 49: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 51: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 52: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 58: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 59: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 60: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 68: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 69: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 70: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 71: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 75: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 76: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 77: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 79: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 80: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 82: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 83: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 85: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 87: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 91: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 92: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 95: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 97: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 99: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 101: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 106: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 107: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 111: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 113: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 116: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 122: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 124: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 125: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 126: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 127: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 129: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 133: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 134: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 137: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 140: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 145: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 146: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 149: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 165: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 170: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 172: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 173: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 178: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 182: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 184: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 186: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 193: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 206: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 208: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 225: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 226: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 230: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 241: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 250: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 257: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 267: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 271: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 274: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 278: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 283: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 284: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 290: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 291: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 296: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 299: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 300: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 301: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 302: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 303: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 304: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 311: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 316: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 319: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 324: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 331: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 337: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 341: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 342: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 343: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 344: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 345: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 346: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 350: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 352: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 354: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 355: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 357: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 360: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 383: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 398: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 399: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 401: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 402: ',' expected after '\"'\n",
      "Skipping line 409: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 428: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 429: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 432: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 437: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 441: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 443: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 445: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 446: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 452: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 459: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 462: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 464: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 466: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 468: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 470: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 472: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 473: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 481: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 484: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 495: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 496: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 497: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 498: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 509: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 510: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 513: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 521: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 533: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 534: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 538: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 540: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 541: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 551: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 553: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 555: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 558: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 565: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 566: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 573: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 576: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 579: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 585: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 589: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 592: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 602: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 612: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 615: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 621: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 625: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 628: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 629: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 630: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 632: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 634: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 645: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 648: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 651: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 656: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 660: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 666: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 684: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 687: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 691: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 693: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 702: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 705: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 708: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 709: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 714: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 718: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 720: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 722: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 724: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 743: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 744: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 753: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 762: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 769: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 771: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 776: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 777: NULL byte detected. This byte cannot be processed in Python's native csv library at the moment, so please pass in engine='c' instead\n",
      "Skipping line 48: Expected 1 fields in line 48, saw 2\n",
      "Skipping line 60: Expected 1 fields in line 60, saw 2\n",
      "Skipping line 65: Expected 1 fields in line 65, saw 2\n",
      "Skipping line 84: Expected 1 fields in line 84, saw 2\n",
      "Skipping line 91: Expected 1 fields in line 91, saw 2\n",
      "Skipping line 92: Expected 1 fields in line 92, saw 2\n",
      "Skipping line 98: Expected 1 fields in line 98, saw 2\n",
      "Skipping line 101: Expected 1 fields in line 101, saw 2\n",
      "Skipping line 104: Expected 1 fields in line 104, saw 2\n",
      "Skipping line 108: Expected 1 fields in line 108, saw 3\n",
      "Skipping line 111: Expected 1 fields in line 111, saw 2\n",
      "Skipping line 120: Expected 1 fields in line 120, saw 2\n",
      "Skipping line 123: Expected 1 fields in line 123, saw 4\n",
      "Skipping line 124: Expected 1 fields in line 124, saw 2\n",
      "Skipping line 125: Expected 1 fields in line 125, saw 2\n",
      "Skipping line 126: Expected 1 fields in line 126, saw 2\n",
      "Skipping line 127: Expected 1 fields in line 127, saw 4\n",
      "Skipping line 128: Expected 1 fields in line 128, saw 2\n",
      "Skipping line 129: Expected 1 fields in line 129, saw 2\n",
      "Skipping line 138: Expected 1 fields in line 138, saw 2\n",
      "Skipping line 145: Expected 1 fields in line 145, saw 2\n",
      "Skipping line 149: Expected 1 fields in line 149, saw 3\n",
      "Skipping line 155: Expected 1 fields in line 155, saw 2\n",
      "Skipping line 156: Expected 1 fields in line 156, saw 2\n",
      "Skipping line 162: Expected 1 fields in line 162, saw 4\n",
      "Skipping line 169: Expected 1 fields in line 169, saw 2\n",
      "Skipping line 174: Expected 1 fields in line 174, saw 4\n",
      "Skipping line 176: Expected 1 fields in line 176, saw 2\n",
      "Skipping line 178: Expected 1 fields in line 178, saw 2\n",
      "Skipping line 180: Expected 1 fields in line 180, saw 2\n",
      "Skipping line 192: Expected 1 fields in line 192, saw 3\n",
      "Skipping line 195: Expected 1 fields in line 195, saw 2\n",
      "Skipping line 201: Expected 1 fields in line 201, saw 2\n",
      "Skipping line 203: Expected 1 fields in line 203, saw 2\n",
      "Skipping line 204: Expected 1 fields in line 204, saw 3\n",
      "Skipping line 206: Expected 1 fields in line 206, saw 2\n",
      "Skipping line 209: Expected 1 fields in line 209, saw 2\n",
      "Skipping line 215: Expected 1 fields in line 215, saw 2\n",
      "Skipping line 216: Expected 1 fields in line 216, saw 2\n",
      "Skipping line 220: Expected 1 fields in line 220, saw 2\n",
      "Skipping line 222: Expected 1 fields in line 222, saw 2\n",
      "Skipping line 225: Expected 1 fields in line 225, saw 2\n",
      "Skipping line 226: Expected 1 fields in line 226, saw 2\n",
      "Skipping line 228: Expected 1 fields in line 228, saw 3\n",
      "Skipping line 231: Expected 1 fields in line 231, saw 2\n",
      "Skipping line 245: Expected 1 fields in line 245, saw 3\n",
      "Skipping line 248: Expected 1 fields in line 248, saw 2\n",
      "Skipping line 254: Expected 1 fields in line 254, saw 2\n",
      "Skipping line 256: Expected 1 fields in line 256, saw 2\n",
      "Skipping line 260: Expected 1 fields in line 260, saw 2\n",
      "Skipping line 263: Expected 1 fields in line 263, saw 3\n",
      "Skipping line 264: Expected 1 fields in line 264, saw 2\n",
      "Skipping line 271: Expected 1 fields in line 271, saw 2\n",
      "Skipping line 277: Expected 1 fields in line 277, saw 2\n",
      "Skipping line 278: Expected 1 fields in line 278, saw 2\n",
      "Skipping line 283: Expected 1 fields in line 283, saw 2\n",
      "Skipping line 298: Expected 1 fields in line 298, saw 2\n",
      "Skipping line 300: Expected 1 fields in line 300, saw 3\n",
      "Skipping line 303: Expected 1 fields in line 303, saw 2\n",
      "Skipping line 306: Expected 1 fields in line 306, saw 2\n",
      "Skipping line 309: Expected 1 fields in line 309, saw 2\n",
      "Skipping line 310: Expected 1 fields in line 310, saw 2\n",
      "Skipping line 313: Expected 1 fields in line 313, saw 2\n",
      "Skipping line 323: Expected 1 fields in line 323, saw 2\n",
      "Skipping line 324: Expected 1 fields in line 324, saw 2\n",
      "Skipping line 332: Expected 1 fields in line 332, saw 2\n",
      "Skipping line 337: Expected 1 fields in line 337, saw 2\n",
      "Skipping line 342: Expected 1 fields in line 342, saw 3\n",
      "Skipping line 343: Expected 1 fields in line 343, saw 2\n",
      "Skipping line 347: Expected 1 fields in line 347, saw 4\n",
      "Skipping line 351: Expected 1 fields in line 351, saw 2\n",
      "Skipping line 352: Expected 1 fields in line 352, saw 4\n",
      "Skipping line 355: Expected 1 fields in line 355, saw 2\n",
      "Skipping line 360: Expected 1 fields in line 360, saw 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 372: Expected 1 fields in line 372, saw 2\n",
      "Skipping line 373: Expected 1 fields in line 373, saw 2\n",
      "Skipping line 378: Expected 1 fields in line 378, saw 2\n",
      "Skipping line 386: Expected 1 fields in line 386, saw 2\n",
      "Skipping line 392: Expected 1 fields in line 392, saw 3\n",
      "Skipping line 394: Expected 1 fields in line 394, saw 2\n",
      "Skipping line 395: Expected 1 fields in line 395, saw 2\n",
      "Skipping line 397: Expected 1 fields in line 397, saw 2\n",
      "Skipping line 414: Expected 1 fields in line 414, saw 2\n",
      "Skipping line 415: Expected 1 fields in line 415, saw 3\n",
      "Skipping line 427: Expected 1 fields in line 427, saw 3\n",
      "Skipping line 429: Expected 1 fields in line 429, saw 2\n",
      "Skipping line 434: Expected 1 fields in line 434, saw 2\n",
      "Skipping line 440: Expected 1 fields in line 440, saw 2\n",
      "Skipping line 441: Expected 1 fields in line 441, saw 2\n",
      "Skipping line 445: Expected 1 fields in line 445, saw 2\n",
      "Skipping line 446: Expected 1 fields in line 446, saw 3\n",
      "Skipping line 451: Expected 1 fields in line 451, saw 3\n",
      "Skipping line 453: Expected 1 fields in line 453, saw 3\n",
      "Skipping line 455: Expected 1 fields in line 455, saw 3\n",
      "Skipping line 462: Expected 1 fields in line 462, saw 2\n",
      "Skipping line 467: Expected 1 fields in line 467, saw 2\n",
      "Skipping line 472: Expected 1 fields in line 472, saw 2\n",
      "Skipping line 486: Expected 1 fields in line 486, saw 2\n",
      "Skipping line 488: Expected 1 fields in line 488, saw 5\n",
      "Skipping line 494: Expected 1 fields in line 494, saw 4\n",
      "Skipping line 499: Expected 1 fields in line 499, saw 2\n",
      "Skipping line 501: Expected 1 fields in line 501, saw 2\n",
      "Skipping line 502: Expected 1 fields in line 502, saw 2\n",
      "Skipping line 504: Expected 1 fields in line 504, saw 2\n",
      "Skipping line 509: Expected 1 fields in line 509, saw 3\n",
      "Skipping line 511: Expected 1 fields in line 511, saw 2\n",
      "Skipping line 513: Expected 1 fields in line 513, saw 2\n",
      "Skipping line 514: Expected 1 fields in line 514, saw 3\n",
      "Skipping line 528: Expected 1 fields in line 528, saw 2\n",
      "Skipping line 529: Expected 1 fields in line 529, saw 3\n",
      "Skipping line 533: Expected 1 fields in line 533, saw 2\n",
      "Skipping line 534: Expected 1 fields in line 534, saw 2\n",
      "Skipping line 545: Expected 1 fields in line 545, saw 2\n",
      "Skipping line 546: Expected 1 fields in line 546, saw 4\n",
      "Skipping line 547: Expected 1 fields in line 547, saw 2\n",
      "Skipping line 554: Expected 1 fields in line 554, saw 2\n",
      "Skipping line 557: Expected 1 fields in line 557, saw 2\n",
      "Skipping line 562: Expected 1 fields in line 562, saw 3\n",
      "Skipping line 563: Expected 1 fields in line 563, saw 4\n",
      "Skipping line 566: Expected 1 fields in line 566, saw 3\n",
      "Skipping line 568: Expected 1 fields in line 568, saw 3\n",
      "Skipping line 576: Expected 1 fields in line 576, saw 2\n",
      "Skipping line 577: Expected 1 fields in line 577, saw 3\n",
      "Skipping line 578: Expected 1 fields in line 578, saw 3\n",
      "Skipping line 583: Expected 1 fields in line 583, saw 2\n"
     ]
    }
   ],
   "source": [
    "pdf = pd.read_csv(\"C:/Users/echain/Desktop/NLP/dataset1.pdf\",error_bad_lines=False, index_col=None, header=0, engine='python')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdfFileObj = open('C:/Users/echain/Desktop/NLP/dataset3.pdf', 'rb')\n",
    "pdfReader = PyPDF2.PdfFileReader(pdfFileObj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "print(pdfReader.numPages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "pageObj = pdfReader.getPage(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "texto2 =pageObj.extractText()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texto2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-11-07 15:14:27,469 [MainThread  ] [ERROR]  Unable to run java; is it installed?\n",
      "2019-11-07 15:14:27,471 [MainThread  ] [ERROR]  Failed to receive startup confirmation from startServer.\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Unable to start Tika server.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-149-af7b7a89450d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtika\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mparser\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mrawText\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'C:/Users/echain/Desktop/NLP/dataset3.pdf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\tika\\parser.py\u001b[0m in \u001b[0;36mfrom_file\u001b[1;34m(filename, serverEndpoint, xmlContent, headers, config_path)\u001b[0m\n\u001b[0;32m     34\u001b[0m     '''\n\u001b[0;32m     35\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mxmlContent\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[0mjsonOutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparse1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'all'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mserverEndpoint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m         jsonOutput = parse1('all', filename, serverEndpoint, services={'meta': '/meta', 'text': '/tika', 'all': '/rmeta/xml'},\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\tika\\tika.py\u001b[0m in \u001b[0;36mparse1\u001b[1;34m(option, urlOrPath, serverEndpoint, verbose, tikaServerJar, responseMimeType, services, rawResponse, headers, config_path)\u001b[0m\n\u001b[0;32m    326\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mservice\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'/tika'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mresponseMimeType\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'text/plain'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    327\u001b[0m     status, response = callServer('put', serverEndpoint, service, open(path, 'rb'),\n\u001b[1;32m--> 328\u001b[1;33m                                   headers, verbose, tikaServerJar, config_path=config_path, rawResponse=rawResponse)\n\u001b[0m\u001b[0;32m    329\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    330\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfile_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'remote'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munlink\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\tika\\tika.py\u001b[0m in \u001b[0;36mcallServer\u001b[1;34m(verb, serverEndpoint, service, data, headers, verbose, tikaServerJar, httpVerbs, classpath, rawResponse, config_path)\u001b[0m\n\u001b[0;32m    520\u001b[0m     \u001b[1;32mglobal\u001b[0m \u001b[0mTikaClientOnly\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    521\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mTikaClientOnly\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 522\u001b[1;33m         \u001b[0mserverEndpoint\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheckTikaServer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscheme\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mserverHost\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mport\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtikaServerJar\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mclasspath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    523\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    524\u001b[0m     \u001b[0mserviceUrl\u001b[0m  \u001b[1;33m=\u001b[0m \u001b[0mserverEndpoint\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mservice\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\miniconda3\\lib\\site-packages\\tika\\tika.py\u001b[0m in \u001b[0;36mcheckTikaServer\u001b[1;34m(scheme, serverHost, port, tikaServerJar, classpath, config_path)\u001b[0m\n\u001b[0;32m    578\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    579\u001b[0m                 \u001b[0mlog\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Failed to receive startup confirmation from startServer.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 580\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Unable to start Tika server.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    581\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mserverEndpoint\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    582\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: Unable to start Tika server."
     ]
    }
   ],
   "source": [
    "from tika import parser\n",
    "\n",
    "rawText = parser.from_file('C:/Users/echain/Desktop/NLP/dataset3.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(rawText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.converter import TextConverter\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "from io import StringIO\n",
    "\n",
    "def convert_pdf_to_txt(path):\n",
    "    rsrcmgr = PDFResourceManager()\n",
    "    retstr = StringIO()\n",
    "    codec = 'UTF-8'\n",
    "    laparams = LAParams()\n",
    "    device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)\n",
    "\n",
    "    fp = file(path, 'rb')\n",
    "    process_pdf(rsrcmgr, device, fp)\n",
    "    fp.close()\n",
    "    device.close()\n",
    "\n",
    "    str = retstr.getvalue()\n",
    "    retstr.close()\n",
    "    return str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'codec'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-da9686c647b1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mconvert_pdf_to_txt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'C:/Users/echain/Desktop/NLP/dataset3.pdf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-16-a254c3af7fa0>\u001b[0m in \u001b[0;36mconvert_pdf_to_txt\u001b[1;34m(path)\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mcodec\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'UTF-8'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[0mlaparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLAParams\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mdevice\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextConverter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrsrcmgr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretstr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcodec\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcodec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlaparams\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlaparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mfp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'codec'"
     ]
    }
   ],
   "source": [
    "convert_pdf_to_txt('C:/Users/echain/Desktop/NLP/dataset3.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'text' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-7d11ff53b18e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"utf-8\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"windows-1252\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"utf-8\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'text' is not defined"
     ]
    }
   ],
   "source": [
    "text.decode(\"utf-8\").encode(\"windows-1252\").decode(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pdfquery' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-65408f814ce5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpdfquery\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPDFQuery\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'C:/Users/echain/Desktop/NLP/dataset3.pdf'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'pdfquery' is not defined"
     ]
    }
   ],
   "source": [
    "pdf = pdfquery.PDFQuery('C:/Users/echain/Desktop/NLP/dataset3.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}