four_TextFileStem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from urllib import request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = \"http://www.gutenberg.org/files/2554/2554-0.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "response = request.urlopen(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw = response.read().decode('utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1176965"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ufeffThe Project Gutenberg EBook of Crime and Punishment, by Fyodor Dostoevsky\\r\\n\\r\\nThis eBook is for the use of anyone anywhere at no cost and with\\r\\nalmost no restrictions whatsoever.  You may copy it, giv'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "blondurl = \"http://news.bbc.co.uk/2/hi/health/2284783.stm\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = request.urlopen(blondurl).read().decode('utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!doctype html public \"-//W3C//DTD HTML 4.0 Transitional//EN\" \"http://www.w3.org/TR/REC-html40/loose.dtd\">\\r\\n<html>\\r\\n<head>\\r\\n<title>BBC NEWS | Health | Blondes \\'to die out in 200 years\\'</title>\\r\\n<meta name=\"keywords\" content=\"BBC, News, BBC News, news online, world, uk, international, foreign, british, online, service\">\\r\\n<meta name=\"OriginalPublicationDate\" content=\"2002/09/27 11:51:55\">\\r\\n<meta name=\"UKFS_URL\" content=\"/1/hi/health/2284783.stm\">\\r\\n<meta name=\"IFS_URL\" content=\"/2/hi/health/2284783.stm\">\\r\\n<meta name=\"HTTP-EQUIV\" content=\"text/html;charset=iso-8859-1\">\\r\\n<meta name=\"Headline\" content=\"Blondes \\'to die out in 200 years\\'\">\\r\\n<meta name=\"Section\" content=\"Health\">\\r\\n<meta name=\"Description\" content=\"Natural blondes are an endangered species and will die out by 2202, a study suggests.\">\\r\\n<!-- GENMaps-->\\r\\n<map name=\"banner\">\\r\\n<area alt=\"BBC NEWS\" coords=\"7,9,167,32\" href=\"http://news.bbc.co.uk/1/hi.html\" shape=\"RECT\">\\r\\n</map>\\r\\n\\r\\n<script src=\"/nol/shared/js/livestats_v1_1.js\" langua'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(html, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "braw = soup.get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "btokens = nltk.word_tokenize(braw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BBC',\n",
       " 'NEWS',\n",
       " '|',\n",
       " 'Health',\n",
       " '|',\n",
       " 'Blondes',\n",
       " \"'to\",\n",
       " 'die',\n",
       " 'out',\n",
       " 'in',\n",
       " '200',\n",
       " \"years'\",\n",
       " 'NEWS',\n",
       " 'SPORT',\n",
       " 'WEATHER',\n",
       " 'WORLD',\n",
       " 'SERVICE',\n",
       " 'A-Z',\n",
       " 'INDEX',\n",
       " 'SEARCH',\n",
       " 'You',\n",
       " 'are',\n",
       " 'in',\n",
       " ':',\n",
       " 'Health',\n",
       " 'News',\n",
       " 'Front',\n",
       " 'Page',\n",
       " 'Africa',\n",
       " 'Americas',\n",
       " 'Asia-Pacific',\n",
       " 'Europe',\n",
       " 'Middle',\n",
       " 'East',\n",
       " 'South',\n",
       " 'Asia',\n",
       " 'UK',\n",
       " 'Business',\n",
       " 'Entertainment',\n",
       " 'Science/Nature',\n",
       " 'Technology',\n",
       " 'Health',\n",
       " 'Medical',\n",
       " 'notes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Talking',\n",
       " 'Point',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Country',\n",
       " 'Profiles',\n",
       " 'In',\n",
       " 'Depth',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Programmes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'SERVICES',\n",
       " 'Daily',\n",
       " 'E-mail',\n",
       " 'News',\n",
       " 'Ticker',\n",
       " 'Mobile/PDAs',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '-',\n",
       " 'Text',\n",
       " 'Only',\n",
       " 'Feedback',\n",
       " 'Help',\n",
       " 'EDITIONS',\n",
       " 'Change',\n",
       " 'to',\n",
       " 'UK']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "btokens[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\vinut\\\\Documents\\\\SyracuseUniversity\\\\CourseWork\\\\Spring2018\\\\NLP\\\\Lab4'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('desert.txt')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "rawtext = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rawtext = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(rawtext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('CrimeandPunishment.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crimetext = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crimetokens = nltk.word_tokenize(crimetext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "porter = nltk.PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lancaster = nltk.LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crimePstem = [porter.stem(t) for t in crimetokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crimeLstem = [lancaster.stem(t) for t in crimetokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['produc',\n",
       " 'by',\n",
       " 'john',\n",
       " 'bicker',\n",
       " ';',\n",
       " 'and',\n",
       " 'dagni',\n",
       " 'crime',\n",
       " 'and',\n",
       " 'punish',\n",
       " 'By',\n",
       " 'fyodor',\n",
       " 'dostoevski',\n",
       " 'translat',\n",
       " 'By',\n",
       " 'constanc',\n",
       " 'garnett',\n",
       " 'translat',\n",
       " \"'S\",\n",
       " 'prefac',\n",
       " 'A',\n",
       " 'few',\n",
       " 'word',\n",
       " 'about',\n",
       " 'dostoevski',\n",
       " 'himself',\n",
       " 'may',\n",
       " 'help',\n",
       " 'the',\n",
       " 'english',\n",
       " 'reader',\n",
       " 'to',\n",
       " 'understand',\n",
       " 'hi',\n",
       " 'work',\n",
       " '.',\n",
       " 'dostoevski',\n",
       " 'wa',\n",
       " 'the',\n",
       " 'son',\n",
       " 'of',\n",
       " 'a',\n",
       " 'doctor',\n",
       " '.',\n",
       " 'hi',\n",
       " 'parent',\n",
       " 'were',\n",
       " 'veri',\n",
       " 'hard-work',\n",
       " 'and',\n",
       " 'deepli',\n",
       " 'religi',\n",
       " 'peopl',\n",
       " ',',\n",
       " 'but',\n",
       " 'so',\n",
       " 'poor',\n",
       " 'that',\n",
       " 'they',\n",
       " 'live',\n",
       " 'with',\n",
       " 'their',\n",
       " 'five',\n",
       " 'children',\n",
       " 'in',\n",
       " 'onli',\n",
       " 'two',\n",
       " 'room',\n",
       " '.',\n",
       " 'the',\n",
       " 'father',\n",
       " 'and',\n",
       " 'mother',\n",
       " 'spent',\n",
       " 'their',\n",
       " 'even',\n",
       " 'in',\n",
       " 'read',\n",
       " 'aloud',\n",
       " 'to',\n",
       " 'their',\n",
       " 'children',\n",
       " ',',\n",
       " 'gener',\n",
       " 'from',\n",
       " 'book',\n",
       " 'of',\n",
       " 'a',\n",
       " 'seriou',\n",
       " 'charact',\n",
       " '.',\n",
       " 'though',\n",
       " 'alway',\n",
       " 'sickli',\n",
       " 'and',\n",
       " 'delic',\n",
       " 'dostoevski',\n",
       " 'came',\n",
       " 'out',\n",
       " 'third',\n",
       " 'in',\n",
       " 'the',\n",
       " 'final',\n",
       " 'examin',\n",
       " 'of',\n",
       " 'the',\n",
       " 'petersburg',\n",
       " 'school',\n",
       " 'of',\n",
       " 'engin',\n",
       " '.',\n",
       " 'there',\n",
       " 'he',\n",
       " 'had',\n",
       " 'alreadi',\n",
       " 'begun',\n",
       " 'hi',\n",
       " 'first',\n",
       " 'work',\n",
       " ',',\n",
       " '``',\n",
       " 'poor',\n",
       " 'folk',\n",
       " '.',\n",
       " \"''\",\n",
       " 'thi',\n",
       " 'stori',\n",
       " 'wa',\n",
       " 'publish',\n",
       " 'by',\n",
       " 'the',\n",
       " 'poet',\n",
       " 'nekrassov',\n",
       " 'in',\n",
       " 'hi',\n",
       " 'review',\n",
       " 'and',\n",
       " 'wa',\n",
       " 'receiv',\n",
       " 'with',\n",
       " 'acclam',\n",
       " '.',\n",
       " 'the',\n",
       " 'shi',\n",
       " ',',\n",
       " 'unknown',\n",
       " 'youth',\n",
       " 'found',\n",
       " 'himself',\n",
       " 'instantli',\n",
       " 'someth',\n",
       " 'of',\n",
       " 'a',\n",
       " 'celebr',\n",
       " '.',\n",
       " 'A',\n",
       " 'brilliant',\n",
       " 'and',\n",
       " 'success',\n",
       " 'career',\n",
       " 'seem',\n",
       " 'to',\n",
       " 'open',\n",
       " 'befor',\n",
       " 'him',\n",
       " ',',\n",
       " 'but',\n",
       " 'those',\n",
       " 'hope',\n",
       " 'were',\n",
       " 'soon',\n",
       " 'dash',\n",
       " '.',\n",
       " 'In',\n",
       " '1849',\n",
       " 'he',\n",
       " 'wa',\n",
       " 'arrest',\n",
       " '.',\n",
       " 'though',\n",
       " 'neither',\n",
       " 'by',\n",
       " 'tempera',\n",
       " 'nor',\n",
       " 'convict',\n",
       " 'a',\n",
       " 'revolutionist',\n",
       " ',',\n",
       " 'dostoevski',\n",
       " 'wa',\n",
       " 'one',\n",
       " 'of',\n",
       " 'a',\n",
       " 'littl',\n",
       " 'group',\n",
       " 'of',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'met']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crimePstem[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['produc',\n",
       " 'by',\n",
       " 'john',\n",
       " 'bick',\n",
       " ';',\n",
       " 'and',\n",
       " 'dagny',\n",
       " 'crim',\n",
       " 'and',\n",
       " 'pun',\n",
       " 'by',\n",
       " 'fyod',\n",
       " 'dostoevsky',\n",
       " 'transl',\n",
       " 'by',\n",
       " 'const',\n",
       " 'garnet',\n",
       " 'transl',\n",
       " \"'s\",\n",
       " 'prefac',\n",
       " 'a',\n",
       " 'few',\n",
       " 'word',\n",
       " 'about',\n",
       " 'dostoevsky',\n",
       " 'himself',\n",
       " 'may',\n",
       " 'help',\n",
       " 'the',\n",
       " 'engl',\n",
       " 'read',\n",
       " 'to',\n",
       " 'understand',\n",
       " 'his',\n",
       " 'work',\n",
       " '.',\n",
       " 'dostoevsky',\n",
       " 'was',\n",
       " 'the',\n",
       " 'son',\n",
       " 'of',\n",
       " 'a',\n",
       " 'doct',\n",
       " '.',\n",
       " 'his',\n",
       " 'par',\n",
       " 'wer',\n",
       " 'very',\n",
       " 'hard-working',\n",
       " 'and',\n",
       " 'deeply',\n",
       " 'religy',\n",
       " 'peopl',\n",
       " ',',\n",
       " 'but',\n",
       " 'so',\n",
       " 'poor',\n",
       " 'that',\n",
       " 'they',\n",
       " 'liv',\n",
       " 'with',\n",
       " 'their',\n",
       " 'fiv',\n",
       " 'childr',\n",
       " 'in',\n",
       " 'on',\n",
       " 'two',\n",
       " 'room',\n",
       " '.',\n",
       " 'the',\n",
       " 'fath',\n",
       " 'and',\n",
       " 'moth',\n",
       " 'spent',\n",
       " 'their',\n",
       " 'ev',\n",
       " 'in',\n",
       " 'read',\n",
       " 'aloud',\n",
       " 'to',\n",
       " 'their',\n",
       " 'childr',\n",
       " ',',\n",
       " 'gen',\n",
       " 'from',\n",
       " 'book',\n",
       " 'of',\n",
       " 'a',\n",
       " 'sery',\n",
       " 'charact',\n",
       " '.',\n",
       " 'though',\n",
       " 'alway',\n",
       " 'sick',\n",
       " 'and',\n",
       " 'del',\n",
       " 'dostoevsky',\n",
       " 'cam',\n",
       " 'out',\n",
       " 'third',\n",
       " 'in',\n",
       " 'the',\n",
       " 'fin',\n",
       " 'examin',\n",
       " 'of',\n",
       " 'the',\n",
       " 'petersburg',\n",
       " 'school',\n",
       " 'of',\n",
       " 'engin',\n",
       " '.',\n",
       " 'ther',\n",
       " 'he',\n",
       " 'had',\n",
       " 'already',\n",
       " 'begun',\n",
       " 'his',\n",
       " 'first',\n",
       " 'work',\n",
       " ',',\n",
       " '``',\n",
       " 'poor',\n",
       " 'folk',\n",
       " '.',\n",
       " \"''\",\n",
       " 'thi',\n",
       " 'story',\n",
       " 'was',\n",
       " 'publ',\n",
       " 'by',\n",
       " 'the',\n",
       " 'poet',\n",
       " 'nekrassov',\n",
       " 'in',\n",
       " 'his',\n",
       " 'review',\n",
       " 'and',\n",
       " 'was',\n",
       " 'receiv',\n",
       " 'with',\n",
       " 'acclam',\n",
       " '.',\n",
       " 'the',\n",
       " 'shy',\n",
       " ',',\n",
       " 'unknown',\n",
       " 'you',\n",
       " 'found',\n",
       " 'himself',\n",
       " 'inst',\n",
       " 'someth',\n",
       " 'of',\n",
       " 'a',\n",
       " 'celebr',\n",
       " '.',\n",
       " 'a',\n",
       " 'bril',\n",
       " 'and',\n",
       " 'success',\n",
       " 'car',\n",
       " 'seem',\n",
       " 'to',\n",
       " 'op',\n",
       " 'bef',\n",
       " 'him',\n",
       " ',',\n",
       " 'but',\n",
       " 'thos',\n",
       " 'hop',\n",
       " 'wer',\n",
       " 'soon',\n",
       " 'dash',\n",
       " '.',\n",
       " 'in',\n",
       " '1849',\n",
       " 'he',\n",
       " 'was',\n",
       " 'arrest',\n",
       " '.',\n",
       " 'though',\n",
       " 'neith',\n",
       " 'by',\n",
       " 'tempera',\n",
       " 'nor',\n",
       " 'convict',\n",
       " 'a',\n",
       " 'revolv',\n",
       " ',',\n",
       " 'dostoevsky',\n",
       " 'was',\n",
       " 'on',\n",
       " 'of',\n",
       " 'a',\n",
       " 'littl',\n",
       " 'group',\n",
       " 'of',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'met']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crimeLstem[:200]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "L- Everything is in lower case and severe in removing letters for some words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wnl = nltk.WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "crimeLemma = [wnl.lemmatize(t) for t in crimetokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Produced',\n",
       " 'by',\n",
       " 'John',\n",
       " 'Bickers',\n",
       " ';',\n",
       " 'and',\n",
       " 'Dagny',\n",
       " 'CRIME',\n",
       " 'AND',\n",
       " 'PUNISHMENT',\n",
       " 'By',\n",
       " 'Fyodor',\n",
       " 'Dostoevsky',\n",
       " 'Translated',\n",
       " 'By',\n",
       " 'Constance',\n",
       " 'Garnett',\n",
       " 'TRANSLATOR',\n",
       " \"'S\",\n",
       " 'PREFACE',\n",
       " 'A',\n",
       " 'few',\n",
       " 'word',\n",
       " 'about',\n",
       " 'Dostoevsky',\n",
       " 'himself',\n",
       " 'may',\n",
       " 'help',\n",
       " 'the',\n",
       " 'English',\n",
       " 'reader',\n",
       " 'to',\n",
       " 'understand',\n",
       " 'his',\n",
       " 'work',\n",
       " '.',\n",
       " 'Dostoevsky',\n",
       " 'wa',\n",
       " 'the',\n",
       " 'son',\n",
       " 'of',\n",
       " 'a',\n",
       " 'doctor',\n",
       " '.',\n",
       " 'His',\n",
       " 'parent',\n",
       " 'were',\n",
       " 'very',\n",
       " 'hard-working',\n",
       " 'and',\n",
       " 'deeply',\n",
       " 'religious',\n",
       " 'people',\n",
       " ',',\n",
       " 'but',\n",
       " 'so',\n",
       " 'poor',\n",
       " 'that',\n",
       " 'they',\n",
       " 'lived',\n",
       " 'with',\n",
       " 'their',\n",
       " 'five',\n",
       " 'child',\n",
       " 'in',\n",
       " 'only',\n",
       " 'two',\n",
       " 'room',\n",
       " '.',\n",
       " 'The',\n",
       " 'father',\n",
       " 'and',\n",
       " 'mother',\n",
       " 'spent',\n",
       " 'their',\n",
       " 'evening',\n",
       " 'in',\n",
       " 'reading',\n",
       " 'aloud',\n",
       " 'to',\n",
       " 'their',\n",
       " 'child',\n",
       " ',',\n",
       " 'generally',\n",
       " 'from',\n",
       " 'book',\n",
       " 'of',\n",
       " 'a',\n",
       " 'serious',\n",
       " 'character',\n",
       " '.',\n",
       " 'Though',\n",
       " 'always',\n",
       " 'sickly',\n",
       " 'and',\n",
       " 'delicate',\n",
       " 'Dostoevsky',\n",
       " 'came',\n",
       " 'out',\n",
       " 'third',\n",
       " 'in',\n",
       " 'the',\n",
       " 'final',\n",
       " 'examination',\n",
       " 'of',\n",
       " 'the',\n",
       " 'Petersburg',\n",
       " 'school',\n",
       " 'of',\n",
       " 'Engineering',\n",
       " '.',\n",
       " 'There',\n",
       " 'he',\n",
       " 'had',\n",
       " 'already',\n",
       " 'begun',\n",
       " 'his',\n",
       " 'first',\n",
       " 'work',\n",
       " ',',\n",
       " '``',\n",
       " 'Poor',\n",
       " 'Folk',\n",
       " '.',\n",
       " \"''\",\n",
       " 'This',\n",
       " 'story',\n",
       " 'wa',\n",
       " 'published',\n",
       " 'by',\n",
       " 'the',\n",
       " 'poet',\n",
       " 'Nekrassov',\n",
       " 'in',\n",
       " 'his',\n",
       " 'review',\n",
       " 'and',\n",
       " 'wa',\n",
       " 'received',\n",
       " 'with',\n",
       " 'acclamation',\n",
       " '.',\n",
       " 'The',\n",
       " 'shy',\n",
       " ',',\n",
       " 'unknown',\n",
       " 'youth',\n",
       " 'found',\n",
       " 'himself',\n",
       " 'instantly',\n",
       " 'something',\n",
       " 'of',\n",
       " 'a',\n",
       " 'celebrity',\n",
       " '.',\n",
       " 'A',\n",
       " 'brilliant',\n",
       " 'and',\n",
       " 'successful',\n",
       " 'career',\n",
       " 'seemed',\n",
       " 'to',\n",
       " 'open',\n",
       " 'before',\n",
       " 'him',\n",
       " ',',\n",
       " 'but',\n",
       " 'those',\n",
       " 'hope',\n",
       " 'were',\n",
       " 'soon',\n",
       " 'dashed',\n",
       " '.',\n",
       " 'In',\n",
       " '1849',\n",
       " 'he',\n",
       " 'wa',\n",
       " 'arrested',\n",
       " '.',\n",
       " 'Though',\n",
       " 'neither',\n",
       " 'by',\n",
       " 'temperament',\n",
       " 'nor',\n",
       " 'conviction',\n",
       " 'a',\n",
       " 'revolutionist',\n",
       " ',',\n",
       " 'Dostoevsky',\n",
       " 'wa',\n",
       " 'one',\n",
       " 'of',\n",
       " 'a',\n",
       " 'little',\n",
       " 'group',\n",
       " 'of',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'met']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crimeLemma[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file0 = nltk.corpus.gutenberg.fileids()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emmatext = nltk.corpus.gutenberg.raw(file0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shorttext = emmatext[:150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[Emma by Jane Austen 1816]\\n\\nVOLUME I\\n\\nCHAPTER I\\n\\n\\nEmma Woodhouse, handsome, clever, and rich, with a comfortable home\\nand happy disposition, seemed to'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shorttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Emma by Jane Austen 1816]\n",
      "\n",
      "VOLUME I\n",
      "\n",
      "CHAPTER I\n",
      "\n",
      "\n",
      "Emma Woodhouse, handsome, clever, and rich, with a comfortable home\n",
      "and happy disposition, seemed to\n"
     ]
    }
   ],
   "source": [
    "print(shorttext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "E\n",
      "m\n",
      "m\n",
      "a\n",
      " \n",
      "b\n",
      "y\n",
      " \n",
      "J\n"
     ]
    }
   ],
   "source": [
    "for char in shorttext[:10]:\n",
    "    print(char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "string = 'monty'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "string1 = 'python'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'montypython'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string + string1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newemmatext = emmatext.replace('\\n','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shorttext = newemmatext[:150]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[Emma by Jane Austen 1816]VOLUME ICHAPTER IEmma Woodhouse, handsome, clever, and rich, with a comfortable homeand happy disposition, seemed to unite s'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shorttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('desert.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "deserttext = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "deserttokens = nltk.word_tokenize(deserttext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "desertporter = nltk.PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "desertlancaster = nltk.LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "desertPstem = [desertporter.stem(t) for t in deserttokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "desertLstem = [desertlancaster.stem(t) for t in deserttokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['three',\n",
       " 'calgarian',\n",
       " 'have',\n",
       " 'found',\n",
       " 'a',\n",
       " 'rather',\n",
       " 'unusu',\n",
       " 'way',\n",
       " 'of',\n",
       " 'leav',\n",
       " 'snow',\n",
       " 'and',\n",
       " 'ice',\n",
       " 'behind',\n",
       " '.',\n",
       " 'they',\n",
       " 'set',\n",
       " 'off',\n",
       " 'thi',\n",
       " 'week',\n",
       " 'on',\n",
       " 'foot',\n",
       " 'and',\n",
       " 'by',\n",
       " 'camel',\n",
       " 'on',\n",
       " 'a',\n",
       " 'gruel',\n",
       " 'trek',\n",
       " 'across',\n",
       " 'the',\n",
       " 'burn',\n",
       " 'arabian',\n",
       " 'desert',\n",
       " '.',\n",
       " 'when',\n",
       " 'they',\n",
       " 'were',\n",
       " 'still',\n",
       " 'in',\n",
       " 'canada',\n",
       " ',',\n",
       " 'plan',\n",
       " 'their',\n",
       " 'trip',\n",
       " ',',\n",
       " 'they',\n",
       " 'expect',\n",
       " 'they',\n",
       " 'would',\n",
       " 'feel',\n",
       " 'lone',\n",
       " '.',\n",
       " 'but',\n",
       " 'after',\n",
       " 'two',\n",
       " 'day',\n",
       " 'into',\n",
       " 'the',\n",
       " '1,200',\n",
       " 'kilometr',\n",
       " 'journey',\n",
       " ',',\n",
       " 'the',\n",
       " 'caravan',\n",
       " 'ha',\n",
       " 'won',\n",
       " 'celebr',\n",
       " 'statu',\n",
       " 'among',\n",
       " 'the',\n",
       " 'nativ',\n",
       " 'bedouin',\n",
       " 'peopl',\n",
       " 'and',\n",
       " 'govern',\n",
       " 'offici',\n",
       " 'of',\n",
       " 'oman',\n",
       " '.',\n",
       " 'some',\n",
       " 'have',\n",
       " 'excitedli',\n",
       " 'tag',\n",
       " 'along',\n",
       " ',',\n",
       " 'say',\n",
       " 'expedit',\n",
       " 'leader',\n",
       " 'jami',\n",
       " 'clark',\n",
       " '.',\n",
       " 'mr.',\n",
       " 'clark',\n",
       " 'is',\n",
       " 'make',\n",
       " 'the',\n",
       " 'trek',\n",
       " 'with',\n",
       " 'hi']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desertPstem[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['three',\n",
       " 'calgarian',\n",
       " 'have',\n",
       " 'found',\n",
       " 'a',\n",
       " 'rather',\n",
       " 'unusu',\n",
       " 'way',\n",
       " 'of',\n",
       " 'leav',\n",
       " 'snow',\n",
       " 'and',\n",
       " 'ice',\n",
       " 'behind',\n",
       " '.',\n",
       " 'they',\n",
       " 'set',\n",
       " 'off',\n",
       " 'thi',\n",
       " 'week',\n",
       " 'on',\n",
       " 'foot',\n",
       " 'and',\n",
       " 'by',\n",
       " 'camel',\n",
       " 'on',\n",
       " 'a',\n",
       " 'gruel',\n",
       " 'trek',\n",
       " 'across',\n",
       " 'the',\n",
       " 'burn',\n",
       " 'arabian',\n",
       " 'desert',\n",
       " '.',\n",
       " 'when',\n",
       " 'they',\n",
       " 'were',\n",
       " 'still',\n",
       " 'in',\n",
       " 'canada',\n",
       " ',',\n",
       " 'plan',\n",
       " 'their',\n",
       " 'trip',\n",
       " ',',\n",
       " 'they',\n",
       " 'expect',\n",
       " 'they',\n",
       " 'would',\n",
       " 'feel',\n",
       " 'lone',\n",
       " '.',\n",
       " 'but',\n",
       " 'after',\n",
       " 'two',\n",
       " 'day',\n",
       " 'into',\n",
       " 'the',\n",
       " '1,200',\n",
       " 'kilometr',\n",
       " 'journey',\n",
       " ',',\n",
       " 'the',\n",
       " 'caravan',\n",
       " 'ha',\n",
       " 'won',\n",
       " 'celebr',\n",
       " 'statu',\n",
       " 'among',\n",
       " 'the',\n",
       " 'nativ',\n",
       " 'bedouin',\n",
       " 'peopl',\n",
       " 'and',\n",
       " 'govern',\n",
       " 'offici',\n",
       " 'of',\n",
       " 'oman',\n",
       " '.',\n",
       " 'some',\n",
       " 'have',\n",
       " 'excitedli',\n",
       " 'tag',\n",
       " 'along',\n",
       " ',',\n",
       " 'say',\n",
       " 'expedit',\n",
       " 'leader',\n",
       " 'jami',\n",
       " 'clark',\n",
       " '.',\n",
       " 'mr.',\n",
       " 'clark',\n",
       " 'is',\n",
       " 'make',\n",
       " 'the',\n",
       " 'trek',\n",
       " 'with',\n",
       " 'hi',\n",
       " 'older',\n",
       " 'brother',\n",
       " 'leigh',\n",
       " 'and',\n",
       " 'their',\n",
       " 'friend',\n",
       " 'bruce',\n",
       " 'kirbi',\n",
       " '.',\n",
       " 'they',\n",
       " 'have']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desertPstem[:111]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['three',\n",
       " 'calg',\n",
       " 'hav',\n",
       " 'found',\n",
       " 'a',\n",
       " 'rath',\n",
       " 'unus',\n",
       " 'way',\n",
       " 'of',\n",
       " 'leav',\n",
       " 'snow',\n",
       " 'and',\n",
       " 'ic',\n",
       " 'behind',\n",
       " '.',\n",
       " 'they',\n",
       " 'set',\n",
       " 'off',\n",
       " 'thi',\n",
       " 'week',\n",
       " 'on',\n",
       " 'foot',\n",
       " 'and',\n",
       " 'by',\n",
       " 'camel',\n",
       " 'on',\n",
       " 'a',\n",
       " 'gruel',\n",
       " 'trek',\n",
       " 'across',\n",
       " 'the',\n",
       " 'burn',\n",
       " 'arab',\n",
       " 'desert',\n",
       " '.',\n",
       " 'when',\n",
       " 'they',\n",
       " 'wer',\n",
       " 'stil',\n",
       " 'in',\n",
       " 'canad',\n",
       " ',',\n",
       " 'plan',\n",
       " 'their',\n",
       " 'trip',\n",
       " ',',\n",
       " 'they',\n",
       " 'expect',\n",
       " 'they',\n",
       " 'would',\n",
       " 'feel',\n",
       " 'lon',\n",
       " '.',\n",
       " 'but',\n",
       " 'aft',\n",
       " 'two',\n",
       " 'day',\n",
       " 'into',\n",
       " 'the',\n",
       " '1,200',\n",
       " 'kilomet',\n",
       " 'journey',\n",
       " ',',\n",
       " 'the',\n",
       " 'carav',\n",
       " 'has',\n",
       " 'won',\n",
       " 'celebr',\n",
       " 'stat',\n",
       " 'among',\n",
       " 'the',\n",
       " 'nat',\n",
       " 'bedouin',\n",
       " 'peopl',\n",
       " 'and',\n",
       " 'govern',\n",
       " 'off',\n",
       " 'of',\n",
       " 'om',\n",
       " '.',\n",
       " 'som',\n",
       " 'hav',\n",
       " 'excit',\n",
       " 'tag',\n",
       " 'along',\n",
       " ',',\n",
       " 'say',\n",
       " 'expedit',\n",
       " 'lead',\n",
       " 'jamy',\n",
       " 'clark',\n",
       " '.',\n",
       " 'mr.',\n",
       " 'clark',\n",
       " 'is',\n",
       " 'mak',\n",
       " 'the',\n",
       " 'trek',\n",
       " 'with',\n",
       " 'his',\n",
       " 'old',\n",
       " 'broth',\n",
       " 'leigh',\n",
       " 'and',\n",
       " 'their',\n",
       " 'friend',\n",
       " 'bruc',\n",
       " 'kirby',\n",
       " '.',\n",
       " 'they',\n",
       " 'hav']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desertLstem[:111]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}