TestCase.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TestCase.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cchu70/ComedianIdentifier/blob/master/TestCase.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "nzituykdKJir",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import sklearn as sk\n",
        "import matplotlib.pyplot as plt\n",
        "import nltk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "58wl8mmJK3_z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KOqgui-QLcsI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "filepath = '/content/gdrive/My Drive/Georgia Tech/Humor Genome Project/ML/ML - categorization/CSV Files/John Oliver/John Oliver Transcripts - John_Oliver.Gene_Editing.csv'\n",
        "testpath = '/content/gdrive/My Drive/Georgia Tech/Spring career fair College of computing'\n",
        "csvFilePath = '/content/gdrive/My Drive/Georgia Tech/Humor Genome Project/ML/ML - categorization/CSV Files/Final CSV/'\n",
        "# reading files from drive: test txt file\n",
        "taylorText = \"\"\n",
        "with open('/content/gdrive/My Drive/1849-Taylor.txt', 'r') as f:\n",
        "  taylorText = \" \".join(line.strip() for line in f) \n",
        "  print(taylorText)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zT8pNROV8Pgj",
        "colab_type": "code",
        "outputId": "a8fe7b8a-66c5-4bac-9e4a-a678e14768f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.tokenize import PunktSentenceTokenizer\n",
        "custom_sent_tokenizer = PunktSentenceTokenizer(johnOliverJoke)\n",
        "tokenized = custom_sent_tokenizer.tokenize(johnOliverJoke)\n",
        "def process_content():\n",
        "    try:\n",
        "        for i in tokenized[:5]:\n",
        "            words = nltk.word_tokenize(i)\n",
        "            tagged = nltk.pos_tag(words)\n",
        "            print(tagged)\n",
        "            chunkGram = r\"\"\"Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}\"\"\"\n",
        "            chunkParser = nltk.RegexpParser(chunkGram)\n",
        "            chunked = chunkParser.parse(tagged)\n",
        "\n",
        "    except Exception as e:\n",
        "        print(str(e))\n",
        "\n",
        "        \n",
        "process_content()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('it', 'PRP'), ('happened', 'VBD'), ('although', 'IN'), ('let', 'VB'), ('us', 'PRP'), ('all', 'DT'), ('agree', 'VBP'), ('on', 'IN'), ('this', 'DT'), ('a', 'DT'), ('bone', 'NN'), ('saw', 'NN'), ('in', 'IN'), ('any', 'DT'), ('context', 'NN'), ('is', 'VBZ'), ('an', 'DT'), ('immediate', 'JJ'), ('red', 'JJ'), ('flag', 'NN'), ('thanks', 'NNS'), ('so', 'RB'), ('much', 'JJ'), ('for', 'IN'), ('inviting', 'VBG'), ('me', 'PRP'), ('over', 'IN'), ('for', 'IN'), ('Thanksgiving', 'NNP'), ('Craig', 'NNP'), ('the', 'DT'), ('turkeys', 'NNS'), ('delicious', 'VBP'), ('the', 'DT'), ('stuffing', 'NN'), ('is', 'VBZ'), ('perfect', 'JJ'), ('that', 'IN'), (\"'s\", 'VBZ'), ('a', 'DT'), ('very', 'RB'), ('large', 'JJ'), ('bone', 'NN'), ('saw', 'NN'), ('and', 'CC'), ('I', 'PRP'), ('just', 'RB'), ('remembered', 'VBD'), ('I', 'PRP'), ('have', 'VBP'), ('to', 'TO'), ('leave', 'VB'), ('now', 'RB')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oe6Vk5r8LfKG",
        "colab_type": "code",
        "outputId": "8a904a72-a241-4d35-8619-6a01c48e69ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "johnOliverJoke = \"\"\"it happened although let us all agree on this a bone saw in any context is an immediate red flag thanks so much for inviting me over for Thanksgiving Craig the turkeys delicious the stuffing is perfect that's a very large bone saw and I just remembered I have to leave now\"\"\"\n",
        "# tokenizing a joke\n",
        "tokens = nltk.word_tokenize(taylorText)\n",
        "tagged = nltk.pos_tag(tokens)\n",
        "print(tagged)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[('Elected', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('American', 'JJ'), ('people', 'NNS'), ('to', 'TO'), ('the', 'DT'), ('highest', 'JJS'), ('office', 'NN'), ('known', 'VBN'), ('to', 'TO'), ('our', 'PRP$'), ('laws', 'NNS'), (',', ','), ('I', 'PRP'), ('appear', 'VBP'), ('here', 'RB'), ('to', 'TO'), ('take', 'VB'), ('the', 'DT'), ('oath', 'NN'), ('prescribed', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('Constitution', 'NNP'), (',', ','), ('and', 'CC'), (',', ','), ('in', 'IN'), ('compliance', 'NN'), ('with', 'IN'), ('a', 'DT'), ('time-honored', 'JJ'), ('custom', 'NN'), (',', ','), ('to', 'TO'), ('address', 'VB'), ('those', 'DT'), ('who', 'WP'), ('are', 'VBP'), ('now', 'RB'), ('assembled', 'VBN'), ('.', '.'), ('The', 'DT'), ('confidence', 'NN'), ('and', 'CC'), ('respect', 'NN'), ('shown', 'VBN'), ('by', 'IN'), ('my', 'PRP$'), ('countrymen', 'NNS'), ('in', 'IN'), ('calling', 'VBG'), ('me', 'PRP'), ('to', 'TO'), ('be', 'VB'), ('the', 'DT'), ('Chief', 'NNP'), ('Magistrate', 'NNP'), ('of', 'IN'), ('a', 'DT'), ('Republic', 'NNP'), ('holding', 'VBG'), ('a', 'DT'), ('high', 'JJ'), ('rank', 'NN'), ('among', 'IN'), ('the', 'DT'), ('nations', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('earth', 'NN'), ('have', 'VBP'), ('inspired', 'VBN'), ('me', 'PRP'), ('with', 'IN'), ('feelings', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('most', 'RBS'), ('profound', 'JJ'), ('gratitude', 'NN'), (';', ':'), ('but', 'CC'), ('when', 'WRB'), ('I', 'PRP'), ('reflect', 'VBP'), ('that', 'IN'), ('the', 'DT'), ('acceptance', 'NN'), ('of', 'IN'), ('the', 'DT'), ('office', 'NN'), ('which', 'WDT'), ('their', 'PRP$'), ('partiality', 'NN'), ('has', 'VBZ'), ('bestowed', 'VBN'), ('imposes', 'VBZ'), ('the', 'DT'), ('discharge', 'NN'), ('of', 'IN'), ('the', 'DT'), ('most', 'RBS'), ('arduous', 'JJ'), ('duties', 'NNS'), ('and', 'CC'), ('involves', 'VBZ'), ('the', 'DT'), ('weightiest', 'JJS'), ('obligations', 'NNS'), (',', ','), ('I', 'PRP'), ('am', 'VBP'), ('conscious', 'JJ'), ('that', 'IN'), ('the', 'DT'), ('position', 'NN'), ('which', 'WDT'), ('I', 'PRP'), ('have', 'VBP'), ('been', 'VBN'), ('called', 'VBN'), ('to', 'TO'), ('fill', 'VB'), (',', ','), ('though', 'RB'), ('sufficient', 'JJ'), ('to', 'TO'), ('satisfy', 'VB'), ('the', 'DT'), ('loftiest', 'JJS'), ('ambition', 'NN'), (',', ','), ('is', 'VBZ'), ('surrounded', 'VBN'), ('by', 'IN'), ('fearful', 'JJ'), ('responsibilities', 'NNS'), ('.', '.'), ('Happily', 'NNP'), (',', ','), ('however', 'RB'), (',', ','), ('in', 'IN'), ('the', 'DT'), ('performance', 'NN'), ('of', 'IN'), ('my', 'PRP$'), ('new', 'JJ'), ('duties', 'NNS'), ('I', 'PRP'), ('shall', 'MD'), ('not', 'RB'), ('be', 'VB'), ('without', 'IN'), ('able', 'JJ'), ('cooperation', 'NN'), ('.', '.'), ('The', 'DT'), ('legislative', 'JJ'), ('and', 'CC'), ('judicial', 'JJ'), ('branches', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('Government', 'NNP'), ('present', 'JJ'), ('prominent', 'JJ'), ('examples', 'NNS'), ('of', 'IN'), ('distinguished', 'VBN'), ('civil', 'JJ'), ('attainments', 'NNS'), ('and', 'CC'), ('matured', 'VBD'), ('experience', 'NN'), (',', ','), ('and', 'CC'), ('it', 'PRP'), ('shall', 'MD'), ('be', 'VB'), ('my', 'PRP$'), ('endeavor', 'NN'), ('to', 'TO'), ('call', 'VB'), ('to', 'TO'), ('my', 'PRP$'), ('assistance', 'NN'), ('in', 'IN'), ('the', 'DT'), ('Executive', 'NNP'), ('Departments', 'NNP'), ('individuals', 'NNS'), ('whose', 'WP$'), ('talents', 'NNS'), (',', ','), ('integrity', 'NN'), (',', ','), ('and', 'CC'), ('purity', 'NN'), ('of', 'IN'), ('character', 'NN'), ('will', 'MD'), ('furnish', 'VB'), ('ample', 'JJ'), ('guaranties', 'NNS'), ('for', 'IN'), ('the', 'DT'), ('faithful', 'NN'), ('and', 'CC'), ('honorable', 'JJ'), ('performance', 'NN'), ('of', 'IN'), ('the', 'DT'), ('trusts', 'NNS'), ('to', 'TO'), ('be', 'VB'), ('committed', 'VBN'), ('to', 'TO'), ('their', 'PRP$'), ('charge', 'NN'), ('.', '.'), ('With', 'IN'), ('such', 'JJ'), ('aids', 'NNS'), ('and', 'CC'), ('an', 'DT'), ('honest', 'NN'), ('purpose', 'NN'), ('to', 'TO'), ('do', 'VB'), ('whatever', 'WDT'), ('is', 'VBZ'), ('right', 'JJ'), (',', ','), ('I', 'PRP'), ('hope', 'VBP'), ('to', 'TO'), ('execute', 'VB'), ('diligently', 'RB'), (',', ','), ('impartially', 'RB'), (',', ','), ('and', 'CC'), ('for', 'IN'), ('the', 'DT'), ('best', 'JJS'), ('interests', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('country', 'NN'), ('the', 'DT'), ('manifold', 'JJ'), ('duties', 'NNS'), ('devolved', 'VBN'), ('upon', 'IN'), ('me', 'PRP'), ('.', '.'), ('In', 'IN'), ('the', 'DT'), ('discharge', 'NN'), ('of', 'IN'), ('these', 'DT'), ('duties', 'NNS'), ('my', 'PRP$'), ('guide', 'NN'), ('will', 'MD'), ('be', 'VB'), ('the', 'DT'), ('Constitution', 'NNP'), (',', ','), ('which', 'WDT'), ('I', 'PRP'), ('this', 'DT'), ('day', 'NN'), ('swear', 'VB'), ('to', 'TO'), ('``', '``'), ('preserve', 'VB'), (',', ','), ('protect', 'VB'), (',', ','), ('and', 'CC'), ('defend', 'NN'), ('.', '.'), (\"''\", \"''\"), ('For', 'IN'), ('the', 'DT'), ('interpretation', 'NN'), ('of', 'IN'), ('that', 'DT'), ('instrument', 'NN'), ('I', 'PRP'), ('shall', 'MD'), ('look', 'VB'), ('to', 'TO'), ('the', 'DT'), ('decisions', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('judicial', 'JJ'), ('tribunals', 'NNS'), ('established', 'VBN'), ('by', 'IN'), ('its', 'PRP$'), ('authority', 'NN'), ('and', 'CC'), ('to', 'TO'), ('the', 'DT'), ('practice', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Government', 'NNP'), ('under', 'IN'), ('the', 'DT'), ('earlier', 'JJR'), ('Presidents', 'NNS'), (',', ','), ('who', 'WP'), ('had', 'VBD'), ('so', 'RB'), ('large', 'JJ'), ('a', 'DT'), ('share', 'NN'), ('in', 'IN'), ('its', 'PRP$'), ('formation', 'NN'), ('.', '.'), ('To', 'TO'), ('the', 'DT'), ('example', 'NN'), ('of', 'IN'), ('those', 'DT'), ('illustrious', 'JJ'), ('patriots', 'NNS'), ('I', 'PRP'), ('shall', 'MD'), ('always', 'RB'), ('defer', 'VB'), ('with', 'IN'), ('reverence', 'NN'), (',', ','), ('and', 'CC'), ('especially', 'RB'), ('to', 'TO'), ('his', 'PRP$'), ('example', 'NN'), ('who', 'WP'), ('was', 'VBD'), ('by', 'IN'), ('so', 'RB'), ('many', 'JJ'), ('titles', 'NNS'), ('``', '``'), ('the', 'DT'), ('Father', 'NNP'), ('of', 'IN'), ('his', 'PRP$'), ('Country', 'NN'), ('.', '.'), (\"''\", \"''\"), ('To', 'TO'), ('command', 'VB'), ('the', 'DT'), ('Army', 'NNP'), ('and', 'CC'), ('Navy', 'NNP'), ('of', 'IN'), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), (';', ':'), ('with', 'IN'), ('the', 'DT'), ('advice', 'NN'), ('and', 'CC'), ('consent', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Senate', 'NNP'), (',', ','), ('to', 'TO'), ('make', 'VB'), ('treaties', 'NNS'), ('and', 'CC'), ('to', 'TO'), ('appoint', 'VB'), ('ambassadors', 'NNS'), ('and', 'CC'), ('other', 'JJ'), ('officers', 'NNS'), (';', ':'), ('to', 'TO'), ('give', 'VB'), ('to', 'TO'), ('Congress', 'NNP'), ('information', 'NN'), ('of', 'IN'), ('the', 'DT'), ('state', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('and', 'CC'), ('recommend', 'VB'), ('such', 'JJ'), ('measures', 'NNS'), ('as', 'IN'), ('he', 'PRP'), ('shall', 'MD'), ('judge', 'VB'), ('to', 'TO'), ('be', 'VB'), ('necessary', 'JJ'), (';', ':'), ('and', 'CC'), ('to', 'TO'), ('take', 'VB'), ('care', 'NN'), ('that', 'IN'), ('the', 'DT'), ('laws', 'NNS'), ('shall', 'MD'), ('be', 'VB'), ('faithfully', 'RB'), ('executed', 'VBN'), ('--', ':'), ('these', 'DT'), ('are', 'VBP'), ('the', 'DT'), ('most', 'RBS'), ('important', 'JJ'), ('functions', 'NNS'), ('intrusted', 'VBN'), ('to', 'TO'), ('the', 'DT'), ('President', 'NNP'), ('by', 'IN'), ('the', 'DT'), ('Constitution', 'NNP'), (',', ','), ('and', 'CC'), ('it', 'PRP'), ('may', 'MD'), ('be', 'VB'), ('expected', 'VBN'), ('that', 'IN'), ('I', 'PRP'), ('shall', 'MD'), ('briefly', 'VB'), ('indicate', 'VB'), ('the', 'DT'), ('principles', 'NNS'), ('which', 'WDT'), ('will', 'MD'), ('control', 'VB'), ('me', 'PRP'), ('in', 'IN'), ('their', 'PRP$'), ('execution', 'NN'), ('.', '.'), ('Chosen', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('body', 'NN'), ('of', 'IN'), ('the', 'DT'), ('people', 'NNS'), ('under', 'IN'), ('the', 'DT'), ('assurance', 'NN'), ('that', 'IN'), ('my', 'PRP$'), ('Administration', 'NNP'), ('would', 'MD'), ('be', 'VB'), ('devoted', 'VBN'), ('to', 'TO'), ('the', 'DT'), ('welfare', 'NN'), ('of', 'IN'), ('the', 'DT'), ('whole', 'JJ'), ('country', 'NN'), (',', ','), ('and', 'CC'), ('not', 'RB'), ('to', 'TO'), ('the', 'DT'), ('support', 'NN'), ('of', 'IN'), ('any', 'DT'), ('particular', 'JJ'), ('section', 'NN'), ('or', 'CC'), ('merely', 'RB'), ('local', 'JJ'), ('interest', 'NN'), (',', ','), ('I', 'PRP'), ('this', 'DT'), ('day', 'NN'), ('renew', 'VBD'), ('the', 'DT'), ('declarations', 'NNS'), ('I', 'PRP'), ('have', 'VBP'), ('heretofore', 'VBN'), ('made', 'VBN'), ('and', 'CC'), ('proclaim', 'VB'), ('my', 'PRP$'), ('fixed', 'JJ'), ('determination', 'NN'), ('to', 'TO'), ('maintain', 'VB'), ('to', 'TO'), ('the', 'DT'), ('extent', 'NN'), ('of', 'IN'), ('my', 'PRP$'), ('ability', 'NN'), ('the', 'DT'), ('Government', 'NNP'), ('in', 'IN'), ('its', 'PRP$'), ('original', 'JJ'), ('purity', 'NN'), ('and', 'CC'), ('to', 'TO'), ('adopt', 'VB'), ('as', 'IN'), ('the', 'DT'), ('basis', 'NN'), ('of', 'IN'), ('my', 'PRP$'), ('public', 'JJ'), ('policy', 'NN'), ('those', 'DT'), ('great', 'JJ'), ('republican', 'JJ'), ('doctrines', 'NNS'), ('which', 'WDT'), ('constitute', 'VBP'), ('the', 'DT'), ('strength', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('national', 'JJ'), ('existence', 'NN'), ('.', '.'), ('In', 'IN'), ('reference', 'NN'), ('to', 'TO'), ('the', 'DT'), ('Army', 'NNP'), ('and', 'CC'), ('Navy', 'NNP'), (',', ','), ('lately', 'RB'), ('employed', 'VBN'), ('with', 'IN'), ('so', 'RB'), ('much', 'JJ'), ('distinction', 'NN'), ('on', 'IN'), ('active', 'JJ'), ('service', 'NN'), (',', ','), ('care', 'VB'), ('shall', 'MD'), ('be', 'VB'), ('taken', 'VBN'), ('to', 'TO'), ('insure', 'VB'), ('the', 'DT'), ('highest', 'JJS'), ('condition', 'NN'), ('of', 'IN'), ('efficiency', 'NN'), (',', ','), ('and', 'CC'), ('in', 'IN'), ('furtherance', 'NN'), ('of', 'IN'), ('that', 'DT'), ('object', 'VBP'), ('the', 'DT'), ('military', 'JJ'), ('and', 'CC'), ('naval', 'JJ'), ('schools', 'NNS'), (',', ','), ('sustained', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('liberality', 'NN'), ('of', 'IN'), ('Congress', 'NNP'), (',', ','), ('shall', 'MD'), ('receive', 'VB'), ('the', 'DT'), ('special', 'JJ'), ('attention', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Executive', 'NNP'), ('.', '.'), ('As', 'IN'), ('American', 'JJ'), ('freemen', 'IN'), ('we', 'PRP'), ('can', 'MD'), ('not', 'RB'), ('but', 'CC'), ('sympathize', 'VB'), ('in', 'IN'), ('all', 'DT'), ('efforts', 'NNS'), ('to', 'TO'), ('extend', 'VB'), ('the', 'DT'), ('blessings', 'NNS'), ('of', 'IN'), ('civil', 'JJ'), ('and', 'CC'), ('political', 'JJ'), ('liberty', 'NN'), (',', ','), ('but', 'CC'), ('at', 'IN'), ('the', 'DT'), ('same', 'JJ'), ('time', 'NN'), ('we', 'PRP'), ('are', 'VBP'), ('warned', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('admonitions', 'NNS'), ('of', 'IN'), ('history', 'NN'), ('and', 'CC'), ('the', 'DT'), ('voice', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), ('beloved', 'VBN'), ('Washington', 'NNP'), ('to', 'TO'), ('abstain', 'VB'), ('from', 'IN'), ('entangling', 'VBG'), ('alliances', 'NNS'), ('with', 'IN'), ('foreign', 'JJ'), ('nations', 'NNS'), ('.', '.'), ('In', 'IN'), ('all', 'DT'), ('disputes', 'NNS'), ('between', 'IN'), ('conflicting', 'VBG'), ('governments', 'NNS'), ('it', 'PRP'), ('is', 'VBZ'), ('our', 'PRP$'), ('interest', 'NN'), ('not', 'RB'), ('less', 'RBR'), ('than', 'IN'), ('our', 'PRP$'), ('duty', 'NN'), ('to', 'TO'), ('remain', 'VB'), ('strictly', 'RB'), ('neutral', 'JJ'), (',', ','), ('while', 'IN'), ('our', 'PRP$'), ('geographical', 'JJ'), ('position', 'NN'), (',', ','), ('the', 'DT'), ('genius', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('institutions', 'NNS'), ('and', 'CC'), ('our', 'PRP$'), ('people', 'NNS'), (',', ','), ('the', 'DT'), ('advancing', 'VBG'), ('spirit', 'NN'), ('of', 'IN'), ('civilization', 'NN'), (',', ','), ('and', 'CC'), (',', ','), ('above', 'IN'), ('all', 'DT'), (',', ','), ('the', 'DT'), ('dictates', 'NNS'), ('of', 'IN'), ('religion', 'NN'), ('direct', 'VBP'), ('us', 'PRP'), ('to', 'TO'), ('the', 'DT'), ('cultivation', 'NN'), ('of', 'IN'), ('peaceful', 'JJ'), ('and', 'CC'), ('friendly', 'JJ'), ('relations', 'NNS'), ('with', 'IN'), ('all', 'DT'), ('other', 'JJ'), ('powers', 'NNS'), ('.', '.'), ('It', 'PRP'), ('is', 'VBZ'), ('to', 'TO'), ('be', 'VB'), ('hoped', 'VBN'), ('that', 'IN'), ('no', 'DT'), ('international', 'JJ'), ('question', 'NN'), ('can', 'MD'), ('now', 'RB'), ('arise', 'VB'), ('which', 'WDT'), ('a', 'DT'), ('government', 'NN'), ('confident', 'NN'), ('in', 'IN'), ('its', 'PRP$'), ('own', 'JJ'), ('strength', 'NN'), ('and', 'CC'), ('resolved', 'VBD'), ('to', 'TO'), ('protect', 'VB'), ('its', 'PRP$'), ('own', 'JJ'), ('just', 'RB'), ('rights', 'NNS'), ('may', 'MD'), ('not', 'RB'), ('settle', 'VB'), ('by', 'IN'), ('wise', 'JJ'), ('negotiation', 'NN'), (';', ':'), ('and', 'CC'), ('it', 'PRP'), ('eminently', 'RB'), ('becomes', 'VBZ'), ('a', 'DT'), ('government', 'NN'), ('like', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), (',', ','), ('founded', 'VBN'), ('on', 'IN'), ('the', 'DT'), ('morality', 'NN'), ('and', 'CC'), ('intelligence', 'NN'), ('of', 'IN'), ('its', 'PRP$'), ('citizens', 'NNS'), ('and', 'CC'), ('upheld', 'JJ'), ('by', 'IN'), ('their', 'PRP$'), ('affections', 'NNS'), (',', ','), ('to', 'TO'), ('exhaust', 'VB'), ('every', 'DT'), ('resort', 'NN'), ('of', 'IN'), ('honorable', 'JJ'), ('diplomacy', 'NN'), ('before', 'IN'), ('appealing', 'VBG'), ('to', 'TO'), ('arms', 'NNS'), ('.', '.'), ('In', 'IN'), ('the', 'DT'), ('conduct', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('foreign', 'JJ'), ('relations', 'NNS'), ('I', 'PRP'), ('shall', 'MD'), ('conform', 'VB'), ('to', 'TO'), ('these', 'DT'), ('views', 'NNS'), (',', ','), ('as', 'IN'), ('I', 'PRP'), ('believe', 'VBP'), ('them', 'PRP'), ('essential', 'JJ'), ('to', 'TO'), ('the', 'DT'), ('best', 'JJS'), ('interests', 'NNS'), ('and', 'CC'), ('the', 'DT'), ('true', 'JJ'), ('honor', 'NN'), ('of', 'IN'), ('the', 'DT'), ('country', 'NN'), ('.', '.'), ('The', 'DT'), ('appointing', 'VBG'), ('power', 'NN'), ('vested', 'VBN'), ('in', 'IN'), ('the', 'DT'), ('President', 'NNP'), ('imposes', 'NNS'), ('delicate', 'VBP'), ('and', 'CC'), ('onerous', 'JJ'), ('duties', 'NNS'), ('.', '.'), ('So', 'RB'), ('far', 'RB'), ('as', 'IN'), ('it', 'PRP'), ('is', 'VBZ'), ('possible', 'JJ'), ('to', 'TO'), ('be', 'VB'), ('informed', 'VBN'), (',', ','), ('I', 'PRP'), ('shall', 'MD'), ('make', 'VB'), ('honesty', 'NN'), (',', ','), ('capacity', 'NN'), (',', ','), ('and', 'CC'), ('fidelity', 'NN'), ('indispensable', 'JJ'), ('prerequisites', 'NNS'), ('to', 'TO'), ('the', 'DT'), ('bestowal', 'NN'), ('of', 'IN'), ('office', 'NN'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('absence', 'NN'), ('of', 'IN'), ('either', 'DT'), ('of', 'IN'), ('these', 'DT'), ('qualities', 'NNS'), ('shall', 'MD'), ('be', 'VB'), ('deemed', 'VBN'), ('sufficient', 'JJ'), ('cause', 'NN'), ('for', 'IN'), ('removal', 'NN'), ('.', '.'), ('It', 'PRP'), ('shall', 'MD'), ('be', 'VB'), ('my', 'PRP$'), ('study', 'NN'), ('to', 'TO'), ('recommend', 'VB'), ('such', 'JJ'), ('constitutional', 'JJ'), ('measures', 'NNS'), ('to', 'TO'), ('Congress', 'NNP'), ('as', 'IN'), ('may', 'MD'), ('be', 'VB'), ('necessary', 'JJ'), ('and', 'CC'), ('proper', 'JJ'), ('to', 'TO'), ('secure', 'VB'), ('encouragement', 'NN'), ('and', 'CC'), ('protection', 'NN'), ('to', 'TO'), ('the', 'DT'), ('great', 'JJ'), ('interests', 'NNS'), ('of', 'IN'), ('agriculture', 'NN'), (',', ','), ('commerce', 'NN'), (',', ','), ('and', 'CC'), ('manufactures', 'NNS'), (',', ','), ('to', 'TO'), ('improve', 'VB'), ('our', 'PRP$'), ('rivers', 'NNS'), ('and', 'CC'), ('harbors', 'NNS'), (',', ','), ('to', 'TO'), ('provide', 'VB'), ('for', 'IN'), ('the', 'DT'), ('speedy', 'JJ'), ('extinguishment', 'NN'), ('of', 'IN'), ('the', 'DT'), ('public', 'JJ'), ('debt', 'NN'), (',', ','), ('to', 'TO'), ('enforce', 'VB'), ('a', 'DT'), ('strict', 'JJ'), ('accountability', 'NN'), ('on', 'IN'), ('the', 'DT'), ('part', 'NN'), ('of', 'IN'), ('all', 'DT'), ('officers', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('Government', 'NNP'), ('and', 'CC'), ('the', 'DT'), ('utmost', 'JJ'), ('economy', 'NN'), ('in', 'IN'), ('all', 'DT'), ('public', 'JJ'), ('expenditures', 'NNS'), (';', ':'), ('but', 'CC'), ('it', 'PRP'), ('is', 'VBZ'), ('for', 'IN'), ('the', 'DT'), ('wisdom', 'NN'), ('of', 'IN'), ('Congress', 'NNP'), ('itself', 'PRP'), (',', ','), ('in', 'IN'), ('which', 'WDT'), ('all', 'DT'), ('legislative', 'JJ'), ('powers', 'NNS'), ('are', 'VBP'), ('vested', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('Constitution', 'NNP'), (',', ','), ('to', 'TO'), ('regulate', 'VB'), ('these', 'DT'), ('and', 'CC'), ('other', 'JJ'), ('matters', 'NNS'), ('of', 'IN'), ('domestic', 'JJ'), ('policy', 'NN'), ('.', '.'), ('I', 'PRP'), ('shall', 'MD'), ('look', 'VB'), ('with', 'IN'), ('confidence', 'NN'), ('to', 'TO'), ('the', 'DT'), ('enlightened', 'JJ'), ('patriotism', 'NN'), ('of', 'IN'), ('that', 'DT'), ('body', 'NN'), ('to', 'TO'), ('adopt', 'VB'), ('such', 'JJ'), ('measures', 'NNS'), ('of', 'IN'), ('conciliation', 'NN'), ('as', 'IN'), ('may', 'MD'), ('harmonize', 'VB'), ('conflicting', 'VBG'), ('interests', 'NNS'), ('and', 'CC'), ('tend', 'VBP'), ('to', 'TO'), ('perpetuate', 'VB'), ('that', 'DT'), ('Union', 'NNP'), ('which', 'WDT'), ('should', 'MD'), ('be', 'VB'), ('the', 'DT'), ('paramount', 'NN'), ('object', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('hopes', 'NNS'), ('and', 'CC'), ('affections', 'NNS'), ('.', '.'), ('In', 'IN'), ('any', 'DT'), ('action', 'NN'), ('calculated', 'VBN'), ('to', 'TO'), ('promote', 'VB'), ('an', 'DT'), ('object', 'NN'), ('so', 'RB'), ('near', 'IN'), ('the', 'DT'), ('heart', 'NN'), ('of', 'IN'), ('everyone', 'NN'), ('who', 'WP'), ('truly', 'VBZ'), ('loves', 'VBZ'), ('his', 'PRP$'), ('country', 'NN'), ('I', 'PRP'), ('will', 'MD'), ('zealously', 'RB'), ('unite', 'VB'), ('with', 'IN'), ('the', 'DT'), ('coordinate', 'NN'), ('branches', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('Government', 'NNP'), ('.', '.'), ('In', 'IN'), ('conclusion', 'NN'), ('I', 'PRP'), ('congratulate', 'VBP'), ('you', 'PRP'), (',', ','), ('my', 'PRP$'), ('fellow-citizens', 'JJ'), (',', ','), ('upon', 'IN'), ('the', 'DT'), ('high', 'JJ'), ('state', 'NN'), ('of', 'IN'), ('prosperity', 'NN'), ('to', 'TO'), ('which', 'WDT'), ('the', 'DT'), ('goodness', 'NN'), ('of', 'IN'), ('Divine', 'NNP'), ('Providence', 'NNP'), ('has', 'VBZ'), ('conducted', 'VBN'), ('our', 'PRP$'), ('common', 'JJ'), ('country', 'NN'), ('.', '.'), ('Let', 'VB'), ('us', 'PRP'), ('invoke', 'VB'), ('a', 'DT'), ('continuance', 'NN'), ('of', 'IN'), ('the', 'DT'), ('same', 'JJ'), ('protecting', 'NN'), ('care', 'NN'), ('which', 'WDT'), ('has', 'VBZ'), ('led', 'VBN'), ('us', 'PRP'), ('from', 'IN'), ('small', 'JJ'), ('beginnings', 'NNS'), ('to', 'TO'), ('the', 'DT'), ('eminence', 'NN'), ('we', 'PRP'), ('this', 'DT'), ('day', 'NN'), ('occupy', 'VB'), (',', ','), ('and', 'CC'), ('let', 'VB'), ('us', 'PRP'), ('seek', 'VB'), ('to', 'TO'), ('deserve', 'VB'), ('that', 'DT'), ('continuance', 'NN'), ('by', 'IN'), ('prudence', 'NN'), ('and', 'CC'), ('moderation', 'NN'), ('in', 'IN'), ('our', 'PRP$'), ('councils', 'NNS'), (',', ','), ('by', 'IN'), ('well-directed', 'JJ'), ('attempts', 'NNS'), ('to', 'TO'), ('assuage', 'VB'), ('the', 'DT'), ('bitterness', 'NN'), ('which', 'WDT'), ('too', 'RB'), ('often', 'RB'), ('marks', 'VBZ'), ('unavoidable', 'JJ'), ('differences', 'NNS'), ('of', 'IN'), ('opinion', 'NN'), (',', ','), ('by', 'IN'), ('the', 'DT'), ('promulgation', 'NN'), ('and', 'CC'), ('practice', 'NN'), ('of', 'IN'), ('just', 'RB'), ('and', 'CC'), ('liberal', 'JJ'), ('principles', 'NNS'), (',', ','), ('and', 'CC'), ('by', 'IN'), ('an', 'DT'), ('enlarged', 'JJ'), ('patriotism', 'NN'), (',', ','), ('which', 'WDT'), ('shall', 'MD'), ('acknowledge', 'VB'), ('no', 'DT'), ('limits', 'NNS'), ('but', 'CC'), ('those', 'DT'), ('of', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), ('widespread', 'JJ'), ('Republic', 'NNP'), ('.', '.')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "kCoJwbouwaiV",
        "colab_type": "code",
        "outputId": "b8efc5ef-d303-4a06-a199-89c87e7a30a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        }
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "# getting rid of commonly used words\n",
        "filtered_sentence = [w for w in tokens if not w in stop_words]\n",
        "filtered_sentence = []\n",
        "\n",
        "for w in tokens:\n",
        "    if w not in stop_words:\n",
        "        filtered_sentence.append(w)\n",
        "        \n",
        "print(filtered_sentence)\n",
        "print(tokens)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-726dc2a9f7e2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'stopwords'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mstop_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# getting rid of commonly used words\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'stopwords' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "uuNHJe9kxriB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from nltk.stem import PorterStemmer\n",
        "ps = PorterStemmer()\n",
        "# Getting the base stem meaning of a word\n",
        "stemmed_words = [];\n",
        "for w in filtered_sentence:\n",
        "  stemmed_words.append(ps.stem(w))\n",
        "print(stemmed_words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0jqN6t049uap",
        "colab_type": "code",
        "outputId": "f84d1527-b717-4f36-cb03-9daf19f23d16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "word_freq_dict = {}\n",
        "for w in stemmed_words:\n",
        "  try:\n",
        "    word_freq_dict[w] += 1\n",
        "  except:\n",
        "    word_freq_dict[w] = 1\n",
        "    \n",
        "print(word_freq_dict)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'elect': 1, 'american': 2, 'peopl': 3, 'highest': 2, 'offic': 5, 'known': 1, 'law': 2, ',': 57, 'I': 18, 'appear': 1, 'take': 2, 'oath': 1, 'prescrib': 1, 'constitut': 6, 'complianc': 1, 'time-honor': 1, 'custom': 1, 'address': 1, 'assembl': 1, '.': 22, 'the': 3, 'confid': 3, 'respect': 1, 'shown': 1, 'countrymen': 1, 'call': 3, 'chief': 1, 'magistr': 1, 'republ': 2, 'hold': 1, 'high': 2, 'rank': 1, 'among': 1, 'nation': 3, 'earth': 1, 'inspir': 1, 'feel': 1, 'profound': 1, 'gratitud': 1, ';': 6, 'reflect': 1, 'accept': 1, 'partial': 1, 'bestow': 2, 'impos': 2, 'discharg': 2, 'arduou': 1, 'duti': 6, 'involv': 1, 'weightiest': 1, 'oblig': 1, 'consciou': 1, 'posit': 2, 'fill': 1, 'though': 1, 'suffici': 2, 'satisfi': 1, 'loftiest': 1, 'ambit': 1, 'surround': 1, 'fear': 1, 'respons': 1, 'happili': 1, 'howev': 1, 'perform': 2, 'new': 1, 'shall': 15, 'without': 1, 'abl': 1, 'cooper': 1, 'legisl': 2, 'judici': 2, 'branch': 2, 'govern': 8, 'present': 1, 'promin': 1, 'exampl': 3, 'distinguish': 1, 'civil': 3, 'attain': 1, 'matur': 1, 'experi': 1, 'endeavor': 1, 'assist': 1, 'execut': 5, 'depart': 1, 'individu': 1, 'whose': 1, 'talent': 1, 'integr': 1, 'puriti': 2, 'charact': 1, 'furnish': 1, 'ampl': 1, 'guaranti': 1, 'faith': 2, 'honor': 3, 'trust': 1, 'commit': 1, 'charg': 1, 'with': 1, 'aid': 1, 'honest': 1, 'purpos': 1, 'whatev': 1, 'right': 2, 'hope': 3, 'dilig': 1, 'imparti': 1, 'best': 2, 'interest': 6, 'countri': 6, 'manifold': 1, 'devolv': 1, 'upon': 2, 'In': 6, 'guid': 1, 'day': 3, 'swear': 1, '``': 2, 'preserv': 1, 'protect': 4, 'defend': 1, \"''\": 2, 'for': 1, 'interpret': 1, 'instrument': 1, 'look': 2, 'decis': 1, 'tribun': 1, 'establish': 1, 'author': 1, 'practic': 2, 'earlier': 1, 'presid': 3, 'larg': 1, 'share': 1, 'format': 1, 'To': 2, 'illustri': 1, 'patriot': 3, 'alway': 1, 'defer': 1, 'rever': 1, 'especi': 1, 'mani': 1, 'titl': 1, 'father': 1, 'command': 1, 'armi': 2, 'navi': 2, 'unit': 2, 'state': 3, 'advic': 1, 'consent': 1, 'senat': 1, 'make': 2, 'treati': 1, 'appoint': 2, 'ambassador': 1, 'give': 1, 'congress': 4, 'inform': 2, 'union': 2, 'recommend': 2, 'measur': 3, 'judg': 1, 'necessari': 2, 'care': 3, '--': 1, 'import': 1, 'function': 1, 'intrust': 1, 'may': 4, 'expect': 1, 'briefli': 1, 'indic': 1, 'principl': 2, 'control': 1, 'chosen': 1, 'bodi': 2, 'assur': 1, 'administr': 1, 'would': 1, 'devot': 1, 'welfar': 1, 'whole': 1, 'support': 1, 'particular': 1, 'section': 1, 'mere': 1, 'local': 1, 'renew': 1, 'declar': 1, 'heretofor': 1, 'made': 1, 'proclaim': 1, 'fix': 1, 'determin': 1, 'maintain': 1, 'extent': 1, 'abil': 1, 'origin': 1, 'adopt': 2, 'basi': 1, 'public': 3, 'polici': 2, 'great': 2, 'republican': 1, 'doctrin': 1, 'strength': 2, 'exist': 1, 'refer': 1, 'late': 1, 'employ': 1, 'much': 1, 'distinct': 1, 'activ': 1, 'servic': 1, 'taken': 1, 'insur': 1, 'condit': 1, 'effici': 1, 'further': 1, 'object': 3, 'militari': 1, 'naval': 1, 'school': 1, 'sustain': 1, 'liber': 2, 'receiv': 1, 'special': 1, 'attent': 1, 'As': 1, 'freemen': 1, 'sympath': 1, 'effort': 1, 'extend': 1, 'bless': 1, 'polit': 1, 'liberti': 1, 'time': 1, 'warn': 1, 'admonit': 1, 'histori': 1, 'voic': 1, 'belov': 1, 'washington': 1, 'abstain': 1, 'entangl': 1, 'allianc': 1, 'foreign': 2, 'disput': 1, 'conflict': 2, 'less': 1, 'remain': 1, 'strictli': 1, 'neutral': 1, 'geograph': 1, 'geniu': 1, 'institut': 1, 'advanc': 1, 'spirit': 1, 'dictat': 1, 'religion': 1, 'direct': 1, 'us': 4, 'cultiv': 1, 'peac': 1, 'friendli': 1, 'relat': 2, 'power': 3, 'It': 2, 'intern': 1, 'question': 1, 'aris': 1, 'resolv': 1, 'settl': 1, 'wise': 1, 'negoti': 1, 'emin': 2, 'becom': 1, 'like': 1, 'found': 1, 'moral': 1, 'intellig': 1, 'citizen': 1, 'upheld': 1, 'affect': 2, 'exhaust': 1, 'everi': 1, 'resort': 1, 'diplomaci': 1, 'appeal': 1, 'arm': 1, 'conduct': 2, 'conform': 1, 'view': 1, 'believ': 1, 'essenti': 1, 'true': 1, 'vest': 2, 'delic': 1, 'oner': 1, 'So': 1, 'far': 1, 'possibl': 1, 'honesti': 1, 'capac': 1, 'fidel': 1, 'indispens': 1, 'prerequisit': 1, 'absenc': 1, 'either': 1, 'qualiti': 1, 'deem': 1, 'caus': 1, 'remov': 1, 'studi': 1, 'proper': 1, 'secur': 1, 'encourag': 1, 'agricultur': 1, 'commerc': 1, 'manufactur': 1, 'improv': 1, 'river': 1, 'harbor': 1, 'provid': 2, 'speedi': 1, 'extinguish': 1, 'debt': 1, 'enforc': 1, 'strict': 1, 'account': 1, 'part': 1, 'utmost': 1, 'economi': 1, 'expenditur': 1, 'wisdom': 1, 'regul': 1, 'matter': 1, 'domest': 1, 'enlighten': 1, 'concili': 1, 'harmon': 1, 'tend': 1, 'perpetu': 1, 'paramount': 1, 'action': 1, 'calcul': 1, 'promot': 1, 'near': 1, 'heart': 1, 'everyon': 1, 'truli': 1, 'love': 1, 'zealous': 1, 'coordin': 1, 'conclus': 1, 'congratul': 1, 'fellow-citizen': 1, 'prosper': 1, 'good': 1, 'divin': 1, 'common': 1, 'let': 2, 'invok': 1, 'continu': 2, 'led': 1, 'small': 1, 'begin': 1, 'occupi': 1, 'seek': 1, 'deserv': 1, 'prudenc': 1, 'moder': 1, 'council': 1, 'well-direct': 1, 'attempt': 1, 'assuag': 1, 'bitter': 1, 'often': 1, 'mark': 1, 'unavoid': 1, 'differ': 1, 'opinion': 1, 'promulg': 1, 'enlarg': 1, 'acknowledg': 1, 'limit': 1, 'widespread': 1}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "outputId": "dbb7bb56-677d-498a-cd31-fd5b053faaf0",
        "id": "I-1Cl22qahA2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.tokenize import PunktSentenceTokenizer\n",
        "custom_sent_tokenizer = PunktSentenceTokenizer(johnOliverJoke)\n",
        "tokenized = custom_sent_tokenizer.tokenize(johnOliverJoke)\n",
        "def process_content(list):\n",
        "    try:\n",
        "        for i in stemmed_words:\n",
        "            words = nltk.word_tokenize(i)\n",
        "            tagged = nltk.pos_tag(words)\n",
        "            list.append(tagged[0])\n",
        "            # chunkGram = r\"\"\"Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}\"\"\"\n",
        "            # chunkParser = nltk.RegexpParser(chunkGram)\n",
        "            # chunked = chunkParser.parse(tagged)\n",
        "          \n",
        "    except Exception as e:\n",
        "        print(str(e))\n",
        "\n",
        "tagged_list = []        \n",
        "process_content(tagged_list)\n",
        "print(tagged_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('elect', 'NN'), ('american', 'JJ'), ('peopl', 'NN'), ('highest', 'JJS'), ('offic', 'NN'), ('known', 'VBN'), ('law', 'NN'), (',', ','), ('I', 'PRP'), ('appear', 'VB'), ('take', 'VB'), ('oath', 'NN'), ('prescrib', 'NN'), ('constitut', 'NN'), (',', ','), (',', ','), ('complianc', 'NN'), ('time-honor', 'NN'), ('custom', 'NN'), (',', ','), ('address', 'NN'), ('assembl', 'NN'), ('.', '.'), ('the', 'DT'), ('confid', 'NN'), ('respect', 'NN'), ('shown', 'VBN'), ('countrymen', 'NNS'), ('call', 'NN'), ('chief', 'NN'), ('magistr', 'NN'), ('republ', 'NN'), ('hold', 'NN'), ('high', 'JJ'), ('rank', 'NN'), ('among', 'IN'), ('nation', 'NN'), ('earth', 'NN'), ('inspir', 'NN'), ('feel', 'NN'), ('profound', 'NN'), ('gratitud', 'NN'), (';', ':'), ('I', 'PRP'), ('reflect', 'NN'), ('accept', 'NN'), ('offic', 'NN'), ('partial', 'JJ'), ('bestow', 'NN'), ('impos', 'NN'), ('discharg', 'NN'), ('arduou', 'NN'), ('duti', 'NN'), ('involv', 'NN'), ('weightiest', 'JJS'), ('oblig', 'NN'), (',', ','), ('I', 'PRP'), ('consciou', 'NN'), ('posit', 'NN'), ('I', 'PRP'), ('call', 'NN'), ('fill', 'NN'), (',', ','), ('though', 'IN'), ('suffici', 'NN'), ('satisfi', 'NN'), ('loftiest', 'JJS'), ('ambit', 'NN'), (',', ','), ('surround', 'NN'), ('fear', 'NN'), ('respons', 'NNS'), ('.', '.'), ('happili', 'NN'), (',', ','), ('howev', 'NN'), (',', ','), ('perform', 'NN'), ('new', 'JJ'), ('duti', 'NN'), ('I', 'PRP'), ('shall', 'MD'), ('without', 'IN'), ('abl', 'NN'), ('cooper', 'NN'), ('.', '.'), ('the', 'DT'), ('legisl', 'NN'), ('judici', 'NN'), ('branch', 'NN'), ('govern', 'NN'), ('present', 'NN'), ('promin', 'NN'), ('exampl', 'NN'), ('distinguish', 'NN'), ('civil', 'JJ'), ('attain', 'NN'), ('matur', 'NN'), ('experi', 'NN'), (',', ','), ('shall', 'MD'), ('endeavor', 'NN'), ('call', 'NN'), ('assist', 'NN'), ('execut', 'NN'), ('depart', 'NN'), ('individu', 'NN'), ('whose', 'WP$'), ('talent', 'NN'), (',', ','), ('integr', 'NN'), (',', ','), ('puriti', 'NN'), ('charact', 'NN'), ('furnish', 'JJ'), ('ampl', 'NN'), ('guaranti', 'NN'), ('faith', 'NN'), ('honor', 'NN'), ('perform', 'NN'), ('trust', 'NN'), ('commit', 'NN'), ('charg', 'NN'), ('.', '.'), ('with', 'IN'), ('aid', 'NN'), ('honest', 'NN'), ('purpos', 'NN'), ('whatev', 'NN'), ('right', 'NN'), (',', ','), ('I', 'PRP'), ('hope', 'NN'), ('execut', 'NN'), ('dilig', 'NN'), (',', ','), ('imparti', 'NN'), (',', ','), ('best', 'JJS'), ('interest', 'NN'), ('countri', 'NN'), ('manifold', 'NN'), ('duti', 'NN'), ('devolv', 'NN'), ('upon', 'IN'), ('.', '.'), ('In', 'IN'), ('discharg', 'NN'), ('duti', 'NN'), ('guid', 'NN'), ('constitut', 'NN'), (',', ','), ('I', 'PRP'), ('day', 'NN'), ('swear', 'NN'), ('``', '``'), ('preserv', 'NN'), (',', ','), ('protect', 'NN'), (',', ','), ('defend', 'NN'), ('.', '.'), (\"''\", \"''\"), ('for', 'IN'), ('interpret', 'NN'), ('instrument', 'NN'), ('I', 'PRP'), ('shall', 'MD'), ('look', 'NN'), ('decis', 'NN'), ('judici', 'NN'), ('tribun', 'NN'), ('establish', 'VB'), ('author', 'NN'), ('practic', 'JJ'), ('govern', 'NN'), ('earlier', 'RBR'), ('presid', 'NN'), (',', ','), ('larg', 'NN'), ('share', 'NN'), ('format', 'NN'), ('.', '.'), ('To', 'TO'), ('exampl', 'NN'), ('illustri', 'NN'), ('patriot', 'NN'), ('I', 'PRP'), ('shall', 'MD'), ('alway', 'RB'), ('defer', 'NN'), ('rever', 'NN'), (',', ','), ('especi', 'NN'), ('exampl', 'NN'), ('mani', 'NN'), ('titl', 'NN'), ('``', '``'), ('father', 'NN'), ('countri', 'NN'), ('.', '.'), (\"''\", \"''\"), ('To', 'TO'), ('command', 'NN'), ('armi', 'NN'), ('navi', 'NN'), ('unit', 'NN'), ('state', 'NN'), (';', ':'), ('advic', 'NN'), ('consent', 'NN'), ('senat', 'NN'), (',', ','), ('make', 'VB'), ('treati', 'NN'), ('appoint', 'NN'), ('ambassador', 'NN'), ('offic', 'NN'), (';', ':'), ('give', 'VB'), ('congress', 'NN'), ('inform', 'NN'), ('state', 'NN'), ('union', 'NN'), ('recommend', 'NN'), ('measur', 'NN'), ('shall', 'MD'), ('judg', 'NN'), ('necessari', 'NN'), (';', ':'), ('take', 'VB'), ('care', 'NN'), ('law', 'NN'), ('shall', 'MD'), ('faith', 'NN'), ('execut', 'NN'), ('--', ':'), ('import', 'NN'), ('function', 'NN'), ('intrust', 'NN'), ('presid', 'NN'), ('constitut', 'NN'), (',', ','), ('may', 'MD'), ('expect', 'VB'), ('I', 'PRP'), ('shall', 'MD'), ('briefli', 'NN'), ('indic', 'NN'), ('principl', 'NN'), ('control', 'NN'), ('execut', 'NN'), ('.', '.'), ('chosen', 'NN'), ('bodi', 'NN'), ('peopl', 'NN'), ('assur', 'NN'), ('administr', 'NN'), ('would', 'MD'), ('devot', 'NN'), ('welfar', 'NN'), ('whole', 'JJ'), ('countri', 'NN'), (',', ','), ('support', 'NN'), ('particular', 'JJ'), ('section', 'NN'), ('mere', 'RB'), ('local', 'JJ'), ('interest', 'NN'), (',', ','), ('I', 'PRP'), ('day', 'NN'), ('renew', 'NN'), ('declar', 'NN'), ('I', 'PRP'), ('heretofor', 'NN'), ('made', 'VBN'), ('proclaim', 'NN'), ('fix', 'NN'), ('determin', 'NN'), ('maintain', 'NN'), ('extent', 'NN'), ('abil', 'NN'), ('govern', 'NN'), ('origin', 'NN'), ('puriti', 'NN'), ('adopt', 'NN'), ('basi', 'NN'), ('public', 'NN'), ('polici', 'NN'), ('great', 'JJ'), ('republican', 'JJ'), ('doctrin', 'NN'), ('constitut', 'NN'), ('strength', 'NN'), ('nation', 'NN'), ('exist', 'NN'), ('.', '.'), ('In', 'IN'), ('refer', 'NN'), ('armi', 'NN'), ('navi', 'NN'), (',', ','), ('late', 'RB'), ('employ', 'NN'), ('much', 'JJ'), ('distinct', 'NN'), ('activ', 'NN'), ('servic', 'NN'), (',', ','), ('care', 'NN'), ('shall', 'MD'), ('taken', 'VBN'), ('insur', 'NN'), ('highest', 'JJS'), ('condit', 'NN'), ('effici', 'NN'), (',', ','), ('further', 'RB'), ('object', 'NN'), ('militari', 'NN'), ('naval', 'NN'), ('school', 'NN'), (',', ','), ('sustain', 'NN'), ('liber', 'NN'), ('congress', 'NN'), (',', ','), ('shall', 'MD'), ('receiv', 'NN'), ('special', 'JJ'), ('attent', 'NN'), ('execut', 'NN'), ('.', '.'), ('As', 'IN'), ('american', 'JJ'), ('freemen', 'NNS'), ('sympath', 'NN'), ('effort', 'NN'), ('extend', 'NN'), ('bless', 'NN'), ('civil', 'JJ'), ('polit', 'NN'), ('liberti', 'NN'), (',', ','), ('time', 'NN'), ('warn', 'NN'), ('admonit', 'NN'), ('histori', 'NN'), ('voic', 'NN'), ('belov', 'NN'), ('washington', 'NN'), ('abstain', 'NN'), ('entangl', 'NN'), ('allianc', 'NN'), ('foreign', 'JJ'), ('nation', 'NN'), ('.', '.'), ('In', 'IN'), ('disput', 'NN'), ('conflict', 'NN'), ('govern', 'NN'), ('interest', 'NN'), ('less', 'RBR'), ('duti', 'NN'), ('remain', 'NN'), ('strictli', 'NN'), ('neutral', 'JJ'), (',', ','), ('geograph', 'NN'), ('posit', 'NN'), (',', ','), ('geniu', 'NN'), ('institut', 'NN'), ('peopl', 'NN'), (',', ','), ('advanc', 'NN'), ('spirit', 'NN'), ('civil', 'JJ'), (',', ','), (',', ','), (',', ','), ('dictat', 'NN'), ('religion', 'NN'), ('direct', 'JJ'), ('us', 'PRP'), ('cultiv', 'NN'), ('peac', 'NN'), ('friendli', 'NN'), ('relat', 'NN'), ('power', 'NN'), ('.', '.'), ('It', 'PRP'), ('hope', 'NN'), ('intern', 'NN'), ('question', 'NN'), ('aris', 'NN'), ('govern', 'NN'), ('confid', 'NN'), ('strength', 'NN'), ('resolv', 'NN'), ('protect', 'NN'), ('right', 'NN'), ('may', 'MD'), ('settl', 'NN'), ('wise', 'NN'), ('negoti', 'NN'), (';', ':'), ('emin', 'NN'), ('becom', 'NN'), ('govern', 'NN'), ('like', 'IN'), (',', ','), ('found', 'NN'), ('moral', 'JJ'), ('intellig', 'NN'), ('citizen', 'NN'), ('upheld', 'NN'), ('affect', 'NN'), (',', ','), ('exhaust', 'NN'), ('everi', 'NN'), ('resort', 'NN'), ('honor', 'NN'), ('diplomaci', 'NN'), ('appeal', 'NN'), ('arm', 'NN'), ('.', '.'), ('In', 'IN'), ('conduct', 'NN'), ('foreign', 'JJ'), ('relat', 'NN'), ('I', 'PRP'), ('shall', 'MD'), ('conform', 'NN'), ('view', 'NN'), (',', ','), ('I', 'PRP'), ('believ', 'NN'), ('essenti', 'NN'), ('best', 'JJS'), ('interest', 'NN'), ('true', 'JJ'), ('honor', 'NN'), ('countri', 'NN'), ('.', '.'), ('the', 'DT'), ('appoint', 'NN'), ('power', 'NN'), ('vest', 'NN'), ('presid', 'NN'), ('impos', 'NN'), ('delic', 'NN'), ('oner', 'NN'), ('duti', 'NN'), ('.', '.'), ('So', 'RB'), ('far', 'RB'), ('possibl', 'NN'), ('inform', 'NN'), (',', ','), ('I', 'PRP'), ('shall', 'MD'), ('make', 'VB'), ('honesti', 'NN'), (',', ','), ('capac', 'NN'), (',', ','), ('fidel', 'NN'), ('indispens', 'NNS'), ('prerequisit', 'NN'), ('bestow', 'NN'), ('offic', 'NN'), (',', ','), ('absenc', 'NN'), ('either', 'DT'), ('qualiti', 'NN'), ('shall', 'MD'), ('deem', 'NN'), ('suffici', 'NN'), ('caus', 'NN'), ('remov', 'NN'), ('.', '.'), ('It', 'PRP'), ('shall', 'MD'), ('studi', 'NN'), ('recommend', 'NN'), ('constitut', 'NN'), ('measur', 'NN'), ('congress', 'NN'), ('may', 'MD'), ('necessari', 'NN'), ('proper', 'NN'), ('secur', 'NN'), ('encourag', 'NN'), ('protect', 'NN'), ('great', 'JJ'), ('interest', 'NN'), ('agricultur', 'NN'), (',', ','), ('commerc', 'NN'), (',', ','), ('manufactur', 'NN'), (',', ','), ('improv', 'NN'), ('river', 'NN'), ('harbor', 'NN'), (',', ','), ('provid', 'NN'), ('speedi', 'NN'), ('extinguish', 'JJ'), ('public', 'NN'), ('debt', 'NN'), (',', ','), ('enforc', 'NN'), ('strict', 'NN'), ('account', 'NN'), ('part', 'NN'), ('offic', 'NN'), ('govern', 'NN'), ('utmost', 'NN'), ('economi', 'NN'), ('public', 'NN'), ('expenditur', 'NN'), (';', ':'), ('wisdom', 'NN'), ('congress', 'NN'), (',', ','), ('legisl', 'NN'), ('power', 'NN'), ('vest', 'NN'), ('constitut', 'NN'), (',', ','), ('regul', 'NN'), ('matter', 'NN'), ('domest', 'NN'), ('polici', 'NN'), ('.', '.'), ('I', 'PRP'), ('shall', 'MD'), ('look', 'NN'), ('confid', 'NN'), ('enlighten', 'VB'), ('patriot', 'NN'), ('bodi', 'NN'), ('adopt', 'NN'), ('measur', 'NN'), ('concili', 'NN'), ('may', 'MD'), ('harmon', 'NN'), ('conflict', 'NN'), ('interest', 'NN'), ('tend', 'NN'), ('perpetu', 'NN'), ('union', 'NN'), ('paramount', 'NN'), ('object', 'NN'), ('hope', 'NN'), ('affect', 'NN'), ('.', '.'), ('In', 'IN'), ('action', 'NN'), ('calcul', 'NN'), ('promot', 'NN'), ('object', 'NN'), ('near', 'IN'), ('heart', 'NN'), ('everyon', 'NN'), ('truli', 'NN'), ('love', 'NN'), ('countri', 'NN'), ('I', 'PRP'), ('zealous', 'JJ'), ('unit', 'NN'), ('coordin', 'NN'), ('branch', 'NN'), ('govern', 'NN'), ('.', '.'), ('In', 'IN'), ('conclus', 'NN'), ('I', 'PRP'), ('congratul', 'NN'), (',', ','), ('fellow-citizen', 'JJ'), (',', ','), ('upon', 'IN'), ('high', 'JJ'), ('state', 'NN'), ('prosper', 'NN'), ('good', 'JJ'), ('divin', 'NN'), ('provid', 'NN'), ('conduct', 'NN'), ('common', 'JJ'), ('countri', 'NN'), ('.', '.'), ('let', 'VB'), ('us', 'PRP'), ('invok', 'NN'), ('continu', 'NN'), ('protect', 'NN'), ('care', 'NN'), ('led', 'VBN'), ('us', 'PRP'), ('small', 'JJ'), ('begin', 'NN'), ('emin', 'NN'), ('day', 'NN'), ('occupi', 'NN'), (',', ','), ('let', 'VB'), ('us', 'PRP'), ('seek', 'NN'), ('deserv', 'NN'), ('continu', 'NN'), ('prudenc', 'NN'), ('moder', 'NN'), ('council', 'NN'), (',', ','), ('well-direct', 'NN'), ('attempt', 'NN'), ('assuag', 'NN'), ('bitter', 'NN'), ('often', 'RB'), ('mark', 'NN'), ('unavoid', 'JJ'), ('differ', 'NN'), ('opinion', 'NN'), (',', ','), ('promulg', 'NN'), ('practic', 'JJ'), ('liber', 'NN'), ('principl', 'NN'), (',', ','), ('enlarg', 'NN'), ('patriot', 'NN'), (',', ','), ('shall', 'MD'), ('acknowledg', 'NN'), ('limit', 'NN'), ('widespread', 'JJ'), ('republ', 'NN'), ('.', '.')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "KUnQ156m-www",
        "colab_type": "code",
        "outputId": "8a979d3b-6e33-4e00-9680-933df55a6867",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "noun_freq_dict = {}\n",
        "for word_type_pair in tagged_list:\n",
        "  word_type = word_type_pair[1]\n",
        "  if (word_type == 'NN'):\n",
        "    w = word_type_pair[0]\n",
        "    try:\n",
        "      noun_freq_dict[w] += 1\n",
        "    except:\n",
        "      noun_freq_dict[w] = 1\n",
        "    \n",
        "print(noun_freq_dict)\n",
        "\n",
        "# Problems: when identifying type, some words that have multiple meanings and usages are counted, even though in context it's not a noun (ie. \"saw\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'elect': 1, 'peopl': 3, 'offic': 5, 'law': 2, 'oath': 1, 'prescrib': 1, 'constitut': 6, 'complianc': 1, 'time-honor': 1, 'custom': 1, 'address': 1, 'assembl': 1, 'confid': 3, 'respect': 1, 'call': 3, 'chief': 1, 'magistr': 1, 'republ': 2, 'hold': 1, 'rank': 1, 'nation': 3, 'earth': 1, 'inspir': 1, 'feel': 1, 'profound': 1, 'gratitud': 1, 'reflect': 1, 'accept': 1, 'bestow': 2, 'impos': 2, 'discharg': 2, 'arduou': 1, 'duti': 6, 'involv': 1, 'oblig': 1, 'consciou': 1, 'posit': 2, 'fill': 1, 'suffici': 2, 'satisfi': 1, 'ambit': 1, 'surround': 1, 'fear': 1, 'happili': 1, 'howev': 1, 'perform': 2, 'abl': 1, 'cooper': 1, 'legisl': 2, 'judici': 2, 'branch': 2, 'govern': 8, 'present': 1, 'promin': 1, 'exampl': 3, 'distinguish': 1, 'attain': 1, 'matur': 1, 'experi': 1, 'endeavor': 1, 'assist': 1, 'execut': 5, 'depart': 1, 'individu': 1, 'talent': 1, 'integr': 1, 'puriti': 2, 'charact': 1, 'ampl': 1, 'guaranti': 1, 'faith': 2, 'honor': 3, 'trust': 1, 'commit': 1, 'charg': 1, 'aid': 1, 'honest': 1, 'purpos': 1, 'whatev': 1, 'right': 2, 'hope': 3, 'dilig': 1, 'imparti': 1, 'interest': 6, 'countri': 6, 'manifold': 1, 'devolv': 1, 'guid': 1, 'day': 3, 'swear': 1, 'preserv': 1, 'protect': 4, 'defend': 1, 'interpret': 1, 'instrument': 1, 'look': 2, 'decis': 1, 'tribun': 1, 'author': 1, 'presid': 3, 'larg': 1, 'share': 1, 'format': 1, 'illustri': 1, 'patriot': 3, 'defer': 1, 'rever': 1, 'especi': 1, 'mani': 1, 'titl': 1, 'father': 1, 'command': 1, 'armi': 2, 'navi': 2, 'unit': 2, 'state': 3, 'advic': 1, 'consent': 1, 'senat': 1, 'treati': 1, 'appoint': 2, 'ambassador': 1, 'congress': 4, 'inform': 2, 'union': 2, 'recommend': 2, 'measur': 3, 'judg': 1, 'necessari': 2, 'care': 3, 'import': 1, 'function': 1, 'intrust': 1, 'briefli': 1, 'indic': 1, 'principl': 2, 'control': 1, 'chosen': 1, 'bodi': 2, 'assur': 1, 'administr': 1, 'devot': 1, 'welfar': 1, 'support': 1, 'section': 1, 'renew': 1, 'declar': 1, 'heretofor': 1, 'proclaim': 1, 'fix': 1, 'determin': 1, 'maintain': 1, 'extent': 1, 'abil': 1, 'origin': 1, 'adopt': 2, 'basi': 1, 'public': 3, 'polici': 2, 'doctrin': 1, 'strength': 2, 'exist': 1, 'refer': 1, 'employ': 1, 'distinct': 1, 'activ': 1, 'servic': 1, 'insur': 1, 'condit': 1, 'effici': 1, 'object': 3, 'militari': 1, 'naval': 1, 'school': 1, 'sustain': 1, 'liber': 2, 'receiv': 1, 'attent': 1, 'sympath': 1, 'effort': 1, 'extend': 1, 'bless': 1, 'polit': 1, 'liberti': 1, 'time': 1, 'warn': 1, 'admonit': 1, 'histori': 1, 'voic': 1, 'belov': 1, 'washington': 1, 'abstain': 1, 'entangl': 1, 'allianc': 1, 'disput': 1, 'conflict': 2, 'remain': 1, 'strictli': 1, 'geograph': 1, 'geniu': 1, 'institut': 1, 'advanc': 1, 'spirit': 1, 'dictat': 1, 'religion': 1, 'cultiv': 1, 'peac': 1, 'friendli': 1, 'relat': 2, 'power': 3, 'intern': 1, 'question': 1, 'aris': 1, 'resolv': 1, 'settl': 1, 'wise': 1, 'negoti': 1, 'emin': 2, 'becom': 1, 'found': 1, 'intellig': 1, 'citizen': 1, 'upheld': 1, 'affect': 2, 'exhaust': 1, 'everi': 1, 'resort': 1, 'diplomaci': 1, 'appeal': 1, 'arm': 1, 'conduct': 2, 'conform': 1, 'view': 1, 'believ': 1, 'essenti': 1, 'vest': 2, 'delic': 1, 'oner': 1, 'possibl': 1, 'honesti': 1, 'capac': 1, 'fidel': 1, 'prerequisit': 1, 'absenc': 1, 'qualiti': 1, 'deem': 1, 'caus': 1, 'remov': 1, 'studi': 1, 'proper': 1, 'secur': 1, 'encourag': 1, 'agricultur': 1, 'commerc': 1, 'manufactur': 1, 'improv': 1, 'river': 1, 'harbor': 1, 'provid': 2, 'speedi': 1, 'debt': 1, 'enforc': 1, 'strict': 1, 'account': 1, 'part': 1, 'utmost': 1, 'economi': 1, 'expenditur': 1, 'wisdom': 1, 'regul': 1, 'matter': 1, 'domest': 1, 'concili': 1, 'harmon': 1, 'tend': 1, 'perpetu': 1, 'paramount': 1, 'action': 1, 'calcul': 1, 'promot': 1, 'heart': 1, 'everyon': 1, 'truli': 1, 'love': 1, 'coordin': 1, 'conclus': 1, 'congratul': 1, 'prosper': 1, 'divin': 1, 'invok': 1, 'continu': 2, 'begin': 1, 'occupi': 1, 'seek': 1, 'deserv': 1, 'prudenc': 1, 'moder': 1, 'council': 1, 'well-direct': 1, 'attempt': 1, 'assuag': 1, 'bitter': 1, 'mark': 1, 'differ': 1, 'opinion': 1, 'promulg': 1, 'enlarg': 1, 'acknowledg': 1, 'limit': 1}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "lXbFzKl6c8jl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Comedian/Joke dictionary Parsing Proof of concept"
      ]
    },
    {
      "metadata": {
        "id": "AeYuU16qdAfc",
        "colab_type": "code",
        "outputId": "eb033493-d53a-41c5-a11a-2712abd4f249",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "# Porting data from CSV file\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "csvFilePath = '/content/gdrive/My Drive/Georgia Tech/Humor Genome Project/ML/ML - categorization/CSV Files/Final CSV/'\n",
        "testpath = '/content/gdrive/My Drive/HumorGenome_FindTheComedian - Sheet1.csv'\n",
        "# reading files from drive: test txt file\n",
        "# Auth: 4/mwCs1plTJyv1OoMi-iYDcH0K7DahhCpolnE8rI_sHhBBeQEKzHrb7-4\n",
        "finalJokesText = \"\"\n",
        "with open(testpath, 'r') as f:\n",
        "  data = f.readlines()\n",
        "  \n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "5rRCbvcCbYKR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Dictionary of comedian and his/her jokes\n",
        "comedianDict = {}\n",
        "# Testing\n",
        "# count = 0\n",
        "for line in data:\n",
        "  try:\n",
        "    comedian, charLen, wordLen, joke = line.split(\",\", 3)\n",
        "    jokeData = (charLen, wordLen, joke)\n",
        "    try:\n",
        "      comedianDict[comedian].append(jokeData)\n",
        "    except:\n",
        "      comedianDict[comedian] = [jokeData]\n",
        "  except:\n",
        "    print(line)\n",
        "    # Weird formatting issues with input jokes for Michelle Wolf\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5MDJZVzYAu70",
        "colab_type": "code",
        "outputId": "4fbe9d88-c2e5-45ee-91da-56dc89594bae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "stop_words = set(stopwords.words('english'))\n",
        "from nltk.tokenize import PunktSentenceTokenizer\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "\n",
        "\n",
        "# bag of words\n",
        "# Get all the words\n",
        "corpus = set()\n",
        "comedian_dict_nouns = {}\n",
        "for key in comedianDict:\n",
        "  joke_nouns = []\n",
        "  for jokeData in comedianDict[key]:\n",
        "    joke = jokeData[2]\n",
        "    # function to parse out nouns\n",
        "    words = set(getNouns(joke))\n",
        "    joke_nouns.append(set(words))\n",
        "    corpus = corpus.union(words)\n",
        "  comedian_dict_nouns[key] = joke_nouns\n",
        "    \n",
        "corpus_vector = list(corpus)\n",
        "print(comedian_dict_nouns)\n",
        "\n",
        "def getNouns(sentence):\n",
        "  tokens = nltk.word_tokenize(sentence)\n",
        "  filtered = [w for w in tokens if not w in stop_words]\n",
        "  tagged = nltk.pos_tag(filtered)\n",
        "  nouns = [w[0] for w in tagged if w[1]==\"NN\"]\n",
        "  return nouns\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "{'Comedian': [{'Joke'}], 'Trevor Noah': [{'anger', 'life'}, {'race', 'footrace', 'campaign', 'question'}, {'weekend'}, {'versus', 'tweeter', 'tan', 'way', 'leader'}, {'credit', 'someone', 'gon'}, {'movie', 'plane', 'gon', 'way'}, {'day', 'night', 'ebony'}, {'person'}, {'criticism', 'gon', 'person', 'lot'}, {'president'}, {'beep', 'figure', 'man', 'lie', 'night', 'vote', 'charge'}, {'Yo', 'dope', 'track', 'yo', 'life', 'anomynish'}, {'point'}, {'language', 'guy'}, {'disaster', 'presidency', 'salad', 'citizenship', 'glad', 'drive-through', 'nobody', 'guy'}], 'Jimmy Kimmel': [set(), set(), {'MEETING'}, set(), set(), {'TV', 'THIS'}, {'GUY'}, set(), set(), set(), {'YOU'}], 'Michelle Wolf': [{'magician', 'feel'}, {'exciting'}, {'home', 'kind'}, {'bathroom', 'her.Like', 'bitch', 'party', 'choice', 'talk', 'room', 'life'}, {'fun', 'hang', 'word', 'someone', 'lady'}, {'brunch', 'fun', 'hang', 'page', 'president', 'woman', 'lady'}, {'fun'}, {'hiding'}, {'caring', 'Men'}, {'love', 'Men'}, {'meeting', 'fact', 'anyone', 'Men'}, {'furniture', 'whoopsie.Like'}, {'call'}, {'bone', 'Everything'}, set(), {'boner', 'sad'}, {'wedding', 'secure', 'man', 'marry', 'princess', 'alliance', 'wedding.We', 'woman', 'guess', 'make', 'adult', 'day', 'you.', 'order'}, {'wedding', 'day', 'princess', 'boy'}, {'problem', 'lot', 'date'}, {'president', 'reason'}, {'bit', 'podium', 'time'}], ' Balls': [{'pay', 'thank', 'thing', 'lot', 'money', 'way'}], 'you have to be like': [{'buddy'}], 'is the one who sometimes gives tours of the White House': [set()], 'Jimmy Fallon': [{'get', 'news', '-Let'}, {'meeting', 'week', 'seat', 'fill'}, {']', 'oh', 'trouble', 'music', 'thinking', 'applause', '%', 'jumping'}, {'wedding', 'applause', 'man', 'night', 'photo', 'side', 'aisle'}, {'thank', 'reason', 'moment', '-Justice', '['}, {'trouble', 'weekend', 'time', 'word', 'fern', 'phone', 'plane', 'rally'}, {'Talk', 'time', 'works.', 'play', 'right'}, {'play', 'mispronounce', 'prize', 'guess', 'right', 'hand', 'Talk', 'word', 'anyone'}, {'pair', '[', 'look', ']'}, {'mask', '[', 'lot', ']', 'owl', 'stuff', 'hockey'}, {'time', 'applause', 'election', 'vote', 'www.vote.gov.And', 'tip', 'okay'}, {'applause', 'time', '[', 'job'}, {'dude', 'laughter', 'somebody', 'hell', ']'}]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "DE9mw1gTfGD8",
        "colab_type": "code",
        "outputId": "78f7f91a-2b8d-46ca-abb1-9f512b6fd181",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "cell_type": "code",
      "source": [
        "# New dictionary of vectors: {'comedian', [[0,1,0,...], [1,1,0,...], ...]}\n",
        "def getWordVector(word_set, corpus_vector):\n",
        "  vector = []\n",
        "  for w in corpus_vector:\n",
        "    if w in word_set:\n",
        "      vector.append(1)\n",
        "    else:\n",
        "      vector.append(0)\n",
        "  return vector\n",
        "    \n",
        "\n",
        "comedian_dict_vector = {}\n",
        "for key in comedian_dict_nouns:\n",
        "  joke_vectors = []\n",
        "  for joke_nouns in comedian_dict_nouns[key]:\n",
        "    vector = getWordVector(joke_nouns, corpus_vector)\n",
        "    joke_vectors.append(vector)\n",
        "  comedian_dict_vector[key] = vectors\n",
        "  \n",
        "  \n",
        "    \n",
        "     "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-29303418e777>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mcomedian_dict_vector\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcomedian_dict_nouns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m   \u001b[0mjoke_vectors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mjoke_nouns\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcomedian_dict_nouns\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'comedian_dict_nouns' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "5bqrtxRmPd3j",
        "colab_type": "code",
        "outputId": "d2e3c6a9-6850-483f-861d-b428ba682207",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 232
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "V = np.array(corpus_vector[0])\n",
        "origin = [0], [0] # origin point\n",
        "\n",
        "plt.quiver(*origin, V[:,0], V[:,1], color=['r','b','g'], scale=21)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-b6644299fcb1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mV\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus_vector\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0morigin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# origin point\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'corpus_vector' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "8Aa3k-Wfk9d4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Next: Re-write so the vector includes frequency? hard to scale with just nouns -> compile all the jokes?"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "g83CWiTOsv-o",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# General Joke Processing\n",
        "(1) Import file \n",
        "(2) clean data \n",
        "(3) extract data of interest (in this case, nouns) \n",
        "(4) create vector "
      ]
    },
    {
      "metadata": {
        "id": "KXEb4d9bs1zp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# 1)\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "\n",
        "# 2)\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "stop_words = set(stopwords.words('english'))\n",
        "import string\n",
        "\n",
        "csvFilePath = '/content/gdrive/My Drive/Georgia Tech/Humor Genome Project/ML/ML - categorization/CSV Files/Final CSV/'\n",
        "\n",
        "# 0) Import file\n",
        "def load_csv(csvFilePath):\n",
        "  with open(csvFilePath, 'r') as f:\n",
        "    data = f.readlines()\n",
        "    return data\n",
        "  \n",
        "# 1) clean joke\n",
        "def clean_doc(text):\n",
        "  # split into tokens by white space\n",
        "\ttokens = doc.split()\n",
        "\t# remove punctuation from each token\n",
        "\ttable = str.maketrans('', '', string.punctuation)\n",
        "\ttokens = [w.translate(table) for w in tokens]\n",
        "\t# remove remaining tokens that are not alphabetic\n",
        "\ttokens = [word for word in tokens if word.isalpha()]\n",
        "\t# filter out stop words\n",
        "\tstop_words = set(stopwords.words('english'))\n",
        "\ttokens = [w for w in tokens if not w in stop_words]\n",
        "\t# filter out short tokens\n",
        "\ttokens = [word for word in tokens if len(word) > 1]\n",
        "\treturn tokens\n",
        "\n",
        "# 2) extract data from joke\n",
        "def extract(tokens, method):\n",
        "  # Add more options\n",
        "  if (method == \"extract_nouns\"):\n",
        "    return getNouns(text)\n",
        "  else:\n",
        "    return tokens\n",
        "  \n",
        "# 2a) get the nouns and frequency\n",
        "def getNouns(tokens):\n",
        "  tagged = nltk.pos_tag(tokens)\n",
        "  nouns = {}\n",
        "  for w in tagged:\n",
        "    if w[1] = \"NN\":\n",
        "      try:\n",
        "        nouns[w[0]] += 1\n",
        "      except:\n",
        "        nouns[w[0]] = 1\n",
        "  return nouns\n",
        "  \n",
        "# 3) create vectors\n",
        "def create_vector(tokens_freq_dict, corpus):\n",
        "  word_set = set(tokens)\n",
        "  vector = []\n",
        "  for w in corpus:\n",
        "    try:\n",
        "      vector.append(tokens_freq_dict[w])\n",
        "    except:\n",
        "      vector.append(0)\n",
        "  return vector"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}