12 Spark NLP & ML for Text Classification.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### Use Spark NLP for Text Classification\n",
        "\n",
        "* Spark NLP is an open-source text processing library for advanced natural language processing for the Python, Java and Scala programming languages.\n",
        "\n",
        "* The library is built on top of Apache Spark and its Spark ML library.\n",
        "\n",
        "* It is currently the most widely used NLP library in the enterprise.\n",
        "\n",
        "* In this tutorial, we will learn how to use Spark NLP for text preprocessing and build machine learning models to classify text into different categories."
      ],
      "metadata": {
        "id": "eXO0IzY8dL0k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Install pyspark\n",
        "! pip install pyspark --quiet\n",
        "\n",
        "# Install Spark NLP\n",
        "! pip install spark-nlp --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "bcqBoeWgPYql",
        "outputId": "66cd9f98-82e5-4eb1-c528-ed23f778a201"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import modules\n",
        "import os\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from pyspark.sql.types import *\n",
        "from pyspark.sql.functions import expr\n",
        "from pyspark.sql import functions as fun\n",
        "from pyspark.sql import Row\n",
        "from pyspark.ml import Pipeline\n",
        "from pyspark.ml.feature import *\n",
        "import sparknlp\n",
        "from sparknlp import DocumentAssembler, Finisher\n",
        "from sparknlp.annotator import *\n",
        "from pyspark.ml.classification import *\n",
        "from pyspark.ml.tuning import *\n",
        "from pyspark.ml.evaluation import *"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "OaI49cH8PqZ4",
        "outputId": "fab687a8-a47a-4e7c-9483-f2316bf8bce0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-E93pX6m2391",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "1a11dcc1-b754-417d-d071-5d2605f5e3e4"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "# Wrap strings when printing in Colab\n",
        "from IPython.display import HTML, display\n",
        "def set_css():\n",
        "  display(HTML('''\n",
        "  <style>\n",
        "    pre {\n",
        "        white-space: pre-wrap;\n",
        "    }\n",
        "  </style>\n",
        "  '''))\n",
        "get_ipython().events.register('pre_run_cell', set_css)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "ZFd_vOA4dLmC",
        "outputId": "a266b459-2d6b-4c62-b7ff-6b51b5142ae3"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "# Mount my google drive to colab\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kPag4KezvY6m"
      },
      "source": [
        "## BBC News Summary Dataset\n",
        "\n",
        "Sources: https://www.kaggle.com/datasets/pariza/bbc-news-summary; http://mlg.ucd.ie/datasets/bbc.html\n",
        "\n",
        "\\\n",
        "The dataset consists of 2225 documents from the BBC news website corresponding to stories in five topical areas from 2004-2005. There are five class Labels: business, entertainment, politics, sport, and tech.\n",
        "\n",
        "\\\n",
        "Dataset structure:\n",
        "\n",
        "\\\n",
        "* News Articles\n",
        "  * business (txt files, e.g., 001.txt)\n",
        "  * entertainment (txt files)\n",
        "  * politics (txt files)\n",
        "  * sport (txt files)\n",
        "  * tech (txt files)\n",
        "\n",
        "\\\n",
        "* Summaries\n",
        "  * business (txt files, e.g., 001.txt)\n",
        "  * entertainment (txt files)\n",
        "  * politics (txt files)\n",
        "  * sport (txt files)\n",
        "  * tech (txt files)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Start the Spark NLP session\n",
        "spark = sparknlp.start()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "RQ2S8EafQqrG",
        "outputId": "e6f8aa52-6578-42ab-b42a-ce3ad476f69f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# In Spark NLP, wholeTextFile() will return an error when it finds a nested folder. \n",
        "# Therefore, we need to create a file path list by traversing all nested folders \n",
        "# and pass all file names with comma separator in order to create a single RDD.\n",
        "root = '/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries'\n",
        "lst_filenames = [os.path.join(path, name) for path, subdirs, files \n",
        "                 in os.walk(root) for name in files]\n",
        "print(len(lst_filenames))\n",
        "lst_filenames[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "id": "GvQi3W-IW3eO",
        "outputId": "c7082661-e7f2-4c93-fd81-23c4b9e8aebd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2225\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries/business/001.txt',\n",
              " '/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries/business/003.txt',\n",
              " '/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries/business/029.txt',\n",
              " '/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries/business/021.txt',\n",
              " '/content/drive/My Drive/YouTube Channel/Datasets/BBC News Summary/Summaries/business/014.txt']"
            ]
          },
          "metadata": {},
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Use Spark NLP to import all txt files in the list (filenames seperated by comma)\n",
        "texts = spark.sparkContext.wholeTextFiles(','.join(lst_filenames))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Q7sbUWjQQdan",
        "outputId": "a19e8da1-b4fe-457c-ccba-1ec4aa9bbd6e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a schema for the imported files\n",
        "schema = StructType([\n",
        "  StructField('filename', StringType()),\n",
        "  StructField('text', StringType())\n",
        "])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "xUwHqIO5RFh2",
        "outputId": "5cd00e23-b7b1-4190-c6b5-0dcb55688a1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a Spark NLP dataframe\n",
        "texts = spark.createDataFrame(texts, schema = schema)\n",
        "texts.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 468
        },
        "id": "Q6BbZZeWSW4V",
        "outputId": "05727018-2a46-45cf-8cb7-3e7191d582df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+--------------------+\n",
            "|            filename|                text|\n",
            "+--------------------+--------------------+\n",
            "|file:/content/dri...|TimeWarner said f...|\n",
            "|file:/content/dri...|Yukos' owner Mena...|\n",
            "|file:/content/dri...|The Vestey Group ...|\n",
            "|file:/content/dri...|Leisure group Ran...|\n",
            "|file:/content/dri...|\"Journalists are ...|\n",
            "|file:/content/dri...|The breakneck pac...|\n",
            "|file:/content/dri...|India's rupee has...|\n",
            "|file:/content/dri...|In 2003, crop pro...|\n",
            "|file:/content/dri...|But officials sai...|\n",
            "|file:/content/dri...|Crude oil prices ...|\n",
            "|file:/content/dri...|On Tuesday, the c...|\n",
            "|file:/content/dri...|The dollar has hi...|\n",
            "|file:/content/dri...|The court ruling ...|\n",
            "|file:/content/dri...|Rod Eddington, BA...|\n",
            "|file:/content/dri...|A US government c...|\n",
            "|file:/content/dri...|The Commerce Depa...|\n",
            "|file:/content/dri...|The affected vehi...|\n",
            "|file:/content/dri...|Ask Jeeves has be...|\n",
            "|file:/content/dri...|Despite the appar...|\n",
            "|file:/content/dri...|The job gains mea...|\n",
            "+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We can convert the Spark NLP dataframe to Pandas dataframe\n",
        "texts.limit(5).toPandas()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "R3xyWf4USoUV",
        "outputId": "c812f80d-81d7-459b-9b40-c76a2bb7ed7d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            filename  \\\n",
              "0  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "1  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "2  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "3  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "4  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "\n",
              "                                                text  \n",
              "0  TimeWarner said fourth quarter sales rose 2% t...  \n",
              "1  Yukos' owner Menatep Group says it will ask Ro...  \n",
              "2  The Vestey Group said it had owned the land si...  \n",
              "3  Leisure group Rank could unveil plans to demer...  \n",
              "4  \"Journalists are the lifeblood of any newspape...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4cfce05e-7cc8-4fd8-8c35-471bf0f8c023\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>The Vestey Group said it had owned the land si...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Leisure group Rank could unveil plans to demer...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>\"Journalists are the lifeblood of any newspape...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4cfce05e-7cc8-4fd8-8c35-471bf0f8c023')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4cfce05e-7cc8-4fd8-8c35-471bf0f8c023 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4cfce05e-7cc8-4fd8-8c35-471bf0f8c023');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 195
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a new column that stores the article topics for the text summarries\n",
        "# Note: getItem will get the item by its index, which starts from 0\n",
        "texts = texts.withColumn(\n",
        "  'topic',\n",
        "  fun.split('filename', '/').getItem(8)\n",
        "  )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "INS0sT4hoSW7",
        "outputId": "4d773ddd-dac0-4cd0-86d2-ec74b818d02b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " # Take a look at the newly created label column\n",
        "texts.limit(5).toPandas()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "O1MZe6-fpToa",
        "outputId": "0029de98-9b83-4051-ff35-bb2943947dc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            filename  \\\n",
              "0  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "1  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "2  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "3  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "4  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "\n",
              "                                                text     topic  \n",
              "0  TimeWarner said fourth quarter sales rose 2% t...  business  \n",
              "1  Yukos' owner Menatep Group says it will ask Ro...  business  \n",
              "2  The Vestey Group said it had owned the land si...  business  \n",
              "3  Leisure group Rank could unveil plans to demer...  business  \n",
              "4  \"Journalists are the lifeblood of any newspape...  business  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b3199112-26eb-4591-a3fb-5bd06edfd30a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>text</th>\n",
              "      <th>topic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
              "      <td>business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
              "      <td>business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>The Vestey Group said it had owned the land si...</td>\n",
              "      <td>business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Leisure group Rank could unveil plans to demer...</td>\n",
              "      <td>business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>\"Journalists are the lifeblood of any newspape...</td>\n",
              "      <td>business</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b3199112-26eb-4591-a3fb-5bd06edfd30a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b3199112-26eb-4591-a3fb-5bd06edfd30a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b3199112-26eb-4591-a3fb-5bd06edfd30a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 197
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Take a look at the number of documents pertaining to each label\n",
        "label_counts = texts.groupBy('topic').count().toPandas()\n",
        "label_counts.plot(kind = 'bar', figsize = (10, 5))\n",
        "plt.xticks(\n",
        "  ticks = range(len(label_counts)),\n",
        "  labels = label_counts['topic']\n",
        " )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 556
        },
        "id": "F6xBDgxUqUw4",
        "outputId": "e0af782a-e11e-4ed9-b93c-3c5f5954de58"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([<matplotlib.axis.XTick at 0x7f1967b9d690>,\n",
              "  <matplotlib.axis.XTick at 0x7f1967b9dad0>,\n",
              "  <matplotlib.axis.XTick at 0x7f1966f71ad0>,\n",
              "  <matplotlib.axis.XTick at 0x7f19672034d0>,\n",
              "  <matplotlib.axis.XTick at 0x7f1967415650>],\n",
              " [Text(0, 0, 'politics'),\n",
              "  Text(0, 0, 'entertainment'),\n",
              "  Text(0, 0, 'business'),\n",
              "  Text(0, 0, 'sport'),\n",
              "  Text(0, 0, 'tech')])"
            ]
          },
          "metadata": {},
          "execution_count": 198
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAFtCAYAAADF8CTsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcpklEQVR4nO3de7hddX3n8ffHAKZAuIVTHiBgYktRnOHWFFF05NI6KFqsBasFpEhNR21px06R1pmxtnbqZUYpjPWRFi04tIiKwuClUgh1xGtQBJH6kNoIQZQYMFCRcvvOH3sFDichZ5/wO1l757xfz7Ofs9Zvrb3397A5J5/z+/3Wb6WqkCRJ0pP3lL4LkCRJ2loYrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKmRbfouAGD33XevxYsX912GJEnStK677rofVtXExo6NRLBavHgxK1as6LsMSZKkaSX57hMdcyhQkiSpEYOVJElSIwYrSZKkRkZijpUkSRpNDz74IKtXr+b+++/vu5Qtbv78+SxatIhtt9126OcYrCRJ0hNavXo1CxYsYPHixSTpu5wtpqpYu3Ytq1evZsmSJUM/z6FASZL0hO6//34WLlw4p0IVQBIWLlw44546g5UkSdqkuRaq1tuc79tgJUmS5qyzzz6b++67r9nrOcdKkiQNbfFZn2z6eqveflzT15ups88+m5NPPpntt9++yevZYyVJkkbahRdeyIEHHshBBx3EKaecwqpVqzj66KM58MADOeaYY7j11lsB+I3f+A0++tGPPvq8HXfcEYBrrrmGI488khNOOIFnPOMZnHTSSVQV55xzDt/73vc46qijOOqoo5rUao+VJEkaWTfddBNve9vb+MIXvsDuu+/OXXfdxamnnvro4wMf+ABnnHEGn/jEJzb5Ol//+te56aab2GuvvTjiiCO49tprOeOMM3j3u9/N8uXL2X333ZvUa7CSJA2l9RDQqOl7SEobd/XVV3PiiSc+Gnx22203vvjFL3LppZcCcMopp3DmmWdO+zqHHXYYixYtAuDggw9m1apVPO95z2ter8FK0hbjP8ySZtM222zDI488AsAjjzzCAw888Oixpz71qY9uz5s3j4ceemhWahhqjlWSVUluTHJ9khVd225JrkxyS/d11649Sc5JsjLJDUkOnZXKJUnSVu/oo4/mIx/5CGvXrgXgrrvu4rnPfS4XX3wxABdddBHPf/7zAVi8eDHXXXcdAJdffjkPPvjgtK+/YMEC7r333mb1zqTH6qiq+uGk/bOAq6rq7UnO6vbfBLwI2K97PBt4X/dVkiRpRp71rGfx5je/mRe84AXMmzePQw45hHPPPZfTTjuNd73rXUxMTPDBD34QgNe+9rUcf/zxHHTQQRx77LHssMMO077+smXLOPbYY9lrr71Yvnz5k643VTX9SckqYOnkYJXk28CRVXVHkj2Ba6pq/yTv77b/bup5T/T6S5curRUrVjzJb0XSqHMocLz5+c1NN998M8985jP7LqM3G/v+k1xXVUs3dv6wyy0U8Nkk1yVZ1rXtMSksfR/Yo9veG7ht0nNXd22SJElbtWGHAp9XVbcn+WngyiT/NPlgVVWS6bu+JukC2jKAfffddyZPlSRJGklD9VhV1e3d1zuBjwOHAT/ohgDpvt7ZnX47sM+kpy/q2qa+5nlVtbSqlk5MTGz+dyBJkjQipg1WSXZIsmD9NvBC4JvA5cCp3WmnApd125cDr+6uDjwcWLep+VWSJGm0DTMfe2u0Od/3MEOBewAf7+7wvA3wt1X1mSRfBS5JcjrwXeAV3fmfAl4MrATuA06bcVWSJGkkzJ8/n7Vr17Jw4UK6LDAnVBVr165l/vz5M3retMGqqr4DHLSR9rXAMRtpL+ANM6pCkiSNpEWLFrF69WrWrFnTdylb3Pz58x9drX1YrrwuSZKe0LbbbsuSJUv6LmNsDLvcgiRJkqZhsJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpkW36LqAPi8/6ZN8lzKpVbz+u7xIkSZqT7LGSJElqxGAlSZLUiMFKkiSpkaGDVZJ5Sb6e5Ipuf0mSLydZmeTDSbbr2p/a7a/sji+endIlSZJGy0x6rH4XuHnS/juA91TVzwJ3A6d37acDd3ft7+nOkyRJ2uoNFaySLAKOA/662w9wNPDR7pQLgJd128d3+3THj+nOlyRJ2qoN22N1NnAm8Ei3vxD4UVU91O2vBvbutvcGbgPojq/rzpckSdqqTbuOVZKXAHdW1XVJjmz1xkmWAcsA9t1331YvK0mSpnD9xi1nmB6rI4BfTrIKuJjBEOBfALskWR/MFgG3d9u3A/sAdMd3BtZOfdGqOq+qllbV0omJiSf1TUiSJI2CaYNVVf1hVS2qqsXAK4Grq+okYDlwQnfaqcBl3fbl3T7d8aurqppWLUmSNIKezDpWbwLemGQlgzlU53ft5wMLu/Y3Amc9uRIlSZLGw4zuFVhV1wDXdNvfAQ7byDn3Ayc2qE2SJGmsuPK6JElSIwYrSZKkRmY0FCiNgq35suFRumRYkjRz9lhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY1MG6ySzE/ylSTfSHJTkrd27UuSfDnJyiQfTrJd1/7Ubn9ld3zx7H4LkiRJo2GYHqt/A46uqoOAg4FjkxwOvAN4T1X9LHA3cHp3/unA3V37e7rzJEmStnrTBqsa+Ndud9vuUcDRwEe79guAl3Xbx3f7dMePSZJmFUuSJI2ooeZYJZmX5HrgTuBK4J+BH1XVQ90pq4G9u+29gdsAuuPrgIUti5YkSRpFQwWrqnq4qg4GFgGHAc94sm+cZFmSFUlWrFmz5sm+nCRJUu9mdFVgVf0IWA48B9glyTbdoUXA7d327cA+AN3xnYG1G3mt86pqaVUtnZiY2MzyJUmSRscwVwVOJNml2/4p4JeAmxkErBO6004FLuu2L+/26Y5fXVXVsmhJkqRRtM30p7AncEGSeQyC2CVVdUWSbwEXJ3kb8HXg/O7884EPJVkJ3AW8chbqliRJGjnTBququgE4ZCPt32Ew32pq+/3AiU2qkyRJGiOuvC5JktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqZFpg1WSfZIsT/KtJDcl+d2ufbckVya5pfu6a9eeJOckWZnkhiSHzvY3IUmSNAqG6bF6CPj9qjoAOBx4Q5IDgLOAq6pqP+Cqbh/gRcB+3WMZ8L7mVUuSJI2gaYNVVd1RVV/rtu8Fbgb2Bo4HLuhOuwB4Wbd9PHBhDXwJ2CXJns0rlyRJGjEzmmOVZDFwCPBlYI+quqM79H1gj257b+C2SU9b3bVJkiRt1YYOVkl2BD4G/F5V3TP5WFUVUDN54yTLkqxIsmLNmjUzeaokSdJIGipYJdmWQai6qKou7Zp/sH6Ir/t6Z9d+O7DPpKcv6toep6rOq6qlVbV0YmJic+uXJEkaGcNcFRjgfODmqnr3pEOXA6d226cCl01qf3V3deDhwLpJQ4aSJElbrW2GOOcI4BTgxiTXd21/BLwduCTJ6cB3gVd0xz4FvBhYCdwHnNa0YkmSpBE1bbCqqs8DeYLDx2zk/ALe8CTrkiRJGjuuvC5JktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1Mi0wSrJB5LcmeSbk9p2S3Jlklu6r7t27UlyTpKVSW5IcuhsFi9JkjRKhumx+hvg2CltZwFXVdV+wFXdPsCLgP26xzLgfW3KlCRJGn3TBquq+hxw15Tm44ELuu0LgJdNar+wBr4E7JJkz1bFSpIkjbLNnWO1R1Xd0W1/H9ij294buG3Seau7NkmSpK3ek568XlUF1Eyfl2RZkhVJVqxZs+bJliFJktS7zQ1WP1g/xNd9vbNrvx3YZ9J5i7q2DVTVeVW1tKqWTkxMbGYZkiRJo2Nzg9XlwKnd9qnAZZPaX91dHXg4sG7SkKEkSdJWbZvpTkjyd8CRwO5JVgNvAd4OXJLkdOC7wCu60z8FvBhYCdwHnDYLNUuSJI2kaYNVVb3qCQ4ds5FzC3jDky1KkiRpHLnyuiRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjRisJEmSGjFYSZIkNWKwkiRJasRgJUmS1IjBSpIkqRGDlSRJUiMGK0mSpEYMVpIkSY0YrCRJkhoxWEmSJDVisJIkSWrEYCVJktSIwUqSJKkRg5UkSVIjBitJkqRGDFaSJEmNGKwkSZIaMVhJkiQ1YrCSJElqxGAlSZLUiMFKkiSpEYOVJElSIwYrSZKkRmYlWCU5Nsm3k6xMctZsvIckSdKoaR6skswD3gu8CDgAeFWSA1q/jyRJ0qiZjR6rw4CVVfWdqnoAuBg4fhbeR5IkaaTMRrDaG7ht0v7qrk2SJGmrlqpq+4LJCcCxVfWb3f4pwLOr6rennLcMWNbt7g98u2kho2V34Id9F6HN4mc33vz8xpuf3/ja2j+7p1XVxMYObDMLb3Y7sM+k/UVd2+NU1XnAebPw/iMnyYqqWtp3HZo5P7vx5uc33vz8xtdc/uxmYyjwq8B+SZYk2Q54JXD5LLyPJEnSSGneY1VVDyX5beDvgXnAB6rqptbvI0mSNGpmYyiQqvoU8KnZeO0xNSeGPLdSfnbjzc9vvPn5ja85+9k1n7wuSZI0V3lLG0mSpEYMVpIkSY0YrCRJkhoxWElTJHnHMG0afUl2TXJg33VoOEmOGKZNGmUGq1mS5J1JdkqybZKrkqxJcnLfdWkov7SRthdt8Sq0WZJc0/3s7QZ8DfirJO/uuy4N5dwh2zRikrw8yS1J1iW5J8m9Se7pu64+zMpyCwLghVV1ZpJfAVYBLwc+B/yfXqvSE0ryOuD1wNOT3DDp0ALg2n6q0mbYuaruSfKbwIVV9ZYpn6dGTJLnAM8FJpK8cdKhnRish6jR907gpVV1c9+F9M1gNXvW/7c9DvhIVa1L0mc9mt7fAp8G/hw4a1L7vVV1Vz8laTNsk2RP4BXAm/suRkPZDtiRwe/NBZPa7wFO6KUizdQPDFUDBqvZc0WSfwJ+ArwuyQRwf881aROqah2wDnhVknnAHgx+RnZMsmNV3dprgRrWnzC488Pnq+qrSZ4O3NJzTdqEqvrHJJ8HDqyqt/Zdj4aX5OXd5ookHwY+Afzb+uNVdWkvhfXIBUJnUTfHY11VPZxkB2BBVX2/77q0ad0tmf4Y+AHwSNdcVeUkaGkWJfliVT2n7zo0vCQf3MThqqrXbLFiRoTBapYkeQNwUVX9qNvfFXhVVf1lv5VpOklWAs+uqrV916KZS/JO4G0Meos/AxwI/Oeqcn7jiEvyPmBv4CPAj9e3z8VeD40vrwqcPa9dH6oAqupu4LU91qPh3cZgSFDj6YVVdQ/wEgYXjvws8Ae9VqRhzQfWAkcDL+0eL+m1Ig0lyQVJdpm0v2uSD/RZU1+cYzV75iVJdV2C3Zyd7XquScP5DnBNkk/y+LkCXrI/HrxwZExV1Wl916DNduDUzoQkh/RZUF/ssZo9nwE+nOSYJMcAf9e1afTdClzJIAgvmPTQeFh/4cjPA1d54cj4SLIoyceT3Nk9PpZkUd91aShP6aa8AI/OMZ6TnTfOsZolSZ4C/BZwTNd0JfDXVfVwf1VpJpJsX1X39V2HZs4LR8ZTkisZLHvyoa7pZOCkqtrYor0aIUleDfwRg/lxACcCf1ZVH3riZ22dDFbSFN1ihecDO1bVvkkOAn6rql7fc2kaQpLtgTcC+1bVsiT7AftX1RU9l6ZpJLm+qg6erk2jKckBDObHAVxdVd/qs56+OBTYWJJLuq83Jrlh6qPv+jSUs4H/yGASLVX1DeA/9FqRZuKDwAMMVvIGuJ3BVYIafWuTnJxkXvc4me7nUGNhN+DHVfW/gTVJlvRdUB/m5PjnLPvd7qtXsoyxqrptyoRnh3DHx89U1a8leRVAVd0XZ6+Pi9cwuDfge7r9awEntI+BJG8BlgL7M/jjZlsGt3CbczfRNlg1VlV3dJuvr6o3TT6W5B3AmzZ8lkbMbUmeC1SSbRmEZW/VMD4eSPJTwPorcn+GSVd3anRV1XeBX+67Dm2WXwEOYXDjc6rqe0nm5EU/DgXOno1NtnzRFq9Cm+M/AW9gsFDh7cDB3b7Gw1sYXIG7T5KLgKuAM/stScNI8vQk/zfJmu6qwMu6WxJp9D3QLS+0/g+aHXqupzdOXm8syeuA1wNPB/550qEFwLVVdXIvhUlzSJKFwOFAgC9V1Q97LklDSPIl4L0MlqcBeCXwO1X17P6q0jCS/BdgPwadCn/OYFj3b6vq3F4L64HBqrEkOwO7Mvgf66xJh+6tqrv6qUoz0U24/B1gMZOGy6vKIYoxkWRv4Gk8/vP7XH8VaRhJbph6T84k36iqg/qqScPpprr8A/BCBn/Q/D3wi1OnxMwFBqvGkuxUVfd06+hswHA1+pJ8g8FyCzfy2E2Yqap/7K0oDa37Bf9rwE08/ibaBuMR1312dwMXMxhS+jUGf6i+C/z9OcqSfK2qDp3StkFQngsMVo0luaKqXpLkXxj8Yph8NVJVlfMFRlySLzv0ML6SfJvB7TWcsD5mut+b663/x2n971B/f44gp79syGAlTZHk1xnMFfgsj79X4Nd6K0pDS/Jp4MSq+te+a9HMJHkF8Jmu1/+/AYcCf+rP3uhy+suGDFaNJTl0U8f9BTH6kvw5cAqDv74mDyUd/cTP0qhI8jHgIAZXA04Oxmf0VpSGsn7oKMnzgD8F/ifw3+1B1jhxHav2/tcmjhWPLfev0XUi8PSqeqDvQrRZLu8eGj/rF+I9DvirqvpkElfN11ixx0qaIskngGVVdWfftUhzSZIrGKwd90sMhgF/AnzFqwI1TgxWs6Rbsft1PHaPuWuA91fVg70VpaEkuQY4EPgqjx9K8qqyEZbkkqp6RZIbeWziMwwmP9dcvDpp3HQ30D4WuLGqbkmyJ/Dvq+qzPZcmDc1gNUuS/DWDeyVd0DWdAjxcVb/ZX1UaRpIXbKzd5RZGW5I9q+qOJE/b2PHudimSNKsMVrNkY4vaudCdNPu6W2n8pKoeSfJzwDOAT9tbLGlL8F6Bs+fh7uavwOAeWDw2MVMjLMnLk9ySZF2Se5Lcm+SevuvS0D4HzO9WX/8sg97iv+m1IklzhlcFzp4/AJYn+U63vxg4rb9yNAPvBF5aVTf3XYg2S6rqviSnA39ZVe9Mcn3fRUmaG+yxmj3XAu9nsA7SXd32F3utSMP6gaFqrCXJc4CTgE92bfN6rEfSHGKP1ey5ELiHwSJ3AL8OfIjBGkkabSuSfBj4BI+/KvDS/krSDPwe8IfAx6vqpm4YfnnPNUmaI5y8PkuSfKuqDpiuTaMnyQc30lxV9ZotXowkaazYYzV7vpbk8Kr6EkCSZwMreq5JQ6gq58KNsSTLefw6VgB4SyJJW4I9VrMkyc3A/sCtXdO+wLeBh3CxwpGU5MxuovO5bPwfZu81NwaS/Pyk3fnArwIPVdWZPZUkaQ6xx2r2HNt3AZqx9RPW7VkcY1V13ZSma5N8pZdiJM059lhJ2qok2W3S7lOApcBfVNX+PZUkaQ6xx0qaIskE8CbgAAZDSYBzdMbIdTw2lPsQsAo4vbdqJM0prmMlbegiBsOCS4C3MviH+at9FqQZOQB4L/AN4JvAp3F4V9IW4lCgNEWS66rq55PcsP4igyRfrapf6Ls2TS/JJQzWkLuoa/p1YJeqcg05SbPOoUBpQ+tv1ntHkuOA7wG7beJ8jZZ/N2W9uOVJvtVbNZLmFIOVtKG3JdkZ+H3gXGAnBqt5azy4hpyk3hispA3dXVXrgHXAUQBJjui3JE0nyY0MJq1vC3whya3d/tOAf+qzNklzh3OspCmSfK2qDp2uTaMlydM2dbyqvrulapE0d9ljJXWSPAd4LjCR5I2TDu0EzOunKg3L4CRpFBispMdsB+zI4OdiwaT2e4ATeqlIkjRWHAqUJkkyD7ikqn6171okSePHBUKlSarqYWCvvuuQJI0nhwKlDV2f5HLgI8CP1zdW1aX9lSRJGgcGK2lD84G1wOR7AxZgsJIkbZJzrCRJkhpxjpU0RZKfS3JVkm92+wcm+a991yVJGn0GK2lDfwX8Id09A6vqBuCVvVYkSRoLBitpQ9tX1VemtD3USyWSpLFisJI29MMkP8NgwjpJTgDu6LckSdI4cPK6NEWSpwPnMbi9zd3AvwAnecsUSdJ0XG5B2lBV1S8m2QF4SlXdm2RJ30VJkkafQ4HShj4GUFU/rqp7u7aP9liPJGlM2GMldZI8A3gWsHOSl086tBODRUMlSdokg5X0mP2BlwC7AC+d1H4v8NpeKpIkjRUnr0tTJHlOVX2x7zokSePHYCVNkWSCQQ/VYib16lbVa/qqSZI0HhwKlDZ0GfD/gH8AHu65FknSGLHHSpoiyfVVdXDfdUiSxo/LLUgbuiLJi/suQpI0fuyxkqZIci+wPfAAgxsxh8GioTv1WpgkaeQ5x0ra0M7AScCSqvqTJPsCe/ZckyRpDNhjJU2R5H3AI8DRVfXMJLsCn62qX+i5NEnSiLPHStrQs6vq0CRfB6iqu5Ns13dRkqTR5+R1aUMPJpkHFDy6rtUj/ZYkSRoHBitpQ+cAHwd+OsmfAZ8H/ke/JUmSxoFzrKSN6G7IfAyDKwKvqqqbey5JkjQGDFaSJEmNOBQoSZLUiMFKkiSpEYOVJElSIwYrSZKkRgxWkiRJjfx/IQpdsHbv7PsAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Take a look at the schema of the Spark NLP dataframe\n",
        "texts.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "-wc9j-Fbq-le",
        "outputId": "3fa9bc1b-c40e-49b6-b419-a503e2dcb0ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- filename: string (nullable = true)\n",
            " |-- text: string (nullable = true)\n",
            " |-- topic: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Randomly split the Spark NLP dataframe into train and test sets\n",
        "train, test = texts.randomSplit([0.8, 0.2], seed = 123)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "7rxEbfcdtpw7",
        "outputId": "e2d3065b-8c9c-496b-a5c5-a0918c5fea7f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the assembler\n",
        "assembler = DocumentAssembler()\\\n",
        "  .setInputCol('text')\\\n",
        "  .setOutputCol('document')\n",
        "docs = assembler.transform(texts)\n",
        "docs.limit(5).toPandas()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "zVpQqdrpt7aM",
        "outputId": "2d9db40f-b3f9-4bbf-8499-3f6936874d3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            filename  \\\n",
              "0  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "1  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "2  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "3  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "4  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "\n",
              "                                                text     topic  \\\n",
              "0  TimeWarner said fourth quarter sales rose 2% t...  business   \n",
              "1  Yukos' owner Menatep Group says it will ask Ro...  business   \n",
              "2  The Vestey Group said it had owned the land si...  business   \n",
              "3  Leisure group Rank could unveil plans to demer...  business   \n",
              "4  \"Journalists are the lifeblood of any newspape...  business   \n",
              "\n",
              "                                            document  \n",
              "0  [(document, 0, 899, TimeWarner said fourth qua...  \n",
              "1  [(document, 0, 689, Yukos' owner Menatep Group...  \n",
              "2  [(document, 0, 943, The Vestey Group said it h...  \n",
              "3  [(document, 0, 908, Leisure group Rank could u...  \n",
              "4  [(document, 0, 1082, \"Journalists are the life...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-52e6a5b1-5d5b-4332-a1ce-f046200875ef\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>text</th>\n",
              "      <th>topic</th>\n",
              "      <th>document</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 899, TimeWarner said fourth qua...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 689, Yukos' owner Menatep Group...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>The Vestey Group said it had owned the land si...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 943, The Vestey Group said it h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Leisure group Rank could unveil plans to demer...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 908, Leisure group Rank could u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>\"Journalists are the lifeblood of any newspape...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 1082, \"Journalists are the life...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-52e6a5b1-5d5b-4332-a1ce-f046200875ef')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-52e6a5b1-5d5b-4332-a1ce-f046200875ef button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-52e6a5b1-5d5b-4332-a1ce-f046200875ef');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 201
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the sentence detector\n",
        "sentence = SentenceDetector()\\\n",
        "  .setInputCols([\"document\"])\\\n",
        "  .setOutputCol(\"sentences\")\n",
        "docs = sentence.transform(docs)\n",
        "docs.limit(5).toPandas()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "s52z029JuRlN",
        "outputId": "d772f638-307e-4b87-da73-01a806185753"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            filename  \\\n",
              "0  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "1  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "2  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "3  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "4  file:/content/drive/My Drive/YouTube Channel/D...   \n",
              "\n",
              "                                                text     topic  \\\n",
              "0  TimeWarner said fourth quarter sales rose 2% t...  business   \n",
              "1  Yukos' owner Menatep Group says it will ask Ro...  business   \n",
              "2  The Vestey Group said it had owned the land si...  business   \n",
              "3  Leisure group Rank could unveil plans to demer...  business   \n",
              "4  \"Journalists are the lifeblood of any newspape...  business   \n",
              "\n",
              "                                            document  \\\n",
              "0  [(document, 0, 899, TimeWarner said fourth qua...   \n",
              "1  [(document, 0, 689, Yukos' owner Menatep Group...   \n",
              "2  [(document, 0, 943, The Vestey Group said it h...   \n",
              "3  [(document, 0, 908, Leisure group Rank could u...   \n",
              "4  [(document, 0, 1082, \"Journalists are the life...   \n",
              "\n",
              "                                           sentences  \n",
              "0  [(document, 0, 899, TimeWarner said fourth qua...  \n",
              "1  [(document, 0, 485, Yukos' owner Menatep Group...  \n",
              "2  [(document, 0, 873, The Vestey Group said it h...  \n",
              "3  [(document, 0, 908, Leisure group Rank could u...  \n",
              "4  [(document, 0, 1082, \"Journalists are the life...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4c2ed1d0-93f8-4c4f-b560-eccdb3303397\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>text</th>\n",
              "      <th>topic</th>\n",
              "      <th>document</th>\n",
              "      <th>sentences</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>TimeWarner said fourth quarter sales rose 2% t...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 899, TimeWarner said fourth qua...</td>\n",
              "      <td>[(document, 0, 899, TimeWarner said fourth qua...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Yukos' owner Menatep Group says it will ask Ro...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 689, Yukos' owner Menatep Group...</td>\n",
              "      <td>[(document, 0, 485, Yukos' owner Menatep Group...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>The Vestey Group said it had owned the land si...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 943, The Vestey Group said it h...</td>\n",
              "      <td>[(document, 0, 873, The Vestey Group said it h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>Leisure group Rank could unveil plans to demer...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 908, Leisure group Rank could u...</td>\n",
              "      <td>[(document, 0, 908, Leisure group Rank could u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>file:/content/drive/My Drive/YouTube Channel/D...</td>\n",
              "      <td>\"Journalists are the lifeblood of any newspape...</td>\n",
              "      <td>business</td>\n",
              "      <td>[(document, 0, 1082, \"Journalists are the life...</td>\n",
              "      <td>[(document, 0, 1082, \"Journalists are the life...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c2ed1d0-93f8-4c4f-b560-eccdb3303397')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4c2ed1d0-93f8-4c4f-b560-eccdb3303397 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4c2ed1d0-93f8-4c4f-b560-eccdb3303397');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 202
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the word tokenizer\n",
        "tokenizer = Tokenizer()\\\n",
        "  .setInputCols(['sentences'])\\\n",
        "  .setOutputCol('tokens')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "WnjIruqwwfPe",
        "outputId": "5445fe35-2809-4db7-d953-fbdd7685a111"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the lemmetizer\n",
        "lemmatizer = LemmatizerModel.pretrained()\\\n",
        "  .setInputCols(['tokens'])\\\n",
        "  .setOutputCol('lemma')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "GdLr6VWS0M8s",
        "outputId": "63b14660-6718-4d53-8b67-b147f8528b71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "lemma_antbnc download started this may take some time.\n",
            "Approximate size to download 907.6 KB\n",
            "[OK!]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the normalizer\n",
        "normalizer = Normalizer()\\\n",
        "  .setInputCols(['lemma'])\\\n",
        "  .setOutputCol('normalized')\\\n",
        "  .setLowercase(True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "CfuPfH5e0Y5E",
        "outputId": "eb5a557f-453f-4b6e-d668-646fe9edb788"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the finisher\n",
        "finisher = Finisher()\\\n",
        "  .setInputCols(['normalized'])\\\n",
        "  .setOutputCols(['normalized'])\\\n",
        "  .setOutputAsArray(True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "yg0QUcrT0jwk",
        "outputId": "d5fd8def-e315-430a-e9e9-8043936944e4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the stop words\n",
        "stopwords = set(StopWordsRemover.loadDefaultStopWords('english'))\n",
        "print(stopwords)\n",
        "\n",
        "# Define the stopword remover\n",
        "sw_remover = StopWordsRemover()\\\n",
        "  .setInputCol('normalized')\\\n",
        "  .setOutputCol('filtered')\\\n",
        "  .setStopWords(list(stopwords))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "id": "OvHzeajl0rtE",
        "outputId": "6bef204b-9729-49de-af54-bfee84200786"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'nor', \"hasn't\", 'own', 'this', \"here's\", 'my', 'be', 'because', 'can', \"where's\", 'but', 'more', 'doing', 'no', 't', \"i've\", 'here', \"you've\", \"won't\", \"who's\", \"they'd\", 'under', 'were', 'myself', \"what's\", 'she', 'has', \"he'd\", 'off', \"mustn't\", 'these', 'until', 'by', 'just', 'whom', 'herself', 'what', 'ourselves', 'they', 'on', 'or', \"he'll\", 'than', 'up', 'been', \"couldn't\", 'about', 'too', 'below', 'which', 'to', 'himself', 'where', 'during', \"shouldn't\", 'does', 'with', 'out', 'few', \"you'll\", \"wouldn't\", \"we've\", 'not', 'same', \"he's\", 'after', \"they'll\", 'into', 'would', 'do', 'from', 'our', 'over', 'above', 'their', 'as', 'his', 'if', 'the', 'further', 'being', 'only', \"she'd\", \"wasn't\", \"i'm\", \"she's\", 'did', 'itself', 'some', \"when's\", 'should', 'them', 'theirs', 'your', 'when', 'have', 'themselves', \"aren't\", 'could', \"it's\", 'both', 'is', 'any', 'was', 'then', \"you're\", \"can't\", 'had', \"that's\", \"haven't\", 'me', 'you', 'i', 'at', 'am', \"they're\", \"isn't\", 'of', 'yourself', 'all', 's', 'yours', 'and', 'how', 'will', 'hers', 'him', 'most', 'a', 'each', 'so', \"shan't\", \"weren't\", 'ours', 'yourselves', 'why', \"hadn't\", 'we', 'through', 'its', 'those', 'against', \"didn't\", 'cannot', 'there', \"she'll\", 'before', 'don', \"how's\", 'such', \"i'll\", 'who', \"doesn't\", 'very', \"don't\", 'other', 'are', \"you'd\", 'he', 'down', 'her', \"there's\", 'that', 'for', \"we'd\", 'while', \"i'd\", 'it', 'once', 'between', 'an', 'ought', \"they've\", 'in', 'having', 'again', \"why's\", 'now', \"we'll\", \"we're\", \"let's\"}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define count vectorizer\n",
        "count_vectorizer = CountVectorizer(\n",
        "  inputCol = 'filtered',\n",
        "  outputCol = 'tf', minDF = 10\n",
        "  )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Dz1aGvsi13jw",
        "outputId": "1a19ca5b-c967-45fe-98e3-967005604df0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define TF-IDF\n",
        "tfidf = IDF(\n",
        "  inputCol = 'tf',\n",
        "  outputCol = 'tfidf', \n",
        "  minDocFreq = 10\n",
        "  )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "bE5SY9IS2TX0",
        "outputId": "9ee8e0e4-fbc4-4e8c-a33a-19aa3fcc42cb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the pipeline\n",
        "text_processing_pipeline = Pipeline(\n",
        "  stages = [\n",
        "    assembler,\n",
        "    sentence,\n",
        "    tokenizer,\n",
        "    lemmatizer,\n",
        "    normalizer,\n",
        "    finisher,\n",
        "    sw_remover,\n",
        "    count_vectorizer,\n",
        "    tfidf\n",
        "   ])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "NAxLStI2259G",
        "outputId": "7d16dd7f-9c4f-45cd-ce23-a9987c087214"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to convert labels to indices and use it to fit the data\n",
        "label_indexer = StringIndexer(\n",
        "  inputCol = 'topic', \n",
        "  outputCol = 'label').fit(texts)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "eBoa0RFu3erM",
        "outputId": "9e54f21f-377b-431e-aa6d-7d7037d32c0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Take a look at the indexed labels\n",
        "label_indexer.labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "mUtN3mB2XWXd",
        "outputId": "3a0360dc-9d25-4b3c-a838-584d92128911"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sport', 'business', 'politics', 'tech', 'entertainment']"
            ]
          },
          "metadata": {},
          "execution_count": 212
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to convert predicted indices to labels\n",
        "prediction_deindexer = IndexToString(\n",
        "  inputCol = 'prediction', \n",
        "  outputCol = 'pred_label', \n",
        "  labels = label_indexer.labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "-Fkuhmx_WZRI",
        "outputId": "90dff568-7273-4e3a-eb05-0bc496845ee2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a model\n",
        "naive_bayes = NaiveBayes(featuresCol = 'tfidf')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "OANuoL_LWu3d",
        "outputId": "a147b488-1433-43dc-b904-dae4b476dfb4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the overall pipeline\n",
        "pipeline = Pipeline(\n",
        "  stages = [\n",
        "    text_processing_pipeline, \n",
        "    label_indexer, \n",
        "    naive_bayes, \n",
        "    prediction_deindexer\n",
        "  ])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "LWOwK6rWW2zm",
        "outputId": "d784bd37-890e-4214-a995-455eb6e903ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Train the model\n",
        "model = pipeline.fit(train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "VEYMUlG7XE3O",
        "outputId": "33a22514-b4f4-46d3-d640-95ea56bccd9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions on the train and test sets\n",
        "train_predicted = model.transform(train)\n",
        "test_predicted = model.transform(test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "__CtfENSZBzj",
        "outputId": "07687344-8c2c-46b4-a806-eafaded36729"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Innitiate the F1 score\n",
        "evaluator = MulticlassClassificationEvaluator(metricName = 'f1')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "E2E0KDXDZL56",
        "outputId": "7c84b5bc-796d-4e88-ac83-eeb856507be8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Report the F1 score on the train set\n",
        "print('f1 on train set', evaluator.evaluate(train_predicted))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "mloq06DuZRyi",
        "outputId": "cb2de2bb-3ac8-4121-a0c2-1f7fd2d9ab39"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "f1 on train set 0.9910445983146179\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Report the F1 score on the test set\n",
        "print('f1 on test set', evaluator.evaluate(test_predicted))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "X7_r4DtmZlR6",
        "outputId": "13f16d3e-3f85-4138-d82e-cb64552791e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "f1 on test set 0.927161970025423\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a function to try out different ML models\n",
        "def fit_model(model):\n",
        "\n",
        "  # Define the type of model \n",
        "  model_type = model(featuresCol = 'tfidf')\n",
        "\n",
        "  # Construct the overall pipeline\n",
        "  pipeline = Pipeline(\n",
        "  stages = [\n",
        "    text_processing_pipeline, \n",
        "    label_indexer, \n",
        "    model_type, \n",
        "    prediction_deindexer\n",
        "  ])\n",
        "\n",
        "  # Train the model\n",
        "  model = pipeline.fit(train)\n",
        "\n",
        "  # Make predictions on the train and test sets\n",
        "  train_predicted = model.transform(train)\n",
        "  test_predicted = model.transform(test)\n",
        "\n",
        "  # Innitiate the F1 score\n",
        "  evaluator = MulticlassClassificationEvaluator(metricName = 'f1')\n",
        "\n",
        "  # Report the F1 score on the train set\n",
        "  print(f'{str(model)}: f1 on train set', evaluator.evaluate(train_predicted))\n",
        "\n",
        "  # Report the F1 score on the test set\n",
        "  print(f'{str(model)}: f1 on test set', evaluator.evaluate(test_predicted))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "DJWZNIZ4a55o",
        "outputId": "5ab671de-38ae-484e-8ba7-af571f49f1f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit a logistic regression\n",
        "fit_model(LogisticRegression)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "-O7hPdjicVX2",
        "outputId": "22654c34-9fce-4696-d3ed-c6b7c9f671df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PipelineModel_85c188ba6293: f1 on train set 1.0\n",
            "PipelineModel_85c188ba6293: f1 on test set 0.922376158577747\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit a random forest classifier\n",
        "fit_model(RandomForestClassifier)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "jFy_2lOedBJs",
        "outputId": "71180cfe-39b9-4b16-c91c-d1f02f832418"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PipelineModel_bbd5b9208695: f1 on train set 0.7941192580748638\n",
            "PipelineModel_bbd5b9208695: f1 on test set 0.7214999913078831\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}