6_GREEKBERT_embedddings_FINAL.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sKgDNzS8LsuA"
      },
      "source": [
        "# A Visual Guide to Using BERT for the First Time\n",
        "https://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thwSfZayPzAQ"
      },
      "source": [
        "config = RobertaConfig.from_pretrained(PreTrained_Model)\n",
        "\n",
        "config.output_hidden_states = False\n",
        "\n",
        "model = TFRobertaModel.from_pretrained(PreTrained_Model, from_pt=True, config = config)\n",
        "\n",
        "embedding_layer = model.roberta(tokens,attention_mask=masks,token_type_ids=segs)[0]\n",
        "\n",
        "Features = embedding_layer[:,0,:].numpy()\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d2NdqLfIQJ7y",
        "outputId": "051f4da1-8f73-4e38-f86c-b89cc91ebd5b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.34.1-py3-none-any.whl (7.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m47.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.4)\n",
            "Collecting huggingface-hub<1.0,>=0.16.4 (from transformers)\n",
            "  Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Collecting tokenizers<0.15,>=0.14 (from transformers)\n",
            "  Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m86.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m65.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Collecting huggingface-hub<1.0,>=0.16.4 (from transformers)\n",
            "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
            "Installing collected packages: safetensors, huggingface-hub, tokenizers, transformers\n",
            "Successfully installed huggingface-hub-0.17.3 safetensors-0.4.0 tokenizers-0.14.1 transformers-4.34.1\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LZL47TlzQJwt"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import cross_val_score\n",
        "import torch\n",
        "import transformers as ppb\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lW36etWDWrkn",
        "outputId": "10496486-deb8-46ef-e795-83f108d3672b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.17.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.66.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.3.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2023.7.22)\n"
          ]
        }
      ],
      "source": [
        "!pip install huggingface_hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aJTrgoHlWk1c"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# # PRE-PROCESSING\n",
        "# import string\n",
        "# from nltk.corpus import stopwords\n",
        "# from nltk.stem import WordNetLemmatizer\n",
        "# import spacy\n",
        "\n",
        "# TRAIN/TEST SET\n",
        "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
        "# from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "from sklearn import model_selection, naive_bayes, svm\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "# nlp = spacy.load(\"el_core_news_sm\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jw-Vxv_5Lnxw"
      },
      "outputs": [],
      "source": [
        "# testing  = pd.read_csv('TWEETS_nolinks.csv')\n",
        "testing  = pd.read_csv('CHATGPT TWEETS_nolinks.csv')\n",
        "\n",
        "# Replace empty strings with NaN\n",
        "testing['class'] = testing['class'].replace(' ', np.nan)\n",
        "final = testing[testing['class'].notnull()]\n",
        "final = final.reset_index()\n",
        "final = final[['Tweet', 'class']]\n",
        "# Use astype to convert to float\n",
        "final['class'] = final['class'].astype(float)\n",
        "# final.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(final[final['class']==0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "smOuyI8AYEkP",
        "outputId": "dae922fd-e9c6-4fea-8556-7746311c9f17"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "287"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(final[final['class']==1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QfcYNM3LYKQQ",
        "outputId": "49249151-8d83-4560-8c06-0e8c14fb927c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "269"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.pie(final['Label'].value_counts().values,\n",
        "        labels = final['Label'].value_counts().index,\n",
        "        autopct='%1.1f%%')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "1W5xLtsyUx3n",
        "outputId": "efc4b1eb-f953-449f-d4af-42e97e356b56"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4s0lEQVR4nO3dd3hUVcIG8PdOy5T0XoFAQu+9KwiC2F0VVwVcsQOK7rd+rquu7u63rOuquPa2YEMQV1DpSEdAaoAAoSak9z5Jpn9/BEejoSUzc+beeX/PkwcdZiZvCvPOPefecySXy+UCERERAJXoAERE5D9YCkRE5MZSICIiN5YCERG5sRSIiMiNpUBERG4sBSIicmMpEBGRG0uBiIjcWApEROTGUiAiIjeWAhERubEUiIjIjaVARERuLAUiInJjKRARkRtLgYiI3FgKRETkxlIgIiI3lgIREbmxFIiIyI2lQEREbiwFIiJyYykQEZEbS4GIiNxYCkRE5MZSICIiN5YCERG5sRSIiMiNpUBERG4sBSIicmMpEBGRG0uBiIjcWApEROTGUiAiIjeWAhERubEUiIjIjaVARERuLAUiInJjKRARkRtLgYiI3FgKRETkxlIgIiI3lgIREbmxFIiIyE0jOgCRpzXZHKgwW1FZb0VVgxU1jTbUNtlQ22hHXZMNTTYnXHDB5QJcLhdcAFwuwPmz/wZc0KpVCNFrEKrXIsygRahBi1C9FqGGlrepVZLYL5jIg1gKJDsNVjtyyhtwtsKM7AozcsrNyKloQHFNEyrqLTBbHT7NE27UIiXCiA6RRqRENv/540diuB4aNQ/IST4kl6v5fRGRvymvt+BgXjVOltYjp9yM7HIzcirMKKm1iI52yTQqCQnhenSINKJzdDD6JIWhT3IYusaF8AiD/BJLgfxCk82BzIIaZORVuz/yqxpFx/Iag1aNHgkh6Jscjj5JYeibHIYuMcFQsShIMJYCCZFX2YAfsiuRkVeFjLxqZBXVwe4M7F9Fk06NXolhGNwpAmPSYzC4UwS0HHoiH2MpkE9Y7A7szq7EpqwybD5RijNlZtGR/J5Jp8bwzlEYkx6NMV1j0CUmWHQkCgAsBfKa/KoGbDpehi3HS7HjdAUafDwBrDRJ4QaM7RqNMekxGJUWjTCDVnQkUiCWAnlURl41Vh4qxKbjZThVWi86jmKpVRJGdI7CDf0TMbl3PEL1LAjyDJYCtVteZQOWHSjA8gMFOFPOYSFf02lUGNctBjf2T8L47rHQa9WiI5GMsRSoTWoabVh5qAjLDuRj79kq8LfIP4QEaTCxVxxu7J+E0WnRPO2VLhtLgS6ZzeHEpqxSLDtQgA1ZpbDanaIj0QVEB+twfb9ETBveEZ05SU2XiOe7ecCbb76JTp06Qa/XY9iwYdi9e/cF77906VJ0794der0effr0wapVq3yUtG0q6i147buTGDFvIx74ZB9WZxazEGSgvN6KBd/n4KpXtmDGf3Zj0/FS8D0gXQyPFNppyZIlmD59Ot555x0MGzYM8+fPx9KlS3H8+HHExsb+6v47duzA2LFjMW/ePFx33XVYtGgRXnzxRezfvx+9e/cW8BWc34mSOny4LRvLMwpgYQkoQudoE6aP6IhbB6cgOIir3NCvsRTaadiwYRgyZAjeeOMNAIDT6URKSgrmzJmDp5566lf3nzp1KsxmM1asWOG+bfjw4ejfvz/eeecdn+U+H5fLhS0nyvDh9mxsO1kuOg55SUiQBr8ZlIx7RnZCp2iT6DjkRzh81A5WqxX79u3DhAkT3LepVCpMmDABO3fubPUxO3fubHF/AJg0adJ57+8rTTYHPt+di6tf3Yp7FuxhIShcncWOhTtyMO7lzbh34R7sO1spOhL5CR4/tkN5eTkcDgfi4uJa3B4XF4esrKxWH1NcXNzq/YuLi72W80Isdgc+25WLtzafQnm9VUgGEsflAjZmlWJjVilGp0Vj7oR0DO4UKToWCcRSCFAOpwv/3ZeP1zacREG1cheeo0u3/VQ5tp8qx6i0KDwxsSsGdWQ5BCKWQjtER0dDrVajpKSkxe0lJSWIj49v9THx8fGXdX9Pc7lcWJ1ZjJfXHcdprj9Erfj+VAW+P7UTV3WPxR8md0P3+FDRkciHOKfQDjqdDoMGDcKGDRvctzmdTmzYsAEjRoxo9TEjRoxocX8AWL9+/Xnv70lbTpThhje+xyOf7Wch0EVtyCrFlNe2Ye7iA8itaBAdh3yEZx+105IlSzBjxgy8++67GDp0KObPn48vvvgCWVlZiIuLw/Tp05GUlIR58+YBaD4l9YorrsA//vEPXHvttVi8eDH+/ve/e/WU1P25VXhxdRZ+yOZkIrWNVi3h3tGpeOyqdBh1HGBQMv5022nq1KkoKyvDc889h+LiYvTv3x9r1qxxTybn5uZCpfrpgGzkyJFYtGgRnnnmGTz99NNIT0/H8uXLvVIIVWYr5q0+hqX78rkMBbWLzeHCu1vO4JuMQjx7XU9M6ZMgOhJ5CY8UFGrp3jzMW52FSjPPKCLPG5MejRdu6MXlMxSIpaAwp0rr8KdlmRwqIq/TqVV4YGxnzB6fxpVZFYSloBBNNgf+veEk3t92BjYHf6TkO0nhBvz5+p64updvzqAj72IpKMCm46V47utM5FXyegMSZ0KPWMy7pS9iQoJER6F2YCnIWG2TDX/++giWHSgQHYUIABBl0uEfv+mLiT3jLn5n8kssBZnaeboC/7P0IK9GJr/026EpePa6njx9VYZYCjJjtTvxr3XH8cG2M3DyJ0d+LDXahFdu74cBHSJER6HLwFKQkTNl9Zjz+QEcKawVHYXokmhUEmaPT8Oc8encGlQmWAoy8d99+Xju60yYrQ7RUYgu24AO4Zg/tT86RnHvBn/HUvBzZosdzy7PxFecTCaZM+nUePn2fpjcm1dD+zOWgh87W2HGzI/24lRpvegoRB4hScCccWl4fGJXSBKHk/wRS8FP7ThVjkcW7Ud1g010FCKPm9AjDvPv6M99ov0QS8EPfbIzBy98exR2nl5ECpYeG4z3pg9GKveI9issBT9idzjx/LdH8OmuXNFRiHwiVK/Bv387AFd2ixUdhc5hKfiJ6gYrHv50P3aeqRAdhcinVBLwh0nd8fCVXURHIbAU/MLJkjrc9/FenOXuVhTAbuyfiJdu7QedhhtCisRSEGzriTLM+mw/6ix20VGIhBuTHo13pw3i8hgCsRQEWpNZhEc/z4DV4RQdhchvDOwQjgX3DEWYUSs6SkBiKQjy1f58/OHLQ3DwDCOiX+keH4KPZw5FbIhedJSAw1IQ4NNdZ/Hs15ncN5noAjpEGvHpzGHoEGUUHSWgsBR87L2tp/H3VVmiYxDJQmxIED6ZOQzd4kNERwkYLAUfemXdcfx74ynRMYhkJcygxX/uGYJBHbkEty+wFHzkbyuO4oPt2aJjEMmSUafGgnuGYFjnKNFRFI+l4AN/WnYYn/3Aq5SJ2iMkSINF9w9Hn+Qw0VEUjVeJeNmLa7JYCEQeUGexY/p/fsDJkjrRURSNpeBF7289g7c3nxYdg0gxqhpsuPvDH5BXyav/vYWl4CVf7svH31cfEx2DSHFKai24+8MfUFrbJDqKIrEUvOC7oyV46r+HeB0CkZecrWjA3R/+gOoGq+goisNS8LDd2ZWYtWg/90Ig8rITJfWY8Z/dqOe6YR7FUvCgo4W1mPnRHljsXMuIyBcO5tfgvo/2wGJ3iI6iGCwFD8mtaMCMBbtR18R3LUS+tOtMJf60LFN0DMVgKXhAvcWO3y3cjbI6i+goRAHpy335+GDbGdExFIGl0E4ulwuPL8nA6TKz6ChEAW3e6ixsPVEmOobssRTa6d8bTmH90RLRMYgCnsPpwpzPDyC7nG/Q2oOl0A7fHS3B/A0nRMcgonNqGm2476M9qGuyiY4iWyyFNjpdVo/Hl2TwWgQiP3O6zIzHFmfAydPC24Sl0Ab1Fjse+Hgv91Um8lMbs0rxz7XHRceQJZbCZeLEMpE8vLPlNL49WCg6huywFC7T6xs5sUwkF08vO8zF8y4TS+Ey7MmpxPzvOLFMJBd1TXY88UUGHJxfuGQshUtUb2n+5eLvFpG87MmpwlubuA3upWIpXKK/fnsUeZWNomMQURu8tuEkMvKqRceQBZbCJVh/tARL9uaJjkFEbWR3ujB38QGYecbgRbEULqK83oI/fnVIdAwiaqeciga88O0R0TH8HkvhIv741WGU13MjDyIl+GJvPtZkFomO4ddYChfwxZ48nn5KpDBPfXUYJdzK87xYCueRV9mAv6w4KjoGEXlYdYMNzy7n/gvnw1I4j6e+OsRt/ogUat3REqw9Uiw6hl9iKbRi5aEifH+qQnQMIvKi5785wrORWsFS+IUGqx1/W8lhIyKlK6ppwr/WcdG8X9KIDuBvXt94CkU1ypyEsteVo3rzQjSe2QeX3QJNeAKipsxFUEI6AMBhrkLV5oVoyjkAZ5MZQSm9EDnhQWgjky74vLV7vkZdxio4asugMoTC2G0UIq6YAUmjAwDUH9mE6i0fwWVthKnPBERedf9PmWpKULLkWSTMmA9VkNF7XzxRKz7akYPfDExG76Qw0VH8BkvhZ86U1ePDbdmiY3iFo6kexZ8+CX2Hvoi97XmojGGwVxVCpQ8G0Lz6a+lXf4Ok0iDmlmeg0hlRu2c5SpY8g8SZb0Ol07f6vOajm1G1ZSGipzyGoKQesFUWoGLVfABA5FX3w9FQg8o1ryNqylxowuNR+uUL0HfsB2PaUABAxbq3EHHFPSwEEsLpAv78zRF8+dAISJIkOo5f4PDRzzz/7VFYHU7RMbyidteX0IRGI/rauQhK7AZteDwMqQOhjUgAANirCmEtPI7Iqx9BUEJXaKOSETnpEbjsVpiPbTnv81oKjkGf3AOmnldCExYHQ+pAGHuMhbXoZPPzVhdDCjLC1GMsghK6Qt+hL2wVzVeHm49ugaTSwNhtpPe/AUTnse9sFf67v0B0DL/BUjhnTWaxojf9bjz1A3Tx6ShbPg95r9+FwgWPoi5jjfvvXY7m7Qt/HPIBAElSQVJrYck//xxLUFIPWIpPw1LYPDZrqy5G4+m9MHQZDADQRCbBZbPAWnIajsY6WItOQBfTCY6melRv+xSREx/yxpdLdFn+sToLtdzCEwCHjwAATTYH/qrwaxJs1cWwHViF0CE3IW7E7bAUnUTVhvcgqbUI7nMVtJHJUIfGoHrLR4icPBsqbRBq93wNR105HPWV531eU88r4WioRfFn/wvABTgdCO5/DcJG3A4AUOuDEX3t4yhf8QpcditMvcfD0HkQyle9hpCB18FeU4LS//4VcNoRNupOmLqP9tF3hOgn5fUWvPbdSTx7XU/RUYRjKQB4a9MpFFQrfAVUlwtB8WmIuGIGAEAX1wW28rOoy1iF4D5XQVJrEHPzn1Cx+jXkv3YHIKmg79Qf+s6DgAssF96Uewg1u75A5NUPIyixG+xVhaj87n1Uf/85wkf9FgBg7DoSxq4jf/aYw7CV5SBy4oMofO8BRF//B6hNESj6+AnoU3pDbQr35neCqFWf7DqLmaNTkRhuEB1FqIAvhbI6C95X6OTyz6mDI6CN7tDiNm1UChqOf+/+/6D4NCT+7nU4LWa4HHaojWEo+vgJ6OLTz/u81ds+RXCv8QjpNwkAoIvpBKfNgso1byBs5FRIUssRSpfdhsp1byPquidgryqCy+mAvkOf5jyRSbAUHYcxbZinvmyiS2a1O/H6xpOYd0tf0VGECvg5hXe2nEajzSE6htcFJfWErTK/xW22ygJoQmN/dV9VkAlqYxhslQWwFp+CMf38L9IumwX4xVkb7iJw/foQo2bHYug7D0RQfBrgcgLOn773LqcdcCpzop/kYenefOSUB/b+6wFdCqW1Tfjsh7OiY/hE6JAbYSk8jpqdX8BWVQjz0c2oP7gGwQOvdd/HnLUdTbmHYKsuRsPJXShZ8iyM6cNhSB3ovk/5ipdRtWWh+/8NaUNRd2AVzEe3NE8yZx9A9bZPYUgbCkmlbpHBWp4Lc9Y2hI++GwCgiUwGJBXqDq5Dw+k9sFXkQ5dw/qMSIm+zO10Bv+VuQA8fvbX5NJpsgfHONCihK2Ju/hOqt3yE6u8/hyYsDhHj70dwr3Hu+zjqK1G18QM4zNVQB0cguNd4hI26o8Xz2GvLgJ8NCYWNvAOAhOptn8JRXwGVIQyGtKGIGDutxeNcLhcq17yBiPH3ua95UGmDEDVlLirXvw2Xw4bIiQ9BExLtvW8C0SX45mAhHhmXhq5xIaKjCCG5XK0c4weA4pomjH1pE6z2wCgFIrp0V/eMw3vTB4uOIUTADh+9uekUC4GIWrXuaAkOBuiezgFZCoXVjViyh3suE9H5BepieQFZCq9vPKXY5SyIyDO2nSzHnpzzX7ipVAFXCnmVDfhyH48SiOjiPth2RnQEnwu4UvhwezZsjoCcWyeiy7T+aAnyKhtEx/CpgCqF2iYblu7lUQIRXRqnq3nPhUASUKXwxZ48mK3Kv3qZiDxnyd68gNq2M2BKwel04aOdOaJjEJHM1DXZ8eW+/IvfUSECphTWHytBXqXCV0IlIq9YuCMHgXKdb8CUwqe7AmONIyLyvOxyMzYdLxUdwycCohRyys3YfqpcdAwikrH/bM8RHcEnAqIUPvvhbGurOBMRXbLtp8pxoqROdAyvU3wpNNkcATVJRETeEwintCu+FNYeKUZVAzfkJqL2++ZgIZxOZQ87KL4Uvj1YKDoCESlESa0Fu85UiI7hVYouhZpGG7ae4AQzEXnO8owC0RG8StGlsDazmKuhEpFHrc4sRpOC93VXdCl8w6EjIvKwuiY7NmYp95oFxZZCeb0FOxU+9kdEYiw/oNwhJMWWwqrDRXAo/CwBIhJj8/Ey1Cj0rEbFlgLPOiIib7E6nFiVWSQ6hlcoshSKahqx92yV6BhEpGArDinzjaciS2HloSIua0FEXrUnu0qR+ywoshS+O1YiOgIRKZzV4cSO08o7mUVxpdBgtWP/2WrRMYgoAGw9USY6gscprhR+yK7kBWtE5BNbWAr+b/tJLmtBRL6RW9mA7HKz6BgexVIgImqHLQrbkU1RpVBa24TjAbAJBhH5D6UNISmqFLjlJhH52q4zlbDYlbNAnrJKgUNHRORjjTYHdmdXio7hMcoqBR4pEJEAStp4RzGlcLKkDqV1FtExiCgAHcqvER3BYxRTCgfyqkVHIKIAdSi/Bi6FrK2jmFLILFBOUxORvNQ02pBT0SA6hkewFIiIPOCgQkYrFFEKDqcLx4p4fQIRiXMwv1p0BI9QRCmcLqtHo4I30iYi/8cjBT9yWEEz/0QkT0eLamFXwGKciiiFzEKWAhGJ1WRzIqtY/sPYiiiFIwW1oiMQESniegXZl4LL5cIRHikQkR84oYAFOWVfCjkVDTBbOclMROKdrZD/3goKKAX5/xCISBnOKuACNtmXQn6l/H8IRKQM+VWNcDrlvdyF/EuhqlF0BCIiAIDV4URhjbxfk2RfCnlVPFIgIv8h9yEk2ZcCjxSIyJ+wFATL45wCEfkRuZ+BJOtSMFvsqGqwiY5BROTGIwWBOJ9ARP7mrMxHL+RdCpWcTyAi/1JS2yQ6QrvIuhQKq1kKRORfahvlPaQt61Ko5nwCEfkZu9OFuib5vjbJuhTk/I0nIuWS8xtWmZeCXXQEIqJfqZHxEJK8S8Ei3288ESkXS0EQHikQkT/i8JEgtSwFIvJDPFIQhBPNROSPqhutoiO0mcxLgUcKROR/ajh8JEY9S4GI/FCDjLcIlm0pOJwuNNrk+40nIuVyuOS7+5psS8Ep4286ESmbnLfklG0pqCVJdAQiolY5WAq+p1KxFIjIP8l5+EgjOkB7qCRAxoVMfuT+5Fx0S8jAMk0jah3y3jmLxIuOHw2gv+gYbSLrUlCrJDgdbAVqv+n67Ug5vALX6oxY1m0MPnFWIb+hWHQskqm+sd1FR2gz2Q4fAYDEeQXygBCNHcmlmwEABmsD7jy8FiuO7sO/dJ3QJ7Sz2HAkSypJvi+t8k0OTjaTZzySlA3JWt/iNrXLgUnHt2LRwc1Y4IjBleE9IIG/b3Rp5FwKsh8+ImqvGzS7Lvj3g3P3YXAukB3TBR+ldMeK2hOwOCw+SkdypFVpRUdoM/nWGQAeKFB7RWjtSCzdekn3TS07jef3r8Ta4mo8ENYH4bowL6cjuTJpTaIjtJmsS0HDIwVqp1lJpyHZLu9so6j6MszJWIl12afxtKkHko3xXkpHchWsDRYdoc1kXQrBelmPfpEfuE69s82PNVgb8NvMtVh5dC9e1nZCX05K0zk8UhAk0qgTHYFkLEZnQ1zJtnY/j8rlxNUntuKzg5vxkSOak9LEUhAlwsRSoLablXQKkr3Ro885MHc/Xj+wFl/Xa3BrRB8EqYM8+vwkDxw+EoRHCtQeU6QdXnvu1LLT+PO5SekHOSkdcEw6HikIEckjBWqj+CArYkq2e/3zRNWXYfa5Sek/mbojhZPSAYFHCoJw+Ijaak7SCUg+vNbAYG3AHZnrsOLoXryi7Yi+oV189rnJ98KDwkVHaDNZlwKPFKitJsF7Q0cXonI5MfHENnx2cBM+tkdhXHhPWV/9Sr8mQUKcMU50jDaT9TmdEZxToDZI0lsQVfy96BgYkHcAA/KAnJgu+DilF77x4yulbVU2FH9RjPpD9XBandDF6ZA8MxmGVMN5H1O9oxrlq8thKbFAbVAjuG8w4qfGQxPc/LJTn1mPwk8KYa+xI2RACJJmJkGlaS5IR4MDp184jU5/6ARdtLz+nUfoI6BV84pmIaKC5fXLQv7h0cTjkJz+s7F6p7LTeG7/SqwrqsLDYX0Q4WeT0g6zA2f+dgaSWkLH33dE+t/TEX9HPFSm8798mE+akf9+PiLGRiD9/9KRMisFjWcaUbigEADgcrqQ924eIsdFovOzndGU04SqzVXux5csLUHkuEjZFQIAxBpjRUdoF1kfKXD4iNpiokv8UUJrIs3leCRjJe7VGvB1tzH42FWD3IYi0bFQtrIM2igtku9Ldt+mi7nwv73GU43QRmsRNTHKff/IKyNRtqoMAOCod8BR50Dk+EiodCqEDAiBpbD5KKnhZAMashuQMC3BS1+Rd8l56AiQ+ZFCfKhedASSmU6GJkSUtP0qZl/Q2xoxNXMdvj26B69qOqKf4Enpuow6GDoZkPtGLo7NOYZTz51C5ebKCz7GkGaAvdKOuoN1cLlcsNfYUbO3BiF9QwAA6hA1NOEa1B+ph9PihPmEGfoUPVx2Fwo/LkTSPUmQZLqMDY8UBDIFaRAdHITyev8chyX/82jiMUgFdtExLonK5cSEk9swAUBGSn8sjInHpuosOF1On+awllpRubESUZOjEHN9DBqzG1H0WREkjYSI0RGtPsaUbkLyg8nIezsPTpsTcAAh/UOQOC0RQPNeKCmPpKD482IUfVaEkL4hiBgTgbKVZTB1N0HSSjjztzOw19sRNSEKUROifPkltwtLQbCOUUaWAl2y8Q7/HDq6mP55GZifB5yN7oyPOzRPSjf5alLaBehT9Yi/tfkaC0NHA5rym1C5qfK8pdBU0ISiRUWIuSEGIX1CYKu2oXhJMQo+KkDyzOZhKFNXE7r8+aejIEuxBdU7qtHlhS7InpeNqIlRCOkbgpN/OglTNxP0KfIYGeDwkWAdo4yiI5BMpJsaEVbyg+gY7dKx/Aye3b8S64oq8UhYH0T64Hx4TbgG+sSWL8hBiUGwVZx/sr5sRRmMaUbETImBPkWPkD4hSJyeiOpt1bBVt/64woWFiL8jHnABTWebEDYkDJpQDUzdTDBnyWff7A6hHURHaBf5l0KkfC8nJ9+ak3AEksshOoZHRJgr8HDGSqw9fRLPGruhoynRa5/LmG6EpbjlUYm12Apt9PlPu3RZXb96dXHPEbSyrXrllkqoTWqEDggFzo2Ouc7tv+5yuOByymcv9i5h8r4wUfalkBrDUqBLc6VNnkNHF6K3NeL2I+vxzZHdmK/piP5emJSOujoKDacbUPptKSwlFlTvrEbl5kpEjf9pnL94aTHy38t3/39I/xDU7qtFxcYKWEutMJ80o+izIhg6G6CNaFkm9lo7yr4tQ8LdzWcbqU1qBCUGoWJdBRpONcB8zAxjujxGBCL1kQjXh4uO0S6yn1NIj5XvGiPkOz2CGxBSukd0DK9RuZy46uQ2XIXmSemPYuKx0UOT0sbORnSY0wElX5ag7Osy6GJ0SLgzAeEjw933sVfbYa2wuv8/YkwEnE1OVH5XieLFxVAb1TD1MCH+9l+v/VT0WRGiJ0e3KIuk+5JQ8H4BKtZXIPqaaBg7y6MUUsNSRUdoN8nlcsnnuKwVTTYHej63BjI6uiQB3k7bjWvy54uO4VO50an4OKUnvq7z4aR0gLut6214bsRzomO0i+yHj/RaNVIi5fEugsQZbfX+iqj+pkN5Np454NtJ6UDXJVze8wmAAkoBANJiOIRE59c3tB7BpftExxDmx0npdadO4FljN3Ty4qR0oFPC8JEiSqFXYqjoCOTHZsdmQmrtlJcAE2Rvap6UzvwBr2k6YGBYmuhIiiP3M48AhZTCgA6tX0BDBACjLFtFR/ArElwYf3I7PsrYiE/tEZgYweW7PSFKH4U4k7wvXAMUUwrhoiOQnxocVgdTWYboGH6rX95BvLJ/DVbUAFMj+sCglsdVw/6od3Tvdj1+69atuP7665GYmAhJkrB8+fKLPmbz5s0YOHAggoKCkJaWhoULF7YrA6CQUgg36pAazesV6NceiT0kOoIspFTk4Jn9K7GusByzQnsjMohH35erV3Svdj3ebDajX79+ePPNNy/p/tnZ2bj22msxbtw4ZGRkYO7cubjvvvuwdu3aduWQ/XUKPxqQEo7scvlcCk++MbyRQ0eXI7yhEg8dXIXfafT4ptsYfCzVI8dcIDqWLPSJ7tOux19zzTW45pprLvn+77zzDlJTU/Hyyy8DAHr06IHt27fj1VdfxaRJk9qcQxFHCgAwoCPf2VBLIyJqYCw/LDqGLAXZm3DbkfX4JnMX/s1J6YuSIKFvTF+ffs6dO3diwoQJLW6bNGkSdu5s39LwyimFlHDREcjPPBzNoaP2kuDCuHOT0p/ZIjAxohfUklp0LL/TOawzQnW+PQuyuLgYcXEtJ7bj4uJQW1uLxsbGNj+vYkqhR0IoDFr+stJPhjZsER1BUfrmH8Qr+1fj2xon7gjnpPTP9YvtJzqCxyimFNQqCX2T/WtvWxLniqgq6CuOio6hSCkVZ/GnAyuxvrAMs0N7I4qT0hgYO9DnnzM+Ph4lJSUtbispKUFoaCgMBkObn1cxpQAAAzmvQOc8GHlQdATFC2uowoMHV2HdySz82dgVqaYk0ZGEGZE4wvefc8QIbNiwocVt69evx4gR7cuiqFIY2UU+W/aRdw2q59CRr+gcFtx65Dt8nbkLr6tTMCgsXXQkn0oLT/PIFpz19fXIyMhARkYGgOZTTjMyMpCbmwsA+OMf/4jp06e77//QQw/hzJkzePLJJ5GVlYW33noLX3zxBR5//PF25VBUKQxLjYJJx3mFQDcxuhJBVcdFxwg4Ely48tT3WJixAYts4bg6QCalPXWUsHfvXgwYMAADBgwAADzxxBMYMGAAnnuuedXVoqIid0EAQGpqKlauXIn169ejX79+ePnll/HBBx+063RUQAFLZ//Sg5/sxdojJRe/IynWkvSNGJb3gegYBCA/sgM+6dQXy+pOotHe9jNi/NnbE97G6KTRomN4jKKOFABgfPf2H8aRvA2o2yQ6Ap2TXJmLP+5fgfUFpZgT2hvRQZGiI3mUTqXD4LjBomN4lOJKYVy3WEiS6BQkypSYcuiqT4uOQb8Q1lCFBw6uwtqTx/CCoSs6ByeLjuQRA+IGQK9R1qm5iiuF2FA9l9IOYDPDD4iOQBegc1hwy9HvsPzwTryhSsZgmU9Kj0jw/VlH3qa4UgCA8d04hBSo+tZy6EgOJLhwxekdWJCxAZ9bwzBJppPS4zqMEx3B4xRZCuM4rxCQboorhbYmR3QMuky9Cw7jX/tXY2W1A3eF94VB0/YLr3wpLTwNncM6i47hcYoshf4p4YgO1omOQT52T+h+0RGoHZIqc/HUgRVYn1+CR0N7+f2k9NWdrhYdwSsUWQqSJOFKDiEFnN7VG0VHIA8Ia6zG/QdXY+3JY/iLoSu6+Omk9NUdWQqycmN/bk4eSG6LL4amLl90DPIgncOCm49+h2WHd+JNVTKG+NGkdJewLugSLv/9mFuj2FIY1SUaCWHKOlWMzm96yD7REchLJLgw9vQO/CdjAxZbwzDZDyallTp0BCi4FFQqCTf2D9wFugKJJLnQs4pnHQWCXgWH8dK5Sem7w/vAqDEKyTGx40Qhn9cXFFsKAHDrIJZCILgzvgjq+kLRMciHkipz8b8HVmJdfjEeC+2FGL3vJqXTwtOQHuE/Q1mepuhSSIsN4R4LAeCuYA4dBaqwxmrcd3A11p446rNJ6ZvTbvb65xBJ0aUAAL8Z6J9nLpBnqCUnulXyrKNAp3VYfzYpnYShYV2983lUWtzQ5QavPLe/UHwp3NAvEVo1F0NSqmkJhVCbuSouNWuelN6JDzO+wxJLCK6J6A2NpPHY84/vMB7h+nCPPZ8/UnwpRJh0GMdrFhTrt6a9oiOQn+pZeAT/3L8KK6vsHpuUviX9Fg8k82+KLwUAuIVDSIqkVbmQXsGhI7qwxKrmSen1+UV4LKQXYvVt26ExKThJkQvg/VJAlMJVPWIRH8prFpTmd4l5UDWUi45BMhHaWIP7Dq3GmhNH8Fd9OtKCUy7r8Ten3QwpANblD4hS0KpVmDGyk+gY5GG3G/aIjkAypHVYcdOxDVh2+Hu8pUrCsEuYlFZLatyUdpP3w/mBgCgFALhzWAfu36wgQSonOpfzgjVqnzGnd+KDS5iUntBxAuJMcT5OJ0bAlEKYQYvbBl/e4SL5r/uScqFqrBQdgxTix0npVVU2TAvvA9MvJqWn9ZwmKJnvBUwpAMDM0alQq5Q/JhgIbtXvFh2BFCihKg9PHliJ9XlFePzcpHTfmL7oF9NPdDSfkVwul0t0CF965LN9WHW4WHQMagej2oEjwbMhWWpERyGFs6m0KLtzERLTlLsA3i8F1JECAMwcrbydkgLNA0lnWQjkE9qITkjsPEF0DJ8KuFIY1DECAzuEi45B7XCzjkNH5COjHgNUgfUyGVhf7Tn3j+HRglyZNA50KONZR+QDoclAvztEp/C5gCyFSb3i0TnGJDoGtcHDSdmQLHWiY1AgGDkbUGtFp/C5gCwFlUrC/1zdTXQMaoObtD+IjkCBwBgFDJwhOoUQAVkKADClTwL6ca8FWQnT2pFUull0DAoEwx8GdGJ2dRMtYEsBAP53cnfREegyzEo6DclqFh2DlC40GRgxW3QKYQK6FEamRWNMerToGHSJrlNz6Ih8YMLzgNYgOoUwAV0KQPPRQgAsfCh7UTobEkq3io5BSpc8FOh7m+gUQgV8KfROCsOUPgmiY9BFzE48BcnWIDoGKZoETJ4nOoRwAV8KAPA/V3eDhmsi+bVrVLtERyCl63s7kDxYdArhWAoAUqNNuH0IV1D1V7FBNsSVbhMdg5RMa2yeSyCWwo/mTkhHiN5zG3yT58xJPA7J3iQ6BinZqMeA0ETRKfwCS+Gc2BA9nuQpqn5pssShI/Ki0OTmUiAALIUW7h7WAQO4WJ5fSdJbEF2yXXSMC5q3zYIh79cjZF4tYl+qw02LG3C83NHiPg9+24gu/66D4f9qEfNSHW5c3ICsX9znQh5a0QjphVrM32Vx32axuzBtWSNC59Wi6+v1+O6MvcVjXvregjmrGtv3xQWCAD8F9ZdYCj8jSRLm3dKHk85+ZE7iCUgOq+gYF7TlrB2zhuiwa6YJ66cZYXMCV3/aALP1p61KBiWqseBGA47NCsbau41wuYCrP2mAw3nx7UyWHbNhV74DiSEtfy/f22fDvkIHds404YFBWtz530b8uD1KdpUT7++34f+u0nv2i1UanoL6KyyFX+geH4qZY1JFx6BzJrp2iI5wUWvuNuGe/jr0ilWjX7waC2/UI7fGhX1FPx0JPDBIh7EdNegUrsLABDX+Nj4IebUu5FRfuBQKap2Ys7oJn91igPYX/1qPlTtwQzcNesWqMWuIDmUNLpQ3ND/fwysb8eKEIIQG8Q3O+fEU1NawFFox96quSInk4aRoHQxNiCzx/1L4pZpzIzyRhtZfkM1WFxYcsCE1XEJK2PlftJ2u5uGhP4xsLpxf6henxvZcBxptLqw9bUdCsIRoo4TPDtmg10i4uUfgrfB5WQbcxVNQW8HTbVph0Knx1xt7454Fe0RHCWiPJmRBKrSJjnFZnC4X5q5pwqgUNXr/4oX8rT1WPLm+CWYb0C1KhfXTTNCpz18KL263QqMCHh2ma/Xv7x2gxaESB3q+VY9oo4QvbjOgqgl4bnMTNs8w4ZmNTVicaUOXSBX+c4MBSaF8D+gW3hGY/A/RKfwSf0vO48pusbi2L690Fukq5/eiI1y2WSubkFnqwOJbf32keVcfLQ48aMKWe4zoGqXC7V82oMne+vDRvkIHXvvBioU3GSCdZx0WrVrCm9cakP1YCPbcH4zRHTT4/bomPDpUhwPFDizPsuPgQ8EYnqTGo2t4Sq+bpAJufhcIChGdxC+xFC7gz9f3RCivXRCii7ER4SXyOhV19qpGrDhpx6YZJiS38q48TC8hPUqNsR01+PJ2A7LKnVh2zN7KMwHbcu0oNbvQ4dV6aP5SC81fanG2xoXfr7Og0/zWNxnalG3HkVIHZg/VYXOOA1PSNTDpJNzeS4vNOZd+ppPijZoLdBwhOoXf4iveBcSG6PF/N/fBnM8PiI4ScOYkHINUII8XMpfLhTmrm7Asy47NM4xIjbj4ey2Xq/nD4mj9SGFaXy0mdG75z3PSpw2Y1leL3/X/9VxBk92FWauaJ6TVKgkOZ/PzA4DNiUs6yykgJPQHxj0tOoVf45HCRVzfLxG3DUoWHSPgjLP797UJPzdrVRM+PWTDolsMCAmSUFzvRHG9E4225hfiM1VOzNtmwb5CB3JrnNiRZ8dtSxth0EqYkv7TC3/3N+qx7FjzHEqUUYXeseoWH1oVEB8soVv0ryed/7rFginpGgxIaP67UR3U+CrLhkMlDryx24pRHfj+DxoDcMv7AbnF5uXgb8oleOHGXtiXW4UzZdzgxRe6BzcgtHS36BiX7O29zS/kV37UchXXBTfqcU9/HfQaYFuuA/N/sKKq0YW4YAljO6qx414jYk0/vS87XuFEjeXy39FnljrwxVE7Mh78ad/xW3tqsDlHgzELzOgWpcKi3wTmLmItTPwLENNVdAq/J7l+vNqFLiizoAa3vLUDVodTdBTFeyNtD67Lf1V0DFKStAnAXV+Cm6dcHIePLlHvpDA8Obmb6BgBYaxVPkNHJAOGSODGN1kIl4ilcBlmjk7Fld1iRMdQtD4hZoSU7hUdg5Tk+teAkHjRKWSDpXAZJEnCv27rh5iQINFRFGtWXCYkcESTPKT/XUDPG0SnkBWWwmWKDg7CK7f345Gol4y2cDMd8pCoNOCaF0WnkB2WQhuMSY/Bo+PTRcdQnIFh9TCV8ZoQ8gBDBHDnF7xquQ1YCm00d0I6JvfiOKUnzYo9zKEjaj+VFpj6KRDVRXQSWWIptJEkSXhlaj90j+c7EU8Z0bhFdARSguvnA51Gi04hWyyFdjDqNHh/+mBEGHmFZHsNC6+FsfyQ6Bgkd6PmAgPuFp1C1lgK7ZQSacS70wZDp+a3sj0ejmEhUDv1uL55a01qF76SecDQ1Ej84zd9RMeQtWENHDqidkjoD9z8Hi9Q8wCWgofcMjAZj17FM5LaYnRkDQwVR0THILkKSQR+uxjQcX0nT2ApeNATE7vixv6JomPIzkNRGaIjkFxpTcCdi4FQbojlKSwFD/vXbf0wjkthXJbB5s2iI5AcSSrgNx8ACf1EJ1EUloKHadUqvH33IIxKixIdRRbGR1VBX3lcdAySo4l/AbpPEZ1CcVgKXqDXqvH+9MEY3DFCdBS/90BkhugIJEfj/gSMnCM6hSKxFLzEqNNgwe+GoG9ymOgofm1g/WbREUhurnoOuOJJ0SkUi6XgRSF6LT6+dyivej6PyTEV0FWdFB2D5GTiX4ExvxedQtFYCl4WbtTh0/uGoXOM6eJ3DjAzw7n4HV2GSfOAUY+KTqF4LAUfiA4OwqL7hqNDJM+j/rn+tZtFRyC5uOYlYMQjolMEBJaCj8SH6fH5A8N5xHDO9bFl0NacER2D/J4EXPsKMOwB0UECBkvBh5LCDfjyoZHox8ln/C6MQ0d0MVLzVppDZooOElBYCj4WadLh8weGY2zXwL7ArW/NRtERyJ9JKuDGN4FBM0QnCTgsBQGMOg0+nDE4YJfE+E1cCTS1uaJjkL+S1MBN7wAD7hKdJCCxFATRqlWYP7U/7h2VKjqKz80I3S86AvkrrRG4bQHQb6roJAGLpSCQJEl47vqe+N/J3UVH8RlJcqFnNYeOqBVhKcC9a4GeN4pOEtBYCn7g4Su74KVb+0KjUv5a8FPji6GpKxAdg/xNh5HAA5uBhL6ikwQ8loKfuG1wCj6+dygiTTrRUbxqWvA+0RHI3wy6B5jxDWCKFp2EAEgul8slOgT9pKC6EQ99sg+HC2pER/E4teTEicj/gdpcLDoK+QOVBpj8D2Do/aKT0M/wSMHPJIUbsPShEbhtULLoKB53d0IRC4GaGaOAactZCH6IpeCH9Fo1XrqtH/52U2/o1Mr5Ef3WtFd0BPIHsb2A+zcBqWNEJ6FWcPjIz+07W4VHPtuHklqL6CjtolW5cDx8LlQNZaKjkEjdrwNufhcIChadhM5DOW9DFWpQxwismDMGQztFio7SLjMS8lkIAU0CrngKmPopC8HPsRRkICYkCIvuH4aHr+wCtUxPW51q5NBRwArvCNyzAhj3R0CS5+9vIOHwkczsO1uF33+RgZyKBtFRLlmQyoljYY9B1VghOgr5lAQMvrd5L2UeHcgGjxRkZlDHCKx6bAymDe8omzdd9yblshACTWgyMG0ZcN0rLASZ4ZGCjG09UYYnvzyE4tom0VEuaGP6l+ic95XoGOQrA+5u3iVNHyo6CbUBS0HmahpteP6bI1h2wD+XjjCoHTgSMgeqpmrRUcjbQhKa9z/oOkl0EmoHDh/JXJhBi1en9sfbdw30yyUy7k86y0IIBH2nAo/sZCEoAI8UFKTKbMVL645j8e5cOP3kp7o5bQk65X8tOgZ5iykGuG4+0OM60UnIQ1gKCpRZUIPnvzmCvWerhOYwaRzINM2CZKkVmoO8QFIB/X4LTPwrYIoSnYY8iKWgYMsO5GPeqiyU1om5Gvr3HU5hTulzQj43eVGX8c2nmcb3EZ2EvICloHBmix3/3ngSC7bnwOpw+vRzb0tbhJT8FT79nORFcX2AiS8AaVeJTkJexFIIEGfK6vHCt0ex5YRvlpoI0dhxyPgIJGu9Tz4feVFoMjD+mebJZBXPTVE6lkKA2XayDK+uP4H9udVe/TxPdTyBh0qe9+rnIC8LCgPGPA4MexjQ6kWnIR9hKQSoLSfK8Np33iuH77t8gqSC1V55bvIytQ4YPBO44knAKO+FGOnysRQCnDfKIUJrx379w5BsZo89J/lIr5uBq/4MRKaKTkKCsBQIgGfL4dlOWZhZ/Jf2hyLfUOuA3rcCI2YB8b1FpyHBWArUwpYTZXhr0yn8kF3Z5ufY1WUh4gvWeTAVeYU+DBj0O2DYQ0Bogug05CdYCtSq48V1+GRXDpbtL4DZ6rjkx8XobNitewiSvdGL6ahdwjsCwx8GBkzjCqb0KywFuqC6Jhu+2l+AT3adxanSi59e+kLqUcwo+psPktHlkYC0CcCQ+4D0q3lqKZ0XS4Eu2Y7T5fhk51msP1oC+3kWV9rd+UPEFm7wcTI6L0NE81LWg2dy8pguCUuBLltxTRM+352L5RkFOPuzHeAS9FbsUD8IySFmWQ06R6UBUscCfW4Det3CawzosrAUqF0O5Fbh64xCrDxchMdj9uLOwnmiIwUmtQ7oPA7oeQPQbQqvL6A2YymQRzicLjSc3YeQI4uArJVAfbHoSMqn0QNdrgJ63gh0m9x8NhFRO7EUyPOcTiB/N3DsW+DYN0B1ruhEyqE1AukTm4sgfRLPHiKPYymQ9xVmAGc2AWd3Anm7gKYa0YnkJTgeSB0D9LgeSJsI6IyiE5GCsRTIt5xOoPQokLsTOLuj+c+6ItGp/IekBuJ6AinDfvqI6Cg6FQUQlgKJV5ndsiQqTolO5DtBoUDy4J8KIHkwEBQiLM68efPw1VdfISsrCwaDASNHjsSLL76Ibt26XfBxS5cuxbPPPoucnBykp6fjxRdfxJQpU3yUmjyJpUD+p760uRwK9gOVp4GKM0BVNmBruPhj/ZkuBIjo1PJIILanX11INnnyZNxxxx0YMmQI7HY7nn76aWRmZuLo0aMwmUytPmbHjh0YO3Ys5s2bh+uuuw6LFi3Ciy++iP3796N3b66lJDcsBZIHl6t5mKnyDFBxuvnPytPNRxmVZ/yjMCQVEJrU/MIf0fHcn6nnPjrJci/jsrIyxMbGYsuWLRg7dmyr95k6dSrMZjNWrPhpl73hw4ejf//+eOedd3wVlTxEIzoA0SWRJCA0sfmj0+hf/31t0U8l0VQNWOoBaz1gqQOs5nP/XQ9Y6879ee42qxnAz94XSermM3x0RkBrALSm5j91xubbf/zQGQGd6VwJnHvRD+8AaHQ++ob4Rk1N80kBkZHnv+5h586deOKJJ1rcNmnSJCxfvtyb0chLWAqkDKEJzR+tFcaFOJ2AzQw47c0v9pog7+STIafTiblz52LUqFEXHAYqLi5GXFxci9vi4uJQXMxrVeSIpUCBTaUSOrHrz2bNmoXMzExs375ddBTyIZYCEf3K7NmzsWLFCmzduhXJyckXvG98fDxKSkpa3FZSUoL4+HhvRiQv8Z/THohIOJfLhdmzZ2PZsmXYuHEjUlMvvrLqiBEjsGFDy5Vx169fjxEjRngrJnkRjxSIyG3WrFlYtGgRvv76a4SEhLjnBcLCwmAwGAAA06dPR1JSEubNa1788LHHHsMVV1yBl19+Gddeey0WL16MvXv34r333hP2dVDb8ZRUInKTJKnV2xcsWIB77rkHAHDllVeiU6dOWLhwofvvly5dimeeecZ98do///lPXrwmUywFIiJy45wCERG5sRSIiMiNpUBERG4sBSIicmMpEBGRG0uBiIjcWApEROTGUiAiIjeWAhERubEUiIjIjaVARERuLAUiInJjKRARkRtLgYiI3FgKRETkxlIgIiI3lgIREbmxFIiIyI2lQEREbiwFIiJyYykQEZEbS4GIiNxYCkRE5MZSICIiN5YCERG5sRSIiMiNpUBERG4sBSIicmMpEBGRG0uBiIjcWApEROTGUiAiIjeWAhERubEUiIjIjaVARERuLAUiInJjKRARkRtLgYiI3FgKRETkxlIgIiK3/wfy74xU+2r0hAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08-wc1aLXRDu"
      },
      "source": [
        "Loading the Pre-trained BERT model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "80d2649073f44c3eb5be4ebb3019f243",
            "271cdf2d5b604e0fa8d895d51992c637",
            "039cf63c73b44a16b3c9e6adfdaf094d",
            "8624338b21554f15876d4bcdc736cd3f",
            "9b61f6b533c04b08a3fc5d0959b82697",
            "68ca276b935c423ea93c223080dbe3d8",
            "e4cb0246af854e8091676559610be5f4",
            "f9ec173ee8774f7880ae0479ec6256eb",
            "9a09f047b6164f8b9479236cbc7aa91c",
            "8a7b0a33fb6145e1bfbf83b7cf8b27f3",
            "4d095418a491433f8b53bbbc05f54456",
            "06cc1a1d3bcd487fad7e623c7bd276ac",
            "e6d45d7a5dfb4a68b2e9cb37e0cbb35f",
            "5e9106e717d4419594a0cf6fc33c6ee0",
            "22fe48dd75fe49f0a68d5b10ae3e6912",
            "df4ee49cb1a94f688708bbdfbf05745b",
            "ed66a21ce71b47ceaee08e22770d66eb",
            "26414fb2827049b391b53a9e2f855d5c",
            "fa2a9ae9a897449790f33ddd9ba60cda",
            "9769fc998e224b69a3212121e38c72e2",
            "019138d27c23407980bccfdf0de5b3c0",
            "7cd275af43ec4f56bcf4981bed1fdb3c",
            "0450238e81c84703a3d4412eb84a3917",
            "fcdbeaf38a7a4b9a868a2492fd4aa565",
            "1be464bf51064963853e5bcd13bfb3ad",
            "00c509489e70477285879909827b8102",
            "72767f75b4c84b69be8ddd2fc3b654f8",
            "7297c8092e054fdb8c9a2d4e044e434b",
            "ee98f9c03c714022804edf03bc088e3b",
            "be8fa7813a80411b8fd2be9c7b0136c8",
            "83eb856bbeb447d0988be09daf37332c",
            "01ce1faab1ad497ab2e395e1950ea24b",
            "48c1d12c24d64522ab48c0a95f19dab7",
            "724c0e83e7624a27b365dfff5cac7c2e",
            "a12e7dffa2324425b39374d8eae0be1e",
            "3300bc1ab3b646b2aba5aca61f721c4e",
            "0eb85797f7014f258ada344325c5d9a2",
            "9cab660a664c4eb5ab776d62e9ebbe37",
            "7709efbd2f6a4c0eabf5230b3d52aa55",
            "039ad0eb3b5c4f3997b501efb5d5fb2e",
            "20c8e11d67034b3296df0e097d809831",
            "fe23771f840248e8ba5abaf1690ee7bb",
            "fdbd536d2a5448e89fed86a4dabfbaea",
            "918acb5d01154b708cd37797e7a731b2",
            "0817103780514c2285bb1de226e3d434",
            "64fb71ff40e64a1c951119a6edde8961",
            "faf310fbcc63418eb131f94d4bd5711a",
            "4606fe1118854e9f8ef40f23ade66c48",
            "48acfd99cf214949ba2756d03b0d3510",
            "684b8bc8715243b1ac6e170aeba6cdc7",
            "80ca82c21fb84f319195ecac4c3e4c38",
            "2b1ccae05f5248d68b0d7caa35e6741c",
            "e06d92fe00164b03be9e5cb0618cc995",
            "9832abd5976b41d8bf0e8b6f7e7a79c8",
            "a29f82f6037f4800894f566a2821e964"
          ]
        },
        "id": "FyewK3VBavPp",
        "outputId": "00a42da8-b49e-43f1-bdb6-a476a11eb41e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/2.00 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "80d2649073f44c3eb5be4ebb3019f243"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/459 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "06cc1a1d3bcd487fad7e623c7bd276ac"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/530k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0450238e81c84703a3d4412eb84a3917"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "724c0e83e7624a27b365dfff5cac7c2e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/454M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0817103780514c2285bb1de226e3d434"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from transformers import AutoTokenizer, AutoModel\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\")\n",
        "model = AutoModel.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\"  )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "doW4cAyQbDNM"
      },
      "source": [
        "Model #1: Preparing the Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tw8rTpnWbNwD"
      },
      "source": [
        "## Tokenization\n",
        "Our first step is to tokenize the sentences -- break them up into word and subwords in the format BERT is comfortable with."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YQDz5IAUTZQc"
      },
      "source": [
        "https://www.kaggle.com/code/aishwarya2210/prediction-of-tweets-using-bert-model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w8NKivrIqvBe"
      },
      "source": [
        "no need to preprocess the text with this tokenizer!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m3c6ml3NbL0r"
      },
      "outputs": [],
      "source": [
        "tokenized = final['Tweet'].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9-JrBBHETNTA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "43db6dca-64d8-49bd-da49-c7e05280791c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0      [101, 239, 9321, 349, 2067, 108, 108, 425, 356...\n",
              "1      [101, 940, 9572, 394, 350, 1856, 351, 362, 426...\n",
              "2      [101, 368, 239, 596, 8422, 3544, 345, 18213, 1...\n",
              "3      [101, 1563, 279, 5418, 269, 8660, 279, 8996, 6...\n",
              "4      [101, 2398, 713, 362, 1711, 360, 3068, 2684, 5...\n",
              "                             ...                        \n",
              "551    [101, 386, 1697, 354, 407, 9744, 380, 29103, 9...\n",
              "552    [101, 366, 377, 374, 1181, 353, 348, 362, 2408...\n",
              "553    [101, 381, 242, 433, 2398, 348, 15922, 2355, 1...\n",
              "554    [101, 631, 239, 110, 1944, 357, 369, 28221, 26...\n",
              "555    [101, 410, 571, 391, 742, 351, 345, 5289, 278,...\n",
              "Name: Tweet, Length: 556, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "tokenized"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lv5tkhWEbj51"
      },
      "source": [
        "## Padding\n",
        "After tokenization, tokenized is a list of sentences -- each sentences is represented as a list of tokens. We want BERT to process our examples all at once (as one batch). It's just faster that way. For that reason, we need to pad all lists to the same size, so we can represent the input as one 2-d array, rather than a list of lists (of different lengths)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mr9Ux8rqbgZE"
      },
      "outputs": [],
      "source": [
        "max_len = 0\n",
        "for i in tokenized.values:\n",
        "    if len(i) > max_len:\n",
        "        max_len = len(i)\n",
        "\n",
        "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qJF5U3oabszD",
        "outputId": "5c8acb63-da20-4424-9ce8-b271273278b6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(556, 91)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "# Our dataset is now in the padded variable, we can view its dimensions below:\n",
        "\n",
        "np.array(padded).shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "17gpzsevTwF9",
        "outputId": "2e5a49cf-974b-49b0-d8ee-c0da01ef746e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 101,  239, 9321, ...,    0,    0,    0],\n",
              "       [ 101,  940, 9572, ...,    0,    0,    0],\n",
              "       [ 101,  368,  239, ...,    0,    0,    0],\n",
              "       ...,\n",
              "       [ 101,  381,  242, ...,    0,    0,    0],\n",
              "       [ 101,  631,  239, ...,    0,    0,    0],\n",
              "       [ 101,  410,  571, ...,    0,    0,    0]])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "padded"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HELs2_BSc1ut"
      },
      "source": [
        "## Masking\n",
        "If we directly send padded to BERT, that would slightly confuse it. We need to create another variable to tell it to ignore (mask) the padding we've added when it's processing its input. That's what attention_mask is:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gKyyO9HkdL6y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ce76f91a-df28-4bf2-a9cc-d676fe325682"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(556, 91)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "attention_mask = np.where(padded != 0, 1, 0) #opou den einai 0 vale 1 alliws 0\n",
        "attention_mask.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cTTVIHU2T1-u",
        "outputId": "b2296fc7-deb5-4fff-9eb9-d67415531cac"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1, 1, 1, ..., 0, 0, 0],\n",
              "       [1, 1, 1, ..., 0, 0, 0],\n",
              "       [1, 1, 1, ..., 0, 0, 0],\n",
              "       ...,\n",
              "       [1, 1, 1, ..., 0, 0, 0],\n",
              "       [1, 1, 1, ..., 0, 0, 0],\n",
              "       [1, 1, 1, ..., 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "attention_mask"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S8N1YW07UD85"
      },
      "outputs": [],
      "source": [
        "# from tensorflow.keras.layers import Input"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cn6gnDDQUG4m"
      },
      "outputs": [],
      "source": [
        "# import tensorflow as tf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1jJJcCC1T5od"
      },
      "outputs": [],
      "source": [
        "# input_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"padded\")\n",
        "# input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n",
        "# embeddings = model(input_ids, attention_mask = input_mask)[0]\n",
        "\n",
        "\n",
        "# embeddings = bert(input_ids,attention_mask = input_mask)[1] #(0 is the last hidden states,1 means pooler_output)\n",
        "# # out = tf.keras.layers.GlobalMaxPool1D()(embeddings)\n",
        "# out = tf.keras.layers.Dropout(0.1)(embeddings)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wt-9QYJNT5SK",
        "outputId": "d2c267a6-b5ee-453a-da31-3f622878c5d3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "attention_mask[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n1ZOg-7JT5Op",
        "outputId": "5672f88a-5497-4eda-d7b5-aeab1b066bf3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([  101,   723,   629,   368,  1106,   433,   351,   600,   374,\n",
              "         239,  3055,   479,   114,   367, 32515,   346,  9269,   267,\n",
              "         344,   349, 29002,   415,  1553,   371, 22326,   265,   108,\n",
              "         361,  3223,   359,  4285,   348, 20290,   278,   364,   422,\n",
              "        1232, 21353,   407, 30145,   411,   148,   100,   102,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "           0,     0,     0])"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "padded[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LgNnUN4PcvZU"
      },
      "source": [
        "The model() function runs our sentences through BERT. The results of the processing will be returned into last_hidden_states."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ySTwcwo-o6GY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "294901d4-ebe8-414e-e95f-f24ac18aec06"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n"
          ]
        }
      ],
      "source": [
        "features = np.ndarray([1,768]) #arxikopoihsh alliws??? fix thisssssssssssss!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
        "#megethos = [ 1, hidden layers] gia na prosthesw apo edw k meta\n",
        "\n",
        "for tweets in padded:\n",
        "  outputs = model(torch.tensor([tweets]))[0]\n",
        "  features = np.vstack([features, outputs[:,0,:].detach().numpy()])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3kffQP_PxwKZ"
      },
      "outputs": [],
      "source": [
        "features = features[1:] #ayta einai ta embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rrNQwMVuI2L5"
      },
      "source": [
        "#Train/Test Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bPTy2V90xwPi"
      },
      "outputs": [],
      "source": [
        "kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)\n",
        "for i, (train_index, test_index) in enumerate(kfold.split(features, final['class'])):\n",
        "        X_train, X_test, Y_train, Y_test = features[train_index], features[test_index], final['class'][train_index], final['class'][test_index]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5_un4QiIJxDd"
      },
      "source": [
        "same precedure as a normal embedding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kIqUospCJfej"
      },
      "source": [
        "## SVM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBrGUtNdxwSU",
        "outputId": "5b1dfdd3-6ba2-4726-dbc1-9aebecdb607a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV] END .........................C=0.1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END .........................C=0.1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END .........................C=0.1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ...................C=0.1, gamma=0.01, kernel=linear; total time=   0.0s\n",
            "[CV] END ...................C=0.1, gamma=0.01, kernel=linear; total time=   0.0s\n",
            "[CV] END ...................C=0.1, gamma=0.01, kernel=linear; total time=   0.0s\n",
            "[CV] END .........................C=100, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END .........................C=100, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END .........................C=100, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END .................C=100, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END .................C=100, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END .................C=100, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ........................C=1, gamma=1, kernel=linear; total time=   0.0s\n",
            "[CV] END ........................C=1, gamma=1, kernel=linear; total time=   0.0s\n",
            "[CV] END ........................C=1, gamma=1, kernel=linear; total time=   0.0s\n",
            "[CV] END ....................C=1000, gamma=0.001, kernel=rbf; total time=   0.0s\n",
            "[CV] END ....................C=1000, gamma=0.001, kernel=rbf; total time=   0.0s\n",
            "[CV] END ....................C=1000, gamma=0.001, kernel=rbf; total time=   0.0s\n",
            "[CV] END ..................C=10, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ..................C=10, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ..................C=10, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ........................C=1000, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ........................C=1000, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ........................C=1000, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ...........................C=1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ...........................C=1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ...........................C=1, gamma=1, kernel=rbf; total time=   0.1s\n",
            "[CV] END ................C=1000, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ................C=1000, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "[CV] END ................C=1000, gamma=0.0001, kernel=linear; total time=   0.0s\n",
            "{'kernel': 'rbf', 'gamma': 0.001, 'C': 1000}\n",
            "SVC(C=1000, class_weight='balanced', gamma=0.001)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.93      0.89      0.91        57\n",
            "         1.0       0.89      0.93      0.91        54\n",
            "\n",
            "    accuracy                           0.91       111\n",
            "   macro avg       0.91      0.91      0.91       111\n",
            "weighted avg       0.91      0.91      0.91       111\n",
            "\n"
          ]
        }
      ],
      "source": [
        "\n",
        "\n",
        "\n",
        "# defining parameter range\n",
        "param_grid = {'C': [0.1, 1, 10, 100, 1000],\n",
        "              'gamma': [1, 0.1, 0.01, 0.001, 0.0001, 'scale'],\n",
        "              'kernel': ['rbf', 'linear']}\n",
        "\n",
        "# grid = GridSearchCV(svm.SVC(), param_grid, refit = True, verbose = 3)\n",
        "grid = RandomizedSearchCV(svm.SVC(class_weight='balanced'),  param_grid, cv=3, random_state=42, refit = True, verbose = 2)\n",
        "\n",
        "# fitting the model for grid search\n",
        "grid.fit(X_train, Y_train)\n",
        "# print best parameter after tuning\n",
        "print(grid.best_params_)\n",
        "# print how our model looks after hyper-parameter tuning\n",
        "print(grid.best_estimator_)\n",
        "\n",
        "grid_predictions = grid.predict(X_test)\n",
        "\n",
        "# print classification report\n",
        "print(classification_report(Y_test, grid_predictions))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FxhOUTD6LVEA"
      },
      "source": [
        "## Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Uv8jia0xweK",
        "outputId": "72580d45-5827-4ce0-89db-09b16cca1416"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=100, min_samples_leaf=2, min_samples_split=10, n_estimators=200; total time=   1.0s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=100, min_samples_leaf=2, min_samples_split=10, n_estimators=200; total time=   1.6s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=100, min_samples_leaf=2, min_samples_split=10, n_estimators=200; total time=   1.5s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=60, min_samples_leaf=4, min_samples_split=10, n_estimators=222; total time=   2.3s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=60, min_samples_leaf=4, min_samples_split=10, n_estimators=222; total time=   1.9s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=60, min_samples_leaf=4, min_samples_split=10, n_estimators=222; total time=   1.8s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=110, min_samples_leaf=2, min_samples_split=2, n_estimators=222; total time=   1.6s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=110, min_samples_leaf=2, min_samples_split=2, n_estimators=222; total time=   1.5s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=110, min_samples_leaf=2, min_samples_split=2, n_estimators=222; total time=   1.5s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=60, min_samples_leaf=1, min_samples_split=5, n_estimators=266; total time=   1.7s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=60, min_samples_leaf=1, min_samples_split=5, n_estimators=266; total time=   1.9s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=60, min_samples_leaf=1, min_samples_split=5, n_estimators=266; total time=   1.7s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=30, min_samples_leaf=1, min_samples_split=10, n_estimators=244; total time=   1.9s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=30, min_samples_leaf=1, min_samples_split=10, n_estimators=244; total time=   2.0s\n",
            "[CV] END bootstrap=False, criterion=entropy, max_depth=30, min_samples_leaf=1, min_samples_split=10, n_estimators=244; total time=   1.9s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=None, min_samples_leaf=1, min_samples_split=10, n_estimators=211; total time=   1.4s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=None, min_samples_leaf=1, min_samples_split=10, n_estimators=211; total time=   1.6s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=None, min_samples_leaf=1, min_samples_split=10, n_estimators=211; total time=   4.1s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=2, n_estimators=300; total time=   3.6s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=2, n_estimators=300; total time=   3.2s\n",
            "[CV] END bootstrap=False, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=2, n_estimators=300; total time=   4.3s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=20, min_samples_leaf=2, min_samples_split=5, n_estimators=200; total time=   3.8s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=20, min_samples_leaf=2, min_samples_split=5, n_estimators=200; total time=   2.6s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=20, min_samples_leaf=2, min_samples_split=5, n_estimators=200; total time=   1.2s\n",
            "[CV] END bootstrap=True, criterion=entropy, max_depth=70, min_samples_leaf=4, min_samples_split=2, n_estimators=255; total time=   1.3s\n",
            "[CV] END bootstrap=True, criterion=entropy, max_depth=70, min_samples_leaf=4, min_samples_split=2, n_estimators=255; total time=   1.3s\n",
            "[CV] END bootstrap=True, criterion=entropy, max_depth=70, min_samples_leaf=4, min_samples_split=2, n_estimators=255; total time=   1.3s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=5, n_estimators=300; total time=   1.4s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=5, n_estimators=300; total time=   1.4s\n",
            "[CV] END bootstrap=True, criterion=gini, max_depth=90, min_samples_leaf=2, min_samples_split=5, n_estimators=300; total time=   1.5s\n",
            "{'n_estimators': 211, 'min_samples_split': 10, 'min_samples_leaf': 1, 'max_depth': None, 'criterion': 'gini', 'bootstrap': False}\n",
            "RandomForestClassifier(bootstrap=False, class_weight='balanced',\n",
            "                       min_samples_split=10, n_estimators=211, random_state=0)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.83      0.95      0.89        57\n",
            "         1.0       0.93      0.80      0.86        54\n",
            "\n",
            "    accuracy                           0.87       111\n",
            "   macro avg       0.88      0.87      0.87       111\n",
            "weighted avg       0.88      0.87      0.87       111\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# defining parameter range\n",
        "\n",
        "# Number of trees in random forest\n",
        "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 300, num = 10)]\n",
        "# Maximum number of levels in tree\n",
        "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
        "max_depth.append(None)\n",
        "# Minimum number of samples required to split a node\n",
        "min_samples_split = [2, 5, 10]\n",
        "# Minimum number of samples required at each leaf node\n",
        "min_samples_leaf = [1, 2, 4]\n",
        "# Method of selecting samples for training each tree\n",
        "bootstrap = [True, False]\n",
        "\n",
        "# Create the random grid\n",
        "random_grid = {'n_estimators': n_estimators,\n",
        "               'criterion': ['gini', 'entropy'],\n",
        "               'max_depth': max_depth,\n",
        "               'min_samples_split': min_samples_split,\n",
        "               'min_samples_leaf': min_samples_leaf,\n",
        "               'bootstrap': bootstrap}\n",
        "\n",
        "\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "RandomForest = RandomForestClassifier(random_state =0, class_weight='balanced')\n",
        "\n",
        "grid = RandomizedSearchCV(RandomForest, random_grid, cv=3, random_state=42, refit = True, verbose = 2)\n",
        "# fitting the model for grid search\n",
        "grid.fit(X_train, Y_train)\n",
        "# print best parameter after tuning\n",
        "print(grid.best_params_)\n",
        "# print how our model looks after hyper-parameter tuning\n",
        "print(grid.best_estimator_)\n",
        "\n",
        "grid_predictions = grid.predict(X_test)\n",
        "\n",
        "# print classification report\n",
        "print(classification_report(Y_test, grid_predictions))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5J0SvcsbLgkT"
      },
      "source": [
        "## SGDClassifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5JpcQ_pcLfnw",
        "outputId": "bb0e5103-34c2-495d-d7ce-568ff1d00697"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV] END alpha=0.001, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END alpha=0.001, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END alpha=0.001, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ...............alpha=0.0001, loss=hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END ...............alpha=0.0001, loss=hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END ...............alpha=0.0001, loss=hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END ........alpha=0.001, loss=hinge, penalty=elasticnet; total time=   0.1s\n",
            "[CV] END ........alpha=0.001, loss=hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ........alpha=0.001, loss=hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ......alpha=0.001, loss=squared_hinge, penalty=none; total time=   0.0s\n",
            "[CV] END ......alpha=0.001, loss=squared_hinge, penalty=none; total time=   0.0s\n",
            "[CV] END ......alpha=0.001, loss=squared_hinge, penalty=none; total time=   0.0s\n",
            "[CV] END ..alpha=0.0001, loss=perceptron, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ..alpha=0.0001, loss=perceptron, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ..alpha=0.0001, loss=perceptron, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ........alpha=0.001, loss=squared_hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END ........alpha=0.001, loss=squared_hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END ........alpha=0.001, loss=squared_hinge, penalty=l2; total time=   0.0s\n",
            "[CV] END alpha=0.0001, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END alpha=0.0001, loss=squared_hinge, penalty=elasticnet; total time=   0.1s\n",
            "[CV] END alpha=0.0001, loss=squared_hinge, penalty=elasticnet; total time=   0.1s\n",
            "[CV] END ..alpha=0.1, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ..alpha=0.1, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END ..alpha=0.1, loss=squared_hinge, penalty=elasticnet; total time=   0.0s\n",
            "[CV] END .................alpha=0.0001, loss=log, penalty=l2; total time=   0.0s\n",
            "[CV] END .................alpha=0.0001, loss=log, penalty=l2; total time=   0.0s\n",
            "[CV] END .................alpha=0.0001, loss=log, penalty=l2; total time=   0.0s\n",
            "[CV] END ......alpha=0.0001, loss=modified_huber, penalty=l2; total time=   0.0s\n",
            "[CV] END ......alpha=0.0001, loss=modified_huber, penalty=l2; total time=   0.0s\n",
            "[CV] END ......alpha=0.0001, loss=modified_huber, penalty=l2; total time=   0.0s\n",
            "{'penalty': 'l2', 'loss': 'hinge', 'alpha': 0.0001}\n",
            "SGDClassifier(class_weight='balanced')\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.89      0.86      0.88        57\n",
            "         1.0       0.86      0.89      0.87        54\n",
            "\n",
            "    accuracy                           0.87       111\n",
            "   macro avg       0.87      0.87      0.87       111\n",
            "weighted avg       0.87      0.87      0.87       111\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.linear_model import SGDClassifier\n",
        "\n",
        "SGDC = SGDClassifier(class_weight='balanced')\n",
        "\n",
        "# Create the random grid\n",
        "random_grid = { \"loss\" : [\"hinge\", \"log\", \"squared_hinge\", \"modified_huber\", \"perceptron\"],\n",
        "                \"alpha\" : [0.0001, 0.001, 0.01, 0.1],\n",
        "                \"penalty\" : [\"l2\", \"l1\", \"elasticnet\", \"none\"]}\n",
        "\n",
        "grid = RandomizedSearchCV(SGDC, random_grid, cv=3, random_state=42, refit = True, verbose = 2)\n",
        "\n",
        "\n",
        "# fitting the model for grid search\n",
        "grid.fit(X_train, Y_train)\n",
        "# print best parameter after tuning\n",
        "print(grid.best_params_)\n",
        "# print how our model looks after hyper-parameter tuning\n",
        "print(grid.best_estimator_)\n",
        "\n",
        "grid_predictions = grid.predict(X_test)\n",
        "\n",
        "# print classification report\n",
        "print(classification_report(Y_test, grid_predictions))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6YFlVLMiLrSK"
      },
      "source": [
        "## XGBoost"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7GHHPAwKLfjy",
        "outputId": "f0309e97-481b-4a81-9e7a-4810e38c5685"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=3, n_estimators=500; total time=   2.1s\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=3, n_estimators=500; total time=   2.3s\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=3, n_estimators=500; total time=   2.2s\n",
            "[CV] END gamma=0.3, learning_rate=0.15, max_depth=6, min_child_weight=7, n_estimators=1500; total time=   3.6s\n",
            "[CV] END gamma=0.3, learning_rate=0.15, max_depth=6, min_child_weight=7, n_estimators=1500; total time=   4.7s\n",
            "[CV] END gamma=0.3, learning_rate=0.15, max_depth=6, min_child_weight=7, n_estimators=1500; total time=   3.5s\n",
            "[CV] END gamma=0.2, learning_rate=0.05, max_depth=10, min_child_weight=1, n_estimators=100; total time=   7.1s\n",
            "[CV] END gamma=0.2, learning_rate=0.05, max_depth=10, min_child_weight=1, n_estimators=100; total time=   7.9s\n",
            "[CV] END gamma=0.2, learning_rate=0.05, max_depth=10, min_child_weight=1, n_estimators=100; total time=   7.7s\n",
            "[CV] END gamma=0.3, learning_rate=0.05, max_depth=15, min_child_weight=5, n_estimators=1500; total time=   6.4s\n",
            "[CV] END gamma=0.3, learning_rate=0.05, max_depth=15, min_child_weight=5, n_estimators=1500; total time=   7.5s\n",
            "[CV] END gamma=0.3, learning_rate=0.05, max_depth=15, min_child_weight=5, n_estimators=1500; total time=   6.8s\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=5, n_estimators=100; total time=   1.1s\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=5, n_estimators=100; total time=   1.2s\n",
            "[CV] END gamma=0.2, learning_rate=0.2, max_depth=5, min_child_weight=5, n_estimators=100; total time=   1.2s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=15, min_child_weight=7, n_estimators=100; total time=   1.1s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=15, min_child_weight=7, n_estimators=100; total time=   1.8s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=15, min_child_weight=7, n_estimators=100; total time=   1.7s\n",
            "[CV] END gamma=0.4, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=1500; total time=   9.9s\n",
            "[CV] END gamma=0.4, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=1500; total time=  12.5s\n",
            "[CV] END gamma=0.4, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=1500; total time=  11.7s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=6, min_child_weight=1, n_estimators=1500; total time=   9.8s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=6, min_child_weight=1, n_estimators=1500; total time=   8.7s\n",
            "[CV] END gamma=0.2, learning_rate=0.15, max_depth=6, min_child_weight=1, n_estimators=1500; total time=  10.0s\n",
            "[CV] END gamma=0.4, learning_rate=0.05, max_depth=5, min_child_weight=7, n_estimators=1100; total time=   4.6s\n",
            "[CV] END gamma=0.4, learning_rate=0.05, max_depth=5, min_child_weight=7, n_estimators=1100; total time=   6.3s\n",
            "[CV] END gamma=0.4, learning_rate=0.05, max_depth=5, min_child_weight=7, n_estimators=1100; total time=   5.6s\n",
            "[CV] END gamma=0.0, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=500; total time=   7.5s\n",
            "[CV] END gamma=0.0, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=500; total time=   6.2s\n",
            "[CV] END gamma=0.0, learning_rate=0.1, max_depth=5, min_child_weight=1, n_estimators=500; total time=   6.7s\n",
            "{'n_estimators': 100, 'min_child_weight': 7, 'max_depth': 15, 'learning_rate': 0.15, 'gamma': 0.2}\n",
            "XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
            "              colsample_bylevel=None, colsample_bynode=None,\n",
            "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
            "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
            "              gamma=0.2, grow_policy=None, importance_type=None,\n",
            "              interaction_constraints=None, learning_rate=0.15, max_bin=None,\n",
            "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
            "              max_delta_step=None, max_depth=15, max_leaves=None,\n",
            "              min_child_weight=7, missing=nan, monotone_constraints=None,\n",
            "              multi_strategy=None, n_estimators=100, n_jobs=None,\n",
            "              num_parallel_tree=None, random_state=None, ...)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.83      0.91      0.87        57\n",
            "         1.0       0.90      0.80      0.84        54\n",
            "\n",
            "    accuracy                           0.86       111\n",
            "   macro avg       0.86      0.85      0.85       111\n",
            "weighted avg       0.86      0.86      0.86       111\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from xgboost import XGBClassifier\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.utils.class_weight import compute_sample_weight\n",
        "\n",
        "sample_weights = compute_sample_weight(\n",
        "    class_weight='balanced',\n",
        "    y=Y_train\n",
        ")\n",
        "\n",
        "xgb = XGBClassifier()\n",
        "\n",
        "# Create the random grid\n",
        "random_grid = { \"learning_rate\" : [0.05,0.1,0.15,0.20],\n",
        "                 \"n_estimators\" : [100, 500, 900, 1100, 1500],\n",
        "                 \"max_depth\" : [ 3, 5, 6, 10, 15],\n",
        "                 \"min_child_weight\" : [ 1, 3, 5, 7 ],\n",
        "                 \"gamma\" : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ]}\n",
        "\n",
        "grid = RandomizedSearchCV(xgb, random_grid, cv=3, random_state=42, refit = True, verbose = 2)\n",
        "\n",
        "# fitting the model for grid search\n",
        "grid.fit(X_train, Y_train, sample_weight=sample_weights)\n",
        "# print best parameter after tuning\n",
        "print(grid.best_params_)\n",
        "# print how our model looks after hyper-parameter tuning\n",
        "print(grid.best_estimator_)\n",
        "\n",
        "grid_predictions = grid.predict(X_test)\n",
        "\n",
        "# print classification report\n",
        "print(classification_report(Y_test, grid_predictions))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MM-c-yWFLfWr"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "veT8NnZUxwj1"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sKvr_zbWqn0Y"
      },
      "source": [
        "https://huggingface.co/docs/transformers/model_doc/roberta"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O8qnEDROo6Jl"
      },
      "outputs": [],
      "source": [
        "# from transformers import RobertaConfig, RobertaModel, TFRobertaModel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UGh1p8n-o64K"
      },
      "outputs": [],
      "source": [
        "# config = RobertaConfig.from_pretrained(model)\n",
        "\n",
        "# config.output_hidden_states = False\n",
        "\n",
        "# model = TFRobertaModel.from_pretrained(model, from_pt=True, config = config)\n",
        "\n",
        "# embedding_layer = model.roberta(input_ids,attention_mask=attention_mask,token_type_ids=segs)[0]\n",
        "\n",
        "# Features = embedding_layer[:,0,:].numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ytOhGGiMQbIs"
      },
      "outputs": [],
      "source": [
        "# config = RobertaConfig.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\")\n",
        "\n",
        "# config.output_hidden_states = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "evT2tgEgQh7n"
      },
      "outputs": [],
      "source": [
        "# model = TFRobertaModel.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\", from_pt=True, config = config)\n",
        "\n",
        "# embedding_layer = model.roberta(padded,attention_mask=list(attention_mask) )[0]\n",
        "\n",
        "# Features = embedding_layer[:,0,:].numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hboRv3uuRSyb"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPYB5B2pl8DykKOT5OIVlpS"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "80d2649073f44c3eb5be4ebb3019f243": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_271cdf2d5b604e0fa8d895d51992c637",
              "IPY_MODEL_039cf63c73b44a16b3c9e6adfdaf094d",
              "IPY_MODEL_8624338b21554f15876d4bcdc736cd3f"
            ],
            "layout": "IPY_MODEL_9b61f6b533c04b08a3fc5d0959b82697"
          }
        },
        "271cdf2d5b604e0fa8d895d51992c637": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68ca276b935c423ea93c223080dbe3d8",
            "placeholder": "​",
            "style": "IPY_MODEL_e4cb0246af854e8091676559610be5f4",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "039cf63c73b44a16b3c9e6adfdaf094d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9ec173ee8774f7880ae0479ec6256eb",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9a09f047b6164f8b9479236cbc7aa91c",
            "value": 2
          }
        },
        "8624338b21554f15876d4bcdc736cd3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a7b0a33fb6145e1bfbf83b7cf8b27f3",
            "placeholder": "​",
            "style": "IPY_MODEL_4d095418a491433f8b53bbbc05f54456",
            "value": " 2.00/2.00 [00:00&lt;00:00, 63.2B/s]"
          }
        },
        "9b61f6b533c04b08a3fc5d0959b82697": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68ca276b935c423ea93c223080dbe3d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4cb0246af854e8091676559610be5f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9ec173ee8774f7880ae0479ec6256eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a09f047b6164f8b9479236cbc7aa91c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8a7b0a33fb6145e1bfbf83b7cf8b27f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d095418a491433f8b53bbbc05f54456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "06cc1a1d3bcd487fad7e623c7bd276ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e6d45d7a5dfb4a68b2e9cb37e0cbb35f",
              "IPY_MODEL_5e9106e717d4419594a0cf6fc33c6ee0",
              "IPY_MODEL_22fe48dd75fe49f0a68d5b10ae3e6912"
            ],
            "layout": "IPY_MODEL_df4ee49cb1a94f688708bbdfbf05745b"
          }
        },
        "e6d45d7a5dfb4a68b2e9cb37e0cbb35f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed66a21ce71b47ceaee08e22770d66eb",
            "placeholder": "​",
            "style": "IPY_MODEL_26414fb2827049b391b53a9e2f855d5c",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "5e9106e717d4419594a0cf6fc33c6ee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fa2a9ae9a897449790f33ddd9ba60cda",
            "max": 459,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9769fc998e224b69a3212121e38c72e2",
            "value": 459
          }
        },
        "22fe48dd75fe49f0a68d5b10ae3e6912": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_019138d27c23407980bccfdf0de5b3c0",
            "placeholder": "​",
            "style": "IPY_MODEL_7cd275af43ec4f56bcf4981bed1fdb3c",
            "value": " 459/459 [00:00&lt;00:00, 19.9kB/s]"
          }
        },
        "df4ee49cb1a94f688708bbdfbf05745b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed66a21ce71b47ceaee08e22770d66eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26414fb2827049b391b53a9e2f855d5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fa2a9ae9a897449790f33ddd9ba60cda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9769fc998e224b69a3212121e38c72e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "019138d27c23407980bccfdf0de5b3c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7cd275af43ec4f56bcf4981bed1fdb3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0450238e81c84703a3d4412eb84a3917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fcdbeaf38a7a4b9a868a2492fd4aa565",
              "IPY_MODEL_1be464bf51064963853e5bcd13bfb3ad",
              "IPY_MODEL_00c509489e70477285879909827b8102"
            ],
            "layout": "IPY_MODEL_72767f75b4c84b69be8ddd2fc3b654f8"
          }
        },
        "fcdbeaf38a7a4b9a868a2492fd4aa565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7297c8092e054fdb8c9a2d4e044e434b",
            "placeholder": "​",
            "style": "IPY_MODEL_ee98f9c03c714022804edf03bc088e3b",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "1be464bf51064963853e5bcd13bfb3ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be8fa7813a80411b8fd2be9c7b0136c8",
            "max": 529930,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83eb856bbeb447d0988be09daf37332c",
            "value": 529930
          }
        },
        "00c509489e70477285879909827b8102": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_01ce1faab1ad497ab2e395e1950ea24b",
            "placeholder": "​",
            "style": "IPY_MODEL_48c1d12c24d64522ab48c0a95f19dab7",
            "value": " 530k/530k [00:00&lt;00:00, 5.00MB/s]"
          }
        },
        "72767f75b4c84b69be8ddd2fc3b654f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7297c8092e054fdb8c9a2d4e044e434b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee98f9c03c714022804edf03bc088e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "be8fa7813a80411b8fd2be9c7b0136c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83eb856bbeb447d0988be09daf37332c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "01ce1faab1ad497ab2e395e1950ea24b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "48c1d12c24d64522ab48c0a95f19dab7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "724c0e83e7624a27b365dfff5cac7c2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a12e7dffa2324425b39374d8eae0be1e",
              "IPY_MODEL_3300bc1ab3b646b2aba5aca61f721c4e",
              "IPY_MODEL_0eb85797f7014f258ada344325c5d9a2"
            ],
            "layout": "IPY_MODEL_9cab660a664c4eb5ab776d62e9ebbe37"
          }
        },
        "a12e7dffa2324425b39374d8eae0be1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7709efbd2f6a4c0eabf5230b3d52aa55",
            "placeholder": "​",
            "style": "IPY_MODEL_039ad0eb3b5c4f3997b501efb5d5fb2e",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "3300bc1ab3b646b2aba5aca61f721c4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20c8e11d67034b3296df0e097d809831",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fe23771f840248e8ba5abaf1690ee7bb",
            "value": 112
          }
        },
        "0eb85797f7014f258ada344325c5d9a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fdbd536d2a5448e89fed86a4dabfbaea",
            "placeholder": "​",
            "style": "IPY_MODEL_918acb5d01154b708cd37797e7a731b2",
            "value": " 112/112 [00:00&lt;00:00, 5.97kB/s]"
          }
        },
        "9cab660a664c4eb5ab776d62e9ebbe37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7709efbd2f6a4c0eabf5230b3d52aa55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "039ad0eb3b5c4f3997b501efb5d5fb2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20c8e11d67034b3296df0e097d809831": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe23771f840248e8ba5abaf1690ee7bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fdbd536d2a5448e89fed86a4dabfbaea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "918acb5d01154b708cd37797e7a731b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0817103780514c2285bb1de226e3d434": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_64fb71ff40e64a1c951119a6edde8961",
              "IPY_MODEL_faf310fbcc63418eb131f94d4bd5711a",
              "IPY_MODEL_4606fe1118854e9f8ef40f23ade66c48"
            ],
            "layout": "IPY_MODEL_48acfd99cf214949ba2756d03b0d3510"
          }
        },
        "64fb71ff40e64a1c951119a6edde8961": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_684b8bc8715243b1ac6e170aeba6cdc7",
            "placeholder": "​",
            "style": "IPY_MODEL_80ca82c21fb84f319195ecac4c3e4c38",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "faf310fbcc63418eb131f94d4bd5711a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b1ccae05f5248d68b0d7caa35e6741c",
            "max": 454248854,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e06d92fe00164b03be9e5cb0618cc995",
            "value": 454248854
          }
        },
        "4606fe1118854e9f8ef40f23ade66c48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9832abd5976b41d8bf0e8b6f7e7a79c8",
            "placeholder": "​",
            "style": "IPY_MODEL_a29f82f6037f4800894f566a2821e964",
            "value": " 454M/454M [00:20&lt;00:00, 27.5MB/s]"
          }
        },
        "48acfd99cf214949ba2756d03b0d3510": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "684b8bc8715243b1ac6e170aeba6cdc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80ca82c21fb84f319195ecac4c3e4c38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2b1ccae05f5248d68b0d7caa35e6741c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e06d92fe00164b03be9e5cb0618cc995": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9832abd5976b41d8bf0e8b6f7e7a79c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a29f82f6037f4800894f566a2821e964": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}