Multiplethread.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "3UovGyzBEbEw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4c3e633b-2868-49a4-d654-4a41048b9be3"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from nltk import tokenize\n",
        "import matplotlib.pyplot as plt\n",
        "import html\n",
        "import re\n",
        "import random\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "#if there is a problem in importing transformer\n",
        "# please install using pip from the code below\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import os.path\n",
        "from dateutil import parser\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VpkgfQbxeBtU",
        "outputId": "e8bb869b-bf9f-4ec7-897b-e392f7fb29b7"
      },
      "source": [
        "url = input(\"Enter the url:\\n\")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter the url:\n",
            "https://arstechnica.com/gadgets/2021/07/morgan-stanley-discloses-data-breach-that-resulted-from-accellion-fta-hacks/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sU-pj5cMglK"
      },
      "source": [
        "# !pip install trafilatura\n",
        "# !pip install transformers\n",
        "# !pip install sentencepiece"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "5H8Syr_1MlLS",
        "outputId": "c1613a42-479f-4454-a3f3-666161b494b3"
      },
      "source": [
        "import trafilatura\n",
        "downloaded = trafilatura.fetch_url(url)\n",
        "trafilatura.extract(downloaded)\n",
        "# outputs main content and comments as plain text ...\n",
        "from trafilatura import bare_extraction\n",
        "bare_extraction(downloaded)['title']"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Morgan Stanley discloses data breach that resulted from Accellion FTA hacks'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O3T9Zu9nMrDi",
        "outputId": "681d4cf9-9a2e-487a-cf79-6f15d3a8517e"
      },
      "source": [
        "#Calculating the number of questions as needed depending upon the length of text\n",
        "import math\n",
        "mohack = []\n",
        "text = bare_extraction(downloaded)['text']\n",
        "mohack=text.splitlines()\n",
        "number_of_threads = math.trunc(len(text)/3200)\n",
        "if number_of_threads == 0:\n",
        "  number_of_threads += 1\n",
        "number_of_threads"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BPU9aGXfMvsV",
        "outputId": "ea03b0ac-23a1-4a96-a4a5-aa949ccdfa0d"
      },
      "source": [
        "!git clone https://github.com/amontgomerie/question_generator"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'question_generator' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ozd5R288Mxqu",
        "outputId": "19f42afc-ca5a-46f7-a862-4059c61a3d5f"
      },
      "source": [
        "%cd question_generator/"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/question_generator\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9jQqqO24M0q8"
      },
      "source": [
        "from questiongenerator import QuestionGenerator\n",
        "qg = QuestionGenerator()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qDZYunNiM1Pm",
        "outputId": "227fe0f9-2d37-437b-dc6c-76a589f3380a"
      },
      "source": [
        "import torch\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "device"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "njYUbd-UM3ak",
        "outputId": "4aa5d8e0-e393-483b-90e9-80cfb8df37ba"
      },
      "source": [
        "lp = qg.generate(text, num_questions=number_of_threads,answer_style='sentences')"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generating questions...\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/models/t5/tokenization_t5.py:191: UserWarning: This sequence already has </s>. In future versions this behavior may lead to duplicated eos tokens being added.\n",
            "  f\"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated eos tokens being added.\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Evaluating QA pairs...\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zToblPOTxW0z",
        "outputId": "179240c2-9806-43ff-8440-6baa9a378492"
      },
      "source": [
        "lp"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'answer': '</s> Guidehouse representatives didn’t immediately respond to an email asking why it took so long for the company to discover the breach, notify customers, and discover if other Guidehouse customers were also compromised.',\n",
              "  'question': 'How long did it take for the company to find out about the breach?'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J4b83GMSM5mE",
        "outputId": "09228c85-7bc7-4a44-f86b-f35de4d4eaf3"
      },
      "source": [
        "for i in range(number_of_threads):\n",
        "  mohack.append(lp[i]['question'])\n",
        "\n",
        "import gensim\n",
        "# upgrade gensim if you can't import softcossim\n",
        "from gensim.matutils import softcossim \n",
        "from gensim import corpora\n",
        "import gensim.downloader as api\n",
        "from gensim.utils import simple_preprocess\n",
        "print(gensim.__version__)\n",
        "#> '3.6.0'"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.6.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZNvZKeFl2GX"
      },
      "source": [
        "mohack.append(bare_extraction(downloaded)['title'])\n",
        "number_of_threads += 1"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CYd80I4lphLA"
      },
      "source": [
        "lp.append({\"question\": bare_extraction(downloaded)['title']})"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y53zLMsRrkKS",
        "outputId": "5aba24fa-5807-4a68-e08d-8456bd0da06a"
      },
      "source": [
        "lp"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'answer': '</s> Guidehouse representatives didn’t immediately respond to an email asking why it took so long for the company to discover the breach, notify customers, and discover if other Guidehouse customers were also compromised.',\n",
              "  'question': 'How long did it take for the company to find out about the breach?'},\n",
              " {'question': 'Morgan Stanley discloses data breach that resulted from Accellion FTA hacks'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vNzlbmt7M9eW"
      },
      "source": [
        "\n",
        "fasttext_model300 = api.load('fasttext-wiki-news-subwords-300')"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0b-5BPJxQuz_"
      },
      "source": [
        "# Prepare a dictionary and a corpus.\n",
        "dictionary = corpora.Dictionary([simple_preprocess(doc) for doc in mohack])\n",
        "\n",
        "# Prepare the similarity matrix\n",
        "similarity_matrix = fasttext_model300.similarity_matrix(dictionary, tfidf=None, threshold=0.0, exponent=2.0, nonzero_limit=100)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "snOmfbPPM_gI",
        "outputId": "8f1fade7-7f81-41c5-9248-e048a32f9a17"
      },
      "source": [
        "sentences = []\n",
        "for i in range(len(mohack)):\n",
        "  sentences.append(dictionary.doc2bow(simple_preprocess(mohack[i])))\n",
        "print(softcossim(sentences[0], sentences[1], similarity_matrix))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.6276910053845753\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BV86GoI9NCQG"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "def create_soft_cossim_matrix(sentences):\n",
        "    len_array = np.arange(len(sentences))\n",
        "    xx, yy = np.meshgrid(len_array, len_array)\n",
        "    cossim_mat = pd.DataFrame([[round(softcossim(sentences[i],sentences[j], similarity_matrix) ,2) for i, j in zip(x,y)] for y, x in zip(xx, yy)])\n",
        "    return cossim_mat\n",
        "\n",
        "matrix =create_soft_cossim_matrix(sentences)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 700
        },
        "id": "QgGdBAZoj9Jn",
        "outputId": "7022b351-bc97-4b5f-fa76-5517b69bc354"
      },
      "source": [
        "matrix"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.00</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.36</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.63</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.15</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.69</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.67</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.37</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.18</td>\n",
              "      <td>0.15</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.14</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.14</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.20</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.63</td>\n",
              "      <td>0.73</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.20</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0.57</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.57</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.70</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.67</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0.46</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.08</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>0.53</td>\n",
              "      <td>0.69</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.64</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0.70</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.31</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0.00</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.05</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0.45</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.12</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.70</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.07</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.32</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>0.47</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.16</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.61</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>0.50</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.14</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.53</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>0.59</td>\n",
              "      <td>0.67</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.03</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0.49</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.14</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.74</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.66</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>0.47</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.13</td>\n",
              "      <td>0.02</td>\n",
              "      <td>0.72</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.67</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.62</td>\n",
              "      <td>0.70</td>\n",
              "      <td>0.03</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.18</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.66</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0.48</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.64</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.06</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.58</td>\n",
              "      <td>0.64</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0.41</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.63</td>\n",
              "      <td>0.57</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.65</td>\n",
              "      <td>0.51</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.29</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>0.53</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.47</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.30</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.30</td>\n",
              "      <td>0.31</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.32</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.24</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.27</td>\n",
              "      <td>0.23</td>\n",
              "      <td>0.29</td>\n",
              "      <td>0.29</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      0     1     2     3     4     5   ...    15    16    17    18    19    20\n",
              "0   1.00  0.63  0.18  0.20  0.63  0.36  ...  0.59  0.49  0.47  0.48  0.41  0.53\n",
              "1   0.63  1.00  0.15  0.13  0.73  0.45  ...  0.67  0.61  0.64  0.60  0.56  0.37\n",
              "2   0.18  0.15  1.00  0.00  0.24  0.45  ...  0.18  0.14  0.13  0.21  0.49  0.07\n",
              "3   0.20  0.13  0.00  1.00  0.20  0.00  ...  0.02  0.00  0.02  0.20  0.00  0.43\n",
              "4   0.63  0.73  0.24  0.20  1.00  0.60  ...  0.75  0.75  0.72  0.64  0.63  0.47\n",
              "5   0.36  0.45  0.45  0.00  0.60  1.00  ...  0.48  0.47  0.50  0.51  0.57  0.19\n",
              "6   0.57  0.62  0.18  0.00  0.71  0.57  ...  0.57  0.65  0.67  0.63  0.55  0.30\n",
              "7   0.00  0.00  0.00  0.00  0.00  0.00  ...  0.00  0.00  0.00  0.00  0.00  0.00\n",
              "8   0.46  0.62  0.16  0.03  0.65  0.48  ...  0.62  0.60  0.62  0.51  0.59  0.30\n",
              "9   0.53  0.69  0.22  0.02  0.74  0.45  ...  0.74  0.72  0.70  0.50  0.55  0.31\n",
              "10  0.00  0.02  0.00  0.13  0.00  0.00  ...  0.07  0.04  0.03  0.06  0.00  0.00\n",
              "11  0.45  0.56  0.12  0.02  0.60  0.41  ...  0.48  0.54  0.61  0.58  0.51  0.32\n",
              "12  0.47  0.57  0.16  0.02  0.65  0.45  ...  0.61  0.56  0.58  0.60  0.47  0.24\n",
              "13  0.50  0.65  0.14  0.02  0.66  0.45  ...  0.71  0.60  0.64  0.50  0.54  0.24\n",
              "14  0.00  0.00  0.00  0.00  0.03  0.00  ...  0.03  0.00  0.18  0.00  0.00  0.00\n",
              "15  0.59  0.67  0.18  0.02  0.75  0.48  ...  1.00  0.74  0.65  0.51  0.54  0.29\n",
              "16  0.49  0.61  0.14  0.00  0.75  0.47  ...  0.74  1.00  0.66  0.58  0.60  0.27\n",
              "17  0.47  0.64  0.13  0.02  0.72  0.50  ...  0.65  0.66  1.00  0.64  0.65  0.23\n",
              "18  0.48  0.60  0.21  0.20  0.64  0.51  ...  0.51  0.58  0.64  1.00  0.51  0.29\n",
              "19  0.41  0.56  0.49  0.00  0.63  0.57  ...  0.54  0.60  0.65  0.51  1.00  0.29\n",
              "20  0.53  0.37  0.07  0.43  0.47  0.19  ...  0.29  0.27  0.23  0.29  0.29  1.00\n",
              "\n",
              "[21 rows x 21 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xWF14khqNEVW",
        "outputId": "b9c9319c-cd93-416c-8439-63b51ffbefbf"
      },
      "source": [
        "#now we will extract the paragraphs which are related to question and then \n",
        "#generate thier summary \n",
        "context = {}\n",
        "for i in range(1,number_of_threads + 1):\n",
        "  lm = []\n",
        "  for j in range(0,len(matrix.iloc[-i])-number_of_threads):\n",
        "    #only strong similarity connection paragraphs would be appended\n",
        "    if matrix.iloc[-i][j] >= 0.53 and matrix.iloc[-i][j] < 1.00 :\n",
        "      lm.append(j) \n",
        "    context[lp[-i]['question']] = lm \n",
        "#context\n",
        "m = []\n",
        "for i,j in context.items():\n",
        "  if j == []:\n",
        "    m.append(i)\n",
        "for k in range(len(m)):\n",
        "  context.pop(m[k])\n",
        "context"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'How long did it take for the company to find out about the breach?': [1,\n",
              "  4,\n",
              "  5,\n",
              "  6,\n",
              "  8,\n",
              "  9,\n",
              "  13,\n",
              "  15,\n",
              "  16,\n",
              "  17],\n",
              " 'Morgan Stanley discloses data breach that resulted from Accellion FTA hacks': [0]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zwIAgk1Yj7ne"
      },
      "source": [
        ""
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcru9JoCNMUs"
      },
      "source": [
        "from transformers import BartTokenizer, BartModel\n",
        "import torch\n",
        "#for the process of abstractive summmarization I have used Bart-cnn from hugging face as pretrained model\n",
        "from typing import List\n",
        "#bart-cnn uses encoder and decoder for summarization\n",
        "#but it's only effective for tokens less than 1300 around\n",
        "from transformers import BartForConditionalGeneration\n",
        "def old_summarization_pipeline(text: List[str]) -> List[str]:\n",
        "    tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')\n",
        "    model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')\n",
        "    input_ids = tokenizer.batch_encode_plus(text, truncation=True, padding=True, return_tensors='pt', max_length=1024)['input_ids']\n",
        "    summary_ids = model.generate(input_ids)\n",
        "    summaries = [tokenizer.decode(s, skip_special_tokens=True, clean_up_tokenization_spaces=False) for s in summary_ids]\n",
        "    return summaries"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o9IsFuZfNObu",
        "outputId": "a6fc1756-c2a4-4002-c9ff-7199311ef5ed"
      },
      "source": [
        "for i,j in context.items():\n",
        "  lm = [] \n",
        "  for k in range(len(j)):\n",
        "    lm.append(old_summarization_pipeline([mohack[j[k]]]))\n",
        "    context[i] = lm\n"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8HPMJmbWNPAy"
      },
      "source": [
        "import re\n",
        "twitterthread = {}\n",
        "for i,j in context.items():\n",
        " finalsummary = []\n",
        " lmo = []\n",
        " for k in range(len(j)):\n",
        "    #print(i,j[k])\n",
        "    lmo = re.split(r'(?<!\\w\\.\\w.)(?<![A-Z][a-z]\\.)(?<=\\.|\\?)\\s', j[k][0])\n",
        "    new_list = [elem for elem in lmo if elem.strip()]\n",
        "    finalsummary.append(new_list)\n",
        "    dict1= {}\n",
        "    d = 0\n",
        "    for l in range(len(finalsummary)):\n",
        "      for m in range(len(finalsummary[l])):\n",
        "        d = d+1\n",
        "        dict1[d] = finalsummary[l][m]\n",
        "    twitterthread[i] = dict1 "
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lY71kyNFuyZQ",
        "outputId": "2dcce25d-dc71-4145-f256-a4d722cd5cbe"
      },
      "source": [
        "twitterthread"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'How long did it take for the company to find out about the breach?': {1: 'The data obtained included names, addresses, dates of birth, Social Security numbers, and affiliated corporate company names.',\n",
              "  2: 'A third-party service called Guidehouse was in possession of the data at the time.',\n",
              "  3: 'Unknown hackers obtained the data by exploiting a series of hacks that came to light in December and January.',\n",
              "  4: 'The Accellion FTA vulnerability that led to this incident was patched in January 2021, within 5 days of the patch becoming available.',\n",
              "  5: 'Although the data was obtained by the unauthorized individual around that time, the vendor did not discover the attack until March of 2021.',\n",
              "  6: 'Guidehouse has informed Morgan Stanley that it found no evidence that Morgan Stanley’s data had been distributed beyond the threat actor.',\n",
              "  7: \"Guidehouse representatives didn't immediately respond to an email asking why it took so long for the company to discover the breach, notify customers, and discover if other Guidehouse customers were also compromised.\",\n",
              "  8: 'This post will be updated if a reply comes after publication.',\n",
              "  9: \"It's unclear if other customers were affected.\",\n",
              "  10: 'Accellion customers use the File Transfer Appliance as a secure alternative to email for sending large data files.',\n",
              "  11: 'Instead of receiving an attachment, email recipients get links to files hosted on the FTA.',\n",
              "  12: 'Although the product is almost 20 years old and Accellion has been transitioning customers to a newer product, the legacy FTA is still used by hundreds of organizations.',\n",
              "  13: 'Hackers exploited the vulnerabilities to install a web shell that gave them a text-based interface to install malware and issue other commands.',\n",
              "  14: 'Many of the hacked organizations later received extortion demands that threatened to publish stolen data on a dark web site affiliated with the Cl0p ransomware group unless they paid a ransom.',\n",
              "  15: 'The earliest detected activity in the hacking campaign came in mid-December.',\n",
              "  16: 'Mandiant identified the hackers exploiting an SQL injection vulnerability in the Accellion FTA.',\n",
              "  17: 'The exploit served as the initial intrusion point.',\n",
              "  18: 'Over time, the attackers exploited additional FTA vulnerabilities to gain enough control to install the web shell.',\n",
              "  19: 'Other organizations that researchers suspect were breached through the vulnerabilities include oil company Shell, security firm Qualys, gasoline retailer RaceTrac Petroleum, and the Reserve Bank of New Zealand.',\n",
              "  20: 'Last month, authorities in Ukraine arrested six suspected Cl0p affiliates.',\n",
              "  21: 'A week later, the dark web site used to publish data stolen through Cl0P ransomware posted new tranches, demonstrating that a core group of members remained active.',\n",
              "  22: 'In-the-wild exploits of the FTA vulnerabilities were first detected in late December.',\n",
              "  23: 'Mandiant initially said that it had notified all affected customers and fixed the zero-day vulnerabilities that enabled the attack within 72 hours of learning of them.',\n",
              "  24: 'Later, Mandiant discovered two additional zero-days.',\n",
              "  25: 'Some customers have complained in the past that Accellion was slow to provide notifications of the vulnerabilities under attack.',\n",
              "  26: 'Some customers have also complained that the company has been slow to notify them of the attacks.',\n",
              "  27: 'Accellions says it is working on a fix for the problem.',\n",
              "  28: '“We were over reliant on Accellion—the supplier of the file transfer application (FTA)—to alert us to any vulnerabilities in their system,” officials with New Zealand’s Reserve Bank said.',\n",
              "  29: '“In this instance, their notifications to us did not leave their system and hence did not reach the Reserve Bank in advance of the breach.',\n",
              "  30: 'We received no advance warning.”'},\n",
              " 'Morgan Stanley discloses data breach that resulted from Accellion FTA hacks': {1: 'Morgan Stanley suffered a data breach that exposed sensitive customer data.',\n",
              "  2: 'It became the latest known casualty of hackers exploiting a series of now-patched vulnerabilities in Accellion FTA, a widely used third-party file-transfer service.',\n",
              "  3: 'The breach exposed sensitive data on Morgan Stanley customers.'}}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m3MqRs4huzjY"
      },
      "source": [
        ""
      ],
      "execution_count": 26,
      "outputs": []
    }
  ]
}