NLTK Summarizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step [1]: Prepare libraries and data\n",
    "### [1.1] Include important libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "import heapq  \n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from nltk.corpus import stopwords\n",
    "punctuation = punctuation + '\\n'\n",
    "from nltk.stem.isri import ISRIStemmer\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import RidgeClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [1.2] Text categories:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories = ['Economy & Business', 'Diverse News', 'Politic', 'Sport', 'Technology']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [1.3] Building the summerizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nltk_summarizer(input_text, number_of_sentence):\n",
    "    stopWords = set(nltk.corpus.stopwords.words(\"arabic\") + nltk.corpus.stopwords.words(\"english\"))\n",
    "    word_frequencies = {}  \n",
    "    for word in nltk.word_tokenize(input_text):  \n",
    "        if word not in stopWords:\n",
    "            if word not in punctuation:\n",
    "                if word not in word_frequencies.keys():\n",
    "                    word_frequencies[word] = 1\n",
    "                else:\n",
    "                    word_frequencies[word] += 1\n",
    "\n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "\n",
    "    for word in word_frequencies.keys():  \n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "\n",
    "    sentence_list = nltk.sent_tokenize(input_text)\n",
    "    sentence_scores = {}  \n",
    "    for sent in sentence_list:  \n",
    "        for word in nltk.word_tokenize(sent.lower()):\n",
    "            if word in word_frequencies.keys():\n",
    "                if len(sent.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word]\n",
    "\n",
    "    summary_sentences = heapq.nlargest(number_of_sentence, sentence_scores, key=sentence_scores.get)\n",
    "\n",
    "    summary = ' '.join(summary_sentences)  \n",
    "    return summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [1.4] Reading dataset file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ArticleId</th>\n",
       "      <th>Text</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1432</th>\n",
       "      <td>1178</td>\n",
       "      <td>labour mps  fears over squabbling if there is ...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>870</td>\n",
       "      <td>voters  reject eu by two to one  british voter...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1360</th>\n",
       "      <td>165</td>\n",
       "      <td>ukip candidate suspended eurosceptic party uki...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>315</td>\n",
       "      <td>two nigerian banks set to merge nigerian banks...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>1170</td>\n",
       "      <td>pakistani women  must not hide  hiding women a...</td>\n",
       "      <td>politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1212</th>\n",
       "      <td>170</td>\n",
       "      <td>stars gear up for bafta ceremony film stars fr...</td>\n",
       "      <td>diverse news</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>2187</td>\n",
       "      <td>battered dollar hits another low the dollar ha...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>428</td>\n",
       "      <td>microsoft takes on desktop search microsoft ha...</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108</th>\n",
       "      <td>790</td>\n",
       "      <td>parker misses england clash tom shanklin will ...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>251</td>\n",
       "      <td>wenger handed summer war chest arsenal boss ar...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArticleId                                               Text  \\\n",
       "1432       1178  labour mps  fears over squabbling if there is ...   \n",
       "556         870  voters  reject eu by two to one  british voter...   \n",
       "1360        165  ukip candidate suspended eurosceptic party uki...   \n",
       "64          315  two nigerian banks set to merge nigerian banks...   \n",
       "480        1170  pakistani women  must not hide  hiding women a...   \n",
       "1212        170  stars gear up for bafta ceremony film stars fr...   \n",
       "996        2187  battered dollar hits another low the dollar ha...   \n",
       "301         428  microsoft takes on desktop search microsoft ha...   \n",
       "1108        790  parker misses england clash tom shanklin will ...   \n",
       "138         251  wenger handed summer war chest arsenal boss ar...   \n",
       "\n",
       "                Category  \n",
       "1432            politics  \n",
       "556             politics  \n",
       "1360            politics  \n",
       "64    economy & business  \n",
       "480             politics  \n",
       "1212        diverse news  \n",
       "996   economy & business  \n",
       "301                 tech  \n",
       "1108               sport  \n",
       "138                sport  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_data = pd.read_csv(r\"dataset/bbc_news_dataset.csv\")\n",
    "en_data = en_data.replace(\"entertainment\", \"diverse news\")\n",
    "en_data = en_data.replace(\"business\", \"economy & business\")\n",
    "\n",
    "ar_data = pd.read_csv(r\"dataset/arabic_dataset.csv\")\n",
    "ar_data = ar_data.replace(\"diverse\", \"diverse news\")\n",
    "ar_data = ar_data.replace(\"culture\", \"diverse news\")\n",
    "ar_data = ar_data.replace(\"politic\", \"politics\")\n",
    "ar_data = ar_data.replace(\"technology\", \"tech\")\n",
    "ar_data = ar_data.replace(\"economy\", \"economy & business\")\n",
    "ar_data = ar_data.replace(\"internationalNews\", \"politics\")\n",
    "ar_data = ar_data[~ar_data['type'].str.contains('localnews')]\n",
    "ar_data = ar_data[~ar_data['type'].str.contains('society')]\n",
    "en_data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>306</td>\n",
       "      <td>\\nتوفي اليوم القائد السابق للمنتخب الغيني و ال...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3341</th>\n",
       "      <td>417</td>\n",
       "      <td>\\nيواجه النجم الرياضي الساحلي اليوم إنطلاقا من...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4196</th>\n",
       "      <td>1272</td>\n",
       "      <td>\\nتمكن فريق الإسعاف التابع للإدارة الجهوية للح...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>14</td>\n",
       "      <td>\\nاستحوذت القروض لفائدة المؤسسات والمهنيين، عل...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4420</th>\n",
       "      <td>40</td>\n",
       "      <td>\\nتلقى حوالي مليوني مستخدم لهواتف \"أندرويد\"تنب...</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4192</th>\n",
       "      <td>1268</td>\n",
       "      <td>\\nأكد الأستاذ أنيس بن ميم محامي الدولي و لاعب ...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4347</th>\n",
       "      <td>1423</td>\n",
       "      <td>\\nصادق المكتب الجامعي لكرة القدم على الروزنامة...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>84</td>\n",
       "      <td>\\nأكد وزير الصناعة والطاقة والمناجم،زكريا حمد ...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3047</th>\n",
       "      <td>123</td>\n",
       "      <td>\\nبعد عدم توفر النصاب القانوني لعقد اجتماع مكت...</td>\n",
       "      <td>sport</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>403</td>\n",
       "      <td>\\nأفاد موقع العربية. نت نقلا عن وكالة \"بلومبير...</td>\n",
       "      <td>diverse news</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               text  \\\n",
       "3230         306  \\nتوفي اليوم القائد السابق للمنتخب الغيني و ال...   \n",
       "3341         417  \\nيواجه النجم الرياضي الساحلي اليوم إنطلاقا من...   \n",
       "4196        1272  \\nتمكن فريق الإسعاف التابع للإدارة الجهوية للح...   \n",
       "613           14  \\nاستحوذت القروض لفائدة المؤسسات والمهنيين، عل...   \n",
       "4420          40  \\nتلقى حوالي مليوني مستخدم لهواتف \"أندرويد\"تنب...   \n",
       "4192        1268  \\nأكد الأستاذ أنيس بن ميم محامي الدولي و لاعب ...   \n",
       "4347        1423  \\nصادق المكتب الجامعي لكرة القدم على الروزنامة...   \n",
       "683           84  \\nأكد وزير الصناعة والطاقة والمناجم،زكريا حمد ...   \n",
       "3047         123  \\nبعد عدم توفر النصاب القانوني لعقد اجتماع مكت...   \n",
       "527          403  \\nأفاد موقع العربية. نت نقلا عن وكالة \"بلومبير...   \n",
       "\n",
       "                    type  \n",
       "3230               sport  \n",
       "3341               sport  \n",
       "4196               sport  \n",
       "613   economy & business  \n",
       "4420                tech  \n",
       "4192               sport  \n",
       "4347               sport  \n",
       "683   economy & business  \n",
       "3047               sport  \n",
       "527         diverse news  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ar_data.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step [2]: Data sterilization\n",
    "### [2.1] Delete links:\n",
    "This will remove all links from the text and it's include the following:\n",
    "- Matches http protocols like [**http:// or https://**].\n",
    "- Match optional whitespaces after http protocols.\n",
    "- Optionally matches including the [**www.**] or not.\n",
    "- Optionally matches whitespaces in the links.\n",
    "- Matches 0 or more of one or more word characters followed by a period.\n",
    "- Matches 0 or more of one or more words (or a dash or a space) followed by [**\\\\**].\n",
    "- Any remaining path at the end of the url followed by an optional ending.\n",
    "- Matches ending query params (even with white spaces, etc)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_links(input_text):\n",
    "    pettern  = r'''(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\".,<>?«»“”‘’]))'''\n",
    "    out_text = re.sub(pettern, ' ', input_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [2.2] Fixing word lengthening:\n",
    "Word lengthening occurs when characters are wrongly repeated. English words have a max of two repeated characters like the words [**wood, school**]. Additional characters need to ripped off, otherwise we might add misleading information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_repeated_characters(input_text):\n",
    "    pattern  = r'(.)\\1{2,}'\n",
    "    out_text = re.sub(pattern, r\"\\1\\1\", input_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_letters(input_text):\n",
    "    replace = {\"أ\": \"ا\",\"ة\": \"ه\",\"إ\": \"ا\",\"آ\": \"ا\",\"\": \"\"}\n",
    "    replace = dict((re.escape(k), v) for k, v in replace.items()) \n",
    "    pattern = re.compile(\"|\".join(replace.keys()))\n",
    "    out_text = pattern.sub(lambda m: replace[re.escape(m.group(0))], input_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [2.3] Delete bad symbols:\n",
    "This method removes unwanted characters from the text, such as question marks, commas, star, plus ...etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(input_text):\n",
    "    replace = r'[/(){}\\[\\]|@âÂ,;\\?\\'\\\"\\*…؟–’،!&\\+-:؛-]'\n",
    "    out_text = re.sub(replace, \" \", input_text)\n",
    "    words = nltk.word_tokenize(out_text)\n",
    "    words = [word for word in words if word.isalpha()]\n",
    "    out_text = ' '.join(words)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_vowelization(input_text):\n",
    "    vowelization = re.compile(\"\"\" ّ|َ|ً|ُ|ٌ|ِ|ٍ|ْ|ـ\"\"\", re.VERBOSE)\n",
    "    out_text = re.sub(vowelization, '', input_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [2.4] Delete stopwords:\n",
    "Like prepositions and hyphens words. for example [**and, in, or ...etc**]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delete_stopwords(input_text):\n",
    "    stop_words = set(nltk.corpus.stopwords.words(\"arabic\") + nltk.corpus.stopwords.words(\"english\"))\n",
    "    tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "    tokens = tokenizer.tokenize(input_text)\n",
    "    wnl = nltk.WordNetLemmatizer()\n",
    "    lemmatizedTokens =[wnl.lemmatize(t) for t in tokens]\n",
    "    out_text = [w for w in lemmatizedTokens if not w in stop_words]\n",
    "    out_text = ' '.join(out_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem_text(input_text):\n",
    "    st = ISRIStemmer()\n",
    "    tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "    tokens = tokenizer.tokenize(input_text)\n",
    "    out_text = [st.stem(w) for w in tokens]\n",
    "    out_text = ' '.join(out_text)\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [2.5] Text prepare:\n",
    "- Applay all previus functions to sterilize the input text.\n",
    "- Convert letters to lowercase to make all words in the text in the same letters sensitivity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_prepare(input_text, ar_text):\n",
    "    out_text = delete_links(input_text)\n",
    "    out_text = delete_repeated_characters(out_text)\n",
    "    out_text = clean_text(out_text)\n",
    "    out_text = delete_stopwords(out_text)\n",
    "    if ar_text:\n",
    "        out_text = replace_letters(out_text)\n",
    "        out_text = remove_vowelization(out_text)\n",
    "        out_text = stem_text(out_text)\n",
    "    else:\n",
    "        out_text = out_text.lower()\n",
    "    return out_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ArticleId</th>\n",
       "      <th>Text</th>\n",
       "      <th>Category</th>\n",
       "      <th>Processed Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>1697</td>\n",
       "      <td>us airways staff agree to pay cut a union repr...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "      <td>u airway staff agree pay cut union representin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>1229</td>\n",
       "      <td>savvy searchers fail to spot ads internet sear...</td>\n",
       "      <td>tech</td>\n",
       "      <td>savvy searcher fail spot ad internet search en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296</th>\n",
       "      <td>1160</td>\n",
       "      <td>microsoft seeking spyware trojan microsoft is ...</td>\n",
       "      <td>tech</td>\n",
       "      <td>microsoft seeking spyware trojan microsoft inv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1429</th>\n",
       "      <td>1920</td>\n",
       "      <td>iraqis win death test case probe the family of...</td>\n",
       "      <td>politics</td>\n",
       "      <td>iraqi win death test case probe family iraqi c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>2179</td>\n",
       "      <td>hollywood ready for oscars night hollywood is ...</td>\n",
       "      <td>diverse news</td>\n",
       "      <td>hollywood ready oscar night hollywood preparin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1394</td>\n",
       "      <td>viewers to be able to shape tv imagine editing...</td>\n",
       "      <td>tech</td>\n",
       "      <td>viewer able shape tv imagine editing titanic w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>1716</td>\n",
       "      <td>roddick to face saulnier in final andy roddick...</td>\n",
       "      <td>sport</td>\n",
       "      <td>roddick face saulnier final andy roddick play ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>893</th>\n",
       "      <td>802</td>\n",
       "      <td>ireland 19-13 england ireland consigned englan...</td>\n",
       "      <td>sport</td>\n",
       "      <td>ireland england ireland consigned england thir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>2141</td>\n",
       "      <td>celts savour grand slam prospect the six natio...</td>\n",
       "      <td>sport</td>\n",
       "      <td>celt savour grand slam prospect six nation ha ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>1955</td>\n",
       "      <td>weak dollar trims cadbury profits the world s ...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "      <td>weak dollar trim cadbury profit world biggest ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArticleId                                               Text  \\\n",
       "991        1697  us airways staff agree to pay cut a union repr...   \n",
       "797        1229  savvy searchers fail to spot ads internet sear...   \n",
       "1296       1160  microsoft seeking spyware trojan microsoft is ...   \n",
       "1429       1920  iraqis win death test case probe the family of...   \n",
       "187        2179  hollywood ready for oscars night hollywood is ...   \n",
       "43         1394  viewers to be able to shape tv imagine editing...   \n",
       "467        1716  roddick to face saulnier in final andy roddick...   \n",
       "893         802  ireland 19-13 england ireland consigned englan...   \n",
       "293        2141  celts savour grand slam prospect the six natio...   \n",
       "909        1955  weak dollar trims cadbury profits the world s ...   \n",
       "\n",
       "                Category                                     Processed Text  \n",
       "991   economy & business  u airway staff agree pay cut union representin...  \n",
       "797                 tech  savvy searcher fail spot ad internet search en...  \n",
       "1296                tech  microsoft seeking spyware trojan microsoft inv...  \n",
       "1429            politics  iraqi win death test case probe family iraqi c...  \n",
       "187         diverse news  hollywood ready oscar night hollywood preparin...  \n",
       "43                  tech  viewer able shape tv imagine editing titanic w...  \n",
       "467                sport  roddick face saulnier final andy roddick play ...  \n",
       "893                sport  ireland england ireland consigned england thir...  \n",
       "293                sport  celt savour grand slam prospect six nation ha ...  \n",
       "909   economy & business  weak dollar trim cadbury profit world biggest ...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_data['Processed Text'] = en_data['Text'].apply(text_prepare, args=(False,))\n",
    "ar_data['Processed Text'] = ar_data['text'].apply(text_prepare, args=(True,))\n",
    "en_data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "      <th>Processed Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>193</td>\n",
       "      <td>\\nأعلنت السلطات الأوكرانية عن نفوق عشرات الدلا...</td>\n",
       "      <td>diverse news</td>\n",
       "      <td>اعل سلط اوكرانيه نفق عشر دلف دين ودس سحل قيل ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3741</th>\n",
       "      <td>817</td>\n",
       "      <td>\\nأكد توماس باخ، رئيس اللجنة الأولمبية الدولية...</td>\n",
       "      <td>sport</td>\n",
       "      <td>اكد ومس باخ رئس لجن لمب دول قئم ريض رفع ثقل يج...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3479</th>\n",
       "      <td>555</td>\n",
       "      <td>\\nصرّح حسن الذوادي، الأمين العام للجنة المنظمة...</td>\n",
       "      <td>sport</td>\n",
       "      <td>حسن ذدي امن عام جنه ظمه قطر حظر نول كحل اما عم...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3938</th>\n",
       "      <td>1014</td>\n",
       "      <td>\\nأعلن نادي ليون الفرنسي عن تعاقده اليوم الهول...</td>\n",
       "      <td>sport</td>\n",
       "      <td>اعل ندي لين رنس عقد اليوم هولند مفس ديب وصل تف...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4039</th>\n",
       "      <td>1115</td>\n",
       "      <td>\\nتمكن النجم الرياضي الرادسي من الترشح إلى الد...</td>\n",
       "      <td>sport</td>\n",
       "      <td>تمك نجم ريض ردس رشح دور نهي سبق كاس ونس لكر سل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3678</th>\n",
       "      <td>754</td>\n",
       "      <td>\\nتدور اليوم الاحد 7 ماي 2017 مباريات الجولة ا...</td>\n",
       "      <td>sport</td>\n",
       "      <td>تدر اليوم احد ماي بري جول تسع و قبل اخر رحل بل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4160</th>\n",
       "      <td>1236</td>\n",
       "      <td>\\nيخوض حارس النجم الرياضي الساحلي و قائدته أيم...</td>\n",
       "      <td>sport</td>\n",
       "      <td>يخض حرس نجم ريض سحل و قئد ايم ثلث لبل يوم احد ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>89</td>\n",
       "      <td>\\nقررت إدارة \"كانتون بازل لاند\" شمال سويسرا تع...</td>\n",
       "      <td>diverse news</td>\n",
       "      <td>قرر دره كنت بزل لند شمل يسر علق جرء جنس عئل لم...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2181</th>\n",
       "      <td>1254</td>\n",
       "      <td>\\nأعلنت وزارة الأمن الإيرانية اليوم الاثنين 20...</td>\n",
       "      <td>politics</td>\n",
       "      <td>اعل وزر امن يرن اليوم اثن جون حبط عدد خطط رهب ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1558</th>\n",
       "      <td>631</td>\n",
       "      <td>\\nوصف المرشد الأعلى لجمهورية إيران الإسلامية آ...</td>\n",
       "      <td>politics</td>\n",
       "      <td>وصف رشد على لجمهوريه اير سلم ايه الل خامنئ ثلا...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               text  \\\n",
       "317          193  \\nأعلنت السلطات الأوكرانية عن نفوق عشرات الدلا...   \n",
       "3741         817  \\nأكد توماس باخ، رئيس اللجنة الأولمبية الدولية...   \n",
       "3479         555  \\nصرّح حسن الذوادي، الأمين العام للجنة المنظمة...   \n",
       "3938        1014  \\nأعلن نادي ليون الفرنسي عن تعاقده اليوم الهول...   \n",
       "4039        1115  \\nتمكن النجم الرياضي الرادسي من الترشح إلى الد...   \n",
       "3678         754  \\nتدور اليوم الاحد 7 ماي 2017 مباريات الجولة ا...   \n",
       "4160        1236  \\nيخوض حارس النجم الرياضي الساحلي و قائدته أيم...   \n",
       "213           89  \\nقررت إدارة \"كانتون بازل لاند\" شمال سويسرا تع...   \n",
       "2181        1254  \\nأعلنت وزارة الأمن الإيرانية اليوم الاثنين 20...   \n",
       "1558         631  \\nوصف المرشد الأعلى لجمهورية إيران الإسلامية آ...   \n",
       "\n",
       "              type                                     Processed Text  \n",
       "317   diverse news  اعل سلط اوكرانيه نفق عشر دلف دين ودس سحل قيل ا...  \n",
       "3741         sport  اكد ومس باخ رئس لجن لمب دول قئم ريض رفع ثقل يج...  \n",
       "3479         sport  حسن ذدي امن عام جنه ظمه قطر حظر نول كحل اما عم...  \n",
       "3938         sport  اعل ندي لين رنس عقد اليوم هولند مفس ديب وصل تف...  \n",
       "4039         sport  تمك نجم ريض ردس رشح دور نهي سبق كاس ونس لكر سل...  \n",
       "3678         sport  تدر اليوم احد ماي بري جول تسع و قبل اخر رحل بل...  \n",
       "4160         sport  يخض حرس نجم ريض سحل و قئد ايم ثلث لبل يوم احد ...  \n",
       "213   diverse news  قرر دره كنت بزل لند شمل يسر علق جرء جنس عئل لم...  \n",
       "2181      politics  اعل وزر امن يرن اليوم اثن جون حبط عدد خطط رهب ...  \n",
       "1558      politics  وصف رشد على لجمهوريه اير سلم ايه الل خامنئ ثلا...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ar_data.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step [3]: Text Splitting and vectorizing\n",
    "### [3.1] Label encoding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ArticleId</th>\n",
       "      <th>Text</th>\n",
       "      <th>Category</th>\n",
       "      <th>Processed Text</th>\n",
       "      <th>Category Encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>577</td>\n",
       "      <td>robots learn  robotiquette  rules robots are l...</td>\n",
       "      <td>tech</td>\n",
       "      <td>robot learn robotiquette rule robot learning l...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>123</td>\n",
       "      <td>safety alert as gm recalls cars the world s bi...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "      <td>safety alert gm recall car world biggest carma...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>488</td>\n",
       "      <td>wolves appoint hoddle as manager glenn hoddle ...</td>\n",
       "      <td>sport</td>\n",
       "      <td>wolf appoint hoddle manager glenn hoddle unvei...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>447</td>\n",
       "      <td>broadband challenges tv viewing the number of ...</td>\n",
       "      <td>tech</td>\n",
       "      <td>broadband challenge tv viewing number european...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>891</td>\n",
       "      <td>wales win in rome wales secured their first aw...</td>\n",
       "      <td>sport</td>\n",
       "      <td>wale win rome wale secured first away win rb s...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1166</th>\n",
       "      <td>1682</td>\n",
       "      <td>fuming robinson blasts officials england coach...</td>\n",
       "      <td>sport</td>\n",
       "      <td>fuming robinson blast official england coach a...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>65</td>\n",
       "      <td>india power shares jump on debut shares in ind...</td>\n",
       "      <td>economy &amp; business</td>\n",
       "      <td>india power share jump debut share india large...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>519</th>\n",
       "      <td>1528</td>\n",
       "      <td>pm apology over jailings tony blair has apolog...</td>\n",
       "      <td>politics</td>\n",
       "      <td>pm apology jailings tony blair ha apologised t...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>207</td>\n",
       "      <td>lib dems highlight problem debt people vulnera...</td>\n",
       "      <td>politics</td>\n",
       "      <td>lib dems highlight problem debt people vulnera...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>413</td>\n",
       "      <td>blair labour s longest-serving pm tony blair h...</td>\n",
       "      <td>politics</td>\n",
       "      <td>blair labour longest serving pm tony blair ha ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ArticleId                                               Text  \\\n",
       "386         577  robots learn  robotiquette  rules robots are l...   \n",
       "393         123  safety alert as gm recalls cars the world s bi...   \n",
       "929         488  wolves appoint hoddle as manager glenn hoddle ...   \n",
       "429         447  broadband challenges tv viewing the number of ...   \n",
       "99          891  wales win in rome wales secured their first aw...   \n",
       "1166       1682  fuming robinson blasts officials england coach...   \n",
       "280          65  india power shares jump on debut shares in ind...   \n",
       "519        1528  pm apology over jailings tony blair has apolog...   \n",
       "836         207  lib dems highlight problem debt people vulnera...   \n",
       "168         413  blair labour s longest-serving pm tony blair h...   \n",
       "\n",
       "                Category                                     Processed Text  \\\n",
       "386                 tech  robot learn robotiquette rule robot learning l...   \n",
       "393   economy & business  safety alert gm recall car world biggest carma...   \n",
       "929                sport  wolf appoint hoddle manager glenn hoddle unvei...   \n",
       "429                 tech  broadband challenge tv viewing number european...   \n",
       "99                 sport  wale win rome wale secured first away win rb s...   \n",
       "1166               sport  fuming robinson blast official england coach a...   \n",
       "280   economy & business  india power share jump debut share india large...   \n",
       "519             politics  pm apology jailings tony blair ha apologised t...   \n",
       "836             politics  lib dems highlight problem debt people vulnera...   \n",
       "168             politics  blair labour longest serving pm tony blair ha ...   \n",
       "\n",
       "      Category Encoded  \n",
       "386                  4  \n",
       "393                  1  \n",
       "929                  3  \n",
       "429                  4  \n",
       "99                   3  \n",
       "1166                 3  \n",
       "280                  1  \n",
       "519                  2  \n",
       "836                  2  \n",
       "168                  2  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_label_encoder = LabelEncoder()\n",
    "en_data['Category Encoded'] = en_label_encoder.fit_transform(en_data['Category'])\n",
    "\n",
    "ar_label_encoder = LabelEncoder()\n",
    "ar_data['Category Encoded'] = ar_label_encoder.fit_transform(ar_data['type'])\n",
    "ar_data['Category Encoded'] = ar_data['Category Encoded'].replace(1, 0)\n",
    "ar_data['Category Encoded'] = ar_data['Category Encoded'].replace(0, 1)\n",
    "\n",
    "en_data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "      <th>Processed Text</th>\n",
       "      <th>Category Encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1496</th>\n",
       "      <td>569</td>\n",
       "      <td>\\nأكدت دول مجلس التعاون الخليجي جميعها في نهاي...</td>\n",
       "      <td>politics</td>\n",
       "      <td>اكد دول جلس تعا خلج جمع نهي قمت سني دوح ثلاثاء...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1420</th>\n",
       "      <td>493</td>\n",
       "      <td>\\n شدد الرئيس الأمريكي دونالد ترامب الأوامر ضد...</td>\n",
       "      <td>politics</td>\n",
       "      <td>شدد رئس امر دونالد رمب ومر ضد هجر جنب ولي تحد ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3256</th>\n",
       "      <td>332</td>\n",
       "      <td>\\nأسفرت مقابلات الجولة الثامنة من مرحلة تفادي ...</td>\n",
       "      <td>sport</td>\n",
       "      <td>سفر قبل جول ثمن رحل فدي نزل بطل قسم وطن الف نت...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>330</td>\n",
       "      <td>\\nيفيد شرب القهوة النساء الأكبر سنا كثيرا فالل...</td>\n",
       "      <td>diverse news</td>\n",
       "      <td>يفد شرب قهه نسء كبر سنا كثر فلل شرب كوب كوب قه...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4436</th>\n",
       "      <td>56</td>\n",
       "      <td>\\nكان الاعتقاد السائد منذ فترة طويلة أن الهوات...</td>\n",
       "      <td>tech</td>\n",
       "      <td>كان عقد سئد فتر طول هتف ذكه يمكن عرض قرص الكتر...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3405</th>\n",
       "      <td>481</td>\n",
       "      <td>\\nتدرك اليوم منافسات مرحلة التتويج من بطولة ال...</td>\n",
       "      <td>sport</td>\n",
       "      <td>تدر اليوم نفس رحل توج بطل قسم وطن الف لكر سله ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4300</th>\n",
       "      <td>1376</td>\n",
       "      <td>\\nأكد العقيد فهمي الرويسي في تصريح للقناة الوط...</td>\n",
       "      <td>sport</td>\n",
       "      <td>اكد عقد فهم روس صرح قنه وطن ولى حدث شغب شهد قب...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2966</th>\n",
       "      <td>42</td>\n",
       "      <td>\\nتتجه الأنظار في مختلف بقاع العالم اليوم إلى ...</td>\n",
       "      <td>sport</td>\n",
       "      <td>تتج ظار خلف بقع علم اليوم لعب كمب نو عقل ندي ر...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3377</th>\n",
       "      <td>453</td>\n",
       "      <td>\\nيسافر النجم الساحلي اليوم بداية من الساعة 10...</td>\n",
       "      <td>sport</td>\n",
       "      <td>سفر نجم سحل اليوم بدي سعه عصم سود خرطوم و رحل ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>141</td>\n",
       "      <td>\\nسهرية فينال 3 NESCAFÉ Comedy Show تتفرجو فيه...</td>\n",
       "      <td>diverse news</td>\n",
       "      <td>سهر ينل NESCAFÉ Comedy Show رجو يله تسع قنه حو...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                               text  \\\n",
       "1496         569  \\nأكدت دول مجلس التعاون الخليجي جميعها في نهاي...   \n",
       "1420         493  \\n شدد الرئيس الأمريكي دونالد ترامب الأوامر ضد...   \n",
       "3256         332  \\nأسفرت مقابلات الجولة الثامنة من مرحلة تفادي ...   \n",
       "454          330  \\nيفيد شرب القهوة النساء الأكبر سنا كثيرا فالل...   \n",
       "4436          56  \\nكان الاعتقاد السائد منذ فترة طويلة أن الهوات...   \n",
       "3405         481  \\nتدرك اليوم منافسات مرحلة التتويج من بطولة ال...   \n",
       "4300        1376  \\nأكد العقيد فهمي الرويسي في تصريح للقناة الوط...   \n",
       "2966          42  \\nتتجه الأنظار في مختلف بقاع العالم اليوم إلى ...   \n",
       "3377         453  \\nيسافر النجم الساحلي اليوم بداية من الساعة 10...   \n",
       "265          141  \\nسهرية فينال 3 NESCAFÉ Comedy Show تتفرجو فيه...   \n",
       "\n",
       "              type                                     Processed Text  \\\n",
       "1496      politics  اكد دول جلس تعا خلج جمع نهي قمت سني دوح ثلاثاء...   \n",
       "1420      politics  شدد رئس امر دونالد رمب ومر ضد هجر جنب ولي تحد ...   \n",
       "3256         sport  سفر قبل جول ثمن رحل فدي نزل بطل قسم وطن الف نت...   \n",
       "454   diverse news  يفد شرب قهه نسء كبر سنا كثر فلل شرب كوب كوب قه...   \n",
       "4436          tech  كان عقد سئد فتر طول هتف ذكه يمكن عرض قرص الكتر...   \n",
       "3405         sport  تدر اليوم نفس رحل توج بطل قسم وطن الف لكر سله ...   \n",
       "4300         sport  اكد عقد فهم روس صرح قنه وطن ولى حدث شغب شهد قب...   \n",
       "2966         sport  تتج ظار خلف بقع علم اليوم لعب كمب نو عقل ندي ر...   \n",
       "3377         sport  سفر نجم سحل اليوم بدي سعه عصم سود خرطوم و رحل ...   \n",
       "265   diverse news  سهر ينل NESCAFÉ Comedy Show رجو يله تسع قنه حو...   \n",
       "\n",
       "      Category Encoded  \n",
       "1496                 2  \n",
       "1420                 2  \n",
       "3256                 3  \n",
       "454                  1  \n",
       "4436                 4  \n",
       "3405                 3  \n",
       "4300                 3  \n",
       "2966                 3  \n",
       "3377                 3  \n",
       "265                  1  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ar_data.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [3.2] Splitting the data to train and text:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_X_train, en_X_test, en_y_train, en_y_test = train_test_split(en_data['Processed Text'], en_data['Category Encoded'], test_size=0.2, random_state=0)\n",
    "ar_X_train, ar_X_test, ar_y_train, ar_y_test = train_test_split(ar_data['Processed Text'], ar_data['Category Encoded'], test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [3.3] TF-IDF vectorizer:\n",
    "The second approach extends the bag-of-words framework by taking into account total frequencies of words in the corpora. It helps to penalize too frequent words and provide better features space. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfidf_features(X_train, X_test, ngram_range):\n",
    "    tfidf_vectorizer = TfidfVectorizer(min_df=2, max_df=0.5, ngram_range=(1, ngram_range))\n",
    "    X_train = tfidf_vectorizer.fit_transform(X_train)\n",
    "    X_test = tfidf_vectorizer.transform(X_test)\n",
    "    return X_train, X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_features_train, en_features_test = tfidf_features(en_X_train, en_X_test, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar_features_train, ar_features_test = tfidf_features(ar_X_train, ar_X_test, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [3.4] Fit model:\n",
    "This function used to generate a model based on it's name then train that model and calculate it's accuracy on test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_model(model_name, ar_text=False):\n",
    "    if model_name == 'ridge_model':\n",
    "        model_name = RidgeClassifier()\n",
    "    elif model_name == 'random_forest_model':\n",
    "        model_name = RandomForestClassifier()\n",
    "    elif model_name == 'logistic_regression_model':\n",
    "        model_name = LogisticRegression()\n",
    "    elif model_name == 'kneighbors_model':\n",
    "        model_name = KNeighborsClassifier()\n",
    "    elif model_name == 'decision_tree_model':\n",
    "        model_name = DecisionTreeClassifier()\n",
    "    elif model_name == 'gaussian_nb_model':\n",
    "        model_name = GaussianNB()\n",
    "    if ar_text:\n",
    "        model_name.fit(ar_features_train.toarray(), ar_y_train)\n",
    "        model_predictions = model_name.predict(ar_features_test.toarray())\n",
    "        print(\"Accuracy on test: \", accuracy_score(ar_y_test, model_predictions))\n",
    "    else:\n",
    "        model_name.fit(en_features_train.toarray(), en_y_train)\n",
    "        model_predictions = model_name.predict(en_features_test.toarray())\n",
    "        print(\"Accuracy on test: \", accuracy_score(en_y_test, model_predictions))\n",
    "    return model_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [3.5] Summerize and predict for input text:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summerize_category(input_text, statements, model_name, ar_text=False):\n",
    "    summary_text = nltk_summarizer(input_text, statements)\n",
    "    print(\"-------------------------------------------------------------------------------------------------------------------------------\")\n",
    "    print(\"Text summary\")\n",
    "    print(\"-------------------------------------------------------------------------------------------------------------------------------\")\n",
    "    print(summary_text)\n",
    "    print(\"-------------------------------------------------------------------------------------------------------------------------------\")\n",
    "    input_text_arr = [text_prepare(input_text, ar_text)]\n",
    "    if ar_text:\n",
    "        features_train, features_test = tfidf_features(ar_X_train, input_text_arr, 2)\n",
    "    else:\n",
    "        features_train, features_test = tfidf_features(en_X_train, input_text_arr, 2)\n",
    "    text_predection = model_name.predict(features_test.toarray())\n",
    "    print(\"Text category:\", categories[text_predection[0]])\n",
    "    print(\"-------------------------------------------------------------------------------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.1] Ridge classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.9832214765100671\n"
     ]
    }
   ],
   "source": [
    "en_ridge_model = fit_model('ridge_model')\n",
    "pickle.dump(en_ridge_model, open('en_ridge_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.2] Random forest classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.9429530201342282\n"
     ]
    }
   ],
   "source": [
    "en_random_forest_model = fit_model('random_forest_model')\n",
    "pickle.dump(en_random_forest_model, open('en_random_forest_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.3]  Logistic regression model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.9899328859060402\n"
     ]
    }
   ],
   "source": [
    "en_logistic_regression_model = fit_model('logistic_regression_model')\n",
    "pickle.dump(en_logistic_regression_model, open('en_logistic_regression_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.4] K-Neighbors classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.959731543624161\n"
     ]
    }
   ],
   "source": [
    "en_kneighbors_model = fit_model('kneighbors_model')\n",
    "pickle.dump(en_kneighbors_model, open('en_kneighbors_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.5] Decision treeClassifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.8791946308724832\n"
     ]
    }
   ],
   "source": [
    "en_decision_tree_model = fit_model('decision_tree_model')\n",
    "pickle.dump(en_decision_tree_model, open('en_decision_tree_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [3.3.6] GaussianNB model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.9463087248322147\n"
     ]
    }
   ],
   "source": [
    "en_gaussian_nb_model = fit_model('gaussian_nb_model')\n",
    "pickle.dump(en_gaussian_nb_model, open('en_gaussian_nb_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step [4]: Test cases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_1 = \"The term machine learning was coined in 1959 by Arthur Samuel, an American IBMer and pioneer in the field of\\\n",
    "computer gaming and artificial intelligence.[8][9] A representative book of the machine learning research during the 1960s\\\n",
    "was the Nilsson's book on Learning Machines, dealing mostly with machine learning for pattern classification.[10] Interest\\\n",
    "related to pattern recognition continued into the 1970s, as described by Duda and Hart in 1973.[11] In 1981 a report was given\\\n",
    "on using teaching strategies so that a neural network learns to recognize 40 characters (26 letters, 10 digits, and 4 special\\\n",
    "symbols) from a computer terminal.[12] Tomamom M. Mitchell provided a widely quoted, more formal definition of the algorithms\\\n",
    "studied in the machine learning field: A computer program is said to learn from experience E with respect to some class of\\\n",
    "tasks T and performance measure P if its performance at tasks in T, as measured by P, improves with experience E.\\\n",
    "[13] This definition of the tasks in which machine learning is concerned offers a fundamentally operational definition rather\\\n",
    "than defining the field in cognitive terms. This follows Alan Turing's proposal in his paper Computing Machinery\\\n",
    "and Intelligence, in which the question Can machines think? is replaced with the question Can machines do what we\\\n",
    "(as thinking entities) can do?.[14] Modern day machine learning has two objectives, one is to classify data based on models\\\n",
    "which have been developed, the other purpose is to make predictions for future outcomes based on these model.\\\n",
    "A hypothetical algorithm specific to classifying data may use computer vision of moles coupled with supervised learning\\\n",
    "in order to train it to classify the cancerous moles. Where as, a machine learning algoritihim for stock trading may inform\\\n",
    "the trader of future potential predictions.[15] Artificial intelligence Machine Learning as subfield of AI[16] Part of Machine\\\n",
    "Learning as subfield of AI or part of AI as subfield of Machine Learning[17] As a scientific endeavor, machine learning grew\\\n",
    "out of the quest for artificial intelligence. In the early days of AI as an academic discipline, some researchers were\\\n",
    "interested in having machines learn from data. They attempted to approach the problem with various symbolic methods,\\\n",
    "as well as what was then termed neural networks; these were mostly perceptrons and other models that were later found\\\n",
    "to be reinventions of the generalized linear models of statistics.[18] Probabilistic reasoning was also employed,\\\n",
    "especially in automated medical diagnosis.[19]:488 However, an increasing emphasis on the logical, knowledge-based approach\\\n",
    "caused a rift between AI and machine learning. Probabilistic systems were plagued by theoretical and practical problems\\\n",
    "of data acquisition and representation.[19]:488 By 1980, expert systems had come to dominate AI, and statistics was out of\\\n",
    "favor.[20] Work on symbolic/knowledge-based learning did continue within AI, leading to inductive logic programming,\\\n",
    "but the more statistical line of research was now outside the field of AI proper, in pattern recognition and information\\\n",
    "retrieval.[19]:708–710; 755 Neural networks research had been abandoned by AI and computer science around the same time.\\\n",
    "This line, too, was continued outside the AI/CS field, as connectionism, by researchers from other disciplines including\\\n",
    "Hopfield, Rumelhart and Hinton. Their main success came in the mid-1980s with the reinvention of backpropagation.\\\n",
    "[19]:25 Machine learning (ML), reorganized as a separate field, started to flourish in the 1990s. The field changed its goal\\\n",
    "from achieving artificial intelligence to tackling solvable problems of a practical nature. It shifted focus away from\\\n",
    "the symbolic approaches it had inherited from AI, and toward methods and models borrowed from statistics and probability\\\n",
    "theory.[20] As of 2020, many sources continue to assert that machine learning remains a subfield of AI.[21][22][16] The main\\\n",
    "disagreement is whether all of ML is part of AI, as this would mean that anyone using ML could claim they are using AI.\\\n",
    "Others have the view that not all of ML is part of AI[23][24][25] where only an intelligent subset of ML is part of AI.\\\n",
    "[26] The question to what is the difference between ML and AI is answered by Judea Pearl in The Book of Why.\\\n",
    "[27] Accordingly ML learns and predicts based on passive observations, whereas AI implies an agent interacting with the\\\n",
    "environment to learn and take actions that maximize its chance of successfully achieving its goals.[30]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text summary\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "[8][9] A representative book of the machine learning research during the 1960swas the Nilsson's book on Learning Machines, dealing mostly with machine learning for pattern classification. The term machine learning was coined in 1959 by Arthur Samuel, an American IBMer and pioneer in the field ofcomputer gaming and artificial intelligence. [19]:25 Machine learning (ML), reorganized as a separate field, started to flourish in the 1990s. [20] As of 2020, many sources continue to assert that machine learning remains a subfield of AI. Where as, a machine learning algoritihim for stock trading may informthe trader of future potential predictions. [19]:488 However, an increasing emphasis on the logical, knowledge-based approachcaused a rift between AI and machine learning. The field changed its goalfrom achieving artificial intelligence to tackling solvable problems of a practical nature. It shifted focus away fromthe symbolic approaches it had inherited from AI, and toward methods and models borrowed from statistics and probabilitytheory. In the early days of AI as an academic discipline, some researchers wereinterested in having machines learn from data. This follows Alan Turing's proposal in his paper Computing Machineryand Intelligence, in which the question Can machines think?\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text category: Technology\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summerize_category(test_1, 10, en_random_forest_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_2 = \"Founded on 6 March 1902 as Madrid Football Club, the club has traditionally worn a white home kit since inception. \\\n",
    "The word real is Spanish for \\\"royal\\\" and was bestowed to the club by King Alfonso XIII in 1920 together with the royal crown\\\n",
    "in the emblem. The team has played its home matches in the 81,044-capacity Santiago Bernabéu Stadium in downtown Madrid since\\\n",
    "1947. Unlike most European sporting entities, Real Madrids members (socios) have owned and operated the club throughout its\\\n",
    "history. The club was estimated to be worth €3.8 billion ($4.2 billion) in 2019, and it was the second highest-earning football\\\n",
    "club in the world, with an annual revenue of €757.3 million in 2019.[7][8] The club is one of the most widely supported teams\\\n",
    "in the world.[9] Real Madrid is one of three founding members of La Liga that have never been relegated from the top division\\\n",
    "since its inception in 1929, along with Athletic Bilbao and Barcelona. The club holds many long-standing rivalries,\\\n",
    "most notably El Clásico with Barcelona and El Derbi with Atlético Madrid. Real Madrid established itself as a major force in\\\n",
    "both Spanish and European football during the 1950s, winning five consecutive European Cups and reaching the final seven times.\\\n",
    "This success was replicated in the league, which the club won five times in the space of seven years. This team, which\\\n",
    "consisted of players Alfredo Di Stéfano, Ferenc Puskás, Francisco Gento, and Raymond Kopa, is considered by some in the sport\\\n",
    "to be the greatest team of all time.[10][11][12] In domestic football, the club has won 66 trophies; a record 34 La Liga\\\n",
    "titles, 19 Copa del Rey, 11 Supercopa de España, a Copa Eva Duarte, and a Copa de la Liga.[13] In European and worldwide\\\n",
    "competitions, Real Madrid have won a record 26 trophies; a record 13 European Cup/UEFA Champions League titles, two UEFA Cups\\\n",
    "and four UEFA Super Cups. In international football, they have achieved a record seven club world championships.[note 1] Real\\\n",
    "Madrid was recognised as the FIFA Club of the 20th Century on 11 December 2000 with 42.35% of the vote,[15] and received\\\n",
    "the FIFA Centennial Order of Merit on 20 May 2004.[16] The club was also awarded Best European Club of the 20th Century by the\\\n",
    "IFFHS on 11 May 2010. In June 2017, the team succeeded in becoming the first club to win consecutive Champions League titles, \\\n",
    "then made it three in a row and four in five seasons in May 2018, extending their lead atop the UEFA club rankings. As of 2020,\\\n",
    "Real Madrid are ranked third behind Bayern Munich and Barcelona.[17][18]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text summary\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Founded on 6 March 1902 as Madrid Football Club, the club has traditionally worn a white home kit since inception. [16] The club was also awarded Best European Club of the 20th Century by theIFFHS on 11 May 2010. In international football, they have achieved a record seven club world championships. The word real is Spanish for \"royal\" and was bestowed to the club by King Alfonso XIII in 1920 together with the royal crownin the emblem. [13] In European and worldwidecompetitions, Real Madrid have won a record 26 trophies; a record 13 European Cup/UEFA Champions League titles, two UEFA Cupsand four UEFA Super Cups.\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text category: Sport\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summerize_category(test_2, 5, en_ridge_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_3 = \"Born and raised in Queens, New York City, Trump attended Fordham University for two years and received a bachelor's \\\n",
    "degree in economics from the Wharton School of the University of Pennsylvania. He became president of his father Fred Trump's \\\n",
    "real estate business in 1971, where he renamed it The Trump Organization, and expanded its operations to building or renovating \\\n",
    "skyscrapers, hotels, casinos, and golf courses. Trump later started various side ventures, mostly by licensing his name. Trump\\\n",
    "and his businesses have been involved in more than 4,000 state and federal legal actions, including six bankruptcies. He owned \\\n",
    "the Miss Universe brand of beauty pageants from 1996 to 2015, and produced and hosted the reality television series The \\\n",
    "Apprentice from 2004 to 2015. Trump's political positions have been described as populist, protectionist, isolationist, and\\\n",
    "nationalist. He entered the 2016 presidential race as a Republican and was elected in a surprise Electoral College victory over\\\n",
    "Democratic nominee Hillary Clinton while losing the popular vote.[a] He became the oldest first-term U.S. president[b] and the\\\n",
    "first without prior military or government service. His election and policies have sparked numerous protests. Trump has made\\\n",
    "many false and misleading statements during his campaigns and presidency, to a degree unprecedented in American politics. Many\\\n",
    "of his comments and actions have been characterized as racially charged or racist. During his presidency, Trump ordered a\\\n",
    "travel ban on citizens from several Muslim-majority countries, citing security concerns; after legal challenges, the Supreme\\\n",
    "Court upheld the policy's third revision. He enacted a tax-cut package for individuals and businesses, rescinding the\\\n",
    "individual health insurance mandate penalty of the Affordable Care Act (ACA), but has failed to repeal and replace the ACA as\\\n",
    "a whole. He appointed Neil Gorsuch, Brett Kavanaugh and Amy Coney Barrett to the Supreme Court. In foreign policy, Trump has\\\n",
    "pursued an America First agenda, renegotiating the North American Free Trade Agreement (NAFTA) as the United\\\n",
    "States–Mexico–Canada Agreement (USMCA) and withdrawing the U.S. from the Trans-Pacific Partnership trade negotiations, the\\\n",
    "Paris Agreement on climate change, and the Iran nuclear deal. He imposed import tariffs which triggered a trade war with China,\\\n",
    "moved the U.S. embassy in Israel to Jerusalem, and withdrew U.S. troops from northern Syria. He met three times with North\\\n",
    "Korean leader Kim Jong-un, but talks on denuclearization broke down in 2019. He reacted slowly to the COVID-19 pandemic,\\\n",
    "downplayed the threat, ignored or contradicted many recommendations from health officials, and promoted false information\\\n",
    "about unproven treatments and the availability of testing. A special counsel investigation led by Robert Mueller found\\\n",
    "that Trump and his campaign benefited from Russian interference in the 2016 presidential election, but did not find sufficient\\\n",
    "evidence to press charges of criminal conspiracy or coordination with Russia.[c] Mueller also investigated Trump for\\\n",
    "obstruction of justice, and his report neither indicted nor exonerated Trump on that offense. Trump later pardoned five people\\\n",
    "who were convicted as a result of the Russia investigation. After Trump solicited Ukraine to investigate his political\\\n",
    "rival Joe Biden, the House of Representatives impeached him in December 2019 for abuse of power and obstruction of Congress.\\\n",
    "The Senate acquitted him of both charges in February 2020. Trump lost the 2020 presidential election to Biden but refused to\\\n",
    "concede defeat. He made unsubstantiated accusations of electoral fraud, mounted a series of unsuccessful legal challenges to\\\n",
    "the results, and ordered government officials not to cooperate in the presidential transition.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text summary\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "He made unsubstantiated accusations of electoral fraud, mounted a series of unsuccessful legal challenges tothe results, and ordered government officials not to cooperate in the presidential transition. During his presidency, Trump ordered atravel ban on citizens from several Muslim-majority countries, citing security concerns; after legal challenges, the SupremeCourt upheld the policy's third revision.\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text category: Politic\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summerize_category(test_3, 2, en_ridge_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.9305711086226204\n"
     ]
    }
   ],
   "source": [
    "ar_ridge_model = fit_model('ridge_model', True)\n",
    "pickle.dump(ar_ridge_model, open('ar_ridge_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.8678611422172452\n"
     ]
    }
   ],
   "source": [
    "ar_random_forest_model = fit_model('random_forest_model', True)\n",
    "pickle.dump(ar_random_forest_model, open('ar_random_forest_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.921612541993281\n"
     ]
    }
   ],
   "source": [
    "ar_logistic_regression_model = fit_model('logistic_regression_model', True)\n",
    "pickle.dump(ar_logistic_regression_model, open('ar_logistic_regression_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.3045912653975364\n"
     ]
    }
   ],
   "source": [
    "ar_kneighbors_model = fit_model('kneighbors_model', True)\n",
    "pickle.dump(ar_kneighbors_model, open('ar_kneighbors_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.8096304591265397\n"
     ]
    }
   ],
   "source": [
    "ar_decision_tree_model = fit_model('decision_tree_model', True)\n",
    "pickle.dump(ar_decision_tree_model, open('ar_decision_tree_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on test:  0.8824188129899216\n"
     ]
    }
   ],
   "source": [
    "ar_gaussian_nb_model = fit_model('gaussian_nb_model', True)\n",
    "pickle.dump(ar_gaussian_nb_model, open('ar_gaussian_nb_model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar_test_1 = \"التكنولوجيا هي كلمة ليست عربية ولكن لها مرادف معرب,\\\n",
    "أقترحه مجمع اللغة العربية بدمشق, واعتمدته الجامعة العربية وبعض الدول العربية, ولكن ليس كلها. و هي كلمة شائعة لها أكثر من مصطلح فالتكنولوجيا \\\n",
    "هي علم تطبيقي يهتم بدراسة الإضافات والتطور في العديد من المجالات مثل الصناعات و الفنون والحرف وكل ما يتعلق بها من مواد ووسائل مستعمل.\\\n",
    "والمفهوم الشّائع لمصطلح التّكنولوجيا هو استعمال الكمبيوتر والأجهزة الحديثة، \\\n",
    "وهذه النّظرة محدودة الرؤية ، فالكمبيوتر نتيجة من نتائج التكنولوجيا، بينما التكنولوجيا الّتي يقصدها هذا المقرّر هي طريقة للتّفكير، وحلّ المشكلات، وهي أسلوب التّفكير الّذي\\\n",
    "يوصل الفرد إلى النتائج المرجوّة أي إنّها وسيلة وليست نتيجة، وإنّها طريقة التّفكير في استخدام المعارف، والمعلومات، والمهارات، بهدف الوصول إلى نتائج لإشباع حاجة الإنسان وزيادة قدراته.\\\n",
    " أوّل فائدة تستحقّ الذكر هي ما أسهمت بهِ الثورة العلميّة والتكنولوجيّة في مجال مٌكافحة الأمراض، وتطوير المضادّات الحيوية وهو ما يُدعى بالتكنولوجيا الطبيّة الحيويّة، وأيضاُ مساهمة الأجهزة\\\n",
    " الطبيّة الحديثة في مجالات تخطيط القلب والتصوير الإشعاعي في تشخيص الأمراض، والوصول إلى نتائج مُذهلة في هذا النطاق، أضِف إلى ذلك إدخال التكنولوجيا في إجراء العمليّات الجراحيةّ\\\n",
    " المعقّدة والدقيقة والجراحة بالمنظار والقسطرة، وكذلك تكنولوجيا صناعة الدواء مما ساعدَ كثيراً في شفاء الحالات المُستعصيّة، وتسهيل العمل على الطاقم البشري من أطبّاء وممرضين وصيادلة.\\\n",
    " كما نذكُر فائدة التكنولوجيا في التسهيل على الناس في التنقّل من خلال وسائل النقل والمواصلات الحديثة؛ كظهُور القطارات الكهربائية، والطائرات، وأنظمة النقل البحريّ المتطوّرة.\\\n",
    " وأيضا تساعد التكنولوجيا علي تسهيل مهامّ البحث العلميّ والوصول إلى المعلومة بأقصر وقت وأقلّ تكلفة من السابق؛ حيث توفّر شبكة الإنترنت.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text summary\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "التكنولوجيا هي كلمة ليست عربية ولكن لها مرادف معرب,أقترحه مجمع اللغة العربية بدمشق, واعتمدته الجامعة العربية وبعض الدول العربية, ولكن ليس كلها. كما نذكُر فائدة التكنولوجيا في التسهيل على الناس في التنقّل من خلال وسائل النقل والمواصلات الحديثة؛ كظهُور القطارات الكهربائية، والطائرات، وأنظمة النقل البحريّ المتطوّرة. وأيضا تساعد التكنولوجيا علي تسهيل مهامّ البحث العلميّ والوصول إلى المعلومة بأقصر وقت وأقلّ تكلفة من السابق؛ حيث توفّر شبكة الإنترنت.\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text category: Diverse News\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summerize_category(ar_test_1, 4, ar_ridge_model, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar_test_2 = \"مع تبوأ بيب غوارديولا دفة الإدارة الفنية لبرشلونة تغيرت أمور كثيرة في الفريق سواء بانضباط اللاعبين أو بأدائهم داخل الملعب. خلال عامه الأول مع برشلونة حقق غوارديولا ما لم يحققه أي مدرب في العالم، ففي تاريخ 2 مايو عام 2009، خاض برشلونة مبارة الكلاسيكو مع غريمه التقليدي ريال مدريد في معقل الأخير، ملعب سانتياغو برنابيو، وحقق برشلونة حينها انتصارًا مدويًا، ففاز بنتيجة 6-2. ضمنت تلك النتيجة إلى حد كبير فوز برشلونة بلقب الليغا، وبعدها بأسبوعين التقى برشلونة مع أتلتيك بلباو في نهائي كأس إسبانيا، وحقق برشلونة اللقب الذي كان غائبا عن خزائنه مدة 13 عامًا، وفي أواخر ذات الشهر حقق برشلونة لقب دوري أبطال أوروبا على حساب نادي مانشستر يونايتد الإنجليزي، وليكون ذلك اللقب الثالث للنادي بتلك البطولة. في شهر أغسطس من ذلك العام ظفر برشلونة ببطولتي كأس السوبر الإسباني على حساب أتلتيك بيلباو للمرة الثامنة بتاريخه وبطولة كأس السوبر الأوروبي على حساب نادي شاختار دونستيك الأوكراني، وفي أواخر العام ذاته شارك النادي كممثل لقارة أوروبا في بطولة كأس العالم لأندية كرة القدم محققًا لقبها لأول مرة في تاريخه بعد انتصاره في المباراة النهائية على نادي إستوديانتيس دو لا بلاتا الأرجنتيني في نهائي مثير امتد لشوطين إضافيين، ليسدل الفريق الستار عن ذلك العام الاستثنائي بإنجاز غير مسبوق، بلغ 6 ألقاب بعام واحد، ويعرف باسم السداسية التاريخية. من أهم النجوم خلال ذلك العام: ليونيل ميسي، تشافي هيرنانديز، أندريس إنيستا رغم أن الغلة خلال عام 2010 لم تكن كسابقتها إلا أن الإبداع والأداء الراقي ظل مستمرًا. أبرز ما حدث في ذلك العام من انجازات كان الظفر ببطولة الليغا للمرة العشرين بتاريخ النادي، وكأس السوبر الإسباني للمرة التاسعة، وخروج برشلونة من دوري أبطال أوروبا أمام إنتر ميلان الإيطالي من الدور نصف النهائي، وخرج كذلك مبكرًا من كأس الملك أمام نادي إشبيلية. أما على صعيد الانتقالات، فقد غادر خلال صيف ذلك العام الفرنسي تيري هنري، وسبقه في الرحيل الكميروني صامويل ايتو في صفقة مبادلة مع نادي الانتر الإيطالي أتى بموجبها اللاعب السويدي إبراهيموفيتش في صفقة تعد الأكبر بتاريخ النادي. لم يمكث ابراهيموفيتش سوى عام واحد أعير بعده إلى لنادي ميلان الإيطالي، وضُم المهاجم الإسباني دافيد فيا من نادي فالنسيا خلال صيف عام 2010. إداريًا فقد جرت انتخابات رئاسة للنادي، وصل على أثرها ساندرو روسيل لرئاسة النادي خلفًا للمحامي خوان لابورتا. في افتتاحية موسم 2010-2011 حقق النادي لقب كأس السوبر الإسباني للمرة التاسعة في تاريخه، وشهد ذات الموسم صراعًا محتدمًا بين برشلونة وغريمه ريال مدريد، ولم يقتصر هذا الصراع على بطولة الليغا، التي حافظ عليها الفريق للمرة الثالثة على التوالي والحادية والعشرين في تاريخ النادي، إذ امتد الصراع ليشمل بطولتي كأس الملك ودوري أبطال أوروبا. وصل برشلونة المباراة النهائية ببطولة كأس إسبانيا ليلتقي الريال الذي استطاع خطف هدف الفوز بالوقت الإضافي الأول، وبعد تلك المباراة بأقل من أسبوع التقى برشلونة مجددًا مع ريال مدريد في الدور نصف النهائي لبطولة دوري أبطال أوروبا واستطاع برشلونة الإطاحة بريال مدريد والوصول للمباراة النهائية بتلك البطولة، والتي جمعت برشلونة مع نادي مانشستر يونايتد الإنجليزي، على ملعب ويمبلي في العاصمة البريطانية لندن، انتصر على إثرها فريق المدرب بيب غوارديولا بنتيجة 3-1 بعد أداء خيالي لنجوم الفريق في المباراة النهائية، محققين اللقب الرابع للكتلان بتلك البطولة، ومحققين أيضًا رقما قياسيا في عدد مرات الوصول لمبارة نهائية في إطار بطولات الأندية الأوروبية. لم تتوقف إنجازات النادي خلال عام 2011 عند ذلك، ففي مستهل الموسم الكروي 2011- 2012 وتحديدًا في منتصف شهر أغسطس، فاز النادي بكأس السوبر الإسباني عندما تفوق على غريمة التقليدي فريق ريال مدريد، ولم يمض أسبوع بعد تحقيق لقب تلك البطولة حتى حقق النادي كأس السوبر الأوروبي على حساب نادي بورتو البرتغالي، لتزداد غلة النادي من الألقاب ويصبح جوسيب غوارديولا أنجح مدرب في تاريخ النادي من حيث عدد الألقاب، وفي أواخر عام 2011 شارك النادي ببطولة كأس العالم للأندية في اليابان وتمكن من الظفر بلقب تلك البطولة بعد تفوقة في المباراة النهائية على فريق سانتوس البرازيلي\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text summary\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "لم يمكث ابراهيموفيتش سوى عام واحد أعير بعده إلى لنادي ميلان الإيطالي، وضُم المهاجم الإسباني دافيد فيا من نادي فالنسيا خلال صيف عام 2010. من أهم النجوم خلال ذلك العام: ليونيل ميسي، تشافي هيرنانديز، أندريس إنيستا رغم أن الغلة خلال عام 2010 لم تكن كسابقتها إلا أن الإبداع والأداء الراقي ظل مستمرًا. إداريًا فقد جرت انتخابات رئاسة للنادي، وصل على أثرها ساندرو روسيل لرئاسة النادي خلفًا للمحامي خوان لابورتا. مع تبوأ بيب غوارديولا دفة الإدارة الفنية لبرشلونة تغيرت أمور كثيرة في الفريق سواء بانضباط اللاعبين أو بأدائهم داخل الملعب.\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n",
      "Text category: Sport\n",
      "-------------------------------------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summerize_category(ar_test_2, 6, ar_ridge_model, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}