text_toxic_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Подготовка"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import sys\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import f1_score\n",
    "import nltk\n",
    "from nltk.corpus import stopwords as nltk_stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "import transformers\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Выгрузим данные из csv и взглянем на них"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tweets = pd.read_csv('/datasets/toxic_comments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>toxic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159566</td>\n",
       "      <td>\":::::And for the second time of asking, when ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159567</td>\n",
       "      <td>You should be ashamed of yourself \\n\\nThat is ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159568</td>\n",
       "      <td>Spitzer \\n\\nUmm, theres no actual article for ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159569</td>\n",
       "      <td>And it looks like it was actually you who put ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159570</td>\n",
       "      <td>\"\\nAnd ... I really don't think you understand...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159571 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  toxic\n",
       "0       Explanation\\nWhy the edits made under my usern...      0\n",
       "1       D'aww! He matches this background colour I'm s...      0\n",
       "2       Hey man, I'm really not trying to edit war. It...      0\n",
       "3       \"\\nMore\\nI can't make any real suggestions on ...      0\n",
       "4       You, sir, are my hero. Any chance you remember...      0\n",
       "...                                                   ...    ...\n",
       "159566  \":::::And for the second time of asking, when ...      0\n",
       "159567  You should be ashamed of yourself \\n\\nThat is ...      0\n",
       "159568  Spitzer \\n\\nUmm, theres no actual article for ...      0\n",
       "159569  And it looks like it was actually you who put ...      0\n",
       "159570  \"\\nAnd ... I really don't think you understand...      0\n",
       "\n",
       "[159571 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Видим, что данным необходима предобработка.  \n",
    "В первую очередь необходимо избавится от спецсимволов, оставив только слова, привести текст к нижнему регистру, а также лемматизировать слова.  \n",
    "Выделим метод get_wordnet_pos для получение правильного pos_tag, а в методе clear_text очистим текст от спецсимволов и лемматизируем каждое слово."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download('wordnet', quiet=True)\n",
    "nltk.download('averaged_perceptron_tagger', quiet=True)\n",
    "nltk.download('stopwords', quiet=True)\n",
    "stopwords = set(nltk_stopwords.words('english'))\n",
    "\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "    if treebank_tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif treebank_tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif treebank_tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif treebank_tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "\n",
    "def clear_text(text):\n",
    "    return re.sub(r\"[^a-zA-Z\\' ]\", ' ', text.lower())\n",
    "\n",
    "def stem_text(text):\n",
    "    return \" \".join([porter.stem(w) for w in text.split()])\n",
    "\n",
    "def lemm_text(text):\n",
    "    text_pos = nltk.pos_tag(text.split())\n",
    "    #meanfull = [x for x in text_pos if x[1].startswith('J') | x[1].startswith('V') | x[1].startswith('N') | x[1].startswith('R')]\n",
    "    return \" \".join([lemmatizer.lemmatize(w[0], get_wordnet_pos(w[1])) for w in text_pos])\n",
    "\n",
    "porter = PorterStemmer()\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Применим clear_text к массиву тексту."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.67 s, sys: 52 ms, total: 1.72 s\n",
      "Wall time: 1.72 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_tweets[\"clear\"] = df_tweets[\"text\"].apply(lambda x: clear_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 9s, sys: 872 ms, total: 4min 10s\n",
      "Wall time: 4min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#необходимо для корректной работы PorterStemmer\n",
    "sys.setrecursionlimit(10000)\n",
    "df_tweets[\"stem\"] = df_tweets[\"clear\"].apply(lambda x: stem_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11min 3s, sys: 4.78 s, total: 11min 8s\n",
      "Wall time: 11min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_tweets[\"lemm\"] = df_tweets[\"clear\"].apply(lambda x: lemm_text(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Посмотрим, что получилось"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>clear</th>\n",
       "      <th>stem</th>\n",
       "      <th>lemm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>explanation why the edits made under my userna...</td>\n",
       "      <td>explan whi the edit made under my usernam hard...</td>\n",
       "      <td>explanation why the edits make under my userna...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>d'aww  he matches this background colour i'm s...</td>\n",
       "      <td>d'aww he match thi background colour i'm seemi...</td>\n",
       "      <td>d'aww he match this background colour i'm seem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>hey man  i'm really not trying to edit war  it...</td>\n",
       "      <td>hey man i'm realli not tri to edit war it' jus...</td>\n",
       "      <td>hey man i'm really not try to edit war it's ju...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>more i can't make any real suggestions on im...</td>\n",
       "      <td>more i can't make ani real suggest on improv i...</td>\n",
       "      <td>more i can't make any real suggestion on impro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>you  sir  are my hero  any chance you remember...</td>\n",
       "      <td>you sir are my hero ani chanc you rememb what ...</td>\n",
       "      <td>you sir be my hero any chance you remember wha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159566</td>\n",
       "      <td>\":::::And for the second time of asking, when ...</td>\n",
       "      <td>0</td>\n",
       "      <td>and for the second time of asking  when ...</td>\n",
       "      <td>and for the second time of ask when your view ...</td>\n",
       "      <td>and for the second time of ask when your view ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159567</td>\n",
       "      <td>You should be ashamed of yourself \\n\\nThat is ...</td>\n",
       "      <td>0</td>\n",
       "      <td>you should be ashamed of yourself   that is a ...</td>\n",
       "      <td>you should be asham of yourself that is a horr...</td>\n",
       "      <td>you should be ashamed of yourself that be a ho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159568</td>\n",
       "      <td>Spitzer \\n\\nUmm, theres no actual article for ...</td>\n",
       "      <td>0</td>\n",
       "      <td>spitzer   umm  theres no actual article for pr...</td>\n",
       "      <td>spitzer umm there no actual articl for prostit...</td>\n",
       "      <td>spitzer umm theres no actual article for prost...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159569</td>\n",
       "      <td>And it looks like it was actually you who put ...</td>\n",
       "      <td>0</td>\n",
       "      <td>and it looks like it was actually you who put ...</td>\n",
       "      <td>and it look like it wa actual you who put on t...</td>\n",
       "      <td>and it look like it be actually you who put on...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159570</td>\n",
       "      <td>\"\\nAnd ... I really don't think you understand...</td>\n",
       "      <td>0</td>\n",
       "      <td>and     i really don't think you understand ...</td>\n",
       "      <td>and i realli don't think you understand i came...</td>\n",
       "      <td>and i really don't think you understand i come...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>159571 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  toxic  \\\n",
       "0       Explanation\\nWhy the edits made under my usern...      0   \n",
       "1       D'aww! He matches this background colour I'm s...      0   \n",
       "2       Hey man, I'm really not trying to edit war. It...      0   \n",
       "3       \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4       You, sir, are my hero. Any chance you remember...      0   \n",
       "...                                                   ...    ...   \n",
       "159566  \":::::And for the second time of asking, when ...      0   \n",
       "159567  You should be ashamed of yourself \\n\\nThat is ...      0   \n",
       "159568  Spitzer \\n\\nUmm, theres no actual article for ...      0   \n",
       "159569  And it looks like it was actually you who put ...      0   \n",
       "159570  \"\\nAnd ... I really don't think you understand...      0   \n",
       "\n",
       "                                                    clear  \\\n",
       "0       explanation why the edits made under my userna...   \n",
       "1       d'aww  he matches this background colour i'm s...   \n",
       "2       hey man  i'm really not trying to edit war  it...   \n",
       "3         more i can't make any real suggestions on im...   \n",
       "4       you  sir  are my hero  any chance you remember...   \n",
       "...                                                   ...   \n",
       "159566        and for the second time of asking  when ...   \n",
       "159567  you should be ashamed of yourself   that is a ...   \n",
       "159568  spitzer   umm  theres no actual article for pr...   \n",
       "159569  and it looks like it was actually you who put ...   \n",
       "159570    and     i really don't think you understand ...   \n",
       "\n",
       "                                                     stem  \\\n",
       "0       explan whi the edit made under my usernam hard...   \n",
       "1       d'aww he match thi background colour i'm seemi...   \n",
       "2       hey man i'm realli not tri to edit war it' jus...   \n",
       "3       more i can't make ani real suggest on improv i...   \n",
       "4       you sir are my hero ani chanc you rememb what ...   \n",
       "...                                                   ...   \n",
       "159566  and for the second time of ask when your view ...   \n",
       "159567  you should be asham of yourself that is a horr...   \n",
       "159568  spitzer umm there no actual articl for prostit...   \n",
       "159569  and it look like it wa actual you who put on t...   \n",
       "159570  and i realli don't think you understand i came...   \n",
       "\n",
       "                                                     lemm  \n",
       "0       explanation why the edits make under my userna...  \n",
       "1       d'aww he match this background colour i'm seem...  \n",
       "2       hey man i'm really not try to edit war it's ju...  \n",
       "3       more i can't make any real suggestion on impro...  \n",
       "4       you sir be my hero any chance you remember wha...  \n",
       "...                                                   ...  \n",
       "159566  and for the second time of ask when your view ...  \n",
       "159567  you should be ashamed of yourself that be a ho...  \n",
       "159568  spitzer umm theres no actual article for prost...  \n",
       "159569  and it look like it be actually you who put on...  \n",
       "159570  and i really don't think you understand i come...  \n",
       "\n",
       "[159571 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Заметим, что после преобразований стемматизированная версия мало отличается от лемматизированной. Вероятно и скор полученный на основе этих колонок будет мало отличаться.  \n",
    "Все ок, можем переходить к обучению."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Обучение"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Выделим feature и target, а также разобьем выборку на тренировочный, тестовый и финальный сет"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = df_tweets.drop(['toxic'], axis=1)\n",
    "target = df_tweets['toxic']\n",
    "\n",
    "features_train, features_test, target_train, target_test = train_test_split(features, \n",
    "                                                                              target, \n",
    "                                                                              test_size=0.2, \n",
    "                                                                              random_state = 12345)\n",
    "\n",
    "\n",
    "features_final, features_test, target_final, target_test = train_test_split(features_test, \n",
    "                                                                              target_test, \n",
    "                                                                              test_size=0.5, \n",
    "                                                                              random_state = 12345)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Проведем векторизацию текста на обучающей выборке"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
       "                input='content', lowercase=True, max_df=1.0, max_features=10000,\n",
       "                min_df=1, ngram_range=(1, 1), norm='l2', preprocessor=None,\n",
       "                smooth_idf=True,\n",
       "                stop_words={'a', 'about', 'above', 'after', 'again', 'against',\n",
       "                            'ain', 'all', 'am', 'an', 'and', 'any', 'are',\n",
       "                            'aren', \"aren't\", 'as', 'at', 'be', 'because',\n",
       "                            'been', 'before', 'being', 'below', 'between',\n",
       "                            'both', 'but', 'by', 'can', 'couldn', \"couldn't\", ...},\n",
       "                strip_accents=None, sublinear_tf=False,\n",
       "                token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "                vocabulary=None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_vectorizer = TfidfVectorizer(\n",
    "    stop_words=stopwords,\n",
    "    ngram_range=(1, 1),\n",
    "    max_features=10000)\n",
    "word_vectorizer.fit(features_train[\"lemm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_word_features = word_vectorizer.transform(features_train[\"lemm\"].values.astype('U'))\n",
    "test_word_features = word_vectorizer.transform(features_test[\"lemm\"].values.astype('U'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Получим наилучшие параметры для LogisticRegression с помощью GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 20 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed: 33.2min finished\n"
     ]
    }
   ],
   "source": [
    "pipe = Pipeline([('classifier' , LogisticRegression())])\n",
    "\n",
    "param_grid = [\n",
    "    {'classifier' : [LogisticRegression()],\n",
    "     'classifier__penalty' : ['l1', 'l2'],\n",
    "    'classifier__C' : np.logspace(-4, 4, 10),\n",
    "    'classifier__solver' : ['liblinear']}\n",
    "]\n",
    "\n",
    "clf = GridSearchCV(pipe, param_grid = param_grid, cv = 5, verbose=True, n_jobs=-1)\n",
    "best_clf = clf.fit(train_word_features, target_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'classifier': LogisticRegression(C=2.782559402207126, class_weight=None, dual=False,\n",
       "                    fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                    max_iter=100, multi_class='warn', n_jobs=None, penalty='l1',\n",
       "                    random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
       "                    warm_start=False),\n",
       " 'classifier__C': 2.782559402207126,\n",
       " 'classifier__penalty': 'l1',\n",
       " 'classifier__solver': 'liblinear'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Построим модель на основе полученных параметров."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression(C=2.782559402207126, class_weight=None, dual=False,\n",
    "                    fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
    "                    max_iter=100, multi_class='warn', n_jobs=None, penalty='l1',\n",
    "                    random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
    "                    warm_start=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=2.782559402207126, class_weight=None, dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=100, multi_class='warn', n_jobs=None, penalty='l1',\n",
       "                   random_state=None, solver='liblinear', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_word_features, target_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7847151142954624"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model.predict(test_word_features)\n",
    "scores = f1_score(pred, target_test)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Полученный скор нас вполне устроит, можем переходить к проверке."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Выводы"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Попробуем применить полученную модель к выборке features_final в разных ситуациях  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_f1_score(text, field):\n",
    "    features = word_vectorizer.transform(features_final[field].values.astype('U'))\n",
    "    pred = model.predict(features)\n",
    "    print(text, f1_score(pred, target_final))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RegExp 0.7671794871794873\n"
     ]
    }
   ],
   "source": [
    "calculate_f1_score(\"RegExp\",\"clear\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stemmatization 0.7671601615074023\n"
     ]
    }
   ],
   "source": [
    "calculate_f1_score(\"Stemmatization\", \"stem\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lemmmatization 0.7810810810810811\n"
     ]
    }
   ],
   "source": [
    "calculate_f1_score(\"Lemmmatization\", \"lemm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Простое отсечение спецсимволов достоточно для получение хорошего скора, однако лемматизация помогает строить более точные модели.  \n",
    "Как оказалось, стемматизация при помощи PorterStemmer дает результат хуже, чем совсем без него, вероятно стоит в дальнейшем оставаться при лемматизации."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}