category_prediction_RF_averagedEmbeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24ba3160",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\envs\\tfgpu\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json, os\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import csv\n",
    "from transformers import AutoTokenizer, TFAutoModel, TFAutoModelForSequenceClassification #, BertModel, BertTokenizer, TFBertForSequenceClassification\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from tensorflow.keras.callbacks import CSVLogger\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import tensorflow.keras.backend as K\n",
    "from collections import OrderedDict\n",
    "import time\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score, \\\n",
    "roc_auc_score, confusion_matrix, classification_report\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import random\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import LSTM, SimpleRNN\n",
    "from tensorflow.keras.layers import GRU\n",
    "from tensorflow.keras.layers import Masking\n",
    "from tensorflow.keras.layers import Embedding, MaxPool1D\n",
    "from tensorflow.keras.callbacks import CSVLogger\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.constraints import max_norm\n",
    "from tensorflow.keras.layers import Bidirectional, BatchNormalization\n",
    "from tensorflow.keras.layers import LeakyReLU\n",
    "from tensorflow.keras.initializers import glorot_uniform, RandomUniform, lecun_uniform, Constant\n",
    "from tensorflow.python.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "from collections import OrderedDict\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D, GlobalMaxPool1D\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.initializers import TruncatedNormal\n",
    "from tensorflow.keras.losses import CategoricalCrossentropy\n",
    "from tensorflow.keras.metrics import CategoricalAccuracy\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.layers import Input, Dense, GlobalMaxPool1D\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "# from imblearn.over_sampling import RandomOverSampler\n",
    "# from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "from sklearn.svm import SVC  \n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import gensim.downloader\n",
    "import io\n",
    "from keras_preprocessing.text import tokenizer_from_json\n",
    "from gensim.models import Word2Vec, FastText\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca0e24c8",
   "metadata": {},
   "source": [
    "Set the seeder to have as stable random operations as possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3cb5b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 123\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "01a6448b",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_algorithms = [\"w2v\", \"ft\", \"py_w2v\", \"py_ft\"]\n",
    "embedding_algorithm = embedding_algorithms[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22f9b6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "representation_form = \"sequences\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3acc7396",
   "metadata": {},
   "outputs": [],
   "source": [
    "# binary or multi-class\n",
    "multi = True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2010727",
   "metadata": {},
   "source": [
    "Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fbf099bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if representation_form == \"bow\":\n",
    "    data = pd.read_csv('bow_data.csv') # bow\n",
    "else:\n",
    "    data = pd.read_csv('sequences_data.csv') # sequences of tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7abc9724",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Vulnerability</th>\n",
       "      <th>Category</th>\n",
       "      <th>Length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f\"str$id\"\"str$id\"\"str$id\"         ...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>client.listentcp()    proxy = proxy(proxy_...</td>\n",
       "      <td>xsrf</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>from django.http import httpresponse, httpresp...</td>\n",
       "      <td>open_redirect</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>def write_preset(conn, queryin, descriptin):\\t...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>update_query = self.up...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Vulnerability       Category  Length\n",
       "0              f\"str$id\"\"str$id\"\"str$id\"         ...  sql_injection       9\n",
       "1      client.listentcp()    proxy = proxy(proxy_...           xsrf       8\n",
       "2  from django.http import httpresponse, httpresp...  open_redirect       9\n",
       "3  def write_preset(conn, queryin, descriptin):\\t...  sql_injection     175\n",
       "4                          update_query = self.up...  sql_injection      14"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7e5d5a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if representation_form == \"bow\":\n",
    "    bow_size = 237 # number of columns that stand as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "76e46cb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "392"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(data[\"Length\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "48846336",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum number of words: 392\n"
     ]
    }
   ],
   "source": [
    "if representation_form == \"sequences\":\n",
    "    word_counts = data[\"Vulnerability\"].apply(lambda x: len(x.split()))\n",
    "    max_length = word_counts.max()\n",
    "    print(\"Maximum number of words:\", max_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "30bbacde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Frequencies:\n",
      " sql_injection            1424\n",
      "xsrf                      976\n",
      "command_injection         721\n",
      "path_disclosure           481\n",
      "open_redirect             442\n",
      "remote_code_execution     334\n",
      "xss                       145\n",
      "Name: Category, dtype: int64\n",
      "Total samples  4523\n"
     ]
    }
   ],
   "source": [
    "label_frequencies = data['Category'].value_counts()\n",
    "print(\"Label Frequencies:\\n\", label_frequencies)\n",
    "print(\"Total samples \", len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b777af2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    }
   ],
   "source": [
    "if multi == True:\n",
    "    n_categories = len(label_frequencies) # 7\n",
    "    print(n_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9738c030",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == True:\n",
    "    \n",
    "    # Convert categories to numerical indexes\n",
    "    category_numerical_indexes, unique_categories = data[\"Category\"].factorize()\n",
    "\n",
    "    # Create a dictionary mapping each category to its numerical index\n",
    "    category_to_index = {category: index for index, category in enumerate(unique_categories)}\n",
    "\n",
    "    # Update the categories in the DataFrame with their numerical indexes\n",
    "    data[\"Category_Index\"] = data[\"Category\"].map(category_to_index)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0984d18f",
   "metadata": {},
   "source": [
    "Word Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f4acbd1",
   "metadata": {},
   "source": [
    "Word2Vec - load pre-trained word2vec embeddings - NL knowledge - static embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4f0d6de6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\iliaskaloup\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "if representation_form == \"sequences\":\n",
    "\n",
    "    # Download the Punkt tokenizer models if not already downloaded\n",
    "    nltk.download('punkt')\n",
    "\n",
    "    tokenized_list = [word_tokenize(sentence) for sentence in data[\"Vulnerability\"].tolist()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09a9edbb",
   "metadata": {},
   "source": [
    "BERT - load pre-trained bert embeddings - NL knowledge - contextual embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9ffe75d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(words, w2v_vectors, dim):\n",
    "    words_vecs = [w2v_vectors[word] for word in words if word in w2v_vectors]\n",
    "    if len(words_vecs) == 0:\n",
    "        return np.zeros(dim)\n",
    "    words_vecs = np.array(words_vecs)\n",
    "    return words_vecs.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c6a86bc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"w2v\":\n",
    "    vectors = gensim.downloader.load('word2vec-google-news-300')\n",
    "    dim = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "55454d02",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"ft\":\n",
    "    vectors = gensim.downloader.load('fasttext-wiki-news-subwords-300')\n",
    "    dim = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "285f35c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"py_w2v\":\n",
    "    vectors = Word2Vec.load(\"w2v_model.model\").wv\n",
    "    dim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b189c4a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"py_ft\":\n",
    "    vectors = FastText.load(\"fast-text_model.model\").wv\n",
    "    dim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4a1971fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\envs\\tfgpu\\lib\\site-packages\\ipykernel_launcher.py:1: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "embs = np.array([vectorize(sentence, vectors, dim) for sentence in np.array(tokenized_list)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acab4fc2",
   "metadata": {},
   "source": [
    "Cross-Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8de3dea",
   "metadata": {},
   "source": [
    "Multi-class Classification: Categorization of all detected vulnerabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "86235386",
   "metadata": {},
   "outputs": [],
   "source": [
    "def runMLCrossVal_multi(X, y, seed, userModel):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X.iloc[train_index, :], X.iloc[test_index, :]\n",
    "        Y_train, Y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "#         Y_train = np.array(Y_train)\n",
    "#         Y_train = Y_train.ravel()\n",
    "#         Y_test = np.array(Y_test)\n",
    "#         Y_test = Y_test.ravel()\n",
    "        \n",
    "#         #sampling\n",
    "#         X_res, Y_res = RandomOverSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "#         #X_res, Y_res = RandomUnderSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "    \n",
    "#         #shuffle dataset\n",
    "#         X_resampled=pd.DataFrame(X_res)\n",
    "#         Y_resampled=pd.DataFrame(Y_res)\n",
    "#         newTrain=X_resampled.assign(Label=Y_resampled.values)\n",
    "#         newTrain = shuffle(newTrain,random_state=seed)\n",
    "#         X_train=np.array(newTrain.iloc[:, 0:-1 ])\n",
    "#         X_train=pd.DataFrame(X_train)\n",
    "#         Y_train=np.array(newTrain.iloc[:, -1 ])\n",
    "#         Y_train=pd.DataFrame(Y_train)\n",
    "        \n",
    "        if userModel == \"SVM\":\n",
    "            myModel = SVC(kernel='rbf', gamma=100)\n",
    "        elif userModel == \"RF\":\n",
    "            myModel = RandomForestClassifier(n_estimators=100, bootstrap = True, max_features = 'sqrt')\n",
    "        elif userModel == \"DT\":\n",
    "            myModel = tree.DecisionTreeClassifier(max_depth=120)\n",
    "        elif userModel == \"NB\":\n",
    "            myModel = GaussianNB()\n",
    "        elif userModel == \"KNN\":\n",
    "            myModel = KNeighborsClassifier(n_neighbors=1)\n",
    "            \n",
    "        myModel.fit(X_train, Y_train.ravel())\n",
    "        \n",
    "        predictions = myModel.predict(X_test)\n",
    "        #predScores = myModel.predict_proba(X_test)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions, average='macro')\n",
    "        recall=recall_score(Y_test, predictions, average='macro')\n",
    "        f1=f1_score(Y_test, predictions, average='macro')\n",
    "        conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "        print(conf_matrix)\n",
    "        #tn, fp, fn, tp = confusion_matrix(Y_test, predictions).ravel()\n",
    "        #acc = ((tp+tn)/(tp+tn+fp+fn))\n",
    "        print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "30994af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "embs = pd.DataFrame(embs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bd4ae015",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "fold number=  1\n",
      "[[106   8  10   6   2   5   6]\n",
      " [  5  85   2   0   1   4   0]\n",
      " [  8   2  29   2   2   0   2]\n",
      " [  1   2   2   9   0   0   0]\n",
      " [  4   1   0   1  26   2   0]\n",
      " [  8   1   4   2   3  52   2]\n",
      " [  3   4   2   0   2   2  35]]\n",
      "Accuracy:75.50%\n",
      "Precision:70.75%\n",
      "Recall:73.16%\n",
      "F1 score:71.62%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.74      0.76       143\n",
      "           1       0.83      0.88      0.85        97\n",
      "           2       0.59      0.64      0.62        45\n",
      "           3       0.45      0.64      0.53        14\n",
      "           4       0.72      0.76      0.74        34\n",
      "           5       0.80      0.72      0.76        72\n",
      "           6       0.78      0.73      0.75        48\n",
      "\n",
      "    accuracy                           0.75       453\n",
      "   macro avg       0.71      0.73      0.72       453\n",
      "weighted avg       0.76      0.75      0.76       453\n",
      "\n",
      "fold number=  2\n",
      "[[111  10   5   2   5   5   5]\n",
      " [  2  90   1   0   0   3   1]\n",
      " [  1   2  36   1   1   2   2]\n",
      " [  1   1   1  11   0   0   0]\n",
      " [  1   5   2   0  24   0   1]\n",
      " [  6   1   2   0   1  60   3]\n",
      " [  7   3   0   0   2   0  36]]\n",
      "Accuracy:81.24%\n",
      "Precision:79.29%\n",
      "Recall:79.84%\n",
      "F1 score:79.46%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.78      0.82       143\n",
      "           1       0.80      0.93      0.86        97\n",
      "           2       0.77      0.80      0.78        45\n",
      "           3       0.79      0.79      0.79        14\n",
      "           4       0.73      0.73      0.73        33\n",
      "           5       0.86      0.82      0.84        73\n",
      "           6       0.75      0.75      0.75        48\n",
      "\n",
      "    accuracy                           0.81       453\n",
      "   macro avg       0.79      0.80      0.79       453\n",
      "weighted avg       0.81      0.81      0.81       453\n",
      "\n",
      "fold number=  3\n",
      "[[107  10  11   0   3   7   5]\n",
      " [  3  85   5   0   0   4   0]\n",
      " [  7   3  26   2   4   0   2]\n",
      " [  1   1   2  10   0   0   1]\n",
      " [  5   4   1   0  23   0   0]\n",
      " [  7   4   2   1   3  54   1]\n",
      " [  8   4   6   0   1   2  28]]\n",
      "Accuracy:73.51%\n",
      "Precision:72.00%\n",
      "Recall:70.01%\n",
      "F1 score:70.63%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.75      0.76       143\n",
      "           1       0.77      0.88      0.82        97\n",
      "           2       0.49      0.59      0.54        44\n",
      "           3       0.77      0.67      0.71        15\n",
      "           4       0.68      0.70      0.69        33\n",
      "           5       0.81      0.75      0.78        72\n",
      "           6       0.76      0.57      0.65        49\n",
      "\n",
      "    accuracy                           0.74       453\n",
      "   macro avg       0.72      0.70      0.71       453\n",
      "weighted avg       0.74      0.74      0.74       453\n",
      "\n",
      "fold number=  4\n",
      "[[122   7   4   1   0   6   3]\n",
      " [  0  92   3   1   0   0   1]\n",
      " [  8   6  26   0   0   2   2]\n",
      " [  1   1   2   8   2   1   0]\n",
      " [  1   1   1   2  27   1   0]\n",
      " [  9   2   0   0   2  58   1]\n",
      " [  2   4   5   2   4   0  31]]\n",
      "Accuracy:80.53%\n",
      "Precision:75.90%\n",
      "Recall:74.22%\n",
      "F1 score:74.81%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.85      0.85       143\n",
      "           1       0.81      0.95      0.88        97\n",
      "           2       0.63      0.59      0.61        44\n",
      "           3       0.57      0.53      0.55        15\n",
      "           4       0.77      0.82      0.79        33\n",
      "           5       0.85      0.81      0.83        72\n",
      "           6       0.82      0.65      0.72        48\n",
      "\n",
      "    accuracy                           0.81       452\n",
      "   macro avg       0.76      0.74      0.75       452\n",
      "weighted avg       0.80      0.81      0.80       452\n",
      "\n",
      "fold number=  5\n",
      "[[117   9   9   0   2   5   0]\n",
      " [  3  83   3   1   3   2   3]\n",
      " [  7   2  27   1   3   3   1]\n",
      " [  1   0   2   8   0   2   2]\n",
      " [  0   1   3   1  25   1   2]\n",
      " [  6   2   0   1   1  61   1]\n",
      " [  4   4   1   0   3   2  34]]\n",
      "Accuracy:78.54%\n",
      "Precision:74.36%\n",
      "Recall:73.30%\n",
      "F1 score:73.64%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.82      0.84       142\n",
      "           1       0.82      0.85      0.83        98\n",
      "           2       0.60      0.61      0.61        44\n",
      "           3       0.67      0.53      0.59        15\n",
      "           4       0.68      0.76      0.71        33\n",
      "           5       0.80      0.85      0.82        72\n",
      "           6       0.79      0.71      0.75        48\n",
      "\n",
      "    accuracy                           0.79       452\n",
      "   macro avg       0.74      0.73      0.74       452\n",
      "weighted avg       0.79      0.79      0.78       452\n",
      "\n",
      "fold number=  6\n",
      "[[113   9   3   1   2   6   8]\n",
      " [  4  88   0   1   1   3   1]\n",
      " [ 11   3  27   0   1   1   1]\n",
      " [  2   2   1   8   1   0   1]\n",
      " [  2   2   1   0  25   0   3]\n",
      " [  8   4   0   2   1  56   1]\n",
      " [  7   4   3   1   2   1  30]]\n",
      "Accuracy:76.77%\n",
      "Precision:74.30%\n",
      "Recall:71.44%\n",
      "F1 score:72.62%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.80      0.78       142\n",
      "           1       0.79      0.90      0.84        98\n",
      "           2       0.77      0.61      0.68        44\n",
      "           3       0.62      0.53      0.57        15\n",
      "           4       0.76      0.76      0.76        33\n",
      "           5       0.84      0.78      0.81        72\n",
      "           6       0.67      0.62      0.65        48\n",
      "\n",
      "    accuracy                           0.77       452\n",
      "   macro avg       0.74      0.71      0.73       452\n",
      "weighted avg       0.77      0.77      0.77       452\n",
      "\n",
      "fold number=  7\n",
      "[[123   4   2   2   3   2   6]\n",
      " [  9  74   3   2   1   6   3]\n",
      " [  8   2  30   2   1   1   0]\n",
      " [  2   1   0  10   1   0   1]\n",
      " [  1   0   2   0  26   1   3]\n",
      " [  6   0   2   2   0  62   0]\n",
      " [  1   4   4   0   0   2  37]]\n",
      "Accuracy:80.09%\n",
      "Precision:76.20%\n",
      "Recall:76.99%\n",
      "F1 score:76.45%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.87      0.84       142\n",
      "           1       0.87      0.76      0.81        98\n",
      "           2       0.70      0.68      0.69        44\n",
      "           3       0.56      0.67      0.61        15\n",
      "           4       0.81      0.79      0.80        33\n",
      "           5       0.84      0.86      0.85        72\n",
      "           6       0.74      0.77      0.76        48\n",
      "\n",
      "    accuracy                           0.80       452\n",
      "   macro avg       0.76      0.77      0.76       452\n",
      "weighted avg       0.80      0.80      0.80       452\n",
      "\n",
      "fold number=  8\n",
      "[[111  10   7   1   4   4   5]\n",
      " [  6  89   1   0   0   0   2]\n",
      " [  3   5  29   0   4   1   2]\n",
      " [  2   1   1  10   0   0   0]\n",
      " [  1   0   0   0  29   0   4]\n",
      " [ 10   3   3   0   0  55   1]\n",
      " [  5   7   3   0   2   4  27]]\n",
      "Accuracy:77.43%\n",
      "Precision:77.26%\n",
      "Recall:74.89%\n",
      "F1 score:75.68%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.78      0.79       142\n",
      "           1       0.77      0.91      0.84        98\n",
      "           2       0.66      0.66      0.66        44\n",
      "           3       0.91      0.71      0.80        14\n",
      "           4       0.74      0.85      0.79        34\n",
      "           5       0.86      0.76      0.81        72\n",
      "           6       0.66      0.56      0.61        48\n",
      "\n",
      "    accuracy                           0.77       452\n",
      "   macro avg       0.77      0.75      0.76       452\n",
      "weighted avg       0.78      0.77      0.77       452\n",
      "\n",
      "fold number=  9\n",
      "[[112   7   6   2   6   3   6]\n",
      " [  8  80   7   0   0   2   1]\n",
      " [  6   5  24   0   1   6   2]\n",
      " [  1   1   2  10   0   0   0]\n",
      " [  4   5   0   0  25   0   0]\n",
      " [  8   6   2   1   0  53   2]\n",
      " [  3   2   6   0   1   6  30]]\n",
      "Accuracy:73.89%\n",
      "Precision:72.42%\n",
      "Recall:70.87%\n",
      "F1 score:71.55%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.79      0.79       142\n",
      "           1       0.75      0.82      0.78        98\n",
      "           2       0.51      0.55      0.53        44\n",
      "           3       0.77      0.71      0.74        14\n",
      "           4       0.76      0.74      0.75        34\n",
      "           5       0.76      0.74      0.75        72\n",
      "           6       0.73      0.62      0.67        48\n",
      "\n",
      "    accuracy                           0.74       452\n",
      "   macro avg       0.72      0.71      0.72       452\n",
      "weighted avg       0.74      0.74      0.74       452\n",
      "\n",
      "fold number=  10\n",
      "[[122   3   6   0   0   5   6]\n",
      " [  3  82   5   1   3   1   3]\n",
      " [  5   4  32   0   2   1   0]\n",
      " [  1   0   0  10   1   1   1]\n",
      " [  5   1   1   0  26   0   1]\n",
      " [  5   5   3   0   1  57   1]\n",
      " [  4   3   3   0   2   2  34]]\n",
      "Accuracy:80.31%\n",
      "Precision:79.43%\n",
      "Recall:77.17%\n",
      "F1 score:78.07%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.86      0.85       142\n",
      "           1       0.84      0.84      0.84        98\n",
      "           2       0.64      0.73      0.68        44\n",
      "           3       0.91      0.71      0.80        14\n",
      "           4       0.74      0.76      0.75        34\n",
      "           5       0.85      0.79      0.82        72\n",
      "           6       0.74      0.71      0.72        48\n",
      "\n",
      "    accuracy                           0.80       452\n",
      "   macro avg       0.79      0.77      0.78       452\n",
      "weighted avg       0.81      0.80      0.80       452\n",
      "\n",
      "Cross Validation is completed after 518\n",
      "accuracy: 77.78% (2.67%)\n",
      "precision: 75.19% (2.82%)\n",
      "recall: 74.19% (2.95%)\n",
      "f1: 74.45% (2.81%)\n"
     ]
    }
   ],
   "source": [
    "if multi == True and representation_form == \"sequences\" and embedding_algorithm == \"w2v\":\n",
    "    runMLCrossVal_multi(embs, data[\"Category_Index\"], seed, \"KNN\")\n",
    "elif multi == True and representation_form == \"sequences\" and embedding_algorithm == \"ft\":\n",
    "    runMLCrossVal_multi(embs, data[\"Category_Index\"], seed, \"KNN\")\n",
    "elif multi == True and representation_form == \"sequences\" and embedding_algorithm == \"py_w2v\":\n",
    "    runMLCrossVal_multi(embs, data[\"Category_Index\"], seed, \"KNN\")\n",
    "elif multi == True and representation_form == \"sequences\" and embedding_algorithm == \"py_ft\":\n",
    "    runMLCrossVal_multi(embs, data[\"Category_Index\"], seed, \"KNN\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}