category_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "24ba3160",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\envs\\tfgpu\\lib\\site-packages\\tqdm\\auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json, os\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import csv\n",
    "from transformers import AutoTokenizer, TFAutoModel, TFAutoModelForSequenceClassification #, BertModel, BertTokenizer, TFBertForSequenceClassification\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from tensorflow.keras.callbacks import CSVLogger\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import tensorflow.keras.backend as K\n",
    "from collections import OrderedDict\n",
    "import time\n",
    "from sklearn.metrics import accuracy_score, recall_score, f1_score, precision_score, \\\n",
    "roc_auc_score, confusion_matrix, classification_report\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import random\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.layers import LSTM, SimpleRNN\n",
    "from tensorflow.keras.layers import GRU\n",
    "from tensorflow.keras.layers import Masking\n",
    "from tensorflow.keras.layers import Embedding, MaxPool1D\n",
    "from tensorflow.keras.callbacks import CSVLogger\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.constraints import max_norm\n",
    "from tensorflow.keras.layers import Bidirectional, BatchNormalization\n",
    "from tensorflow.keras.layers import LeakyReLU\n",
    "from tensorflow.keras.initializers import glorot_uniform, RandomUniform, lecun_uniform, Constant\n",
    "from tensorflow.python.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "from collections import OrderedDict\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D, GlobalMaxPool1D\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.initializers import TruncatedNormal\n",
    "from tensorflow.keras.losses import CategoricalCrossentropy\n",
    "from tensorflow.keras.metrics import CategoricalAccuracy\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.layers import Input, Dense, GlobalMaxPool1D\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "from sklearn.svm import SVC  \n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import gensim.downloader\n",
    "import io\n",
    "from keras_preprocessing.text import tokenizer_from_json\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import FastText\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca0e24c8",
   "metadata": {},
   "source": [
    "Set the seeder to have as stable random operations as possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3cb5b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 123\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "01a6448b",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_algorithms = [\"w2v\", \"ft\", \"w2v_code\", \"ft_code\", \"bert\", \"codebert\"]\n",
    "embedding_algorithm = embedding_algorithms[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22f9b6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "representations = [\"bow\", \"sequences\"]\n",
    "representation_form = representations[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3acc7396",
   "metadata": {},
   "outputs": [],
   "source": [
    "# binary or multi-class\n",
    "multi = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a56a193",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_w2v = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2010727",
   "metadata": {},
   "source": [
    "Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fbf099bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if representation_form == \"bow\":\n",
    "    #data = pd.read_csv('bow_data.csv') # bow\n",
    "    data = pd.read_csv('preprocessed_data.csv')\n",
    "else:\n",
    "    data = pd.read_csv('sequences_data.csv') # sequences of tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7abc9724",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Vulnerability</th>\n",
       "      <th>Category</th>\n",
       "      <th>Length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f\"str$id\"\"str$id\"\"str$id\"         ...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>client.listentcp()    proxy = proxy(proxy_...</td>\n",
       "      <td>xsrf</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>from django.http import httpresponse, httpresp...</td>\n",
       "      <td>open_redirect</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>def write_preset(conn, queryin, descriptin):\\t...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>update_query = self.up...</td>\n",
       "      <td>sql_injection</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Vulnerability       Category  Length\n",
       "0              f\"str$id\"\"str$id\"\"str$id\"         ...  sql_injection       9\n",
       "1      client.listentcp()    proxy = proxy(proxy_...           xsrf       8\n",
       "2  from django.http import httpresponse, httpresp...  open_redirect       9\n",
       "3  def write_preset(conn, queryin, descriptin):\\t...  sql_injection     175\n",
       "4                          update_query = self.up...  sql_injection      14"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "76e46cb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "392"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(data[\"Length\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "48846336",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum number of words: 392\n"
     ]
    }
   ],
   "source": [
    "if representation_form == \"sequences\":\n",
    "    word_counts = data[\"Vulnerability\"].apply(lambda x: len(x.split()))\n",
    "    max_length = word_counts.max()\n",
    "    print(\"Maximum number of words:\", max_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "30bbacde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Frequencies:\n",
      " sql_injection            1424\n",
      "xsrf                      976\n",
      "command_injection         721\n",
      "path_disclosure           481\n",
      "open_redirect             442\n",
      "remote_code_execution     334\n",
      "xss                       145\n",
      "Name: Category, dtype: int64\n",
      "Total samples  4523\n"
     ]
    }
   ],
   "source": [
    "label_frequencies = data['Category'].value_counts()\n",
    "print(\"Label Frequencies:\\n\", label_frequencies)\n",
    "print(\"Total samples \", len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b777af2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == True:\n",
    "    n_categories = len(label_frequencies) # 7\n",
    "    n_categories"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0984d18f",
   "metadata": {},
   "source": [
    "Word Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f4acbd1",
   "metadata": {},
   "source": [
    "Word2Vec - load pre-trained word2vec embeddings - NL knowledge - static embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4f0d6de6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\iliaskaloup\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "if representation_form == \"sequences\":\n",
    "\n",
    "    # Download the Punkt tokenizer models if not already downloaded\n",
    "    nltk.download('punkt')\n",
    "\n",
    "    tokenized_list = [word_tokenize(sentence) for sentence in data[\"Vulnerability\"].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ae0f0abb",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"w2v\" and representation_form == \"sequences\" and train_w2v == False:\n",
    "\n",
    "    w2v_vectors = gensim.downloader.load('word2vec-google-news-300')\n",
    "    dim = w2v_vectors.vector_size\n",
    "\n",
    "#     fileEmb = 'w2v_embeddingsIters.txt'\n",
    "#     w2v_vectors.save_word2vec_format(fileEmb, binary=False)\n",
    "    \n",
    "#     embeddings_index = {}\n",
    "#     f = open(os.path.join('', fileEmb), encoding=\"utf-8\")\n",
    "#     for line in f:    \n",
    "#         values = line.split()\n",
    "#         word = values[0]\n",
    "#         coefs = np.asarray(values[1:])\n",
    "#         embeddings_index[word] = coefs   \n",
    "#     f.close()\n",
    "    \n",
    "    \n",
    "    tokenizer_obj = Tokenizer()   \n",
    "    tokenizer_obj.fit_on_texts(tokenized_list)\n",
    "    \n",
    "    tokenizer_json = tokenizer_obj.to_json()\n",
    "    tokenizerFile = 'w2v_tokenizer.json'\n",
    "    with io.open(tokenizerFile, 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "        \n",
    "    with open(tokenizerFile) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "        \n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    word_index = tokenizer_obj.word_index\n",
    "    \n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)\n",
    "    \n",
    "    \n",
    "    num_words = len(word_index) + 1 # +1 for the unknown-zeros\n",
    "    embedding_matrix = np.zeros((num_words, dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i > num_words:\n",
    "            continue\n",
    "        #embedding_vector = embeddings_index.get(word)\n",
    "        embedding_vector = w2v_vectors[word] if word in w2v_vectors else None\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2c4e52e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"ft\" and representation_form == \"sequences\":\n",
    "\n",
    "    fastText_vectors = gensim.downloader.load('fasttext-wiki-news-subwords-300')\n",
    "    dim = fastText_vectors.vector_size   \n",
    "    \n",
    "    tokenizer_obj = Tokenizer()   \n",
    "    tokenizer_obj.fit_on_texts(tokenized_list)\n",
    "    \n",
    "    tokenizer_json = tokenizer_obj.to_json()\n",
    "    tokenizerFile = 'fast_tokenizer.json'\n",
    "    with io.open(tokenizerFile, 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "        \n",
    "    with open(tokenizerFile) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "        \n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    word_index = tokenizer_obj.word_index\n",
    "    \n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)    \n",
    "    \n",
    "    num_words = len(word_index) + 1 # +1 for the unknown-zeros\n",
    "    embedding_matrix = np.zeros((num_words, dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i > num_words:\n",
    "            continue\n",
    "        #embedding_vector = embeddings_index.get(word)\n",
    "        embedding_vector = fastText_vectors[word] if word in fastText_vectors else None\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f67fba7",
   "metadata": {},
   "source": [
    "Pre-trained Word2Vec embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b6354f78",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"w2v_code\" and representation_form == \"sequences\":\n",
    "\n",
    "    embeddings_index = {}\n",
    "    f = open(os.path.join('', 'w2v_embeddings.txt'), encoding=\"utf-8\")\n",
    "    for line in f:    \n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:])\n",
    "        embeddings_index[word] = coefs   \n",
    "    f.close() \n",
    "    \n",
    "    dim = 100\n",
    "    \n",
    "    tokenizer_obj = Tokenizer()   \n",
    "    tokenizer_obj.fit_on_texts(tokenized_list)\n",
    "    \n",
    "    tokenizer_json = tokenizer_obj.to_json()\n",
    "    tokenizerFile = 'w2v_new_tokenizer.json'\n",
    "    with io.open(tokenizerFile, 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "        \n",
    "    with open(tokenizerFile) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "        \n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    word_index = tokenizer_obj.word_index\n",
    "    \n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)    \n",
    "    \n",
    "    num_words = len(word_index) + 1 # +1 for the unknown-zeros\n",
    "    embedding_matrix = np.zeros((num_words, dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i > num_words:\n",
    "            continue\n",
    "        #embedding_vector = embeddings_index.get(word)\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5a8f31f",
   "metadata": {},
   "source": [
    "Pre-trained FastText embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9794262d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"ft_code\" and representation_form == \"sequences\":\n",
    "\n",
    "    embeddings_index = {}\n",
    "    f = open(os.path.join('', 'fast-text_embeddings.txt'), encoding=\"utf-8\")\n",
    "    for line in f:    \n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:])\n",
    "        embeddings_index[word] = coefs   \n",
    "    f.close() \n",
    "    \n",
    "    dim = 100\n",
    "    \n",
    "    tokenizer_obj = Tokenizer()   \n",
    "    tokenizer_obj.fit_on_texts(tokenized_list)\n",
    "    \n",
    "    tokenizer_json = tokenizer_obj.to_json()\n",
    "    tokenizerFile = 'fastCode_tokenizer.json'\n",
    "    with io.open(tokenizerFile, 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "        \n",
    "    with open(tokenizerFile) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "        \n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    word_index = tokenizer_obj.word_index\n",
    "    \n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)    \n",
    "    \n",
    "    num_words = len(word_index) + 1 # +1 for the unknown-zeros\n",
    "    embedding_matrix = np.zeros((num_words, dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i > num_words:\n",
    "            continue\n",
    "        #embedding_vector = embeddings_index.get(word)\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "            \n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09a9edbb",
   "metadata": {},
   "source": [
    "BERT - load pre-trained bert embeddings - NL knowledge - contextual embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a6b99ea5",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"bert\" and representation_form == \"sequences\": \n",
    "    model_variation = \"bert-base-uncased\" # \"roberta-base\"\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_variation)\n",
    "    bert = TFAutoModel.from_pretrained(model_variation)\n",
    "    \n",
    "    bert_embeddings = bert.get_input_embeddings()\n",
    "    embedding_matrix = bert_embeddings.weights[0].numpy()\n",
    "    num_words = len(embedding_matrix)\n",
    "    print(num_words)\n",
    "    dim = len(embedding_matrix[0])\n",
    "    print(dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e9c734b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"bert\" and representation_form == \"sequences\":\n",
    "    sentences = data[\"Vulnerability\"].tolist()\n",
    "    sequences = [tokenizer.encode(sente, truncation=True, add_special_tokens=False, return_tensors=\"tf\").numpy() for sente in sentences] # Tokenize the complete sentences\n",
    "    \n",
    "    lines_pad = []\n",
    "    for seq in sequences:\n",
    "        lines_pad.append(seq[0])\n",
    "    \n",
    "    lines_pad = pad_sequences(lines_pad, padding = 'post', maxlen = 512)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb41a30e",
   "metadata": {},
   "source": [
    "CodeBERT - load pre-trained codebert embeddings - PL knowledge - contextual embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8be0cc47",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"codebert\" and representation_form == \"sequences\": \n",
    "    model_variation = \"microsoft/codebert-base\"\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_variation, do_lower_case=True)\n",
    "    codebert = TFAutoModel.from_pretrained(model_variation)\n",
    "    \n",
    "    codebert_embeddings = codebert.get_input_embeddings()\n",
    "    embedding_matrix = codebert_embeddings.weights[0].numpy()\n",
    "    num_words = len(embedding_matrix)\n",
    "    print(num_words)\n",
    "    dim = len(embedding_matrix[0])\n",
    "    print(dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f596d80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def padSequences(sequences, max_len):\n",
    "    lines_pad = []\n",
    "    for sequence in sequences:\n",
    "        seq = sequence['input_ids'].numpy()[0]\n",
    "        if len(seq) < max_len:\n",
    "            for i in range(len(seq), max_len):\n",
    "                seq = np.append(seq, 0)\n",
    "        lines_pad.append(seq)\n",
    "    return lines_pad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "02135a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "if embedding_algorithm == \"codebert\" and representation_form == \"sequences\": \n",
    "\n",
    "    sentences = data[\"Vulnerability\"].tolist()\n",
    "    sequences = [tokenizer(sente, return_tensors=\"tf\", truncation=True, add_special_tokens=False) for sente in sentences]\n",
    "    \n",
    "    lines_pad = padSequences(sequences, 512)\n",
    "    lines_pad = [arr.tolist() for arr in lines_pad]\n",
    "    lines_pad = np.array(lines_pad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0307192",
   "metadata": {},
   "source": [
    "Visualize Text Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c271ce97",
   "metadata": {},
   "source": [
    "RNN model, LSTM specifically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7222be80",
   "metadata": {},
   "outputs": [],
   "source": [
    "def buildLstm(max_len, top_words, dim, seed, embedding_matrix, multi, n_outputs):\n",
    "    model=Sequential()\n",
    "    #model.add(Embedding(input_dim=top_words+1, output_dim=dim, input_length=None, mask_zero=True))\n",
    "    model.add(Embedding(input_dim=top_words, output_dim=dim, input_length=None, weights=[embedding_matrix], mask_zero=True, trainable=False))\n",
    "    #model.add(SimpleRNN(300, dropout=0.3, stateful=False))\n",
    "    model.add(LSTM(100, dropout=0.2, return_sequences=True, stateful=False))\n",
    "    model.add(LSTM(50, dropout=0.1, stateful=False))\n",
    "    #model.add(Bidirectional(LSTM(300, dropout=0.3, stateful=False)))\n",
    "    #model.add(GRU(300, dropout=0.3, stateful=False))\n",
    "    model.add(Activation('relu')) #dropout=0.2, recurrent_dropout=0.2, kernel_constraint=max_norm(3), bias_constraint=max_norm(3)\n",
    "    model.add(BatchNormalization(momentum=0.0))\n",
    "    if multi == False:\n",
    "        model.add(Dense(n_outputs, activation='sigmoid'))\n",
    "        model.compile(loss='binary_crossentropy', optimizer='adam')  \n",
    "    else: \n",
    "        model.add(Dense(n_outputs, activation='softmax'))\n",
    "        model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f8f9769",
   "metadata": {},
   "source": [
    "CNN model 1-d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "688dbee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def buildCnn(max_len, top_words, dim, seed, embedding_matrix, multi, n_outputs):\n",
    "    cnn_model = Sequential()\n",
    "    cnn_model.add(Embedding(top_words, dim, input_length=None, weights=[embedding_matrix], mask_zero=True, trainable=False))\n",
    "    cnn_model.add(Conv1D(filters = 128, kernel_size = 1, activation = 'relu'))\n",
    "    '''cnn_model.add(MaxPooling1D(pool_size = 5))\n",
    "    cnn_model.add(Conv1D(filters = 128, kernel_size = 5, activation = 'relu'))\n",
    "    cnn_model.add(MaxPooling1D(pool_size = 5))\n",
    "    cnn_model.add(Conv1D(filters = 128, kernel_size = 5, activation = 'relu'))'''\n",
    "    cnn_model.add(GlobalMaxPool1D())\n",
    "    #cnn_model.add(Dense(units = 128, activation = 'relu'))\n",
    "    if multi == False:\n",
    "        cnn_model.add(Dense(n_outputs, activation = 'sigmoid'))\n",
    "        cnn_model.compile(loss = \"binary_crossentropy\", optimizer = \"adam\")\n",
    "    else: \n",
    "        model.add(Dense(n_outputs, activation='softmax'))\n",
    "        model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')\n",
    "    return cnn_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "098df0c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def buildLstmCNN(max_len, top_words, dim, seed, embedding_matrix, multi, n_outputs):\n",
    "    model = Sequential()\n",
    "    kernel_initializer = glorot_uniform()\n",
    "\n",
    "    # Embedding Layer\n",
    "    model.add(Embedding(input_dim=top_words, output_dim=dim, input_length=None, weights=[embedding_matrix], mask_zero=True, trainable=False))\n",
    "\n",
    "    # LSTM/GRU Layers\n",
    "    model.add(GRU(500, activation='tanh', dropout=0.2, return_sequences=True, stateful=False, kernel_constraint=max_norm(3), bias_constraint=max_norm(3), kernel_initializer=kernel_initializer))\n",
    "    model.add(GRU(100, activation='tanh', dropout=0.1, return_sequences=True, stateful=False, kernel_initializer=kernel_initializer))\n",
    "    model.add(GRU(200, activation='tanh', dropout=0.1, return_sequences=True, stateful=False, kernel_initializer=kernel_initializer))\n",
    "    \n",
    "    # CNN Layers\n",
    "    model.add(Conv1D(filters=128, kernel_size=5, activation='relu', kernel_initializer=kernel_initializer)) \n",
    "    model.add(GlobalMaxPool1D())\n",
    "\n",
    "    # Fully Connected Layers\n",
    "    #model.add(Dense(64, activation='relu', kernel_initializer=kernel_initializer))\n",
    "    # Classification layer\n",
    "    model.add(Dense(n_outputs, activation='softmax', kernel_initializer=kernel_initializer))\n",
    "\n",
    "    optimizer = optimizers.Adam(learning_rate=0.001)\n",
    "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acab4fc2",
   "metadata": {},
   "source": [
    "Cross-Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cd95cd0",
   "metadata": {},
   "source": [
    "Binary Classification: Recognition of Injection Vulnerabilities (command_injection and sql_injection merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ab9f8af8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to determine if the category is an injection or not\n",
    "def is_injection(category):\n",
    "    if category in ['sql_injection', 'command_injection']:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "if multi == False:\n",
    "    data['Injection'] = data['Category'].apply(is_injection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fcdff955",
   "metadata": {},
   "outputs": [],
   "source": [
    "# BoW and ML\n",
    "\n",
    "def runMLCrossVal_binary(X, y, seed, userModel):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1', 'roc_auc']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X.iloc[train_index, :], X.iloc[test_index, :]\n",
    "        Y_train, Y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "#         Y_train = np.array(Y_train)\n",
    "#         Y_train = Y_train.ravel()\n",
    "#         Y_test = np.array(Y_test)\n",
    "#         Y_test = Y_test.ravel()\n",
    "        \n",
    "#         #sampling\n",
    "#         X_res, Y_res = RandomOverSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "#         #X_res, Y_res = RandomUnderSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "    \n",
    "#         #shuffle dataset\n",
    "#         X_resampled=pd.DataFrame(X_res)\n",
    "#         Y_resampled=pd.DataFrame(Y_res)\n",
    "#         newTrain=X_resampled.assign(Label=Y_resampled.values)\n",
    "#         newTrain = shuffle(newTrain,random_state=seed)\n",
    "#         X_train=np.array(newTrain.iloc[:, 0:-1 ])\n",
    "#         X_train=pd.DataFrame(X_train)\n",
    "#         Y_train=np.array(newTrain.iloc[:, -1 ])\n",
    "#         Y_train=pd.DataFrame(Y_train)\n",
    "        \n",
    "        if userModel == \"svm\":\n",
    "            myModel = SVC(kernel='rbf', gamma=100)\n",
    "        elif userModel == \"RF\":\n",
    "            myModel = RandomForestClassifier(n_estimators=100, bootstrap = True, max_features = 'sqrt')\n",
    "        elif userModel == \"DT\":\n",
    "            myModel = tree.DecisionTreeClassifier(max_depth=120)\n",
    "        elif userModel == \"NB\":\n",
    "            myModel = GaussianNB()\n",
    "\n",
    "        myModel.fit(X_train, Y_train.ravel())\n",
    "        \n",
    "        predictions = myModel.predict(X_test)\n",
    "        #predScores = myModel.predict_proba(X_test)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions)\n",
    "        recall=recall_score(Y_test, predictions)\n",
    "        f1=f1_score(Y_test, predictions)\n",
    "        roc_auc=roc_auc_score(Y_test, predictions)\n",
    "        print(confusion_matrix(Y_test, predictions, labels=[0, 1]))\n",
    "        tn, fp, fn, tp = confusion_matrix(Y_test, predictions).ravel()\n",
    "        acc = ((tp+tn)/(tp+tn+fp+fn))\n",
    "        print(\"Accuracy:%.2f%%\"%(acc*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(\"Roc_Auc score:%.2f%%\"%(roc_auc*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        score_dict['roc_auc'] = np.append(score_dict['roc_auc'], roc_auc)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n",
    "    print(\"roc_auc: %.2f%% (%.2f%%)\" % (score_dict['roc_auc'].mean()*100, score_dict['roc_auc'].std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d4da8420",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == False and representation_form == \"bow\":\n",
    "    runMLCrossVal_binary(data.iloc[:, 0:bow_size], data[\"Injection\"], seed, \"RF\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "246db4fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sequences of tokens and DL\n",
    "\n",
    "def runDLCrossVal_binary(X, y, max_len, num_words, dim, seed, embedding_matrix, userModel, multi):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1', 'roc_auc']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        Y_train, Y_test = y[train_index], y[test_index]\n",
    "#         Y_train = np.array(Y_train)\n",
    "#         Y_train = Y_train.ravel()\n",
    "#         Y_test = np.array(Y_test)\n",
    "#         Y_test = Y_test.ravel()\n",
    "        \n",
    "#         #sampling\n",
    "#         X_res, Y_res = RandomOverSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "#         #X_res, Y_res = RandomUnderSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "    \n",
    "#         #shuffle dataset\n",
    "#         X_resampled=pd.DataFrame(X_res)\n",
    "#         Y_resampled=pd.DataFrame(Y_res)\n",
    "#         newTrain=X_resampled.assign(Label=Y_resampled.values)\n",
    "#         newTrain = shuffle(newTrain,random_state=seed)\n",
    "#         X_train=np.array(newTrain.iloc[:, 0:-1 ])\n",
    "#         X_train=pd.DataFrame(X_train)\n",
    "#         Y_train=np.array(newTrain.iloc[:, -1 ])\n",
    "#         Y_train=pd.DataFrame(Y_train)\n",
    "        \n",
    "        if userModel == \"cnn\":\n",
    "            myModel = buildCnn(max_len, num_words, dim, seed, embedding_matrix, multi, 1) \n",
    "        elif userModel == \"lstm\":\n",
    "            myModel = buildLstm(max_len, num_words, dim, seed, embedding_matrix, multi, 1)\n",
    "        print(\"model summary\\m\",myModel.summary())\n",
    "        csv_logger = CSVLogger('log.csv', append=True, separator=',')\n",
    "        es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "        mc = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)\n",
    "        history = myModel.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs = nb_epoch, batch_size = BS, shuffle=False, verbose=1, callbacks=[csv_logger,es,mc])\n",
    "        \n",
    "        #load best model\n",
    "        #model = load_model('best_model.h5')\n",
    "        myModel.load_weights(\"best_model.h5\")\n",
    "        \n",
    "        scores = myModel.evaluate(X_test, Y_test, verbose=0)\n",
    "        #predictions = myModel.predict_classes(X_test, verbose=0)\n",
    "        predictions = (myModel.predict(X_test) > 0.5).astype(\"int32\")\n",
    "        predScores = myModel.predict(X_test)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions)\n",
    "        recall=recall_score(Y_test, predictions)\n",
    "        f1=f1_score(Y_test, predictions)\n",
    "        roc_auc=roc_auc_score(Y_test, predictions)\n",
    "        print(confusion_matrix(Y_test, predictions, labels=[0, 1]))\n",
    "        tn, fp, fn, tp = confusion_matrix(Y_test, predictions).ravel()\n",
    "        acc = ((tp+tn)/(tp+tn+fp+fn))\n",
    "        print(\"Accuracy:%.2f%%\"%(acc*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(\"Roc_Auc score:%.2f%%\"%(roc_auc*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        score_dict['roc_auc'] = np.append(score_dict['roc_auc'], roc_auc)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n",
    "    print(\"roc_auc: %.2f%% (%.2f%%)\" % (score_dict['roc_auc'].mean()*100, score_dict['roc_auc'].std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ced8e7dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == False and representation_form == \"sequences\":\n",
    "    runDLCrossVal_binary(lines_pad, data[\"Injection\"].values, 512, num_words, dim, seed, embedding_matrix, \"lstm\", multi)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8de3dea",
   "metadata": {},
   "source": [
    "Multi-class Classification: Categorization of all detected vulnerabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ab8dd793",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == True:\n",
    "    \n",
    "    # Convert categories to numerical indexes\n",
    "    category_numerical_indexes, unique_categories = data[\"Category\"].factorize()\n",
    "\n",
    "    # Create a dictionary mapping each category to its numerical index\n",
    "    category_to_index = {category: index for index, category in enumerate(unique_categories)}\n",
    "\n",
    "    # Update the categories in the DataFrame with their numerical indexes\n",
    "    data[\"Category_Index\"] = data[\"Category\"].map(category_to_index)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e9e8cf6",
   "metadata": {},
   "source": [
    "Use ML models and BoW code representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "613fd054",
   "metadata": {},
   "outputs": [],
   "source": [
    "# No sampling\n",
    "def runMLCrossVal_multi(X, y, seed, userModel):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X.iloc[train_index, :], X.iloc[test_index, :]\n",
    "        Y_train, Y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        \n",
    "        X_train_list = X_train[\"Vulnerability\"].tolist()\n",
    "        X_test_list = X_test[\"Vulnerability\"].tolist()\n",
    "\n",
    "        vectorizer = CountVectorizer() #min_df=1\n",
    "        x_train = vectorizer.fit_transform(X_train_list)\n",
    "        x_test = vectorizer.transform(X_test_list)\n",
    "        \n",
    "#         Y_train = np.array(Y_train)\n",
    "#         Y_train = Y_train.ravel()\n",
    "#         Y_test = np.array(Y_test)\n",
    "#         Y_test = Y_test.ravel()\n",
    "        \n",
    "#         #sampling\n",
    "#         class_counts = pd.Series(Y_train).value_counts()\n",
    "#         majority_class = class_counts.idxmax()\n",
    "#         sampling_strategy = {\n",
    "#             majority_class: class_counts.iloc[1]  # Set the second largest class as the target count\n",
    "#         }\n",
    "        \n",
    "#         #X_res, Y_res = RandomOverSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "#         x_resampled, y_resampled = RandomUnderSampler(random_state=seed, sampling_strategy=sampling_strategy).fit_resample(x_train, Y_train)\n",
    "    \n",
    "#         #shuffle dataset\n",
    "#         x_train, Y_train = shuffle(x_resampled, y_resampled, random_state=seed)\n",
    "#         x_train = pd.DataFrame(x_train)\n",
    "        \n",
    "#         # Convert dataframes to numpy arrays if they are not already\n",
    "#         x_train = np.array(x_train)\n",
    "#         Y_train = np.array(Y_train)\n",
    "\n",
    "        \n",
    "        if userModel == \"SVM\":\n",
    "            myModel = SVC(kernel='rbf', gamma=100)\n",
    "        elif userModel == \"RF\":\n",
    "            myModel = RandomForestClassifier(n_estimators=100, bootstrap = True, max_features = 'sqrt')\n",
    "        elif userModel == \"DT\":\n",
    "            myModel = tree.DecisionTreeClassifier(max_depth=120)\n",
    "        elif userModel == \"NB\":\n",
    "            myModel = GaussianNB()\n",
    "        elif userModel == \"KNN\":\n",
    "            myModel = KNeighborsClassifier(n_neighbors=1)\n",
    "            \n",
    "        myModel.fit(x_train, Y_train.ravel())\n",
    "        \n",
    "        predictions = myModel.predict(x_test)\n",
    "        #predScores = myModel.predict_proba(X_test)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions, average='macro')\n",
    "        recall=recall_score(Y_test, predictions, average='macro')\n",
    "        f1=f1_score(Y_test, predictions, average='macro')\n",
    "        conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "        print(conf_matrix)\n",
    "        #tn, fp, fn, tp = confusion_matrix(Y_test, predictions).ravel()\n",
    "        #acc = ((tp+tn)/(tp+tn+fp+fn))\n",
    "        print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3127b79d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # under-sampling\n",
    "# def runMLCrossVal_multi(X, y, seed, userModel):\n",
    "#     ############## cross validation\n",
    "#     scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "#     values = [np.array([]) for i in range(0, len(scores))]\n",
    "#     score_dict = OrderedDict(zip(scores, values))\n",
    "#     k=10\n",
    "#     f=0\n",
    "#     kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "#     nb_epoch = 100\n",
    "#     BS = 64\n",
    "#     print(\"Training...\")\n",
    "#     milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "#     for train_index, test_index in kfold.split(X, y):\n",
    "#         f = f + 1\n",
    "#         print('fold number= ',f)\n",
    "#         X_train, X_test = X.iloc[train_index, :], X.iloc[test_index, :]\n",
    "#         Y_train, Y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        \n",
    "#         X_train_list = X_train[\"Vulnerability\"].tolist()\n",
    "#         X_test_list = X_test[\"Vulnerability\"].tolist()\n",
    "\n",
    "#         vectorizer = CountVectorizer() #min_df=1\n",
    "#         x_train = vectorizer.fit_transform(X_train_list)\n",
    "#         x_test = vectorizer.transform(X_test_list)\n",
    "        \n",
    "#         # Apply under-sampling with the specified strategy\n",
    "#         class_counts = pd.Series(Y_train).value_counts()\n",
    "#         majority_class = class_counts.idxmax() # majority class\n",
    "#         target_count = class_counts.iloc[1]  # Number of samples in the second largest class\n",
    "        \n",
    "#         sampling_strategy = {majority_class: target_count}\n",
    "        \n",
    "#         rus = RandomUnderSampler(random_state=seed, sampling_strategy=sampling_strategy)\n",
    "#         x_train_resampled, Y_train_resampled = rus.fit_resample(x_train, Y_train)        \n",
    "#         # Shuffle the resampled data while preserving the correspondence between features and labels\n",
    "#         x_train_resampled, Y_train_resampled = shuffle(x_train_resampled, Y_train_resampled, random_state=seed)\n",
    "        \n",
    "#         x_train = x_train_resampled\n",
    "#         Y_train = Y_train_resampled\n",
    "        \n",
    "#         if userModel == \"SVM\":\n",
    "#             myModel = SVC(kernel='rbf', gamma=100)\n",
    "#         elif userModel == \"RF\":\n",
    "#             myModel = RandomForestClassifier(n_estimators=100, bootstrap = True, max_features = 'sqrt')\n",
    "#         elif userModel == \"DT\":\n",
    "#             myModel = tree.DecisionTreeClassifier(max_depth=120)\n",
    "#         elif userModel == \"NB\":\n",
    "#             myModel = GaussianNB()\n",
    "            \n",
    "#         myModel.fit(x_train, Y_train)\n",
    "        \n",
    "#         predictions = myModel.predict(x_test)\n",
    "#         #predScores = myModel.predict_proba(X_test)\n",
    "#         accuracy=accuracy_score(Y_test, predictions)\n",
    "#         precision=precision_score(Y_test, predictions, average='macro')\n",
    "#         recall=recall_score(Y_test, predictions, average='macro')\n",
    "#         f1=f1_score(Y_test, predictions, average='macro')\n",
    "#         conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "#         print(conf_matrix)\n",
    "#         #tn, fp, fn, tp = confusion_matrix(Y_test, predictions).ravel()\n",
    "#         #acc = ((tp+tn)/(tp+tn+fp+fn))\n",
    "#         print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "#         print(\"Precision:%.2f%%\"%(precision*100))\n",
    "#         print(\"Recall:%.2f%%\"%(recall*100))\n",
    "#         print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "#         print(classification_report(Y_test, predictions))\n",
    "#         del myModel\n",
    "#         score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "#         score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "#         score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "#         score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "#     milli_sec2 = int(round(time.time() * 1000))\n",
    "#     print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "#     print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "#     print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "#     print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "#     print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f8488cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "if multi == True and representation_form == \"bow\":\n",
    "    runMLCrossVal_multi(pd.DataFrame(data[\"Vulnerability\"]), data[\"Category_Index\"], seed, \"RF\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28251b1",
   "metadata": {},
   "source": [
    "Use DL models and sequences of tokens code representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "881e21ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# no sampling\n",
    "def runDLCrossVal_multi(X, y, max_len, num_words, dim, seed, embedding_matrix, userModel, multi):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        Y_train, Y_test = y[train_index], y[test_index]\n",
    "        \n",
    "        if userModel == \"cnn\":\n",
    "            myModel = buildCnn(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories) \n",
    "        elif userModel == \"lstm\":\n",
    "            myModel = buildLstmCNN(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories)\n",
    "        print(\"model summary\\m\",myModel.summary())\n",
    "        csv_logger = CSVLogger('log.csv', append=True, separator=',')\n",
    "        es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "        mc = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)\n",
    "        history = myModel.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs = nb_epoch, batch_size = BS, shuffle=False, verbose=1, callbacks=[csv_logger,es,mc])\n",
    "        \n",
    "        #load best model\n",
    "        #model = load_model('best_model.h5')\n",
    "        myModel.load_weights(\"best_model.h5\")\n",
    "        \n",
    "        scores = myModel.evaluate(X_test, Y_test, verbose=0)\n",
    "        #predictions = myModel.predict_classes(X_test, verbose=0)\n",
    "        predScores = myModel.predict(X_test)\n",
    "        predictions = np.argmax(predScores, axis=1)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions, average='macro')\n",
    "        recall=recall_score(Y_test, predictions, average='macro')\n",
    "        f1=f1_score(Y_test, predictions, average='macro')\n",
    "        conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "        print(conf_matrix)\n",
    "        print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bd4555d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # over and undersampling\n",
    "# def runDLCrossVal_multi(X, y, max_len, num_words, dim, seed, embedding_matrix, userModel, multi):\n",
    "#     ############## cross validation\n",
    "#     scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "#     values = [np.array([]) for i in range(0, len(scores))]\n",
    "#     score_dict = OrderedDict(zip(scores, values))\n",
    "#     k=10\n",
    "#     f=0\n",
    "#     kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "#     nb_epoch = 100\n",
    "#     BS = 64\n",
    "#     print(\"Training...\")\n",
    "#     milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "#     for train_index, test_index in kfold.split(X, y):\n",
    "#         f = f + 1\n",
    "#         print('fold number= ',f)\n",
    "#         X_train, X_test = X[train_index], X[test_index]\n",
    "#         Y_train, Y_test = y[train_index], y[test_index]\n",
    "\n",
    "#         # Apply under-sampling with the specified strategy\n",
    "#         class_counts = pd.Series(Y_train).value_counts()\n",
    "#         majority_class = class_counts.idxmax() # majority class\n",
    "#         target_count = class_counts.iloc[1]  # Number of samples in the second largest class\n",
    "        \n",
    "#         # under\n",
    "#         sampling_strategy = {majority_class: target_count}        \n",
    "#         rus = RandomUnderSampler(random_state=seed, sampling_strategy=sampling_strategy)\n",
    "#         x_train_resampled, Y_train_resampled = rus.fit_resample(X_train, Y_train)  \n",
    "        \n",
    "#         # over\n",
    "#         for mc in range(2, len(class_counts)):\n",
    "#             minority_class = class_counts.iloc[mc]\n",
    "#             sampling_strategy = {minority_class: target_count}\n",
    "#             ros = RandomOverSampler(random_state=seed, sampling_strategy=sampling_strategy)\n",
    "#             x_train_resampled, Y_train_resampled = rus.fit_resample(x_train_resampled, Y_train_resampled)\n",
    "        \n",
    "#         # Shuffle the resampled data while preserving the correspondence between features and labels\n",
    "#         x_train_resampled, Y_train_resampled = shuffle(x_train_resampled, Y_train_resampled, random_state=seed)\n",
    "        \n",
    "#         X_train = x_train_resampled\n",
    "#         Y_train = Y_train_resampled\n",
    "        \n",
    "#         if userModel == \"cnn\":\n",
    "#             myModel = buildCnn(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories) \n",
    "#         elif userModel == \"lstm\":\n",
    "#             myModel = buildLstm(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories)\n",
    "#         print(\"model summary\\m\",myModel.summary())\n",
    "#         csv_logger = CSVLogger('log.csv', append=True, separator=',')\n",
    "#         es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "#         mc = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)\n",
    "#         history = myModel.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs = nb_epoch, batch_size = BS, shuffle=False, verbose=1, callbacks=[csv_logger,es,mc])\n",
    "        \n",
    "#         #load best model\n",
    "#         #model = load_model('best_model.h5')\n",
    "#         myModel.load_weights(\"best_model.h5\")\n",
    "        \n",
    "#         scores = myModel.evaluate(X_test, Y_test, verbose=0)\n",
    "#         #predictions = myModel.predict_classes(X_test, verbose=0)\n",
    "#         predScores = myModel.predict(X_test)\n",
    "#         predictions = np.argmax(predScores, axis=1)\n",
    "#         accuracy=accuracy_score(Y_test, predictions)\n",
    "#         precision=precision_score(Y_test, predictions, average='macro')\n",
    "#         recall=recall_score(Y_test, predictions, average='macro')\n",
    "#         f1=f1_score(Y_test, predictions, average='macro')\n",
    "#         conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "#         print(conf_matrix)\n",
    "#         print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "#         print(\"Precision:%.2f%%\"%(precision*100))\n",
    "#         print(\"Recall:%.2f%%\"%(recall*100))\n",
    "#         print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "#         print(classification_report(Y_test, predictions))\n",
    "#         del myModel\n",
    "#         score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "#         score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "#         score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "#         score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "#     milli_sec2 = int(round(time.time() * 1000))\n",
    "#     print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "#     print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "#     print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "#     print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "#     print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "bd4ae015",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training...\n",
      "fold number=  1\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru (GRU)                    (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d (Global (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 24s 115ms/step - loss: 1.4122 - val_loss: 1.2831\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.28307, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.1372 - val_loss: 1.2111- ETA: 0s \n",
      "\n",
      "Epoch 00002: val_loss improved from 1.28307 to 1.21106, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.0055 - val_loss: 1.1168\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.21106 to 1.11677, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.8835 - val_loss: 1.0608\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.11677 to 1.06078, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7895 - val_loss: 1.0349\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.06078 to 1.03488, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7051 - val_loss: 1.0594\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.03488\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.6331 - val_loss: 1.0030\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.03488 to 1.00301, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5679 - val_loss: 1.0707\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.00301\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5163 - val_loss: 1.0723\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.00301\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4715 - val_loss: 1.0825\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.00301\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4422 - val_loss: 1.1162\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.00301\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4157 - val_loss: 1.2182\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.00301\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3990 - val_loss: 1.1655\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 1.00301\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3606 - val_loss: 1.2223\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 1.00301\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3344 - val_loss: 1.2043\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 1.00301\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3340 - val_loss: 1.2138\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 1.00301\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3038 - val_loss: 1.3141\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 1.00301\n",
      "Epoch 00017: early stopping\n",
      "[[112   9   3   1   8   2   8]\n",
      " [ 11  74   5   2   2   2   1]\n",
      " [  5   9  21   0   6   3   1]\n",
      " [  3   3   0   4   3   0   1]\n",
      " [  3   4   2   2  22   0   1]\n",
      " [ 10   5   2   0   2  48   5]\n",
      " [  4  10   2   0   2   1  29]]\n",
      "Accuracy:68.43%\n",
      "Precision:63.24%\n",
      "Recall:60.23%\n",
      "F1 score:60.97%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.78      0.77       143\n",
      "           1       0.65      0.76      0.70        97\n",
      "           2       0.60      0.47      0.52        45\n",
      "           3       0.44      0.29      0.35        14\n",
      "           4       0.49      0.65      0.56        34\n",
      "           5       0.86      0.67      0.75        72\n",
      "           6       0.63      0.60      0.62        48\n",
      "\n",
      "    accuracy                           0.68       453\n",
      "   macro avg       0.63      0.60      0.61       453\n",
      "weighted avg       0.69      0.68      0.68       453\n",
      "\n",
      "fold number=  2\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_3 (GRU)                  (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_4 (GRU)                  (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_5 (GRU)                  (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 112ms/step - loss: 1.4291 - val_loss: 1.2208\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.22080, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 1.1737 - val_loss: 1.0879\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.22080 to 1.08793, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 1.0115 - val_loss: 0.9835\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.08793 to 0.98353, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.9016 - val_loss: 0.9416\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.98353 to 0.94164, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.7944 - val_loss: 0.9083\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.94164 to 0.90829, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.7013 - val_loss: 0.9532\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.90829\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.6346 - val_loss: 0.8580\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.90829 to 0.85805, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5579 - val_loss: 0.9056\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.85805\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5244 - val_loss: 0.9643\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.85805\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4722 - val_loss: 0.9431\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.85805\n",
      "Epoch 11/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4278 - val_loss: 0.9351\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.85805\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4010 - val_loss: 0.9276\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.85805\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3682 - val_loss: 0.9616\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.85805\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3505 - val_loss: 0.9585\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.85805\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3462 - val_loss: 1.0515\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.85805\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3319 - val_loss: 1.0118\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.85805\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3168 - val_loss: 1.0392\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.85805\n",
      "Epoch 00017: early stopping\n",
      "[[130   5   3   0   3   1   1]\n",
      " [ 15  73   1   0   1   4   3]\n",
      " [ 10  16  17   0   0   1   1]\n",
      " [  2   3   0   7   0   2   0]\n",
      " [  8   3   2   0  20   0   0]\n",
      " [ 11   2   1   1   5  51   2]\n",
      " [ 13   5   0   1   2   2  25]]\n",
      "Accuracy:71.30%\n",
      "Precision:73.12%\n",
      "Recall:62.36%\n",
      "F1 score:65.88%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.91      0.78       143\n",
      "           1       0.68      0.75      0.72        97\n",
      "           2       0.71      0.38      0.49        45\n",
      "           3       0.78      0.50      0.61        14\n",
      "           4       0.65      0.61      0.62        33\n",
      "           5       0.84      0.70      0.76        73\n",
      "           6       0.78      0.52      0.62        48\n",
      "\n",
      "    accuracy                           0.71       453\n",
      "   macro avg       0.73      0.62      0.66       453\n",
      "weighted avg       0.72      0.71      0.70       453\n",
      "\n",
      "fold number=  3\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_6 (GRU)                  (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_7 (GRU)                  (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_8 (GRU)                  (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 113ms/step - loss: 1.4373 - val_loss: 1.3091TA: 0s - loss: 1.43\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.30914, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 1.1410 - val_loss: 1.2167\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.30914 to 1.21673, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 1.0026 - val_loss: 1.1102\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.21673 to 1.11020, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.9008 - val_loss: 1.0853\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.11020 to 1.08530, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.7924 - val_loss: 1.0495\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.08530 to 1.04946, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.7053 - val_loss: 1.0070\n",
      "\n",
      "Epoch 00006: val_loss improved from 1.04946 to 1.00695, saving model to best_model.h5\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.6191 - val_loss: 1.0777\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 1.00695\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.5564 - val_loss: 0.9865\n",
      "\n",
      "Epoch 00008: val_loss improved from 1.00695 to 0.98649, saving model to best_model.h5\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.5021 - val_loss: 1.0185\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.98649\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4792 - val_loss: 1.0769\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.98649\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4479 - val_loss: 1.0823\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.98649\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.4352 - val_loss: 1.0441\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.98649\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3956 - val_loss: 1.0066\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.98649\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3785 - val_loss: 1.0935\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.98649\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.3412 - val_loss: 1.1202\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.98649\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3259 - val_loss: 1.1276\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.98649\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.2953 - val_loss: 1.1156\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.98649\n",
      "Epoch 18/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.2851 - val_loss: 1.1200\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.98649\n",
      "Epoch 00018: early stopping\n",
      "[[113   7   6   0   5  10   2]\n",
      " [ 11  77   2   0   5   1   1]\n",
      " [ 10   2  23   1   6   2   0]\n",
      " [  8   0   1   6   0   0   0]\n",
      " [  4   1   2   0  23   1   2]\n",
      " [  9   4   3   0   1  54   1]\n",
      " [ 10   8   2   0   1   5  23]]\n",
      "Accuracy:70.42%\n",
      "Precision:71.48%\n",
      "Recall:63.19%\n",
      "F1 score:65.36%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.79      0.73       143\n",
      "           1       0.78      0.79      0.79        97\n",
      "           2       0.59      0.52      0.55        44\n",
      "           3       0.86      0.40      0.55        15\n",
      "           4       0.56      0.70      0.62        33\n",
      "           5       0.74      0.75      0.74        72\n",
      "           6       0.79      0.47      0.59        49\n",
      "\n",
      "    accuracy                           0.70       453\n",
      "   macro avg       0.71      0.63      0.65       453\n",
      "weighted avg       0.71      0.70      0.70       453\n",
      "\n",
      "fold number=  4\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_9 (GRU)                  (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_10 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_11 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_3 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64/64 [==============================] - 14s 117ms/step - loss: 1.4367 - val_loss: 1.1970\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.19699, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 1.1612 - val_loss: 1.0727\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.19699 to 1.07266, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.9943 - val_loss: 1.0121\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.07266 to 1.01208, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.8887 - val_loss: 1.0063\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.01208 to 1.00626, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7761 - val_loss: 0.9593\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.00626 to 0.95932, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.7099 - val_loss: 0.9640\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.95932\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.6234 - val_loss: 0.9624\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.95932\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5796 - val_loss: 0.9774\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.95932\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5167 - val_loss: 1.0082\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.95932\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4986 - val_loss: 1.0301\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.95932\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4553 - val_loss: 1.0101\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.95932\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4131 - val_loss: 1.0138\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.95932\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3898 - val_loss: 1.0339\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.95932\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3539 - val_loss: 1.1404\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.95932\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - ETA: 0s - loss: 0.354 - 5s 77ms/step - loss: 0.3548 - val_loss: 1.0790\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.95932\n",
      "Epoch 00015: early stopping\n",
      "[[118  12   1   0   2   9   1]\n",
      " [  4  84   1   2   0   3   3]\n",
      " [  6  10  20   0   2   3   3]\n",
      " [  1   4   1   7   1   1   0]\n",
      " [  2   8   0   3  15   3   2]\n",
      " [  5   3   4   0   2  57   1]\n",
      " [  8   5   5   1   3   6  20]]\n",
      "Accuracy:71.02%\n",
      "Precision:65.88%\n",
      "Recall:61.07%\n",
      "F1 score:62.46%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.83      0.82       143\n",
      "           1       0.67      0.87      0.75        97\n",
      "           2       0.62      0.45      0.53        44\n",
      "           3       0.54      0.47      0.50        15\n",
      "           4       0.60      0.45      0.52        33\n",
      "           5       0.70      0.79      0.74        72\n",
      "           6       0.67      0.42      0.51        48\n",
      "\n",
      "    accuracy                           0.71       452\n",
      "   macro avg       0.66      0.61      0.62       452\n",
      "weighted avg       0.71      0.71      0.70       452\n",
      "\n",
      "fold number=  5\n",
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_4 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_12 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_13 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_14 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_4 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 113ms/step - loss: 1.4179 - val_loss: 1.1687\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.16873, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.1550 - val_loss: 1.0927\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.16873 to 1.09267, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.0049 - val_loss: 1.0192A: 0s - loss: 1.0\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.09267 to 1.01916, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.8849 - val_loss: 1.0191\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.01916 to 1.01907, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7882 - val_loss: 1.0248 - loss: 0\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 1.01907\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7144 - val_loss: 1.0220\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.01907\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.6301 - val_loss: 0.9856\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.01907 to 0.98556, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5784 - val_loss: 1.0039\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.98556\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5299 - val_loss: 0.9999: 0s - loss: 0.529\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.98556\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4943 - val_loss: 1.0343\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.98556\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4629 - val_loss: 1.0402\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.98556\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.4164 - val_loss: 1.1012\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.98556\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3979 - val_loss: 1.1242\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.98556\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3695 - val_loss: 1.1494\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.98556\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3542 - val_loss: 1.0831\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.98556\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3360 - val_loss: 1.0762\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.98556\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3156 - val_loss: 1.0691\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.98556\n",
      "Epoch 00017: early stopping\n",
      "[[124   6   4   2   1   2   3]\n",
      " [ 12  76   5   1   0   1   3]\n",
      " [  9  10  23   1   0   0   1]\n",
      " [  5   0   1   6   1   1   1]\n",
      " [  3   4   4   0  19   1   2]\n",
      " [ 17   5   4   1   1  42   2]\n",
      " [  7   6   2   0   2   1  30]]\n",
      "Accuracy:70.80%\n",
      "Precision:69.60%\n",
      "Recall:62.22%\n",
      "F1 score:64.89%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.87      0.78       142\n",
      "           1       0.71      0.78      0.74        98\n",
      "           2       0.53      0.52      0.53        44\n",
      "           3       0.55      0.40      0.46        15\n",
      "           4       0.79      0.58      0.67        33\n",
      "           5       0.88      0.58      0.70        72\n",
      "           6       0.71      0.62      0.67        48\n",
      "\n",
      "    accuracy                           0.71       452\n",
      "   macro avg       0.70      0.62      0.65       452\n",
      "weighted avg       0.72      0.71      0.70       452\n",
      "\n",
      "fold number=  6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_15 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_16 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_17 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_5 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_5 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 114ms/step - loss: 1.4443 - val_loss: 1.2263\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.22629, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 1.1487 - val_loss: 1.1420\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.22629 to 1.14202, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 1.0031 - val_loss: 1.0437\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.14202 to 1.04366, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 0.8823 - val_loss: 1.0176\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.04366 to 1.01759, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 0.7808 - val_loss: 1.0386\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 1.01759\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.7110 - val_loss: 1.0409\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.01759\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.6325 - val_loss: 0.9612\n",
      "\n",
      "Epoch 00007: val_loss improved from 1.01759 to 0.96116, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5632 - val_loss: 1.0373\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.96116\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.5218 - val_loss: 0.9916\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.96116\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4553 - val_loss: 1.0447\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.96116\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4373 - val_loss: 1.0842\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.96116\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4367 - val_loss: 1.0801\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.96116\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3855 - val_loss: 1.1237\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.96116\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3548 - val_loss: 1.1510\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.96116\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3380 - val_loss: 1.1535\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.96116\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 0.3243 - val_loss: 1.2499\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.96116\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 76ms/step - loss: 0.3200 - val_loss: 1.3304\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.96116\n",
      "Epoch 00017: early stopping\n",
      "[[115   8   5   2   3   5   4]\n",
      " [  9  77   2   3   2   3   2]\n",
      " [  3   8  21   3   3   4   2]\n",
      " [  4   3   0   4   2   1   1]\n",
      " [  3   5   0   2  20   0   3]\n",
      " [  8   1   2   1   1  57   2]\n",
      " [ 12   3   2   1   0   4  26]]\n",
      "Accuracy:70.80%\n",
      "Precision:63.60%\n",
      "Recall:61.13%\n",
      "F1 score:62.04%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.81      0.78       142\n",
      "           1       0.73      0.79      0.76        98\n",
      "           2       0.66      0.48      0.55        44\n",
      "           3       0.25      0.27      0.26        15\n",
      "           4       0.65      0.61      0.62        33\n",
      "           5       0.77      0.79      0.78        72\n",
      "           6       0.65      0.54      0.59        48\n",
      "\n",
      "    accuracy                           0.71       452\n",
      "   macro avg       0.64      0.61      0.62       452\n",
      "weighted avg       0.70      0.71      0.70       452\n",
      "\n",
      "fold number=  7\n",
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_6 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_18 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_19 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_20 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_6 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 15s 114ms/step - loss: 1.4407 - val_loss: 1.2284\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.22836, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.1583 - val_loss: 1.0786\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.22836 to 1.07861, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 1.0176 - val_loss: 0.9752\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.07861 to 0.97516, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.8965 - val_loss: 0.9484\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.97516 to 0.94843, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.8020 - val_loss: 0.9240\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.94843 to 0.92404, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.7200 - val_loss: 0.9214\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.92404 to 0.92142, saving model to best_model.h5\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.6491 - val_loss: 0.8956\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.92142 to 0.89559, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5750 - val_loss: 0.9340\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.89559\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.5188 - val_loss: 0.9656\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.89559\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4748 - val_loss: 0.9866\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.89559\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4518 - val_loss: 0.9825\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.89559\n",
      "Epoch 12/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64/64 [==============================] - 5s 82ms/step - loss: 0.4257 - val_loss: 0.9453\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.89559\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 77ms/step - loss: 0.3974 - val_loss: 0.9844\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.89559\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3489 - val_loss: 1.0426\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.89559\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3332 - val_loss: 1.0976\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.89559\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3251 - val_loss: 1.1579\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.89559\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3025 - val_loss: 1.1955oss: 0.30\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.89559\n",
      "Epoch 00017: early stopping\n",
      "[[120  14   3   0   0   3   2]\n",
      " [  8  78   2   3   1   3   3]\n",
      " [ 10   6  23   0   0   3   2]\n",
      " [  3   1   0   9   1   1   0]\n",
      " [  4   4   1   1  15   6   2]\n",
      " [ 13   3   1   0   0  54   1]\n",
      " [  2   7   3   1   0   3  32]]\n",
      "Accuracy:73.23%\n",
      "Precision:73.77%\n",
      "Recall:66.21%\n",
      "F1 score:68.69%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.85      0.79       142\n",
      "           1       0.69      0.80      0.74        98\n",
      "           2       0.70      0.52      0.60        44\n",
      "           3       0.64      0.60      0.62        15\n",
      "           4       0.88      0.45      0.60        33\n",
      "           5       0.74      0.75      0.74        72\n",
      "           6       0.76      0.67      0.71        48\n",
      "\n",
      "    accuracy                           0.73       452\n",
      "   macro avg       0.74      0.66      0.69       452\n",
      "weighted avg       0.74      0.73      0.73       452\n",
      "\n",
      "fold number=  8\n",
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_7 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_21 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_22 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_23 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_7 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_7 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 113ms/step - loss: 1.4301 - val_loss: 1.2309\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.23088, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 1.1527 - val_loss: 1.1111\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.23088 to 1.11109, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.9994 - val_loss: 1.0205\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.11109 to 1.02053, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.8649 - val_loss: 0.9975\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.02053 to 0.99745, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.7743 - val_loss: 1.0215\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.99745\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.6937 - val_loss: 1.0006\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.99745\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.6179 - val_loss: 1.0503\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.99745\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.5405 - val_loss: 1.0310\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.99745\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.5074 - val_loss: 1.1023\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.99745\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4755 - val_loss: 1.1311\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.99745\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4307 - val_loss: 1.1274\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.99745\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3941 - val_loss: 1.1061\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.99745\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3892 - val_loss: 1.1352\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.99745\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3571 - val_loss: 1.0737\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.99745\n",
      "Epoch 00014: early stopping\n",
      "[[120  12   3   0   3   3   1]\n",
      " [ 12  73   1   2   0   6   4]\n",
      " [ 12   7  21   0   1   2   1]\n",
      " [  5   2   1   5   0   1   0]\n",
      " [  3   3   4   0  23   1   0]\n",
      " [ 13   5   1   1   1  47   4]\n",
      " [  8   9   3   1   4   2  21]]\n",
      "Accuracy:68.58%\n",
      "Precision:66.84%\n",
      "Recall:59.87%\n",
      "F1 score:62.34%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.85      0.76       142\n",
      "           1       0.66      0.74      0.70        98\n",
      "           2       0.62      0.48      0.54        44\n",
      "           3       0.56      0.36      0.43        14\n",
      "           4       0.72      0.68      0.70        34\n",
      "           5       0.76      0.65      0.70        72\n",
      "           6       0.68      0.44      0.53        48\n",
      "\n",
      "    accuracy                           0.69       452\n",
      "   macro avg       0.67      0.60      0.62       452\n",
      "weighted avg       0.68      0.69      0.68       452\n",
      "\n",
      "fold number=  9\n",
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_24 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_25 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_26 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_8 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_8 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 116ms/step - loss: 1.4262 - val_loss: 1.2586\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.25864, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 1.1503 - val_loss: 1.1399\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.25864 to 1.13986, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 1.0052 - val_loss: 1.0723\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.13986 to 1.07226, saving model to best_model.h5\n",
      "Epoch 4/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64/64 [==============================] - 5s 79ms/step - loss: 0.8864 - val_loss: 1.0501\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.07226 to 1.05008, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.7683 - val_loss: 1.0489\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.05008 to 1.04887, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.6944 - val_loss: 1.0775\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 1.04887\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.6194 - val_loss: 1.0656\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 1.04887\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.5643 - val_loss: 1.0822\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 1.04887\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.5134 - val_loss: 1.0746\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 1.04887\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.4732 - val_loss: 1.1255\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 1.04887\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.4268 - val_loss: 1.0945\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 1.04887\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.3920 - val_loss: 1.2220\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 1.04887\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.3826 - val_loss: 1.1299\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 1.04887\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 80ms/step - loss: 0.3462 - val_loss: 1.2826\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 1.04887\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 79ms/step - loss: 0.3215 - val_loss: 1.2479ET\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 1.04887\n",
      "Epoch 00015: early stopping\n",
      "[[114  17   4   0   2   4   1]\n",
      " [ 11  78   3   0   0   1   5]\n",
      " [  8  13  19   0   0   1   3]\n",
      " [  4   3   1   4   0   1   1]\n",
      " [  9   8   0   0  13   2   2]\n",
      " [ 11   9   1   0   0  49   2]\n",
      " [  4   3   5   1   1   6  28]]\n",
      "Accuracy:67.48%\n",
      "Precision:70.34%\n",
      "Recall:56.61%\n",
      "F1 score:60.16%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.80      0.75       142\n",
      "           1       0.60      0.80      0.68        98\n",
      "           2       0.58      0.43      0.49        44\n",
      "           3       0.80      0.29      0.42        14\n",
      "           4       0.81      0.38      0.52        34\n",
      "           5       0.77      0.68      0.72        72\n",
      "           6       0.67      0.58      0.62        48\n",
      "\n",
      "    accuracy                           0.67       452\n",
      "   macro avg       0.70      0.57      0.60       452\n",
      "weighted avg       0.69      0.67      0.67       452\n",
      "\n",
      "fold number=  10\n",
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (None, None, 100)         763000    \n",
      "_________________________________________________________________\n",
      "gru_27 (GRU)                 (None, None, 500)         903000    \n",
      "_________________________________________________________________\n",
      "gru_28 (GRU)                 (None, None, 100)         180600    \n",
      "_________________________________________________________________\n",
      "gru_29 (GRU)                 (None, None, 200)         181200    \n",
      "_________________________________________________________________\n",
      "conv1d_9 (Conv1D)            (None, None, 128)         128128    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_9 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 2,156,831\n",
      "Trainable params: 1,393,831\n",
      "Non-trainable params: 763,000\n",
      "_________________________________________________________________\n",
      "model summary\\m None\n",
      "Epoch 1/100\n",
      "64/64 [==============================] - 14s 115ms/step - loss: 1.4036 - val_loss: 1.1850\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 1.18499, saving model to best_model.h5\n",
      "Epoch 2/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 1.1600 - val_loss: 1.0815\n",
      "\n",
      "Epoch 00002: val_loss improved from 1.18499 to 1.08149, saving model to best_model.h5\n",
      "Epoch 3/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 1.0102 - val_loss: 1.0081\n",
      "\n",
      "Epoch 00003: val_loss improved from 1.08149 to 1.00810, saving model to best_model.h5\n",
      "Epoch 4/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.8970 - val_loss: 1.0031\n",
      "\n",
      "Epoch 00004: val_loss improved from 1.00810 to 1.00309, saving model to best_model.h5\n",
      "Epoch 5/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.7972 - val_loss: 0.9362\n",
      "\n",
      "Epoch 00005: val_loss improved from 1.00309 to 0.93625, saving model to best_model.h5\n",
      "Epoch 6/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.7142 - val_loss: 0.9900\n",
      "\n",
      "Epoch 00006: val_loss did not improve from 0.93625\n",
      "Epoch 7/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.6448 - val_loss: 0.9040\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.93625 to 0.90404, saving model to best_model.h5\n",
      "Epoch 8/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.6087 - val_loss: 0.9910\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.90404\n",
      "Epoch 9/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.5353 - val_loss: 0.9434\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.90404\n",
      "Epoch 10/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4803 - val_loss: 0.9947\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.90404\n",
      "Epoch 11/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4362 - val_loss: 0.9520\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.90404\n",
      "Epoch 12/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.4173 - val_loss: 1.0790\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.90404\n",
      "Epoch 13/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3947 - val_loss: 1.0431\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.90404\n",
      "Epoch 14/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3885 - val_loss: 1.0250\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.90404\n",
      "Epoch 15/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3583 - val_loss: 1.0634\n",
      "\n",
      "Epoch 00015: val_loss did not improve from 0.90404\n",
      "Epoch 16/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3390 - val_loss: 1.1031\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.90404\n",
      "Epoch 17/100\n",
      "64/64 [==============================] - 5s 78ms/step - loss: 0.3285 - val_loss: 1.1324\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.90404\n",
      "Epoch 00017: early stopping\n",
      "[[122   7   5   1   2   4   1]\n",
      " [ 17  73   3   0   0   2   3]\n",
      " [ 10   9  23   0   0   2   0]\n",
      " [  3   4   0   4   2   1   0]\n",
      " [  3   4   2   0  22   0   3]\n",
      " [ 11   5   2   0   1  52   1]\n",
      " [  8  10   1   0   2   1  26]]\n",
      "Accuracy:71.24%\n",
      "Precision:73.63%\n",
      "Recall:61.76%\n",
      "F1 score:65.32%\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.86      0.77       142\n",
      "           1       0.65      0.74      0.70        98\n",
      "           2       0.64      0.52      0.57        44\n",
      "           3       0.80      0.29      0.42        14\n",
      "           4       0.76      0.65      0.70        34\n",
      "           5       0.84      0.72      0.78        72\n",
      "           6       0.76      0.54      0.63        48\n",
      "\n",
      "    accuracy                           0.71       452\n",
      "   macro avg       0.74      0.62      0.65       452\n",
      "weighted avg       0.72      0.71      0.71       452\n",
      "\n",
      "Cross Validation is completed after 977530\n",
      "accuracy: 70.33% (1.61%)\n",
      "precision: 69.15% (3.82%)\n",
      "recall: 61.47% (2.34%)\n",
      "f1: 63.81% (2.50%)\n"
     ]
    }
   ],
   "source": [
    "if multi == True and representation_form == \"sequences\" and train_w2v == False:\n",
    "    runDLCrossVal_multi(lines_pad, data[\"Category_Index\"].values, 512, num_words, dim, seed, embedding_matrix, \"lstm\", multi)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e31b9aa3",
   "metadata": {},
   "source": [
    "One more option for training your own word2vec vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "55864532",
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainWord2Vec(tokenized_list, max_length, dim):\n",
    "    # this should be executed only in the training set during cross-validation\n",
    "    w2v_model = Word2Vec(sentences=tokenized_list, vector_size=dim, window=5, min_count=1, workers=4, epochs=4)\n",
    "    #w2v_model.save(\"python_word2vec.model\")\n",
    "    #w2v_model = Word2Vec.load(\"python_word2vec.model\")\n",
    "\n",
    "    #     fileEmb = 'w2v_embeddingsIters.txt'\n",
    "    #     w2v_vectors.save_word2vec_format(fileEmb, binary=False)\n",
    "\n",
    "    #     embeddings_index = {}\n",
    "    #     f = open(os.path.join('', fileEmb), encoding=\"utf-8\")\n",
    "    #     for line in f:    \n",
    "    #         values = line.split()\n",
    "    #         word = values[0]\n",
    "    #         coefs = np.asarray(values[1:])\n",
    "    #         embeddings_index[word] = coefs   \n",
    "    #     f.close()\n",
    "\n",
    "\n",
    "    tokenizer_obj = Tokenizer()   \n",
    "    tokenizer_obj.fit_on_texts(tokenized_list)\n",
    "\n",
    "    tokenizer_json = tokenizer_obj.to_json()\n",
    "    tokenizerFile = 'w2v_new_tokenizer.json'\n",
    "    with io.open(tokenizerFile, 'w', encoding='utf-8') as f:\n",
    "        f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "\n",
    "    with open(tokenizerFile) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "\n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    word_index = tokenizer_obj.word_index\n",
    "\n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)\n",
    "\n",
    "\n",
    "    num_words = len(word_index) + 1 # +1 for the unknown-zeros\n",
    "    embedding_matrix = np.zeros((num_words, dim))\n",
    "    for word, i in word_index.items():\n",
    "        if i > num_words:\n",
    "            continue\n",
    "        #embedding_vector = embeddings_index.get(word)\n",
    "        embedding_vector = w2v_model.wv[word] if word in w2v_model.wv else None\n",
    "        if embedding_vector is not None:\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "\n",
    "    return lines_pad, embedding_matrix, num_words, tokenizerFile\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6e99615c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def testWord2Vec(tokenized_list, tokenizer_path, max_length):\n",
    "    \n",
    "    with open(tokenizer_path) as f:\n",
    "        dataTokenizer = json.load(f)\n",
    "        tokenizer_obj = tokenizer_from_json(dataTokenizer)\n",
    "        \n",
    "    sequences = tokenizer_obj.texts_to_sequences(tokenized_list)\n",
    "    \n",
    "    lines_pad = pad_sequences(sequences, padding = 'post', maxlen = max_length)\n",
    "    \n",
    "    return lines_pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "275db8c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def runDLCrossValW2v_multi(X, y, max_len, dim, seed, userModel):\n",
    "    ############## cross validation\n",
    "    scores=['accuracy', 'precision', 'recall', 'f1']\n",
    "    values = [np.array([]) for i in range(0, len(scores))]\n",
    "    score_dict = OrderedDict(zip(scores, values))\n",
    "    k=10\n",
    "    f=0\n",
    "    kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "    \n",
    "    nb_epoch = 100\n",
    "    BS = 64\n",
    "    print(\"Training...\")\n",
    "    milli_sec1 = int(round(time.time() * 1000))\n",
    "    \n",
    "    for train_index, test_index in kfold.split(X, y):\n",
    "        f = f + 1\n",
    "        print('fold number= ',f)\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        Y_train, Y_test = y[train_index], y[test_index]\n",
    "        \n",
    "        X_train, embedding_matrix, num_words, tokenizerFile = trainWord2Vec(X_train, max_len, dim)\n",
    "        \n",
    "        X_test = testWord2Vec(X_test, tokenizerFile, max_len)\n",
    "        \n",
    "#         Y_train = np.array(Y_train)\n",
    "#         Y_train = Y_train.ravel()\n",
    "#         Y_test = np.array(Y_test)\n",
    "#         Y_test = Y_test.ravel()\n",
    "        \n",
    "#         #sampling\n",
    "#         X_res, Y_res = RandomOverSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "#         #X_res, Y_res = RandomUnderSampler(random_state=seed, sampling_strategy=0.5).fit_resample(X_train, Y_train)\n",
    "    \n",
    "#         #shuffle dataset\n",
    "#         X_resampled=pd.DataFrame(X_res)\n",
    "#         Y_resampled=pd.DataFrame(Y_res)\n",
    "#         newTrain=X_resampled.assign(Label=Y_resampled.values)\n",
    "#         newTrain = shuffle(newTrain,random_state=seed)\n",
    "#         X_train=np.array(newTrain.iloc[:, 0:-1 ])\n",
    "#         X_train=pd.DataFrame(X_train)\n",
    "#         Y_train=np.array(newTrain.iloc[:, -1 ])\n",
    "#         Y_train=pd.DataFrame(Y_train)\n",
    "        \n",
    "        if userModel == \"cnn\":\n",
    "            myModel = buildCnn(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories) \n",
    "        elif userModel == \"lstm\":\n",
    "            myModel = buildLstm(max_len, num_words, dim, seed, embedding_matrix, multi, n_categories)\n",
    "        print(\"model summary\\m\",myModel.summary())\n",
    "        csv_logger = CSVLogger('log.csv', append=True, separator=',')\n",
    "        es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "        mc = ModelCheckpoint('best_model.h5', monitor='val_loss', mode='min', verbose=1, save_best_only=True)\n",
    "        \n",
    "        history = myModel.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs = nb_epoch, batch_size = BS, shuffle=False, verbose=1, callbacks=[csv_logger,es,mc])\n",
    "        \n",
    "        \n",
    "        #load best model\n",
    "        #model = load_model('best_model.h5')\n",
    "        myModel.load_weights(\"best_model.h5\")\n",
    "        \n",
    "        scores = myModel.evaluate(X_test, Y_test, verbose=0)\n",
    "        #predictions = myModel.predict_classes(X_test, verbose=0)\n",
    "        predScores = myModel.predict(X_test)\n",
    "        predictions = np.argmax(predScores, axis=1)\n",
    "        accuracy=accuracy_score(Y_test, predictions)\n",
    "        precision=precision_score(Y_test, predictions, average='macro')\n",
    "        recall=recall_score(Y_test, predictions, average='macro')\n",
    "        f1=f1_score(Y_test, predictions, average='macro')\n",
    "        conf_matrix = confusion_matrix(Y_test, predictions)\n",
    "        print(conf_matrix)\n",
    "        print(\"Accuracy:%.2f%%\"%(accuracy*100))\n",
    "        print(\"Precision:%.2f%%\"%(precision*100))\n",
    "        print(\"Recall:%.2f%%\"%(recall*100))\n",
    "        print(\"F1 score:%.2f%%\"%(f1*100))\n",
    "        print(classification_report(Y_test, predictions))\n",
    "        del myModel\n",
    "        score_dict['accuracy'] = np.append(score_dict['accuracy'], accuracy)\n",
    "        score_dict['precision'] = np.append(score_dict['precision'], precision)\n",
    "        score_dict['recall'] = np.append(score_dict['recall'], recall)\n",
    "        score_dict['f1'] = np.append(score_dict['f1'], f1)\n",
    "        \n",
    "    milli_sec2 = int(round(time.time() * 1000))\n",
    "    print(\"Cross Validation is completed after\", milli_sec2-milli_sec1)\n",
    "    \n",
    "    print(\"accuracy: %.2f%% (%.2f%%)\" % (score_dict['accuracy'].mean()*100, score_dict['accuracy'].std()*100))\n",
    "    print(\"precision: %.2f%% (%.2f%%)\" % (score_dict['precision'].mean()*100, score_dict['precision'].std()*100))\n",
    "    print(\"recall: %.2f%% (%.2f%%)\" % (score_dict['recall'].mean()*100, score_dict['recall'].std()*100))\n",
    "    print(\"f1: %.2f%% (%.2f%%)\" % (score_dict['f1'].mean()*100, score_dict['f1'].std()*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "de802e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "if train_w2v == True and multi == True and representation_form == \"sequences\":\n",
    "    dim = 300\n",
    "    runDLCrossValW2v_multi(np.array(tokenized_list), data[\"Category_Index\"].values, 512, dim, seed, \"lstm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4504cdb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}