lab10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 10\n",
    "\n",
    "## TODO:\n",
    "\n",
    "- [] Decide whether you will be using a computer vision data set in this and next lab. If so, either pick one of your choosing (but not benchmark data sets like MNIST or FASNION MNIST), or use a simple smiley face data set provided on Canvas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sklearn\n",
    "import random\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, SensitivityAtSpecificity, SpecificityAtSensitivity, Precision, Recall, AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>failures</th>\n",
       "      <th>higher</th>\n",
       "      <th>studytime</th>\n",
       "      <th>Medu</th>\n",
       "      <th>Fedu</th>\n",
       "      <th>Dalc</th>\n",
       "      <th>age</th>\n",
       "      <th>reason_reputation</th>\n",
       "      <th>school</th>\n",
       "      <th>address</th>\n",
       "      <th>internet</th>\n",
       "      <th>G3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1043</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1044 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      failures  higher  studytime  Medu  Fedu  Dalc  age  reason_reputation  \\\n",
       "0            0       1          2     4     4     1   18                  0   \n",
       "1            0       1          2     1     1     1   17                  0   \n",
       "2            3       1          2     1     1     2   15                  0   \n",
       "3            0       1          3     4     2     1   15                  0   \n",
       "4            0       1          2     3     3     1   16                  0   \n",
       "...        ...     ...        ...   ...   ...   ...  ...                ...   \n",
       "1039         1       1          3     2     3     1   19                  0   \n",
       "1040         0       1          2     3     1     1   18                  0   \n",
       "1041         0       1          2     1     1     1   18                  0   \n",
       "1042         0       1          1     3     1     3   17                  0   \n",
       "1043         0       1          1     3     2     3   18                  0   \n",
       "\n",
       "      school  address  internet  G3  \n",
       "0          0        0         0   6  \n",
       "1          0        0         1   6  \n",
       "2          0        0         1  10  \n",
       "3          0        0         1  15  \n",
       "4          0        0         0  10  \n",
       "...      ...      ...       ...  ..  \n",
       "1039       1        1         1  10  \n",
       "1040       1        0         1  16  \n",
       "1041       1        0         0   9  \n",
       "1042       1        0         1  10  \n",
       "1043       1        1         1  11  \n",
       "\n",
       "[1044 rows x 12 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "df3 = pd.read_csv(\"Data/data3.csv\")\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>failures</th>\n",
       "      <th>higher</th>\n",
       "      <th>studytime</th>\n",
       "      <th>Medu</th>\n",
       "      <th>Fedu</th>\n",
       "      <th>Dalc</th>\n",
       "      <th>age</th>\n",
       "      <th>reason_reputation</th>\n",
       "      <th>school</th>\n",
       "      <th>address</th>\n",
       "      <th>internet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1043</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1044 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      failures  higher  studytime  Medu  Fedu  Dalc  age  reason_reputation  \\\n",
       "0            0       1          2     4     4     1   18                  0   \n",
       "1            0       1          2     1     1     1   17                  0   \n",
       "2            3       1          2     1     1     2   15                  0   \n",
       "3            0       1          3     4     2     1   15                  0   \n",
       "4            0       1          2     3     3     1   16                  0   \n",
       "...        ...     ...        ...   ...   ...   ...  ...                ...   \n",
       "1039         1       1          3     2     3     1   19                  0   \n",
       "1040         0       1          2     3     1     1   18                  0   \n",
       "1041         0       1          2     1     1     1   18                  0   \n",
       "1042         0       1          1     3     1     3   17                  0   \n",
       "1043         0       1          1     3     2     3   18                  0   \n",
       "\n",
       "      school  address  internet  \n",
       "0          0        0         0  \n",
       "1          0        0         1  \n",
       "2          0        0         1  \n",
       "3          0        0         1  \n",
       "4          0        0         0  \n",
       "...      ...      ...       ...  \n",
       "1039       1        1         1  \n",
       "1040       1        0         1  \n",
       "1041       1        0         0  \n",
       "1042       1        0         1  \n",
       "1043       1        1         1  \n",
       "\n",
       "[1044 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df3.copy()\n",
    "y = X.pop(\"G3\") # y becomes the G3 column and deletes it from df3_copy\n",
    "y_binary = [0 if y_i > np.mean(y) else 1 for y_i in y] #making the target class into binary (over/under average score)\n",
    "\n",
    "# display(X)\n",
    "# print(y_binary)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scale\n",
    "scaler = MinMaxScaler()\n",
    "X = scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras & Smile Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "smiley_X, smiley_y = np.load(\"./Data/smiley_X.npy\"), np.load(\"./Data/smiley_y.npy\")\n",
    "X_train, X_test, y_train, y_test = train_test_split(smiley_X, smiley_y, test_size=1/3, stratify=smiley_y, random_state=42,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAADCUlEQVR4nO3csYrDMBQAwdPh//9lpUkXEUHA8cbMlHYhgVkeuHhjzvkH9PxffQFgTZwQJU6IEidEiROijs17v3LhfGP10OSEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkToo6rL3CmMcbpZ8w5Tz/jl/kGnzM5IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQtTYLOS957ZeaFlu3jY5IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihKjj6gv8ujGWK0d52uxF5g2TE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlToi69d7ab+yUtZf1Pd/gcyYnRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcEDU2Oz/vuRAUWpbLfU1OiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcEHVs3o+v3AJ4YXJClDghSpwQJU6IEidEiROiHvSOHM4vlf74AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "i = random.randint(0,95)\n",
    "print(y_train[i])\n",
    "plt.imshow(X_train[i], cmap=\"binary\")\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 3ms/step - loss: 0.0121 - accuracy: 1.0000 - true_positives_15: 24.0000 - true_negatives_15: 24.0000 - false_positives_15: 0.0000e+00 - false_negatives_15: 0.0000e+00 - precision_11: 1.0000 - recall_11: 1.0000 - specificity_at_sensitivity_7: 1.0000 - auc_6: 1.0000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWgUlEQVR4nO3df4zc9X3n8efbu2uvKV5+GJMEr4NNICQmSggxXHrJtVzTFuM2Ic1JV3O6S0Vyh5DIXU89tSWt7nJVdVLucr3+EGktlEM06hWrVRLhVgRKc01TndqCCZBgE4IDCV6MamNKbBJ7vTPzvj9mZj07u/bO7n7X48/4+ZBWu9/vfGf28xnLL7/9mc/n843MRJJUvhX9boAkqRoGuiQNCANdkgaEgS5JA8JAl6QBMdyvX3zJJZfkxo0b+/XrJalIjz/++CuZuW6ux/oW6Bs3bmT37t39+vWSVKSI+N6pHnPIRZIGhIEuSQPCQJekAWGgS9KAMNAlaUDMG+gRcW9EHIyIp0/xeETE70XEvoj4RkRcV30zJUnz6aVCvw/YeprHbwauan3dDvzB0pslSVqoeeehZ+bXImLjaS65Bfh8Nvfh/buIuDAi3pSZL1fVyIX4wuMTfO/wD3q69t2XX8Q/v/rSZW5R/zz09MvsPXCk382Q1GXLxov5sbfOuTZoSapYWLQe2N9xPNE6NyvQI+J2mlU8b37zmyv41TMdn6rzn/70qdbvOv21mbDh4tX8za/8ROXtOFv86he+yfePTc37Xkg6s+748bectYE+V1zMedeMzLwHuAdgy5Ytld9Z48jxKQB+88Pv4N+89/LTXvtfd+3hi1+fqLoJZ41GIzlyfIr/8IGr+KWfemu/myPpDKhilssEsKHjeBw4UMHrLtiRYzUAxkbn/3dqbHSYo5M1Go3BvGPT0ckamb29F5IGQxWBvgv4aGu2y3uB7/dr/LxdoY+tHpn32rHVI2TC6ydqy92svjhyrPf3QtJgmLd8i4j7gRuBSyJiAvgUMAKQmTuAB4FtwD7gh8Bty9XY+UyH2GgPgd665sixqZ6uL830P24D2DdJc+tllsut8zyewJ2VtWgJjh5vVtsXrO5hyKV1Tfs5g6bdr7Ee3gtJg2GgVoq2q9I1PVSlazoq9EG0kP+tSBoMgxXo0x+KLmDIZUAr9Ha/DHTp3DFYgX58ipGhYHRk/m61hyIGvkJ3yEU6ZxT/tz0z+e2/fI4Drx3jiRf/kbHREaKHlTTtyvWP/v57/O3zh1kR8LH3b+JtbxyrpF1/sns/j77waiWvtRjPvNxcIXr+quL/iCX1qPi/7d8/NsXvfeU5xkaHWTM6wgfe3ttS/gtWj/CjV6zlxVd/yMEjk7z02jEuPG8lv7atmkD/nUe+zWvHprjovJWVvN5i/OTbL2V4aKD+EybpNIoP9BP1BgC/vPVt864O7bRiRXD/7e+dPr7+v/0lR49XN/xy5HiNn79+A5/64DWVvaYknU7x5Vut3lzpuXJoaRuWjI0OT3+oulS1eoPXJ2t+ICnpjCo+0KdaFfrwiqV1ZWz1yPS0x6V6fbI9B9xAl3TmDECgNyv04SVX6COVTWGcXtTjPiqSzqDiA73WaFboI0v88G/N6DBHK5rC+P1jvS9wkqSqlB/orQp9qYFe5ZDLyU3CrNAlnTnFB3p7lkslQy7HajS3plmahaxYlaSqFB/o0xX6kj8UHeZEvcFkrbHkNrUr9Av8UFTSGTQAgV5dhQ7w7z6/m/2v/nBRr3HwyHE+ft9j7Pjqd2a8piSdCcUHenvIZalj6D/6lrVcv/Ei/ua5V/i75w8v6jWe2P8aX/nWQVavHOJfXDfuGLqkM6r4QD/5oejSKvS3rDufz330emDxOzC2h2t+d/u1/Na/fFdPe8pIUlXKD/RGNQuLAM4fXdoOjCdagb5qeGjJbZGkhSo+0KcqqtABhlYEa1YNL/ouRpO1OgCrhot/WyUVqPjkmapoDL1tzejwouejT05ZoUvqn+IDvVbR0v+2sdUjix5yaY+hr+rhBhuSVLXik2eqoqX/bc09XRYb6M0hl5XuQS6pD4pPnqqW/reNrV78NrqTtQYrh1awYoWzWySdecUH+lRFC4vaxkZHODq5+DF0PxCV1C/Fp89URUv/28ZWj/DSPx7jxs/8Ff/z4WcX9NzJWt3xc0l9U3z6VLX0v+0j163nlmvXM1VPHtn7Dwt67mSt4QwXSX1TfKBPNVqzXCoat37n+IX89s9fyz99y9oFfzjaDPTi31JJhSo+fabqDUaGovJl9mOrRxa8wGhyqs5KA11SnxSfPrV6o5Jl/93GRkd4fbI2PaTTCyt0Sf1UfPpM1bOy8fNOa1r7urRv+NyLyVrdMXRJfVN8oNcajWVZyDPWujnFQuakT9YaznKR1DfFp89UbXkq9LH2zosL+GD0hEMukvqo+PSZaizTGHq7Ql9AoDttUVI/FR/otXpWsnVutzXTe6MvdAy9+LdUUqGKvkfavoNH2fXUAa645Ecqf+0Lz1sJwJ1//HWGepwSeaLe4P1XWqFL6o+iA/2FV5o3c/7Idesrf+3LLhjlUx/czMGjkz0/J4APv7v6tkhSL4oO9PZ2tTdd88bKXzsiuO19myp/XUlaLkUP+HqHIEk6qexA9w5BkjSt6CT0psySdFJPSRgRWyPi2YjYFxF3zfH4RRHxpYj4RkQ8GhHvqL6ps7UrdDfEkqQeAj0ihoDPAjcDm4FbI2Jz12W/BjyZme8EPgr8btUNnUt7DN17eEpSbxX6DcC+zHw+M08AO4Fbuq7ZDHwFIDO/BWyMiDdU2tI5TNbqDK8Ihg10Seop0NcD+zuOJ1rnOj0FfAQgIm4ALgfGu18oIm6PiN0RsfvQoUOLa3EHt6uVpJN6ScO5lklm1/GngYsi4kng3wNPALPWzGfmPZm5JTO3rFu3bqFtneVErcGqEacsShL0trBoAtjQcTwOHOi8IDOPALcBRPPWQS+0vpaVe6dI0km9pOFjwFURsSkiVgLbgV2dF0TEha3HAP4t8LVWyC8rh1wk6aR5K/TMrEXEJ4CHgSHg3szcExF3tB7fAbwd+HxE1IG9wMeXsc3TJqfcrlaS2nrayyUzHwQe7Dq3o+PnvwWuqrZp85us1V0lKkktRafhZG15bj8nSSUqOg29h6cknVR0GjZnuTiGLklQeqBPOctFktqKTsMTdQNdktqKTsPJqYY7LUpSS9FpWM9kaEXRXZCkyhSdho1GsmKunWYk6RxUdKA3K3QTXZKg8EBvVugGuiRB6YGeGOiS1FJ0oNcbiSv/Jamp6DhsZLLCMXRJAgYh0B1ykSSg+ECHIQNdkoDCA73uPHRJmlZsoDcazftUO4YuSU3lBno2A90hF0lqKjbQ62mFLkmdig30RqP53VkuktRUbqC3h1yK7YEkVavYOJwecrFClySg4EBPh1wkaYZiA70+PeRioEsSlBzo7Xno5rkkAQUHejptUZJmKDbQ6y4skqQZyg30hrNcJKlTsYHeKtAdcpGklmIDvV2hu7BIkpqKjcOGC4skaQYDXZIGRLGBXnelqCTNUGyguzmXJM1UbBw6bVGSZio20B1Dl6SZCg705nc355KkpmIDve5NoiVphp4CPSK2RsSzEbEvIu6a4/ELIuLPIuKpiNgTEbdV39SZTg65LPdvkqQyzBvoETEEfBa4GdgM3BoRm7suuxPYm5nvAm4EfisiVlbc1hkaDTfnkqROvVToNwD7MvP5zDwB7ARu6bomgTUREcD5wKtArdKWdqm7fa4kzdBLoK8H9nccT7TOdbobeDtwAPgm8IuZ7ZvEnRQRt0fE7ojYfejQoUU2uWl6cy4rdEkCegv0uRIzu45vAp4ELgOuBe6OiLFZT8q8JzO3ZOaWdevWLbCpM7k5lyTN1EscTgAbOo7HaVbinW4DvphN+4AXgLdV08S51Z2HLkkz9BLojwFXRcSm1ged24FdXde8CHwAICLeAFwNPF9lQ7ulgS5JMwzPd0Fm1iLiE8DDwBBwb2buiYg7Wo/vAH4TuC8ivklziOZXM/OVZWz39OZcLiySpKZ5Ax0gMx8EHuw6t6Pj5wPAT1fbtNNzLxdJmqnYjxSnh1yK7YEkVavYOGx/KOrCIklqKjbQG94kWpJmKDfQHUOXpBmKDfS6e7lI0gzFBnp7t0XzXJKaig9056FLUlOxge7CIkmaqdhAd8hFkmYqPtD9UFSSmooN9JPb5xrokgQFB3p7YVFYoUsSUHKgW6FL0gzlBrpj6JI0Q7GBXneWiyTNUGygO+QiSTOVG+itD0UdcpGkpmIDvT1t0TyXpKZiA32q3mBkKJy2KEktxQb6ZK3BquGhfjdDks4aBQd6nVXDxTZfkipXbCJOTjUMdEnqUGwiTtYarBpxyEWS2goOdIdcJKlTsYl4ouaQiyR1KjYRneUiSTOVHegjxTZfkipXbCI6hi5JMxWbiJNTDVYa6JI0rdhEdAxdkmYqONAdcpGkTsUm4qTTFiVphmITcXLKlaKS1KnIQM9Mh1wkqUuRiVhrJI3EQJekDkUm4mStAeAsF0nqUGagT9UBXCkqSR2KTMR2hb5yqMjmS9Ky6CkRI2JrRDwbEfsi4q45Hv/liHiy9fV0RNQj4uLqm9vUvkH00ArvJypJbfMGekQMAZ8FbgY2A7dGxObOazLzM5l5bWZeC3wS+OvMfHUZ2gsY6JI0l14q9BuAfZn5fGaeAHYCt5zm+luB+6to3KnU00CXpG69BPp6YH/H8UTr3CwRcR6wFfjC0pt2ao1Whb4iDHRJausl0OdKzTzFtR8E/t+phlsi4vaI2B0Ruw8dOtRrG2exQpek2XoJ9AlgQ8fxOHDgFNdu5zTDLZl5T2Zuycwt69at672VXWp1A12SuvUS6I8BV0XEpohYSTO0d3VfFBEXAD8OPFBtE2drtCt0h1wkadrwfBdkZi0iPgE8DAwB92bmnoi4o/X4jtalPwf8RWb+YNla2+IsF0mabd5AB8jMB4EHu87t6Dq+D7ivqoadTrtCX2GgS9K0Ipda1psLRR1ykaQOhQZ6u0Lvc0Mk6SxSZCS2A33YRJekaUUm4sl56H1uiCSdRYqMRFeKStJsRQa60xYlabYyAz2t0CWpW5GB3rBCl6RZigx0N+eSpNnKDHQrdEmapexAdwxdkqaVHehW6JI0rchAd3MuSZqtyEB3cy5Jmq3MQE8355KkbkVGYr1Vors5lySdVGQitm4p6pCLJHUoMtAb7ocuSbMUGYmuFJWk2coMdLfPlaRZigx0N+eSpNmKDPTpIRcrdEmaVmagN5IIV4pKUqdiA93qXJJmKjPQM63OJalLkYHesEKXpFmKDPR6wxkuktStyEBvZGKeS9JMRQZ6rdFgeKjIpkvSsikyFesNV4lKUrciA73RSCzQJWmmImOxns5ykaRuRQZ6o+E8dEnqVmSg1zOdtihJXYoM9FrDQJekbkUGuitFJWm2IgO9boUuSbMM97sBi9FcKWqgS1qcqakpJiYmOH78eL+bckqjo6OMj48zMjLS83OKDHQrdElLMTExwZo1a9i4cSNxFhaHmcnhw4eZmJhg06ZNPT+vpyGXiNgaEc9GxL6IuOsU19wYEU9GxJ6I+OueW7AI9fTmFpIW7/jx46xdu/asDHOAiGDt2rUL/h/EvBV6RAwBnwV+CpgAHouIXZm5t+OaC4HfB7Zm5osRcemCWrFAx6fqjA4XOfwv6SxxtoZ522La10sq3gDsy8znM/MEsBO4peuafwV8MTNfBMjMgwtuyQIcOTbF2Orex5Uk6VzQS6CvB/Z3HE+0znV6K3BRRHw1Ih6PiI/O9UIRcXtE7I6I3YcOHVpci4Gjx2uMjRroksr10EMPcfXVV3PllVfy6U9/upLX7CXQ56r7s+t4GHgP8DPATcB/joi3znpS5j2ZuSUzt6xbt27BjW1rVuhFfp4rSdTrde68806+/OUvs3fvXu6//3727t07/xPn0UsqTgAbOo7HgQNzXPNKZv4A+EFEfA14F/DtJbewS6ORvH6ixhordEkV+I0/28PeA0cqfc3Nl43xqQ9ec8rHH330Ua688kquuOIKALZv384DDzzA5s2bl/R7e6nQHwOuiohNEbES2A7s6rrmAeCfRcRwRJwH/BPgmSW17BSOTtbIhLFRK3RJZXrppZfYsOFknTw+Ps5LL7205NedNxUzsxYRnwAeBoaAezNzT0Tc0Xp8R2Y+ExEPAd8AGsDnMvPpJbduDkeOTQH4oaikSpyukl4umd2j1tXMuumpzM3MB4EHu87t6Dr+DPCZJbdoHkeOtwLdIRdJhRofH2f//pNzTSYmJrjsssuW/LrFTeY+erwG4Ieikop1/fXX89xzz/HCCy9w4sQJdu7cyYc+9KElv25xqTg95GKFLqlQw8PD3H333dx0003U63U+9rGPcc01Sx/6KS7Q156/kpvf8UbWrVnV76ZI0qJt27aNbdu2VfqaxQX6ey6/mPdcfnG/myFJZ53ixtAlSXMz0CWdk+aaOng2WUz7DHRJ55zR0VEOHz581oZ6ez/00dHRBT2vuDF0SVqq8fFxJiYmWMomgcutfceihTDQJZ1zRkZGFnQnoFI45CJJA8JAl6QBYaBL0oCIfn3KGxGHgO8t8umXAK9U2JwS2Odzg30+Nyylz5dn5px3COpboC9FROzOzC39bseZZJ/PDfb53LBcfXbIRZIGhIEuSQOi1EC/p98N6AP7fG6wz+eGZelzkWPokqTZSq3QJUldDHRJGhDFBXpEbI2IZyNiX0Tc1e/2VCUi7o2IgxHxdMe5iyPikYh4rvX9oo7HPtl6D56NiJv60+qliYgNEfFXEfFMROyJiF9snR/YfkfEaEQ8GhFPtfr8G63zA9tngIgYiognIuLPW8cD3V+AiPhuRHwzIp6MiN2tc8vb78ws5gsYAr4DXAGsBJ4CNve7XRX17ceA64CnO879D+Cu1s93Af+99fPmVt9XAZta78lQv/uwiD6/Cbiu9fMa4Nutvg1sv4EAzm/9PAL8PfDeQe5zqx+/BPwx8Oet44Hub6sv3wUu6Tq3rP0urUK/AdiXmc9n5glgJ3BLn9tUicz8GvBq1+lbgD9s/fyHwIc7zu/MzMnMfAHYR/O9KUpmvpyZX2/9fBR4BljPAPc7m15vHY60vpIB7nNEjAM/A3yu4/TA9ncey9rv0gJ9PbC/43iidW5QvSEzX4Zm+AGXts4P3PsQERuBd9OsWAe6363hhyeBg8AjmTnoff4d4FeARse5Qe5vWwJ/ERGPR8TtrXPL2u/S9kOPOc6di/MuB+p9iIjzgS8A/zEzj0TM1b3mpXOcK67fmVkHro2IC4EvRcQ7TnN50X2OiJ8FDmbm4xFxYy9PmeNcMf3t8r7MPBARlwKPRMS3TnNtJf0urUKfADZ0HI8DB/rUljPhHyLiTQCt7wdb5wfmfYiIEZph/n8y84ut0wPfb4DMfA34KrCVwe3z+4APRcR3aQ6R/kRE/BGD299pmXmg9f0g8CWaQyjL2u/SAv0x4KqI2BQRK4HtwK4+t2k57QJ+ofXzLwAPdJzfHhGrImITcBXwaB/atyTRLMX/N/BMZv6vjocGtt8Rsa5VmRMRq4GfBL7FgPY5Mz+ZmeOZuZHm39f/m5n/mgHtb1tE/EhErGn/DPw08DTL3e9+fxK8iE+Ot9GcDfEd4Nf73Z4K+3U/8DIwRfNf648Da4GvAM+1vl/ccf2vt96DZ4Gb+93+Rfb5/TT/W/kN4MnW17ZB7jfwTuCJVp+fBv5L6/zA9rmjHzdycpbLQPeX5ky8p1pfe9pZtdz9dum/JA2I0oZcJEmnYKBL0oAw0CVpQBjokjQgDHRJGhAGuiQNCANdkgbE/wd03KmUgHWXAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 24ms/step - loss: 6.7634e-04 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 2.1181e-04 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 2.6671e-05 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 7.9819e-06 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 26ms/step - loss: 1.2031e-05 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 3.1655e-06 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 16ms/step - loss: 1.7914e-06 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 31ms/step - loss: 2.2886e-07 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 24ms/step - loss: 1.2168e-07 - accuracy: 1.0000\n",
      "1/1 [==============================] - 0s 28ms/step - loss: 6.4966e-08 - accuracy: 1.0000\n",
      "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
      "[0.0006763405981473625, 0.00021180622570682317, 2.667050648597069e-05, 7.98188011685852e-06, 1.2031421647407115e-05, 3.165470388921676e-06, 1.7913844203576446e-06, 2.28859590833963e-07, 1.2167849661182117e-07, 6.49659384066581e-08]\n"
     ]
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[9, 9]),\n",
    "    keras.layers.Dense(25, activation=\"relu\"),\n",
    "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "])\n",
    "\n",
    "model.compile(loss=\"binary_crossentropy\",\n",
    "              optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\",\n",
    "                       tf.keras.metrics.TruePositives(),\n",
    "                       tf.keras.metrics.TrueNegatives(),\n",
    "                       tf.keras.metrics.FalsePositives(),\n",
    "                       tf.keras.metrics.FalseNegatives(), \n",
    "                       tf.keras.metrics.Precision(),\n",
    "                       tf.keras.metrics.Recall(),\n",
    "                       tf.keras.metrics.SpecificityAtSensitivity(0.5),\n",
    "                       tf.keras.metrics.AUC()])\n",
    "\n",
    "history = model.fit(X_train, y_train, validation_split=0.5, epochs=500, verbose=0,)\n",
    "pd.DataFrame(history.history[\"val_accuracy\"]).plot()\n",
    "\n",
    "loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under = model.evaluate(X_test, y_test)\n",
    "\n",
    "\n",
    "data = pd.DataFrame([[\"TF\", 2, \"Adam\", loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under, 0.5]], \n",
    "                    columns=[\"Model\", \"Layers\", \"LossFn\", \"Loss\", \"Accuracy\",\"TP\", \"TN\", \"FP\", \"FN\", \"Precision\", \"Recall\", \"SpecificityAtSensitivity\", \"AUC\", \"split\"])\n",
    "table = pd.concat([table, data], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 2ms/step - loss: 0.0012 - accuracy: 1.0000 - true_positives_16: 24.0000 - true_negatives_16: 24.0000 - false_positives_16: 0.0000e+00 - false_negatives_16: 0.0000e+00 - precision_12: 1.0000 - recall_12: 1.0000 - specificity_at_sensitivity_8: 1.0000 - auc_7: 1.0000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXf0lEQVR4nO3dbYxc1X3H8e/P+4CDMc+LBV4Tm+ISTBoI3TiRaAkpAWy31E1eVKaqEpFEliUTkVZK4yRSoygvShs1SirculZqEdQYvwkWTuTYoLQJUpQEr4vBD+CwsQleltYLKKGlJTsP/76YO7t3Z2d3r3dnWfvM7yONdu69586cs5J/e3zuufcoIjAzs3QtmO8KmJnZ3HLQm5klzkFvZpY4B72ZWeIc9GZmieuc7wo0c/nll8fy5cvnuxpmZueMgwcPvhoRPc2OnZVBv3z5cvr7++e7GmZm5wxJv5zsmIduzMwS56A3M0ucg97MLHEOejOzxDnozcwSN23QS9oh6bSkI5Mcl6R/kDQg6VlJN+eOrZF0PDu2pZUVNzOzYor06B8C1kxxfC2wMnttBP4JQFIHsDU7vgq4R9Kq2VTWzMzO3LTz6CPiSUnLpyiyHng4as87/qmkiyVdCSwHBiLiBICkXVnZY7Ou9RnY+bOX+M9f/9/b+ZVmZjNy/nmdbPrgb7X8c1txw9RS4FRuezDb12z/+yf7EEkbqf2PgKuvvroF1YLX3xzhC7sPZ5/fko80M5szl19w3lkb9M0iNKbY31REbAe2A/T19bVkNZTflCsA/M1Hf4d7Vrfmj4eZ2bmmFUE/CCzLbfcCQ0D3JPvfNuVK7e9FV4cnF5lZ+2pFAu4BPpbNvvkA8OuIeAU4AKyUtEJSN7AhK/u2GalUAejq8LiNmbWvaXv0kh4BbgMulzQIfAnoAoiIbcBeYB0wAPwvcG92rCzpPmA/0AHsiIijc9CGSblHb2ZWbNbNPdMcD2DzJMf2UvtDMC9KWY++c4F79GbWvpLu6taDvqsz6WaamU0p6QQs1YduFiTdTDOzKSWdgGVfjDUzSzvo67NuOn0x1szaWNIJWJ910+2gN7M2lnQCjs668dCNmbWxpIN+7IappJtpZjalpBNw7IYp9+jNrH0lHfQl9+jNzBIP+mqtR+8xejNrZ2kHfbnWo/esGzNrZ0knYLnqefRmZkknYMkXY83MUg/67GKsn3VjZm0s6QQsVap0LBAL/JhiM2tjSQd9uRIetjGztpd00I9Uqh62MbO2l3QKlivhRUfMrO0lnYKlStXLCJpZ2ysU9JLWSDouaUDSlibHL5G0W9Kzkp6S9O7csRclHZZ0SFJ/Kys/nVIl/PgDM2t70y4OLqkD2ArcAQwCByTtiYhjuWJfAA5FxEckvSsrf3vu+Ici4tUW1ruQUqXqi7Fm1vaKdHdXAwMRcSIiRoBdwPqGMquAHwBExPPAcklLWlrTGShXq+7Rm1nbK5KCS4FTue3BbF/eM8BHASStBt4J9GbHAnhc0kFJGyf7EkkbJfVL6h8eHi5a/ymNlMOPPzCztlckBZuNfUTD9gPAJZIOAZ8GngbK2bFbIuJmYC2wWdKtzb4kIrZHRF9E9PX09BSq/HTK1SrdHroxszY37Rg9tR78stx2LzCULxARbwD3AkgScDJ7ERFD2c/TknZTGwp6ctY1L6BUqbpHb2Ztr0gKHgBWSlohqRvYAOzJF5B0cXYM4FPAkxHxhqRFkhZnZRYBdwJHWlf9qZV8Z6yZ2fQ9+ogoS7oP2A90ADsi4qikTdnxbcD1wMOSKsAx4JPZ6UuA3bVOPp3AzojY1/pmNFeqVLngvCL/aTEzS1ehFIyIvcDehn3bcu9/Aqxsct4J4MZZ1nHGatMrPXRjZu0t6RQsV8J3xppZ20s66EcqVT/rxszaXtIpWK4EXe7Rm1mbSzroPUZvZpZ80PvOWDOzpFOwVPGdsWZmSQd92UM3ZmZpB72HbszMEg76iKDkh5qZmaUb9JVqEIF79GbW9pJNwXK19iRlj9GbWbtLNgVHKlUAP73SzNpeskFfrrhHb2YGCQd9KevRd7pHb2ZtLvmgd4/ezNpdsilYGh26cY/ezNpbskFfdo/ezAxIOOjrs246FyTbRDOzQpJNwfqsm+5OD92YWXtLNuhL7tGbmQEFg17SGknHJQ1I2tLk+CWSdkt6VtJTkt5d9Ny5MuIxejMzoEDQS+oAtgJrgVXAPZJWNRT7AnAoIt4DfAz4xhmcOydGyr4z1swMivXoVwMDEXEiIkaAXcD6hjKrgB8ARMTzwHJJSwqeOyfu33UIgIVdHW/H15mZnbWKBP1S4FRuezDbl/cM8FEASauBdwK9Bc8lO2+jpH5J/cPDw8VqP4VqNVi8sJPrr7xw1p9lZnYuKxL0zcY+omH7AeASSYeATwNPA+WC59Z2RmyPiL6I6Ovp6SlQralVI/jTvmV0LPDQjZm1t84CZQaBZbntXmAoXyAi3gDuBZAk4GT2On+6c+dKqRq+EGtmRrEe/QFgpaQVkrqBDcCefAFJF2fHAD4FPJmF/7TnzpVSpeoLsWZmFOjRR0RZ0n3AfqAD2BERRyVtyo5vA64HHpZUAY4Bn5zq3Llpypj66lLu0ZuZFRu6ISL2Ansb9m3Lvf8JsLLouXPNjyg2MxuTZJe3HvTd7tGbmaUa9LWJPZ2ecWNmlmbQjz6iuDPJ5pmZnZEkk3D0OTd+oJmZWZpBP7owuB9RbGaWZtD7EcVmZmOSTMKx9WKTbJ6Z2RlJMglLFT+i2MysLsmgL1e96IiZWV2SSThSzubRu0dvZpZm0PvOWDOzMUkmYX3optNBb2aWZtDXh258MdbMLNGg98VYM7MxSSbh2PTKJJtnZnZGkkxCP73SzGxMokGfzbrx0yvNzNIM+rIfgWBmNirJJPRSgmZmYwoFvaQ1ko5LGpC0pcnxiyR9V9Izko5Kujd37EVJhyUdktTfyspPpj5G7xumzMwKLA4uqQPYCtwBDAIHJO2JiGO5YpuBYxFxt6Qe4Likb0fESHb8QxHxaqsrP5mxxxS7R29mVqTLuxoYiIgTWXDvAtY3lAlgsSQBFwCvA+WW1rSg35QrfO2JnwPQ4aA3MysU9EuBU7ntwWxf3oPA9cAQcBi4PyKq2bEAHpd0UNLGyb5E0kZJ/ZL6h4eHCzeg0Su/eguAa6+4gNrfHTOz9lYk6JulZTRs3wUcAq4CbgIelHRhduyWiLgZWAtslnRrsy+JiO0R0RcRfT09PUXq3lT9rtjPfHjljD/DzCwlRYJ+EFiW2+6l1nPPuxd4NGoGgJPAuwAiYij7eRrYTW0oaM6MPqLYywiamQHFgv4AsFLSCkndwAZgT0OZl4DbASQtAa4DTkhaJGlxtn8RcCdwpFWVb6beo+/2wuBmZkCBWTcRUZZ0H7Af6AB2RMRRSZuy49uArwAPSTpMbajncxHxqqRrgN3ZWHknsDMi9s1RWwAvDG5m1mjaoAeIiL3A3oZ923Lvh6j11hvPOwHcOMs6npGxRxQ76M3MIME7Y8ceUeyhGzMzSDDo/YhiM7PxkkvD0UcUu0dvZgYkGfReGNzMLC+5NCyP9uiTa5qZ2Ywkl4YjFV+MNTPLSy7oveiImdl4yaWhZ92YmY2XXBqWPHRjZjZOgkHvoRszs7zk0rDsoRszs3GSS8NSpYrk1aXMzOqSC/qRSrg3b2aWk1wilitVutybNzMblVzQlypVujqTa5aZ2Ywll4ilanjRETOznOQSsVSu0u059GZmo5IL+nI1/EAzM7Oc5BJxpFL1XbFmZjmFgl7SGknHJQ1I2tLk+EWSvivpGUlHJd1b9NxWK1eqnl5pZpYzbSJK6gC2AmuBVcA9klY1FNsMHIuIG4HbgL+X1F3w3JYqeR69mdk4RRJxNTAQESciYgTYBaxvKBPAYkkCLgBeB8oFz22pUqXqZQTNzHKKBP1S4FRuezDbl/cgcD0wBBwG7o+IasFzAZC0UVK/pP7h4eGC1Z+o5KEbM7NxiiRis+5xNGzfBRwCrgJuAh6UdGHBc2s7I7ZHRF9E9PX09BSoVnO1oRv36M3M6ooE/SCwLLfdS63nnncv8GjUDAAngXcVPLelfDHWzGy8Iol4AFgpaYWkbmADsKehzEvA7QCSlgDXAScKnttSIxXfGWtmltc5XYGIKEu6D9gPdAA7IuKopE3Z8W3AV4CHJB2mNlzzuYh4FaDZuXPTlJpypUp3p4duzMzqpg16gIjYC+xt2Lct934IuLPouXOpVKm6R29mlpNcInoevZnZeMklYslDN2Zm4yQX9GU/ptjMbJzkErFU9vRKM7O85BKxVPXTK83M8tILel+MNTMbJ6lErFaDSjX8UDMzs5ykgr5UrQK4R29mlpNUIpYqteeleYzezGxMUkFfrrhHb2bWKKlEHMmC3ouDm5mNSSoRy9nQTbeHbszMRiUV9KV6j953xpqZjUoqEUcvxnYm1Swzs1lJKhHrPfquBR66MTOrSyroy6PTK5NqlpnZrCSViGOzbtyjNzOrSyro6/Pou92jNzMblVQi1i/Geh69mdmYQokoaY2k45IGJG1pcvyzkg5lryOSKpIuzY69KOlwdqy/1Q3IG70Y66EbM7NR0y4OLqkD2ArcAQwCByTtiYhj9TIR8VXgq1n5u4G/iIjXcx/zoYh4taU1b6LkRyCYmU1QJBFXAwMRcSIiRoBdwPopyt8DPNKKyp2pkmfdmJlNUCQRlwKnctuD2b4JJJ0PrAG+k9sdwOOSDkraONmXSNooqV9S//DwcIFqTVSuetaNmVmjIkHfLDVjkrJ3Az9uGLa5JSJuBtYCmyXd2uzEiNgeEX0R0dfT01OgWhNVqrVqdchBb2ZWVyToB4Flue1eYGiSshtoGLaJiKHs52lgN7WhoDkxGvS+M9bMbFSRoD8ArJS0QlI3tTDf01hI0kXAB4HHcvsWSVpcfw/cCRxpRcWbiez/GQsc9GZmo6addRMRZUn3AfuBDmBHRByVtCk7vi0r+hHg8Yh4M3f6EmC3akMpncDOiNjXygbkVbKkd86bmY2ZNugBImIvsLdh37aG7YeAhxr2nQBunFUNz4DH6M3MJkpqHmK13qN3l97MbFRaQe8evZnZBEkFfaV+MdZBb2Y2Kqmgr/fovZKgmdmYpCKxPuvG8+jNzMYkFfSjF2M9dGNmNiqtoK866M3MGiUV9NlTij10Y2aWk1bQ+85YM7MJkgr6iGCBQB66MTMblVTQV6rh8XkzswZpBX2EH39gZtYgqaCvVsOPPzAza5BU0FeqnnFjZtYoqaCvRuAOvZnZeMkFvXv0ZmbjJRX0FY/Rm5lNkFTQVz3rxsxsgrSCvupFR8zMGiUV9JXszlgzMxtTKOglrZF0XNKApC1Njn9W0qHsdURSRdKlRc5tpWrVQzdmZo2mDXpJHcBWYC2wCrhH0qp8mYj4akTcFBE3AZ8HfhQRrxc5t5UqnnVjZjZBkR79amAgIk5ExAiwC1g/Rfl7gEdmeO6sVMNj9GZmjYoE/VLgVG57MNs3gaTzgTXAd2Zw7kZJ/ZL6h4eHC1RromrVN0yZmTUqEvTNojMmKXs38OOIeP1Mz42I7RHRFxF9PT09Bao1UaXqoRszs0ZFgn4QWJbb7gWGJim7gbFhmzM9d9Zqs24c9GZmeUWC/gCwUtIKSd3UwnxPYyFJFwEfBB4703NbJXwx1sxsgs7pCkREWdJ9wH6gA9gREUclbcqOb8uKfgR4PCLenO7cVjeizguPmJlNNG3QA0TEXmBvw75tDdsPAQ8VOXeuVALPozcza5DUnbG1hUfmuxZmZmeXtILeY/RmZhMkFfSVaiCP0ZuZjZNU0FfDz6M3M2uUVND7hikzs4nSCnrPujEzmyCpoA8/j97MbIKkgt5rxpqZTZRc0HvoxsxsvEJ3xp4rPOvGzGajVCoxODjIW2+9Nd9VmdTChQvp7e2lq6ur8DmJBT2edWNmMzY4OMjixYtZvnz5WXlPTkTw2muvMTg4yIoVKwqfl9TQjRceMbPZeOutt7jsssvOypAHkMRll112xv/jSCrovWasmc3W2RrydTOpX1pB71k3ZmYTJBX04RumzCwB+/bt47rrruPaa6/lgQcemPXnJRX0tYVH5rsWZmYzV6lU2Lx5M9///vc5duwYjzzyCMeOHZvVZyY168Zj9GbWKl/+7lGODb3R0s9cddWFfOnuG6Ys89RTT3HttddyzTXXALBhwwYee+wxVq1aNePvTapHX/VSgmZ2jnv55ZdZtmzZ6HZvby8vv/zyrD4zqR69Fx4xs1aZruc9VyJiwr7ZzgQq1KOXtEbScUkDkrZMUuY2SYckHZX0o9z+FyUdzo71z6q20/Di4GZ2ruvt7eXUqVOj24ODg1x11VWz+sxpe/SSOoCtwB3AIHBA0p6IOJYrczHwj8CaiHhJ0hUNH/OhiHh1VjUtoBo46M3snPa+972PF154gZMnT7J06VJ27drFzp07Z/WZRYZuVgMDEXECQNIuYD2Qvwz8Z8CjEfESQEScnlWtZqi28Mh8fLOZWWt0dnby4IMPctddd1GpVPjEJz7BDTfMbhipSNAvBU7ltgeB9zeU+W2gS9IPgcXANyLi4exYAI9LCuCfI2J7sy+RtBHYCHD11VcXbkDeXTcs4forL5zRuWZmZ4t169axbt26ln1ekaBvNhbSeLWgE/hd4HbgHcBPJP00In4O3BIRQ9lwzhOSno+IJyd8YO0PwHaAvr6+iVcjCvj6hvfO5DQzs6QVGegYBJbltnuBoSZl9kXEm9lY/JPAjQARMZT9PA3spjYUZGZmb5MiQX8AWClphaRuYAOwp6HMY8DvS+qUdD61oZ3nJC2StBhA0iLgTuBI66pvZtZazaY3nk1mUr9ph24ioizpPmA/0AHsiIijkjZlx7dFxHOS9gHPAlXgmxFxRNI1wO5sDmgnsDMi9p1xLc3M3gYLFy7ktddeO2sfVVx/Hv3ChQvP6DydjX+9+vr6or9/Tqfcm5lNcC6vMCXpYET0NTsnqTtjzcxmo6ur64xWbjpXeNa5mVniHPRmZolz0JuZJe6svBgraRj45QxPvxyY8+fqnGXc5vbgNreHmbb5nRHR0+zAWRn0syGpf7Irz6lym9uD29we5qLNHroxM0ucg97MLHEpBn3Tp2Mmzm1uD25ze2h5m5Mbozczs/FS7NGbmVmOg97MLHHJBH2RBczPRZJ2SDot6Uhu36WSnpD0Qvbzktyxz2e/g+OS7pqfWs+OpGWS/l3Sc9li8/dn+5Ntt6SFkp6S9EzW5i9n+5Ntc52kDklPS/petp10myW9KOmwpEOS+rN9c9vmiDjnX9Qen/wL4BqgG3gGWDXf9WpR224FbgaO5Pb9HbAle78F+Nvs/aqs7ecBK7LfScd8t2EGbb4SuDl7vxj4eda2ZNtNbSW3C7L3XcDPgA+k3OZc2/8S2Al8L9tOus3Ai8DlDfvmtM2p9OhHFzCPiBGgvoD5OS9qyy6+3rB7PfCt7P23gD/J7d8VEb+JiJPAAOfgil4R8UpE/Ef2/r+B56itXZxsu6Pmf7LNruwVJNxmAEm9wB8C38ztTrrNk5jTNqcS9M0WMF86T3V5OyyJiFegForAFdn+5H4PkpYD76XWw0263dkQxiHgNPBERCTfZuDrwF9RW7CoLvU2B/C4pIOSNmb75rTNqTyPvsgC5u0gqd+DpAuA7wCfiYg3pljxJ4l2R0QFuEnSxdRWZnv3FMXP+TZL+iPgdEQclHRbkVOa7Dun2py5JSKGJF0BPCHp+SnKtqTNqfToiyxgnpL/knQlQPbzdLY/md+DpC5qIf/tiHg02518uwEi4lfAD4E1pN3mW4A/lvQiteHWP5D0r6TdZiJiKPt5GthNbShmTtucStAXWcA8JXuAj2fvP05tcfb6/g2SzpO0AlgJPDUP9ZsV1bru/wI8FxFfyx1Ktt2SerKePJLeAXwYeJ6E2xwRn4+I3ohYTu3f7L9FxJ+TcJslLZK0uP4euBM4wly3eb6vQLfwSvY6arMzfgF8cb7r08J2PQK8ApSo/XX/JHAZ8APgheznpbnyX8x+B8eBtfNd/xm2+feo/ff0WeBQ9lqXcruB9wBPZ20+Avx1tj/ZNje0/zbGZt0k22ZqMwOfyV5H61k11232IxDMzBKXytCNmZlNwkFvZpY4B72ZWeIc9GZmiXPQm5klzkFvZpY4B72ZWeL+H5NSvpCSABqGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[9, 9]),\n",
    "    keras.layers.Dense(25, activation=\"relu\"),\n",
    "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "])\n",
    "\n",
    "model.compile(loss=\"binary_crossentropy\",\n",
    "              optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\",\n",
    "                       tf.keras.metrics.TruePositives(),\n",
    "                       tf.keras.metrics.TrueNegatives(),\n",
    "                       tf.keras.metrics.FalsePositives(),\n",
    "                       tf.keras.metrics.FalseNegatives(), \n",
    "                       tf.keras.metrics.Precision(),\n",
    "                       tf.keras.metrics.Recall(),\n",
    "                       tf.keras.metrics.SpecificityAtSensitivity(0.5),\n",
    "                       tf.keras.metrics.AUC()])\n",
    "\n",
    "history = model.fit(X_train, y_train, validation_split=0.2, epochs=500, verbose=0,)\n",
    "pd.DataFrame(history.history[\"val_accuracy\"]).plot()\n",
    "\n",
    "loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under = model.evaluate(X_test, y_test)\n",
    "\n",
    "\n",
    "data = pd.DataFrame([[\"TF\", 2, \"Adam\", loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under, 0.2]], \n",
    "                    columns=[\"Model\", \"Layers\", \"LossFn\", \"Loss\", \"Accuracy\",\"TP\", \"TN\", \"FP\", \"FN\", \"Precision\", \"Recall\", \"SpecificityAtSensitivity\", \"AUC\", \"split\"])\n",
    "table = pd.concat([table, data], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s 3ms/step - loss: 0.1009 - accuracy: 0.9583 - true_positives_17: 23.0000 - true_negatives_17: 23.0000 - false_positives_17: 1.0000 - false_negatives_17: 1.0000 - precision_13: 0.9583 - recall_13: 0.9583 - specificity_at_sensitivity_9: 1.0000 - auc_8: 0.9983\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAarklEQVR4nO3df3Dc9X3n8efLsuRfMv4hywQsBxus/HAIECKckB8NlybB0AQf17Q1mV7a/BiONmRyc3NtnGQumU7nZppm2kmv0Dq+HkOba/HlEihu6oPQXAJtrwGLYH4YsCUbitcmXlkiRissrX68749dmfV6Za3lXX13V6/HjMb7/X4/Wr0/mvHLH3/2+/18FBGYmVn9m5d0AWZmVhkOdDOzBuFANzNrEA50M7MG4UA3M2sQ85P6watWrYp169Yl9ePNzOrS448/fjwi2ktdSyzQ161bR3d3d1I/3sysLkn616muecrFzKxBTBvoku6SlJb0zBTXJem/SeqV9JSkqytfppmZTaecEfrdwOazXL8B6Mx/3Qr8+fmXZWZm52raOfSIeETSurM02QL8VeTWEPiJpOWSLoqIl8+1mNHRUVKpFMPDw+f6rbNq4cKFdHR00NzcnHQpZmanVOJD0TXA4YLjVP7cOQd6KpVi6dKlrFu3DkkVKK3yIoL+/n5SqRTr169Puhwzs1Mq8aFoqeQtueKXpFsldUvq7uvrO+P68PAwbW1tNRvmAJJoa2ur+f9FmNncU4lATwFrC447gKOlGkbEjojoioiu9vaSt1HWdJhPqocazWzuqcSUyy7gdkk7gXcBJ2Yyf27JefH4EPc+cQS8lLLZrOhat5JfeFPpQe35mDbQJd0DXAeskpQCvgY0A0TEdmA3cCPQC7wGfKriVc6yBx54gC984QuMj4/z2c9+lm3btiVdUlV965GD3PPYYfwfD7PZcdsHLksm0CPilmmuB/C5ilWUsPHxcT73uc/x0EMP0dHRwTXXXMNNN93Exo0bky6tanqOZdi0fiXf+Q/XJl2KmZ0HPyla5LHHHmPDhg1ceumltLS0sHXrVu6///6ky6qaiKAnnaFzdWvSpZjZeUpsLZfp/N7f7ePZo69W9D03XnwBX/vY287a5siRI6xd+/pnvB0dHTz66KMVraMcz//sVb56/z5Gxyeq+nMmAk6cHHWgmzWAmg30pJTaYzWJu1p+sO8Yj70wwPs7V1X9Z33orav5xbdeWPWfY2bVVbOBPt1Iulo6Ojo4fPj156RSqRQXX3zxrNfRm87QsWIR3/7Mu2b9Z5tZffIcepFrrrmGnp4eXnjhBbLZLDt37uSmm26atZ8/MREMjYxx4Nigp0HM7JzU7Ag9KfPnz+eOO+7g+uuvZ3x8nE9/+tO87W2z97+F37x7D48cyD1F+4Eq3NZkZo3LgV7CjTfeyI033jjrP3d8Inj0UD/vuayND75lNTddOftTPWZWvxzoNeTIKycZGZvgpisvZuumNyZdjpnVGc+h15Ce9CAAnRd67tzMzl3NBXqp2wZrTbVq7ElnANjQvrQq729mja2mAn3hwoX09/fXdKhProe+cOHCir93bzrD6qULWLbYG2eY2bmrqTn0jo4OUqkUpdZKryWTOxZVWk864+kWM5uxmgr05ubmObsLUETQe2yQX+laO31jM7MSamrKZS57+cQwQ9lxNvhhIjObIQd6jZj8QNRPh5rZTDnQa0TPsclbFn2Hi5nNTE3Noc8lvekMv/3XjzMyllsed2AoS9uSFlYuaUm4MjOrVw70hPxTTx8HjmX46BUXMX9ebnned1/alnBVZlbPHOgJ6UlnWLaomT+95R2JrLduZo3Hc+gJmdz2zWFuZpVSVqBL2ixpv6ReSdtKXF8h6T5JT0l6TNLllS+1sfT6ISIzq7BpA11SE3AncAOwEbhF0saiZl8G9kbEFcAngT+pdKGNpD8zwsBQlg2rfUeLmVVOOSP0TUBvRByKiCywE9hS1GYj8EOAiHgeWCfJm1ROwfecm1k1lBPoa4DDBcep/LlCTwL/DkDSJuAS4IzFTiTdKqlbUnetr9dSLRHBtx4+CHiZXDOrrHICvdSndsXLIf4BsELSXuDzwBPA2BnfFLEjIroioqu9fW5ur3bgWIYf7e9DgjdcUPkVG81s7irntsUUULhiVAdwtLBBRLwKfApAuds2Xsh/WZHnf/YqAPf99nt9h4uZVVQ5I/Q9QKek9ZJagK3ArsIGkpbnrwF8FngkH/JWpDedYZ7grRf5A1Ezq6xpR+gRMSbpduBBoAm4KyL2Sbotf3078FbgrySNA88Cn6lizXVneHScr92/j8GRUZ48fIJ1bUtYML8p6bLMrMGU9aRoROwGdhed217w+l+AzsqW1jiePnKC/9V9mDXLF7G4pYmbry7+TNnM7Pz50f9Z0J/JAvCtf/9OLl+zLOFqzKxR+dH/WTAwlAv0tlavpGhm1eNAnwUDQyMAXhrXzKrKgT4L+oeytC6Y7w9CzayqHOizYGAo69G5mVWdA73KxieCPS8MONDNrOoc6FX23ccPc/TEMBdesCDpUsyswTnQq+zpIycA+K83vz3hSsys0TnQq6znWIar37icVa0eoZtZdTnQq+xgX4ZOb2RhZrPAgV5FA0NZjmeyXvfczGaFA72KevM7E23wzkRmNgsc6FXUkx4EoPNCT7mYWfU50KtkYCjLV+57hkXNTVy8zDsTmVn1OdCrZO/hVwC47s3t3pnIzGaFA71KJpfM3XbDWxKuxMzmCgd6lUwumetH/s1stjjQq2TgtSwtTfNoXeA9RMxsdjjQq2Qgk1th0fPnZjZbHOhVMjCU9Q5FZjarygp0SZsl7ZfUK2lbievLJP2dpCcl7ZP0qcqXWj+GR8d5+ECf58/NbFZNG+iSmoA7gRuAjcAtkjYWNfsc8GxEXAlcB/yRpDmbZnf/vxcZmwg6VixOuhQzm0PKGaFvAnoj4lBEZIGdwJaiNgEsVW7CuBUYAMYqWmkdefboqwB89aPF/+6ZmVVPOYG+BjhccJzKnyt0B/BW4CjwNPCFiJgofiNJt0rqltTd19c3w5JrX086w795czuLWryHqJnNnnICvdRtGlF0fD2wF7gYuAq4Q9IFZ3xTxI6I6IqIrvb29nMstfYNDo/y9Qeezy2Z6/VbzGyWlRPoKWBtwXEHuZF4oU8B90ZOL/ACMOcekfyH547x5z8+yOKWJt7fuSrpcsxsjinnqZc9QKek9cARYCvwiaI2LwG/CPyjpAuBNwOHKlloPeg5lmH+PPHYlz9Ey3zfEWpms2vaQI+IMUm3Aw8CTcBdEbFP0m3569uB3wfulvQ0uSmaL0bE8SrWXZN60xnWrVriMDezRJT1XHpE7AZ2F53bXvD6KPCRypZWX370fJofPHuMGy5/Q9KlmNkc5aFkhfz90y8D8Mlr1yVbiJnNWQ70CulJZ3jvhjauvawt6VLMbI5yoJ+HiGB8IhgYynIwnaFztW9VNLPkeG3X8/Dl+57hnsdeOnX8Jt97bmYJcqCfh8Iw/8OPX8FHr7gowWrMbK5zoFfIr3atnb6RmVkVeQ59hvYe/nnSJZiZncaBPkO/+90nT72+ZdMbE6zEzCzHUy4zMDY+wYvHX+Pmd6zhD3757TTP87+LZpY8B/oMHH7lJNnxCa69rI0F871ErpnVBg8tZ6Dn2CAAnatbE67EzOx1DvQZSL1yEoBL2pYkXImZ2esc6DMwMJRlnmD5ouakSzEzO8WBPgP9Q1lWLG5h3rxSmzmZmSXDgT4DA0MjrFzSknQZZmancaDPwMBQ1oFuZjXHgT4D/UNZ2lod6GZWWxzoM+ARupnVIgf6OcqOTXDi5ChtSxYkXYqZ2Wkc6Ofoxf4hIuDSdt+Dbma1paxAl7RZ0n5JvZK2lbj+O5L25r+ekTQuaWXly01ez7EMABv8lKiZ1ZhpA11SE3AncAOwEbhF0sbCNhHxjYi4KiKuAr4EPBwRA1WoN1Fj4xN88x8OAHBZuwPdzGpLOSP0TUBvRByKiCywE9hylva3APdUorha85NDA/SkM3SsWMTCZi/KZWa1pZxAXwMcLjhO5c+dQdJiYDPwvSmu3yqpW1J3X1/fudaauAP5Rbnu/a33JFyJmdmZygn0Us+3xxRtPwb881TTLRGxIyK6IqKrvb293BprRk86w/LFzbQv9R0uZlZ7ygn0FFC4YWYHcHSKtltp0OkWgN70IJ2rW5G8houZ1Z5yAn0P0ClpvaQWcqG9q7iRpGXAB4D7K1ti7Tj26ggXL1+UdBlmZiVNu2NRRIxJuh14EGgC7oqIfZJuy1/fnm96M/CDiBiqWrUJy4yMsXShN3kys9pUVjpFxG5gd9G57UXHdwN3V6qwWpQZHqN1gddAN7Pa5CdFyzQyNk52fMIjdDOrWQ70MmWGxwBoXeBAN7Pa5EAvU2bEgW5mtc2BXqbByRG6p1zMrEY50Ms0OUJf6hG6mdUoB3qZMh6hm1mNc6CXyXPoZlbrHOhl2Hv459z7xBHAI3Qzq11OpzL80Q/280+9x7msfQkrFnsvUTOrTQ70MvSmM9x81Rr++NeuSroUM7MpecplGoPDo7x8YpgNF3qHIjOrbQ70afSm83uIess5M6txDvRp9A2OAHDRMi+ba2a1zYE+jVMPFPnuFjOrcQ70aZy6/9yBbmY1zoE+jUGvsmhmdcKBPo3MyBjNTWLBfP+qzKy2OaWmkdulaL43hjazmudAn8bQyJjnz82sLpQV6JI2S9ovqVfStinaXCdpr6R9kh6ubJnJGRzxPqJmVh+mHXpKagLuBD4MpIA9knZFxLMFbZYDfwZsjoiXJK2uUr2zLjM85jXQzawulDNC3wT0RsShiMgCO4EtRW0+AdwbES8BRES6smUmJ+MpFzOrE+UE+hrgcMFxKn+u0JuAFZJ+LOlxSZ8s9UaSbpXULam7r69vZhXPogPHBnn6yAnfsmhmdaGcQC91e0cUHc8H3gn8EnA98F8kvemMb4rYERFdEdHV3t5+zsXOtr959CUA3nNZW8KVmJlNr5yhZwpYW3DcARwt0eZ4RAwBQ5IeAa4EDlSkyoT0pjNc2bGMrZvemHQpZmbTKmeEvgfolLReUguwFdhV1OZ+4P2S5ktaDLwLeK6ypc6+nvQgl632KotmVh+mHaFHxJik24EHgSbgrojYJ+m2/PXtEfGcpAeAp4AJ4C8i4plqFl5tJ06OcuzVETpXL026FDOzspT1aV9E7AZ2F53bXnT8DeAblSstWZProHd6hG5mdcJPik6hNz0IQKd3KjKzOuFAn0JvOsOC+fPoWLE46VLMzMriQJ/Cwb4h1q9aQtM8L8plZvXBgT6FvsERLrxgYdJlmJmVzYE+hYGhLG1LWpIuw8ysbA70KfQPjbDSgW5mdcSBXsJr2TGGRydY2epAN7P64UAvYWAoC+ApFzOrKw70EiYDfeWSBQlXYmZWPgd6Cf2nAt0jdDOrHw70ElKvnATgomW+bdHM6ocDvYTeY4MsaWlyoJtZXXGgl9CTzrDhwqVIfkrUzOqHA72E3nTGqyyaWd1xoBc58doo6cERB7qZ1R0HepHePi+ba2b1aU5vZz82PsFTR04wNv76ntcPH0gDeKciM6s7czrQv/fTFF/83tNnnL9g4XzWLF+UQEVmZjM3pwM99cpJJPj2p99F4Q0ta5YvYp7XQTezOjOnA71/KMuKxS28r3NV0qWYmZ23sj4UlbRZ0n5JvZK2lbh+naQTkvbmv75a+VIrbyCT9eP9ZtYwph2hS2oC7gQ+DKSAPZJ2RcSzRU3/MSI+WoUaq2ZgyIFuZo2jnBH6JqA3Ig5FRBbYCWypblmzo39oxEvkmlnDKCfQ1wCHC45T+XPFrpX0pKT/I+ltpd5I0q2SuiV19/X1zaDcynrltVGP0M2sYZQT6KVu94ii458Cl0TElcCfAn9b6o0iYkdEdEVEV3t7+zkVWmn3PZHyvqFm1lDKCfQUsLbguAM4WtggIl6NiEz+9W6gWVJN3zrynT0pAD7ytjckXImZWWWUE+h7gE5J6yW1AFuBXYUNJL1B+aUJJW3Kv29/pYutpJ50hl95ZweXr1mWdClmZhUx7V0uETEm6XbgQaAJuCsi9km6LX99O/Bx4LckjQEnga0RUTwtUzN+dmKY45kRr9diZg2lrAeL8tMou4vObS94fQdwR2VLq47vPp7iP//vJwHovNDrtZhZ45hzT4o+eqifZYua+Z3r38z7NtT0NL+Z2TmZc4Hek86w8aIL+PV3X5J0KWZmFdUw66Ef6svw4/1pJiZKT93v/9kgDzzzcm43Is+dm1kDapgR+q9+6184nsny7c9s4v2dp9/jHhHc8t9/wsBQFoC3+84WM2tADRHoJ7PjHM/kwvr5lwfPCPS+wREGhrJ8/oMb2HLVxVy6yiN0M2s8DRHoB/syp173pAfPuN6Tzl2/9tI2NngnIjNrUHUf6I8e6ufPfnwQgFWtC3jkwHG+ct/puxBNBv4Gz52bWQOr+0Df8cgh/rn3OFd2LONjV17M9ocP8uC+n53R7tpL22hvXZBAhWZms6PuA/34UJZrL2vj2595FwCfff+lCVdkZpaMur9tccBrmpuZAY0Q6JksK5d4KsXMrK4DfXh0nKHsOG2tHqGbmdV1oE8+KORdh8zMHOhmZg2jrgO9LzMC4A9Fzcyo80A/1DcEwPpVSxKuxMwseXUd6L3pQVYuaaHNDwyZmdV3oPccy7Ch3Y/zm5lBnQf6SwOvsW7V4qTLMDOrCXUd6IPDYyxb1Jx0GWZmNaGsQJe0WdJ+Sb2Stp2l3TWSxiV9vHIlljY2PsHJ0XFaFzjQzcygjECX1ATcCdwAbARukbRxinZfBx6sdJGlDI2MA9C6sO7XFzMzq4hyRuibgN6IOBQRWWAnsKVEu88D3wPSFaxvSoMjowAsXeBANzOD8gJ9DXC44DiVP3eKpDXAzcD2s72RpFsldUvq7uvrO9daT5MZGQM8Qjczm1ROoKvEuSg6/ibwxYgYP9sbRcSOiOiKiK729vazNZ1WZjgf6B6hm5kB5W1wkQLWFhx3AEeL2nQBOyUBrAJulDQWEX9biSJLGfQI3czsNOWk4R6gU9J64AiwFfhEYYOIWD/5WtLdwPerGebw+gjdc+hmZjnTpmFEjEm6ndzdK03AXRGxT9Jt+etnnTevFs+hm5mdrqw0jIjdwO6icyWDPCJ+8/zLmp7n0M3MTle3T4r+/GSWeYIlLQ50MzOo40A/1DfEurYlzJtX6iYcM7O5p24DvSedYcNqr7RoZjapLgN9dHyCF48POdDNzArUZaC//PNhxiaCdd6pyMzslLoM9ONDub1E271TkZnZKXUZ6AOZLAArvTm0mdkp9RnoQw50M7Ni9Rnor+UCva3VgW5mNqk+A30oy4L581jU3JR0KWZmNaMuA70/k6VtSQv51R3NzIw6DfSBoRFWerrFzOw0dRnog8NjLFvkzaHNzArVZaBnRsa8yqKZWZG6DPTB4TFaF3iEbmZWqC4DPTMyxlJvbGFmdpq6C/SI8JSLmVkJdRfow6MTjE+Et54zMytSd4E+ODIKeOs5M7NiZQW6pM2S9kvqlbStxPUtkp6StFdSt6T3Vb7UnMm9RD2HbmZ2umlTUVITcCfwYSAF7JG0KyKeLWj2Q2BXRISkK4DvAG+pRsGZEW8ObWZWSjkj9E1Ab0QciogssBPYUtggIjIREfnDJUBQJZMjdAe6mdnpygn0NcDhguNU/txpJN0s6Xng74FPV6a8Mw1OjtA95WJmdppyAr3UClhnjMAj4r6IeAvwb4HfL/lG0q35Ofbuvr6+cyp00qrWFm64/A2s8m5FZmanKWeYmwLWFhx3AEenahwRj0i6TNKqiDhedG0HsAOgq6trRtMy77xkJe+8ZOVMvtXMrKGVM0LfA3RKWi+pBdgK7CpsIGmD8mvZSroaaAH6K12smZlNbdoRekSMSbodeBBoAu6KiH2Sbstf3w78MvBJSaPASeDXCj4kNTOzWaCkcrerqyu6u7sT+dlmZvVK0uMR0VXqWt09KWpmZqU50M3MGoQD3cysQTjQzcwahAPdzKxBJHaXi6Q+4F9n+O2rgOPTtmos7vPc4D7PDefT50sior3UhcQC/XxI6p7qtp1G5T7PDe7z3FCtPnvKxcysQTjQzcwaRL0G+o6kC0iA+zw3uM9zQ1X6XJdz6GZmdqZ6HaGbmVkRB7qZWYOou0CXtFnSfkm9krYlXU+lSLpLUlrSMwXnVkp6SFJP/s8VBde+lP8d7Jd0fTJVnx9JayX9SNJzkvZJ+kL+fMP2W9JCSY9JejLf59/Ln2/YPkNus3lJT0j6fv64ofsLIOlFSU9L2iupO3+uuv2OiLr5Irce+0HgUnKbaDwJbEy6rgr17ReAq4FnCs79IbAt/3ob8PX86435vi8A1ud/J01J92EGfb4IuDr/eilwIN+3hu03uS0dW/Ovm4FHgXc3cp/z/fhPwN8A388fN3R/8315EVhVdK6q/a63EfomoDciDkVEFtgJbEm4poqIiEeAgaLTW4C/zL/+S3L7tU6e3xkRIxHxAtBL7ndTVyLi5Yj4af71IPAcuQ3IG7bfkZPJHzbnv4IG7rOkDuCXgL8oON2w/Z1GVftdb4G+BjhccJzKn2tUF0bEy5ALP2B1/nzD/R4krQPeQW7E2tD9zk8/7AXSwEMR0eh9/ibwu8BEwblG7u+kAH4g6XFJt+bPVbXf5WwSXUtU4txcvO+yoX4PklqB7wH/MSJezW9PW7JpiXN11++IGAeukrQcuE/S5WdpXtd9lvRRIB0Rj0u6rpxvKXGubvpb5L0RcVTSauAhSc+fpW1F+l1vI/QUsLbguAM4mlAts+GYpIsA8n+m8+cb5vcgqZlcmP91RNybP93w/QaIiJ8DPwY207h9fi9wk6QXyU2RflDS/6Rx+3tKRBzN/5kG7iM3hVLVftdboO8BOiWtl9QCbAV2JVxTNe0CfiP/+jeA+wvOb5W0QNJ6oBN4LIH6zotyQ/H/ATwXEX9ccKlh+y2pPT8yR9Ii4EPA8zRonyPiSxHRERHryP19/b8R8es0aH8nSVoiaenka+AjwDNUu99JfxI8g0+ObyR3N8RB4CtJ11PBft0DvAyMkvvX+jNAG/BDoCf/58qC9l/J/w72AzckXf8M+/w+cv+tfArYm/+6sZH7DVwBPJHv8zPAV/PnG7bPBf24jtfvcmno/pK7E+/J/Ne+yayqdr/96L+ZWYOotykXMzObggPdzKxBONDNzBqEA93MrEE40M3MGoQD3cysQTjQzcwaxP8HT1mQqTFNd2IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Flatten(input_shape=[9, 9]),\n",
    "    keras.layers.Dense(25, activation=\"relu\"),\n",
    "    keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "])\n",
    "\n",
    "model.compile(loss=\"binary_crossentropy\",\n",
    "              optimizer=\"adam\",\n",
    "              metrics=[\"accuracy\",\n",
    "                       tf.keras.metrics.TruePositives(),\n",
    "                       tf.keras.metrics.TrueNegatives(),\n",
    "                       tf.keras.metrics.FalsePositives(),\n",
    "                       tf.keras.metrics.FalseNegatives(), \n",
    "                       tf.keras.metrics.Precision(),\n",
    "                       tf.keras.metrics.Recall(),\n",
    "                       tf.keras.metrics.SpecificityAtSensitivity(0.5),\n",
    "                       tf.keras.metrics.AUC()])\n",
    "\n",
    "history = model.fit(X_train, y_train, validation_split=0.8, epochs=500, verbose=0,)\n",
    "pd.DataFrame(history.history[\"val_accuracy\"]).plot()\n",
    "\n",
    "loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under = model.evaluate(X_test, y_test)\n",
    "\n",
    "\n",
    "data = pd.DataFrame([[\"TF\", 2, \"Adam\", loss, accuracy, TP, TN, FP, FN, Precision, Recall, SpecificityAtSensitivity, area_under, 0.8]], \n",
    "                    columns=[\"Model\", \"Layers\", \"LossFn\", \"Loss\", \"Accuracy\",\"TP\", \"TN\", \"FP\", \"FN\", \"Precision\", \"Recall\", \"SpecificityAtSensitivity\", \"AUC\", \"split\"])\n",
    "table = pd.concat([table, data], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Layers</th>\n",
       "      <th>LossFn</th>\n",
       "      <th>Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>TP</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>SpecificityAtSensitivity</th>\n",
       "      <th>AUC</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TF</td>\n",
       "      <td>2</td>\n",
       "      <td>Adam</td>\n",
       "      <td>0.012068</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>24.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TF</td>\n",
       "      <td>2</td>\n",
       "      <td>Adam</td>\n",
       "      <td>0.001169</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>24.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TF</td>\n",
       "      <td>2</td>\n",
       "      <td>Adam</td>\n",
       "      <td>0.100932</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>23.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.998264</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Model  Layers LossFn      Loss  Accuracy    TP    TN   FP   FN  Precision  \\\n",
       "0    TF       2   Adam  0.012068  1.000000  24.0  24.0  0.0  0.0   1.000000   \n",
       "1    TF       2   Adam  0.001169  1.000000  24.0  24.0  0.0  0.0   1.000000   \n",
       "2    TF       2   Adam  0.100932  0.958333  23.0  23.0  1.0  1.0   0.958333   \n",
       "\n",
       "     Recall  SpecificityAtSensitivity       AUC  split  \n",
       "0  1.000000                       1.0  1.000000    0.5  \n",
       "1  1.000000                       1.0  1.000000    0.2  \n",
       "2  0.958333                       1.0  0.998264    0.8  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "7f3991beda84519ddbebd03ee67fce523f4a815cc55d3e3530775fe1c7f2ada7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}