lab11.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F9z5SYcyHX37"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "tTtOFnAFJ8dR"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import os\n",
        "import random\n",
        "\n",
        "from sklearn.model_selection import KFold, train_test_split\n",
        "import tensorflow as tf\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow import keras\n",
        "from keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, SensitivityAtSpecificity, SpecificityAtSensitivity, Precision, Recall, AUC\n",
        "from keras.utils import to_categorical\n",
        "import cv2\n",
        "from PIL import Image\n",
        "from keras.optimizers import Adam\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D, Dense, MaxPool2D, Dropout, Flatten, Activation, AveragePooling2D"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OCPorEhBEUyR",
        "outputId": "88063d81-0475-4022-c767-710cfd84f62f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
          ]
        }
      ],
      "source": [
        "print(tf.config.list_physical_devices())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BvDZkVbVHjvP"
      },
      "source": [
        "## Classes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "pdvhUFd_Dj9v"
      },
      "outputs": [],
      "source": [
        "classes = { 0:'Speed limit (20km/h)',\n",
        "            1:'Speed limit (30km/h)', \n",
        "            2:'Speed limit (50km/h)', \n",
        "            3:'Speed limit (60km/h)', \n",
        "            4:'Speed limit (70km/h)', \n",
        "            5:'Speed limit (80km/h)', \n",
        "            6:'End of speed limit (80km/h)', \n",
        "            7:'Speed limit (100km/h)', \n",
        "            8:'Speed limit (120km/h)', \n",
        "            9:'No passing', \n",
        "            10:'No passing veh over 3.5 tons', \n",
        "            11:'Right-of-way at intersection', \n",
        "            12:'Priority road', \n",
        "            13:'Yield', \n",
        "            14:'Stop', \n",
        "            15:'No vehicles', \n",
        "            16:'Veh > 3.5 tons prohibited', \n",
        "            17:'No entry', \n",
        "            18:'General caution', \n",
        "            19:'Dangerous curve left', \n",
        "            20:'Dangerous curve right', \n",
        "            21:'Double curve', \n",
        "            22:'Bumpy road', \n",
        "            23:'Slippery road', \n",
        "            24:'Road narrows on the right', \n",
        "            25:'Road work', \n",
        "            26:'Traffic signals', \n",
        "            27:'Pedestrians', \n",
        "            28:'Children crossing', \n",
        "            29:'Bicycles crossing', \n",
        "            30:'Beware of ice/snow',\n",
        "            31:'Wild animals crossing', \n",
        "            32:'End speed + passing limits', \n",
        "            33:'Turn right ahead', \n",
        "            34:'Turn left ahead', \n",
        "            35:'Ahead only', \n",
        "            36:'Go straight or right', \n",
        "            37:'Go straight or left', \n",
        "            38:'Keep right', \n",
        "            39:'Keep left', \n",
        "            40:'Roundabout mandatory', \n",
        "            41:'End of no passing', \n",
        "            42:'No passing for > 3.5 tons' }\n",
        "\n",
        "NUMBER_OF_CLASSES = len(classes)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KPvehp-LHrkk"
      },
      "source": [
        "## Getting the data from the images"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "01AsZImgE7OL",
        "outputId": "2fe6b974-ecad-4c24-e0ed-c3f78ac4d08c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(39209, 30, 30, 3) (39209,)\n"
          ]
        }
      ],
      "source": [
        "X = []\n",
        "y = []\n",
        "\n",
        "for i in range(NUMBER_OF_CLASSES):\n",
        "    path = \"./Data/archive//train/\" + str(i)\n",
        "    images = os.listdir(path)\n",
        "\n",
        "    for img in images:\n",
        "      image = cv2.imread(path + '/' + img)\n",
        "      image_fromarray = Image.fromarray(image, 'RGB')\n",
        "      resize_image = image_fromarray.resize((30, 30))\n",
        "      X.append(np.array(resize_image))\n",
        "      y.append(i)\n",
        "\n",
        "\n",
        "# putting it into numpy\n",
        "X = np.array(X)\n",
        "y = np.array(y)\n",
        "\n",
        "print(X.shape, y.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6nyihYwvH6_y"
      },
      "source": [
        "## Splitting and scaling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "clEaWki-H9Pr",
        "outputId": "12984795-7ccd-4456-a1d8-101049a26552"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Turn right ahead\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAATWElEQVR4nO3cSZPk5nUF0AcgM2tiUyQ1mLIXtkMr//8/4p2ttS2FQiJp9lBdVZkYtGj7hVbqvAi2TCrOWT8AH8Zbuag7bNu2FQBU1fj/vQAAfjyEAgBNKADQhAIATSgA0IQCAE0oANCEAgDtcO3gv/7m36IdHw9DvJjp6tV8sFV2jGk6ZgeoqnXN/rdvWeZo/nCYovmqqmnMsnwJz2E6hDeiquaXl2j+84f7aH7Y8efLm3eP0fzT0zmaH3YsahqzZ/Z4kz2zw5C/d0P4PM1rtv/pcJNtUFU3p2yb9D9w5yU8iapaluwoa3iMbblE8x8Okn1vfvsf//7RGb8UAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaFeX3CwVdoWseQfLtoRdRmGPzLIs0XxV3qmSlvSkfSpVVcucdaSkRxjC6/phm2x+C5+nacw7osYpW1TaATRs+XVK78aUdhnt6GMaw06wNbx364737vLynG0Q3rt1T29V2FOW9r89P2X9YVVVNeTfj4/xSwGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoVxfi1ZaVWm07CvFqvH45H+wpJEuPkB1jyJvhcmFJ2pCWZm1h+WFVnY7ZvUs797YdaxriMrms8Gzd8ps3hOexhvNTWG5XVbWG7+qQlhOu+b2blznbIOzcy1dU8XtX4fOxpOdcVcOuD8hf55cCAE0oANCEAgBNKADQhAIATSgA0IQCAE0oANCEAgBNKADQhAIALeg+yjo24t6ZypuM0u6ZPT0hY5qbYc/QkJYAfdgqm95xL1Jr2G+zrtl1mqb875f4bqf3bs91DbdJO3r2PONpv1Jc+bSjIyq9e/Hzt2tNmS09hz09Rp/gPPxSAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgCQUAmlAAoF3ffRT27UzjFC5lR41H2J+z7eljCvtw8raaHbkcn0Z2jD1NSWmXUTo/ht02VXn3TNpLNEx7+r0+7d9h445nPH3v0p6hcdxzztl5pOcdPxuVd2mln4499+5T9Jr5pQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoANKEAQBMKADShAEATCgC0oBAvs/0NCsymKSzdG/IMTMvCxjEsqIoL9/JirrThbtnyezeG2yzpIda8YHGcTtH8dFii+W3HdUr/DovrGHe8d0P4Xkxhwd2e0rahsvu9huWE65KvaU3vdzi/p9ruh6/D80sBgL8gFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgHZ199EY9pcs6xwvZgp7g4Yx7T7KxvfYwrKkXb0w4TZpD9W6497dHLI1ncK+nW3O/36Zwr95jlM2v8YFTlVbpX042fOUdnVVVYWnXcOUVaatWaXUh23C3qAtfLnTvqeqqjG8F+njse25UDu+Hx/jlwIATSgA0IQCAE0oANCEAgBNKADQhAIATSgA0IQCAE0oANCEAgDt6hKTKewZ2tPBMkxhp0/cX5JnYNo9k0p7iaoq7jtJ+5gO4X2oqnp1k/XhnOas5+V4cxPNV1XNh2M0/xLei3nJu2qWsN9mDbuS1l2Pa/g8he9E2mNUVbWExxjSzrFo+v+Okc2nX5sdX4Jd39mP8UsBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaFe3mH3+6iHa8bKjqGkJC8bSQrK0GK6qaghL9+KirR1rmsJmruMxKzO8uz9F81VV90NW5zVcHqP5045StXHL7t0WltWddrSqbcdsozV8/rYhLw48z9maXsIyw3Hcc6HSkr7QjiLK9CzGKXvvxrB0tKpq+QSNeH4pANCEAgBNKADQhAIATSgA0IQCAE0oANCEAgBNKADQhAIATSgA0K7uPro7fRbteB3zvJnnrI/k6Snrz1nWczRfVRVWz9QYVpFMeVtS3R+zRb26O0bzpylf0/L8lG4Qjb+8/j7bf1Vdtku2QdgpNex4xqcp65U6nm6j+dN93p/zfMiej7RuZx52PE9hNVHag7ZW3n20pecRfjx+LH+h/1jWAcCPgFAAoAkFAJpQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQDa1d1Hb7/Pej9uP7uPFzMewq6QKSthWbe872QI+3OGytZ0F55zVdWrU7bN7ZCdw/nt22i+qmo9v4/mpzXrqpl2dNUcwi6jQ3gvpmPWGVRVVWPWffQc9oHNj6+j+aqqw33Wr3QzXf3Z+F95H9M8Z8/HEpYlbduev4ezbdY1e/72LGn6BH/W+6UAQBMKADShAEATCgA0oQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoAtKubrc6XrODus3qIFzPPc7bBkJWL1ZCV+lVV1ZIVc90cs/KvV3d5qdrtmF2nl8fvo/n1+Smar6oah2xNW3gr9ty6LS1AnLJ7d//lF9n+q+r+y3+I5t++fY7m//DH30XzVVX1mN3vu7tX0fw4ZoV7VVXnLXuexsoekDG811VVS9hYt6xpiWNe+riGxZLX8EsBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAdnX30WXOukWmtHemqk5hRcr5OetHWcMeo6qq05jl5t3N1Ze0qqpupi2ar6qa37/LNri8ROPHHX8qbGH3zBb2vIyHvPzoMGXdWHNlPVTvLtm9rqp6uP0qmv8y7MZaxrxL65s//Hc0Pzxl793NzY73bsoewm3IuozW/LWrLfx+DEN2kCH81lTVnrqkj/JLAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgHZ1ecvddBPt+GHIu2qent9E88vlOZrPV1R1mrJOlbQ36HJ+zDaoqvWcnfcYFqQMQ16oso3Z1R0PWUfPV7/KOoOqqh6++EW2weGzbH68z+arapyybQ6HrBDs61//SzRfVXUK/zb89vf/Fc2vc/a8VlVNQ/a9OYSdT5c1/3t4GLPuo2Nl344hfIeqqtZdX7W/zi8FAJpQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgCQUAmlAAoF1diHc7vY12fLPkeXP7Kiu1enyfFU7N8xbNV1VNYUnVVNkxtiUr2aqqOkzhtQ377YYtv07DmK3pF1//YzT/9T//JpqvqlpO2fPx9v130fy4ZgWOVVV3dw/R/PGQFeh988dvo/mqqnV+ieandc72v+NbMIxXf5qqqmocsmd2mnYUyQ3Z85S+d8uSF1Gua/79+Bi/FABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhXF4w8DK+zHW9Zj1FV1du3WafKEnaw7Ok7OYSdPocx62BZw3OoqtrmczQ/DVmnStjwUlVVDz97Fc1/8fU/RfOPz3kvzO9/+5/R/Dj+KZr/1S++iuarqh4fs/s9jM/R/PfffRPNV1Wd32YdTtuSPX9r2B9WVbWF3Ufpt+A8551Bl7CbaEj/5t7RObbVjg6nj/BLAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgHZ1wcjd8Dba8bhk3SVVVV/+/GfR/JvHrL9kec67Raaw+2jYLtH8uOadPoe07iTsVBnD/qaqqs+//CKavxxP0fyffpf181RVXd68RPNf3oetT++yDqCqqpfzt9H8ubJzePfmf6L5qqrl3ffR/G2FHUBb/rfnGvYMLZV9C9Y1X9OypMcIvwVh31NV/n26ah0/+B4B+MkSCgA0oQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoANKEAQBMKALSrG5im8V225x0lWNNwjObHMSskW8NiuKqqbctKqsa0rC7eoGrIerlqDI+xp2PrdMrK5NYtW9M659dpDUsZn5+yE1+352i+qurp/VM0/7x8F82/XPKSvkNYcLcN6XXa896l22Tz25YXUQ5DdozjMXv+hl1/o+fvxcf4pQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoANKEAQBMKADShAEC7upxjmLJum2nKeow+bPMQzY9jtqZhyLtqlvMlmr9sWTHR/Y7rVMscjQ9hV82YlitV1c3pNpp/CqtttjXvzxkqez4u5+y8L+eXaL6qalmzNdWaddvchh1AVVVD2I21hb1Vh2P2bFRVVXidlrDKaNvRGZQ+gtuSPU/HQ9aVVJW/29fwSwGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYB2ddnGL3/562jH67u8F+b1m6wrZF6zrpBpuonmq6rmLes+msMuknXHmoY57OgJz2HbwiKZqlrH7F6MYffMsKOrZgj/5tm2rG9n3fK/qdLeoCHuMsq7j1JLeC+W6z8z7RJ2Pl3m7LzzJ7xqHNKutbBTak8f054T+Qi/FABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYB2dVPVu8fX0Y5vsg62D9u8uovmD+esDWoNS7Oqqs7hJs9DVv71cMxKtqqqtnmO5p/n7Dp9/vOvovmqquHmNpqfz8/R/Da/j+Y/SB/CsKwuLEjbI+xU29WHlz1NVTWdsv1vx/QINYdlg2HP4K6CxdQQlmPWkN+89RM04vmlAEATCgA0oQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoANKEAQLu6qOfp3bfRjvf0wpwOWX/OuGT7X7e8J2QNS1Vewp6h52PWlVRVdXOTdURtla3pfInbcOrxzbtofj5nN++4ZvuvqtrGl2h+mbM1xb1EVTWM2UZL2G0TvhIfthnDd/WUPX8vS959tIVrGsPeoHXZ0Rm0ZcfYwjWNOx6ocdB9BMAnJBQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYB2dfHOIewNSvt2qqqW5SmaX7esN2gb8gwcKuw+umTn/brO0XxV1We32Xmfwq6a+f37aL6q6rvffRPNH9IemfkSzVdV1Zp1OIVLqqq836vCYyzhMxv3GFXV5XiK5p+GcH7JO32W9GaE4+sa3+xa16xZagq/HXv+RB/TE/80ywDg75VQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgCQUAmlAAoF3drDbGZXJ5UdMWbpOX7uUZmPZmbVlnVr0LS9uqquawmOurh7RA71U0X1X1+t1zdowhu7DheFVVLVtWDpf2tqVliVVV0/EYzS/hM7uM2f6rqs5D9ny8D7sJw47Iqqra1rCAMy0aXMIXtarWNTvxMSwn3LGkHd/lK/b5g+8RgJ8soQBAEwoANKEAQBMKADShAEATCgA0oQBAEwoANKEAQBMKALSrS0+GMetHGba88GQIu0KGIZuvHV018TZhF8myo9Pn/SUsn3nKDnJ7vMv2X1XzmG1zmsLuowrPuaqWJdtm3vIeqtRxvI3mL2mnz5q+E1WXNXvGlyXsJUoLxKri6rQ1fJH2rGkYsuu0hd+OXdfpE/xZ75cCAE0oANCEAgBNKADQhAIATSgA0IQCAE0oANCEAgBNKADQhAIA7ZN1H407ajyW5RjND0O4ph2L2sLY3MKqpGVH38lS2XV6PGddNe8vz9F8VdVhyjp33ofnPY55b9WwZc/HEnbVLGve77U9LdF8eoR1T+dY2tc1Z+dQf4M1feqOsj3bxF1GYbfSh2Pk1/Zj/FIAoAkFAJpQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgCQUA2vWNYUNWwnbYUe709vkSzQ+HUzR/GPMMnIewcCo8723ZUWi1pUVbWVldVV7Sd7nM4RbZee/ow6tTWNI3hQWLW1xXV7XndifWHQWL6b1YwxK2YU87ZvgMruE7saeIMm273MJvwZi+ppUXdl61jh9+lwD8VAkFAJpQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgXV/2Mj5EOx7W53Qt9flD1q/0VFm3yOO8o0BnyHJzC3thxh09Q2EFS23bJy7cqap1XaL5ISwz2tK+px3bjGlv1Y7rui7ZddrC/qZ91ymbT+/dNOV/e6YdTul7t+c6pa/qGv7NfQi/NVX5M3vVPn/wPQLwkyUUAGhCAYAmFABoQgGAJhQAaEIBgCYUAGhCAYAmFABoQgGAdnX30TbdRzse6hIv5jy/j+bXQ9gLM9xE81V/g16Y8Byqqtaw52VI+1GGvBdmSguZwiKZ8LJWVdUUnnd6jD2tM0vaEZUeYE+lT3jvxnBVaS9RVX7eab3Snu6jtI8pPYdxR/fRD9985JcCAH9BKADQhAIATSgA0IQCAE0oANCEAgBNKADQhAIATSgA0IQCAE0oANCuL8Q7ZGVya96HV/OSlYWtV6/+g7izraq2sGEsLWGrKTyJqrpcnqP5tKRvz3WqsOhtDK/TniWNx6xs8OaU3YstvddV9fySvhjhvduy+5AfoWoY0mc2v05pYd2yhKV7O0r6pvg0smOs6TlU1eGQfz8+xi8FAJpQAKAJBQCaUACgCQUAmlAAoAkFAJpQAKAJBQCaUACgCQUA2rClJSMA/N3ySwGAJhQAaEIBgCYUAGhCAYAmFABoQgGAJhQAaEIBgPZnx7WHTuEEuP4AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "i = random.randint(0,X.shape[0])\n",
        "plt.imshow(X[i])\n",
        "print(classes.get(y[i]))\n",
        "plt.axis('off')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mt0271M_JJcS"
      },
      "source": [
        "## Convert targets into categorical values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "y = keras.utils.to_categorical(y, NUMBER_OF_CLASSES)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwZSsodxJgam"
      },
      "source": [
        "# Build CNNs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "VtD6G9jHATaj"
      },
      "outputs": [],
      "source": [
        "#data augmentation for better results\n",
        "img_aug = ImageDataGenerator(\n",
        "    rotation_range=7,\n",
        "    zoom_range=0.14,\n",
        "    width_shift_range=0.05,\n",
        "    height_shift_range=0.2,\n",
        "    shear_range=0.1,\n",
        "    horizontal_flip=False,\n",
        "    vertical_flip=False,\n",
        "    fill_mode=\"nearest\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "nTDnQfa1D9Ba"
      },
      "outputs": [],
      "source": [
        "kf = KFold(10, shuffle=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "EYkT13Uv-0hX"
      },
      "outputs": [],
      "source": [
        "table = pd.DataFrame()\n",
        "table = pd.read_csv(\"./log.csv\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oL49z4H0Mktn"
      },
      "source": [
        "## LeNet5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BmC0PE7H-9gt",
        "outputId": "5d037ddc-44a3-4408-a11f-9e1300387b54"
      },
      "outputs": [],
      "source": [
        "checkpoint_path = \"LeNet/cp.ckpt\"\n",
        "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "\n",
        "# Create a callback that saves the model's weights\n",
        "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n",
        "                                                 save_weights_only=True,\n",
        "                                                 verbose=1)\n",
        "\n",
        "metric_history = pd.DataFrame()\n",
        "i=1\n",
        "model = keras.models.Sequential()\n",
        "for train_index, test_index in kf.split(X):\n",
        "    X_kf, X_cv = X[train_index], X[test_index]\n",
        "    y_kf, y_cv = y[train_index], y[test_index]\n",
        "\n",
        "    print(i)\n",
        "    i=i+1\n",
        "\n",
        "    model = keras.models.Sequential([\n",
        "        Conv2D(filters=6, kernel_size=(5,5), padding='same', activation='tanh', input_shape=(30, 30, 3)),\n",
        "        AveragePooling2D(strides=(2,2)),\n",
        "        Conv2D(filters=16, kernel_size=(5,5), padding='valid', activation='tanh'),\n",
        "        AveragePooling2D(strides=(2,2)),\n",
        "        Conv2D(filters=120, kernel_size=(1,1), padding='valid', activation='tanh'),\n",
        "        Flatten(),\n",
        "        Dense(84, activation='relu'),\n",
        "        Dense(NUMBER_OF_CLASSES, activation='softmax'),\n",
        "    ])\n",
        "    \n",
        "    model.compile(loss=\"categorical_crossentropy\",\n",
        "              optimizer=\"Adam\",\n",
        "              metrics=[\"accuracy\", TruePositives(), FalsePositives(), TrueNegatives(), FalseNegatives(), SensitivityAtSpecificity(0.5), SpecificityAtSensitivity(0.5), Precision(), Recall(), AUC()])\n",
        "\n",
        "\n",
        "    history = model.fit(img_aug.flow(X_kf, y_kf, batch_size=64), validation_data=[X_cv, y_cv], epochs=20, verbose=1, callbacks=[cp_callback],)\n",
        "\n",
        "    loss, accuracy, tp, fp, tn, fn, sensitivity, specificity, precision, recall, auc = model.evaluate(X_cv, y_cv)\n",
        "    metrics = pd.DataFrame([[loss, accuracy, tp, fp, tn, fn, sensitivity, specificity, precision, recall, auc]], columns=[\"Loss\", \"Accuracy\", \"TP\", \"FP\", \"TN\", \"FN\", \"Sensitivity\",\"Specificity\", \"Precision\", \"Recall\", \"AUC\"])\n",
        "    metric_history = pd.concat([metric_history, metrics], ignore_index=True)\n",
        "print(metric_history.mean())\n",
        "info = pd.DataFrame([[\"LeNet 5\"]], columns=[\"Topology\", ])\n",
        "info = pd.concat([info, pd.DataFrame(metric_history.mean()).T], axis=1,)\n",
        "table = pd.concat([table, info], ignore_index=True)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gFK670Zgv7Cc"
      },
      "outputs": [],
      "source": [
        "table.to_csv(\"./log.csv\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TXdUErdSVYQj"
      },
      "source": [
        "## AlexNet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qSM5ibAZRdOA",
        "outputId": "0d8a0bee-8908-48d5-a822-20004de8a3f9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1\n",
            "Epoch 1/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 3.6739 - accuracy: 0.3987 - true_positives: 7364.0000 - false_positives: 3427.0000 - true_negatives: 1478669.0000 - false_negatives: 27924.0000 - sensitivity_at_specificity: 0.9481 - specificity_at_sensitivity: 0.9724 - precision: 0.6824 - recall: 0.2087 - auc: 0.8972\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 16s 46ms/step - loss: 3.6739 - accuracy: 0.3987 - true_positives: 7364.0000 - false_positives: 3427.0000 - true_negatives: 1478669.0000 - false_negatives: 27924.0000 - sensitivity_at_specificity: 0.9481 - specificity_at_sensitivity: 0.9724 - precision: 0.6824 - recall: 0.2087 - auc: 0.8972 - val_loss: 0.9860 - val_accuracy: 0.7133 - val_true_positives: 2041.0000 - val_false_positives: 289.0000 - val_true_negatives: 164393.0000 - val_false_negatives: 1880.0000 - val_sensitivity_at_specificity: 0.9944 - val_specificity_at_sensitivity: 0.9986 - val_precision: 0.8760 - val_recall: 0.5205 - val_auc: 0.9863\n",
            "Epoch 2/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 1.1115 - accuracy: 0.6828 - true_positives: 18380.0000 - false_positives: 3664.0000 - true_negatives: 1473056.0000 - false_negatives: 16780.0000 - sensitivity_at_specificity: 0.9830 - specificity_at_sensitivity: 0.9979 - precision: 0.8338 - recall: 0.5228 - auc: 0.9774\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 11s 41ms/step - loss: 1.1106 - accuracy: 0.6832 - true_positives: 18463.0000 - false_positives: 3676.0000 - true_negatives: 1478420.0000 - false_negatives: 16825.0000 - sensitivity_at_specificity: 0.9829 - specificity_at_sensitivity: 0.9979 - precision: 0.8340 - recall: 0.5232 - auc: 0.9774 - val_loss: 0.5507 - val_accuracy: 0.8324 - val_true_positives: 2909.0000 - val_false_positives: 287.0000 - val_true_negatives: 164395.0000 - val_false_negatives: 1012.0000 - val_sensitivity_at_specificity: 0.9962 - val_specificity_at_sensitivity: 0.9998 - val_precision: 0.9102 - val_recall: 0.7419 - val_auc: 0.9947\n",
            "Epoch 3/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.7326 - accuracy: 0.7922 - true_positives: 24300.0000 - false_positives: 3362.0000 - true_negatives: 1478734.0000 - false_negatives: 10988.0000 - sensitivity_at_specificity: 0.9865 - specificity_at_sensitivity: 0.9995 - precision: 0.8785 - recall: 0.6886 - auc: 0.9874\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.7326 - accuracy: 0.7922 - true_positives: 24300.0000 - false_positives: 3362.0000 - true_negatives: 1478734.0000 - false_negatives: 10988.0000 - sensitivity_at_specificity: 0.9865 - specificity_at_sensitivity: 0.9995 - precision: 0.8785 - recall: 0.6886 - auc: 0.9874 - val_loss: 0.3452 - val_accuracy: 0.9005 - val_true_positives: 3312.0000 - val_false_positives: 217.0000 - val_true_negatives: 164465.0000 - val_false_negatives: 609.0000 - val_sensitivity_at_specificity: 0.9967 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9385 - val_recall: 0.8447 - val_auc: 0.9970\n",
            "Epoch 4/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.5366 - accuracy: 0.8492 - true_positives: 27388.0000 - false_positives: 2877.0000 - true_negatives: 1473843.0000 - false_negatives: 7772.0000 - sensitivity_at_specificity: 0.9890 - specificity_at_sensitivity: 0.9998 - precision: 0.9049 - recall: 0.7790 - auc: 0.9914\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 11s 42ms/step - loss: 0.5360 - accuracy: 0.8493 - true_positives: 27493.0000 - false_positives: 2885.0000 - true_negatives: 1479211.0000 - false_negatives: 7795.0000 - sensitivity_at_specificity: 0.9890 - specificity_at_sensitivity: 0.9998 - precision: 0.9050 - recall: 0.7791 - auc: 0.9915 - val_loss: 0.3628 - val_accuracy: 0.8959 - val_true_positives: 3356.0000 - val_false_positives: 276.0000 - val_true_negatives: 164406.0000 - val_false_negatives: 565.0000 - val_sensitivity_at_specificity: 0.9926 - val_specificity_at_sensitivity: 0.9999 - val_precision: 0.9240 - val_recall: 0.8559 - val_auc: 0.9949\n",
            "Epoch 5/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.4346 - accuracy: 0.8784 - true_positives: 29098.0000 - false_positives: 2539.0000 - true_negatives: 1479557.0000 - false_negatives: 6190.0000 - sensitivity_at_specificity: 0.9910 - specificity_at_sensitivity: 0.9999 - precision: 0.9197 - recall: 0.8246 - auc: 0.9934\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.4346 - accuracy: 0.8784 - true_positives: 29098.0000 - false_positives: 2539.0000 - true_negatives: 1479557.0000 - false_negatives: 6190.0000 - sensitivity_at_specificity: 0.9910 - specificity_at_sensitivity: 0.9999 - precision: 0.9197 - recall: 0.8246 - auc: 0.9934 - val_loss: 0.2103 - val_accuracy: 0.9424 - val_true_positives: 3608.0000 - val_false_positives: 149.0000 - val_true_negatives: 164533.0000 - val_false_negatives: 313.0000 - val_sensitivity_at_specificity: 0.9964 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9603 - val_recall: 0.9202 - val_auc: 0.9977\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3757 - accuracy: 0.8932 - true_positives: 30113.0000 - false_positives: 2375.0000 - true_negatives: 1479721.0000 - false_negatives: 5175.0000 - sensitivity_at_specificity: 0.9912 - specificity_at_sensitivity: 0.9999 - precision: 0.9269 - recall: 0.8533 - auc: 0.9941\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.3757 - accuracy: 0.8932 - true_positives: 30113.0000 - false_positives: 2375.0000 - true_negatives: 1479721.0000 - false_negatives: 5175.0000 - sensitivity_at_specificity: 0.9912 - specificity_at_sensitivity: 0.9999 - precision: 0.9269 - recall: 0.8533 - auc: 0.9941 - val_loss: 0.2035 - val_accuracy: 0.9449 - val_true_positives: 3589.0000 - val_false_positives: 135.0000 - val_true_negatives: 164547.0000 - val_false_negatives: 332.0000 - val_sensitivity_at_specificity: 0.9959 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9637 - val_recall: 0.9153 - val_auc: 0.9975\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3114 - accuracy: 0.9144 - true_positives: 31186.0000 - false_positives: 2003.0000 - true_negatives: 1480093.0000 - false_negatives: 4102.0000 - sensitivity_at_specificity: 0.9925 - specificity_at_sensitivity: 0.9999 - precision: 0.9396 - recall: 0.8838 - auc: 0.9952\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.3114 - accuracy: 0.9144 - true_positives: 31186.0000 - false_positives: 2003.0000 - true_negatives: 1480093.0000 - false_negatives: 4102.0000 - sensitivity_at_specificity: 0.9925 - specificity_at_sensitivity: 0.9999 - precision: 0.9396 - recall: 0.8838 - auc: 0.9952 - val_loss: 0.1926 - val_accuracy: 0.9452 - val_true_positives: 3617.0000 - val_false_positives: 140.0000 - val_true_negatives: 164542.0000 - val_false_negatives: 304.0000 - val_sensitivity_at_specificity: 0.9959 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9627 - val_recall: 0.9225 - val_auc: 0.9975\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2802 - accuracy: 0.9226 - true_positives: 31644.0000 - false_positives: 1910.0000 - true_negatives: 1480186.0000 - false_negatives: 3644.0000 - sensitivity_at_specificity: 0.9927 - specificity_at_sensitivity: 0.9999 - precision: 0.9431 - recall: 0.8967 - auc: 0.9955\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2802 - accuracy: 0.9226 - true_positives: 31644.0000 - false_positives: 1910.0000 - true_negatives: 1480186.0000 - false_negatives: 3644.0000 - sensitivity_at_specificity: 0.9927 - specificity_at_sensitivity: 0.9999 - precision: 0.9431 - recall: 0.8967 - auc: 0.9955 - val_loss: 0.1083 - val_accuracy: 0.9745 - val_true_positives: 3780.0000 - val_false_positives: 77.0000 - val_true_negatives: 164605.0000 - val_false_negatives: 141.0000 - val_sensitivity_at_specificity: 0.9964 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9800 - val_recall: 0.9640 - val_auc: 0.9981\n",
            "Epoch 9/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2579 - accuracy: 0.9293 - true_positives: 31873.0000 - false_positives: 1795.0000 - true_negatives: 1474925.0000 - false_negatives: 3287.0000 - sensitivity_at_specificity: 0.9930 - specificity_at_sensitivity: 0.9999 - precision: 0.9467 - recall: 0.9065 - auc: 0.9958\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2581 - accuracy: 0.9293 - true_positives: 31988.0000 - false_positives: 1801.0000 - true_negatives: 1480295.0000 - false_negatives: 3300.0000 - sensitivity_at_specificity: 0.9930 - specificity_at_sensitivity: 0.9999 - precision: 0.9467 - recall: 0.9065 - auc: 0.9958 - val_loss: 0.1554 - val_accuracy: 0.9587 - val_true_positives: 3705.0000 - val_false_positives: 124.0000 - val_true_negatives: 164558.0000 - val_false_negatives: 216.0000 - val_sensitivity_at_specificity: 0.9959 - val_specificity_at_sensitivity: 0.9999 - val_precision: 0.9676 - val_recall: 0.9449 - val_auc: 0.9977\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2343 - accuracy: 0.9359 - true_positives: 32370.0000 - false_positives: 1676.0000 - true_negatives: 1480420.0000 - false_negatives: 2918.0000 - sensitivity_at_specificity: 0.9931 - specificity_at_sensitivity: 0.9999 - precision: 0.9508 - recall: 0.9173 - auc: 0.9960\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2343 - accuracy: 0.9359 - true_positives: 32370.0000 - false_positives: 1676.0000 - true_negatives: 1480420.0000 - false_negatives: 2918.0000 - sensitivity_at_specificity: 0.9931 - specificity_at_sensitivity: 0.9999 - precision: 0.9508 - recall: 0.9173 - auc: 0.9960 - val_loss: 0.1047 - val_accuracy: 0.9737 - val_true_positives: 3790.0000 - val_false_positives: 80.0000 - val_true_negatives: 164602.0000 - val_false_negatives: 131.0000 - val_sensitivity_at_specificity: 0.9959 - val_specificity_at_sensitivity: 0.9999 - val_precision: 0.9793 - val_recall: 0.9666 - val_auc: 0.9978\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2236 - accuracy: 0.9387 - true_positives: 32431.0000 - false_positives: 1591.0000 - true_negatives: 1475129.0000 - false_negatives: 2729.0000 - sensitivity_at_specificity: 0.9932 - specificity_at_sensitivity: 0.9999 - precision: 0.9532 - recall: 0.9224 - auc: 0.9961\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.2240 - accuracy: 0.9385 - true_positives: 32543.0000 - false_positives: 1600.0000 - true_negatives: 1480496.0000 - false_negatives: 2745.0000 - sensitivity_at_specificity: 0.9932 - specificity_at_sensitivity: 0.9999 - precision: 0.9531 - recall: 0.9222 - auc: 0.9961 - val_loss: 0.0994 - val_accuracy: 0.9770 - val_true_positives: 3804.0000 - val_false_positives: 72.0000 - val_true_negatives: 164610.0000 - val_false_negatives: 117.0000 - val_sensitivity_at_specificity: 0.9964 - val_specificity_at_sensitivity: 0.9999 - val_precision: 0.9814 - val_recall: 0.9702 - val_auc: 0.9981\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2111 - accuracy: 0.9432 - true_positives: 32747.0000 - false_positives: 1493.0000 - true_negatives: 1480603.0000 - false_negatives: 2541.0000 - sensitivity_at_specificity: 0.9936 - specificity_at_sensitivity: 0.9999 - precision: 0.9564 - recall: 0.9280 - auc: 0.9963\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2111 - accuracy: 0.9432 - true_positives: 32747.0000 - false_positives: 1493.0000 - true_negatives: 1480603.0000 - false_negatives: 2541.0000 - sensitivity_at_specificity: 0.9936 - specificity_at_sensitivity: 0.9999 - precision: 0.9564 - recall: 0.9280 - auc: 0.9963 - val_loss: 0.1012 - val_accuracy: 0.9712 - val_true_positives: 3788.0000 - val_false_positives: 93.0000 - val_true_negatives: 164589.0000 - val_false_negatives: 133.0000 - val_sensitivity_at_specificity: 0.9972 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9760 - val_recall: 0.9661 - val_auc: 0.9985\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2024 - accuracy: 0.9449 - true_positives: 32727.0000 - false_positives: 1472.0000 - true_negatives: 1475248.0000 - false_negatives: 2433.0000 - sensitivity_at_specificity: 0.9935 - specificity_at_sensitivity: 0.9999 - precision: 0.9570 - recall: 0.9308 - auc: 0.9963\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2023 - accuracy: 0.9448 - true_positives: 32844.0000 - false_positives: 1477.0000 - true_negatives: 1480619.0000 - false_negatives: 2444.0000 - sensitivity_at_specificity: 0.9935 - specificity_at_sensitivity: 0.9999 - precision: 0.9570 - recall: 0.9307 - auc: 0.9963 - val_loss: 0.0845 - val_accuracy: 0.9799 - val_true_positives: 3825.0000 - val_false_positives: 65.0000 - val_true_negatives: 164617.0000 - val_false_negatives: 96.0000 - val_sensitivity_at_specificity: 0.9964 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9833 - val_recall: 0.9755 - val_auc: 0.9981\n",
            "Epoch 14/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1995 - accuracy: 0.9457 - true_positives: 32929.0000 - false_positives: 1484.0000 - true_negatives: 1480612.0000 - false_negatives: 2359.0000 - sensitivity_at_specificity: 0.9936 - specificity_at_sensitivity: 0.9999 - precision: 0.9569 - recall: 0.9332 - auc: 0.9964\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1995 - accuracy: 0.9457 - true_positives: 32929.0000 - false_positives: 1484.0000 - true_negatives: 1480612.0000 - false_negatives: 2359.0000 - sensitivity_at_specificity: 0.9936 - specificity_at_sensitivity: 0.9999 - precision: 0.9569 - recall: 0.9332 - auc: 0.9964 - val_loss: 0.1190 - val_accuracy: 0.9699 - val_true_positives: 3785.0000 - val_false_positives: 100.0000 - val_true_negatives: 164582.0000 - val_false_negatives: 136.0000 - val_sensitivity_at_specificity: 0.9939 - val_specificity_at_sensitivity: 0.9999 - val_precision: 0.9743 - val_recall: 0.9653 - val_auc: 0.9968\n",
            "Epoch 15/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1923 - accuracy: 0.9498 - true_positives: 33125.0000 - false_positives: 1410.0000 - true_negatives: 1480686.0000 - false_negatives: 2163.0000 - sensitivity_at_specificity: 0.9933 - specificity_at_sensitivity: 0.9999 - precision: 0.9592 - recall: 0.9387 - auc: 0.9963\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1923 - accuracy: 0.9498 - true_positives: 33125.0000 - false_positives: 1410.0000 - true_negatives: 1480686.0000 - false_negatives: 2163.0000 - sensitivity_at_specificity: 0.9933 - specificity_at_sensitivity: 0.9999 - precision: 0.9592 - recall: 0.9387 - auc: 0.9963 - val_loss: 0.0797 - val_accuracy: 0.9770 - val_true_positives: 3818.0000 - val_false_positives: 70.0000 - val_true_negatives: 164612.0000 - val_false_negatives: 103.0000 - val_sensitivity_at_specificity: 0.9974 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9820 - val_recall: 0.9737 - val_auc: 0.9986\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1841 - accuracy: 0.9521 - true_positives: 33219.0000 - false_positives: 1364.0000 - true_negatives: 1480732.0000 - false_negatives: 2069.0000 - sensitivity_at_specificity: 0.9934 - specificity_at_sensitivity: 0.9999 - precision: 0.9606 - recall: 0.9414 - auc: 0.9964\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1841 - accuracy: 0.9521 - true_positives: 33219.0000 - false_positives: 1364.0000 - true_negatives: 1480732.0000 - false_negatives: 2069.0000 - sensitivity_at_specificity: 0.9934 - specificity_at_sensitivity: 0.9999 - precision: 0.9606 - recall: 0.9414 - auc: 0.9964 - val_loss: 0.0838 - val_accuracy: 0.9768 - val_true_positives: 3821.0000 - val_false_positives: 76.0000 - val_true_negatives: 164606.0000 - val_false_negatives: 100.0000 - val_sensitivity_at_specificity: 0.9974 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9805 - val_recall: 0.9745 - val_auc: 0.9986\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1828 - accuracy: 0.9519 - true_positives: 33260.0000 - false_positives: 1365.0000 - true_negatives: 1480731.0000 - false_negatives: 2028.0000 - sensitivity_at_specificity: 0.9931 - specificity_at_sensitivity: 0.9999 - precision: 0.9606 - recall: 0.9425 - auc: 0.9962\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1828 - accuracy: 0.9519 - true_positives: 33260.0000 - false_positives: 1365.0000 - true_negatives: 1480731.0000 - false_negatives: 2028.0000 - sensitivity_at_specificity: 0.9931 - specificity_at_sensitivity: 0.9999 - precision: 0.9606 - recall: 0.9425 - auc: 0.9962 - val_loss: 0.0786 - val_accuracy: 0.9806 - val_true_positives: 3830.0000 - val_false_positives: 59.0000 - val_true_negatives: 164623.0000 - val_false_negatives: 91.0000 - val_sensitivity_at_specificity: 0.9967 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9848 - val_recall: 0.9768 - val_auc: 0.9983\n",
            "Epoch 18/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1746 - accuracy: 0.9553 - true_positives: 33262.0000 - false_positives: 1270.0000 - true_negatives: 1475450.0000 - false_negatives: 1898.0000 - sensitivity_at_specificity: 0.9933 - specificity_at_sensitivity: 0.9999 - precision: 0.9632 - recall: 0.9460 - auc: 0.9964\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1744 - accuracy: 0.9553 - true_positives: 33385.0000 - false_positives: 1274.0000 - true_negatives: 1480822.0000 - false_negatives: 1903.0000 - sensitivity_at_specificity: 0.9933 - specificity_at_sensitivity: 0.9999 - precision: 0.9632 - recall: 0.9461 - auc: 0.9964 - val_loss: 0.0921 - val_accuracy: 0.9773 - val_true_positives: 3816.0000 - val_false_positives: 75.0000 - val_true_negatives: 164607.0000 - val_false_negatives: 105.0000 - val_sensitivity_at_specificity: 0.9964 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9807 - val_recall: 0.9732 - val_auc: 0.9981\n",
            "Epoch 19/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1837 - accuracy: 0.9524 - true_positives: 33273.0000 - false_positives: 1346.0000 - true_negatives: 1480750.0000 - false_negatives: 2015.0000 - sensitivity_at_specificity: 0.9930 - specificity_at_sensitivity: 0.9999 - precision: 0.9611 - recall: 0.9429 - auc: 0.9962\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1837 - accuracy: 0.9524 - true_positives: 33273.0000 - false_positives: 1346.0000 - true_negatives: 1480750.0000 - false_negatives: 2015.0000 - sensitivity_at_specificity: 0.9930 - specificity_at_sensitivity: 0.9999 - precision: 0.9611 - recall: 0.9429 - auc: 0.9962 - val_loss: 0.0754 - val_accuracy: 0.9814 - val_true_positives: 3840.0000 - val_false_positives: 64.0000 - val_true_negatives: 164618.0000 - val_false_negatives: 81.0000 - val_sensitivity_at_specificity: 0.9972 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9836 - val_recall: 0.9793 - val_auc: 0.9985\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1609 - accuracy: 0.9583 - true_positives: 33535.0000 - false_positives: 1186.0000 - true_negatives: 1480910.0000 - false_negatives: 1753.0000 - sensitivity_at_specificity: 0.9940 - specificity_at_sensitivity: 0.9999 - precision: 0.9658 - recall: 0.9503 - auc: 0.9967\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1609 - accuracy: 0.9583 - true_positives: 33535.0000 - false_positives: 1186.0000 - true_negatives: 1480910.0000 - false_negatives: 1753.0000 - sensitivity_at_specificity: 0.9940 - specificity_at_sensitivity: 0.9999 - precision: 0.9658 - recall: 0.9503 - auc: 0.9967 - val_loss: 0.0750 - val_accuracy: 0.9791 - val_true_positives: 3831.0000 - val_false_positives: 73.0000 - val_true_negatives: 164609.0000 - val_false_negatives: 90.0000 - val_sensitivity_at_specificity: 0.9974 - val_specificity_at_sensitivity: 1.0000 - val_precision: 0.9813 - val_recall: 0.9770 - val_auc: 0.9987\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0750 - accuracy: 0.9791 - true_positives: 3831.0000 - false_positives: 73.0000 - true_negatives: 164609.0000 - false_negatives: 90.0000 - sensitivity_at_specificity: 0.9974 - specificity_at_sensitivity: 1.0000 - precision: 0.9813 - recall: 0.9770 - auc: 0.9987\n",
            "2\n",
            "Epoch 1/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 3.1029 - accuracy: 0.5149 - true_positives_1: 12191.0000 - false_positives_1: 3972.0000 - true_negatives_1: 1472748.0000 - false_negatives_1: 22969.0000 - sensitivity_at_specificity_1: 0.9476 - specificity_at_sensitivity_1: 0.9901 - precision_1: 0.7543 - recall_1: 0.3467 - auc_1: 0.9305\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 3.0952 - accuracy: 0.5156 - true_positives_1: 12262.0000 - false_positives_1: 3985.0000 - true_negatives_1: 1478111.0000 - false_negatives_1: 23026.0000 - sensitivity_at_specificity_1: 0.9478 - specificity_at_sensitivity_1: 0.9905 - precision_1: 0.7547 - recall_1: 0.3475 - auc_1: 0.9307 - val_loss: 0.6667 - val_accuracy: 0.7983 - val_true_positives_1: 2702.0000 - val_false_positives_1: 386.0000 - val_true_negatives_1: 164296.0000 - val_false_negatives_1: 1219.0000 - val_sensitivity_at_specificity_1: 0.9939 - val_specificity_at_sensitivity_1: 0.9994 - val_precision_1: 0.8750 - val_recall_1: 0.6891 - val_auc_1: 0.9920\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.6748 - accuracy: 0.8053 - true_positives_1: 25199.0000 - false_positives_1: 3381.0000 - true_negatives_1: 1478715.0000 - false_negatives_1: 10089.0000 - sensitivity_at_specificity_1: 0.9878 - specificity_at_sensitivity_1: 0.9996 - precision_1: 0.8817 - recall_1: 0.7141 - auc_1: 0.9890\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.6748 - accuracy: 0.8053 - true_positives_1: 25199.0000 - false_positives_1: 3381.0000 - true_negatives_1: 1478715.0000 - false_negatives_1: 10089.0000 - sensitivity_at_specificity_1: 0.9878 - specificity_at_sensitivity_1: 0.9996 - precision_1: 0.8817 - recall_1: 0.7141 - auc_1: 0.9890 - val_loss: 0.3678 - val_accuracy: 0.8799 - val_true_positives_1: 3269.0000 - val_false_positives_1: 323.0000 - val_true_negatives_1: 164359.0000 - val_false_negatives_1: 652.0000 - val_sensitivity_at_specificity_1: 0.9949 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9101 - val_recall_1: 0.8337 - val_auc_1: 0.9959\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.4257 - accuracy: 0.8790 - true_positives_1: 29196.0000 - false_positives_1: 2616.0000 - true_negatives_1: 1474104.0000 - false_negatives_1: 5964.0000 - sensitivity_at_specificity_1: 0.9912 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9178 - recall_1: 0.8304 - auc_1: 0.9937\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.4259 - accuracy: 0.8789 - true_positives_1: 29298.0000 - false_positives_1: 2627.0000 - true_negatives_1: 1479469.0000 - false_negatives_1: 5990.0000 - sensitivity_at_specificity_1: 0.9912 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9177 - recall_1: 0.8303 - auc_1: 0.9937 - val_loss: 0.2075 - val_accuracy: 0.9368 - val_true_positives_1: 3604.0000 - val_false_positives_1: 181.0000 - val_true_negatives_1: 164501.0000 - val_false_negatives_1: 317.0000 - val_sensitivity_at_specificity_1: 0.9964 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9522 - val_recall_1: 0.9192 - val_auc_1: 0.9977\n",
            "Epoch 4/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2930 - accuracy: 0.9176 - true_positives_1: 31351.0000 - false_positives_1: 1939.0000 - true_negatives_1: 1480157.0000 - false_negatives_1: 3937.0000 - sensitivity_at_specificity_1: 0.9933 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9418 - recall_1: 0.8884 - auc_1: 0.9957\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2930 - accuracy: 0.9176 - true_positives_1: 31351.0000 - false_positives_1: 1939.0000 - true_negatives_1: 1480157.0000 - false_negatives_1: 3937.0000 - sensitivity_at_specificity_1: 0.9933 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9418 - recall_1: 0.8884 - auc_1: 0.9957 - val_loss: 0.1304 - val_accuracy: 0.9640 - val_true_positives_1: 3738.0000 - val_false_positives_1: 115.0000 - val_true_negatives_1: 164567.0000 - val_false_negatives_1: 183.0000 - val_sensitivity_at_specificity_1: 0.9982 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9702 - val_recall_1: 0.9533 - val_auc_1: 0.9989\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2392 - accuracy: 0.9336 - true_positives_1: 32115.0000 - false_positives_1: 1680.0000 - true_negatives_1: 1475040.0000 - false_negatives_1: 3045.0000 - sensitivity_at_specificity_1: 0.9937 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9503 - recall_1: 0.9134 - auc_1: 0.9962\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2388 - accuracy: 0.9337 - true_positives_1: 32235.0000 - false_positives_1: 1685.0000 - true_negatives_1: 1480411.0000 - false_negatives_1: 3053.0000 - sensitivity_at_specificity_1: 0.9937 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9503 - recall_1: 0.9135 - auc_1: 0.9962 - val_loss: 0.1000 - val_accuracy: 0.9717 - val_true_positives_1: 3784.0000 - val_false_positives_1: 89.0000 - val_true_negatives_1: 164593.0000 - val_false_negatives_1: 137.0000 - val_sensitivity_at_specificity_1: 0.9974 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9770 - val_recall_1: 0.9651 - val_auc_1: 0.9986\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2078 - accuracy: 0.9426 - true_positives_1: 32682.0000 - false_positives_1: 1496.0000 - true_negatives_1: 1480600.0000 - false_negatives_1: 2606.0000 - sensitivity_at_specificity_1: 0.9940 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9562 - recall_1: 0.9262 - auc_1: 0.9965\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2078 - accuracy: 0.9426 - true_positives_1: 32682.0000 - false_positives_1: 1496.0000 - true_negatives_1: 1480600.0000 - false_negatives_1: 2606.0000 - sensitivity_at_specificity_1: 0.9940 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9562 - recall_1: 0.9262 - auc_1: 0.9965 - val_loss: 0.0809 - val_accuracy: 0.9765 - val_true_positives_1: 3810.0000 - val_false_positives_1: 74.0000 - val_true_negatives_1: 164608.0000 - val_false_negatives_1: 111.0000 - val_sensitivity_at_specificity_1: 0.9980 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9809 - val_recall_1: 0.9717 - val_auc_1: 0.9989\n",
            "Epoch 7/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1832 - accuracy: 0.9487 - true_positives_1: 32936.0000 - false_positives_1: 1366.0000 - true_negatives_1: 1475354.0000 - false_negatives_1: 2224.0000 - sensitivity_at_specificity_1: 0.9949 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9602 - recall_1: 0.9367 - auc_1: 0.9971\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1833 - accuracy: 0.9487 - true_positives_1: 33055.0000 - false_positives_1: 1372.0000 - true_negatives_1: 1480724.0000 - false_negatives_1: 2233.0000 - sensitivity_at_specificity_1: 0.9948 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9601 - recall_1: 0.9367 - auc_1: 0.9971 - val_loss: 0.1206 - val_accuracy: 0.9707 - val_true_positives_1: 3784.0000 - val_false_positives_1: 98.0000 - val_true_negatives_1: 164584.0000 - val_false_negatives_1: 137.0000 - val_sensitivity_at_specificity_1: 0.9954 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9748 - val_recall_1: 0.9651 - val_auc_1: 0.9975\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1768 - accuracy: 0.9521 - true_positives_1: 33229.0000 - false_positives_1: 1312.0000 - true_negatives_1: 1480784.0000 - false_negatives_1: 2059.0000 - sensitivity_at_specificity_1: 0.9945 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9620 - recall_1: 0.9417 - auc_1: 0.9969\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1768 - accuracy: 0.9521 - true_positives_1: 33229.0000 - false_positives_1: 1312.0000 - true_negatives_1: 1480784.0000 - false_negatives_1: 2059.0000 - sensitivity_at_specificity_1: 0.9945 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9620 - recall_1: 0.9417 - auc_1: 0.9969 - val_loss: 0.0660 - val_accuracy: 0.9829 - val_true_positives_1: 3843.0000 - val_false_positives_1: 55.0000 - val_true_negatives_1: 164627.0000 - val_false_negatives_1: 78.0000 - val_sensitivity_at_specificity_1: 0.9982 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9859 - val_recall_1: 0.9801 - val_auc_1: 0.9990\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1608 - accuracy: 0.9552 - true_positives_1: 33360.0000 - false_positives_1: 1219.0000 - true_negatives_1: 1480877.0000 - false_negatives_1: 1928.0000 - sensitivity_at_specificity_1: 0.9950 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9647 - recall_1: 0.9454 - auc_1: 0.9972\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1608 - accuracy: 0.9552 - true_positives_1: 33360.0000 - false_positives_1: 1219.0000 - true_negatives_1: 1480877.0000 - false_negatives_1: 1928.0000 - sensitivity_at_specificity_1: 0.9950 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9647 - recall_1: 0.9454 - auc_1: 0.9972 - val_loss: 0.0747 - val_accuracy: 0.9804 - val_true_positives_1: 3833.0000 - val_false_positives_1: 65.0000 - val_true_negatives_1: 164617.0000 - val_false_negatives_1: 88.0000 - val_sensitivity_at_specificity_1: 0.9980 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9833 - val_recall_1: 0.9776 - val_auc_1: 0.9989\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1668 - accuracy: 0.9552 - true_positives_1: 33429.0000 - false_positives_1: 1274.0000 - true_negatives_1: 1480822.0000 - false_negatives_1: 1859.0000 - sensitivity_at_specificity_1: 0.9941 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9633 - recall_1: 0.9473 - auc_1: 0.9968\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1668 - accuracy: 0.9552 - true_positives_1: 33429.0000 - false_positives_1: 1274.0000 - true_negatives_1: 1480822.0000 - false_negatives_1: 1859.0000 - sensitivity_at_specificity_1: 0.9941 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9633 - recall_1: 0.9473 - auc_1: 0.9968 - val_loss: 0.0570 - val_accuracy: 0.9839 - val_true_positives_1: 3848.0000 - val_false_positives_1: 53.0000 - val_true_negatives_1: 164629.0000 - val_false_negatives_1: 73.0000 - val_sensitivity_at_specificity_1: 0.9982 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9864 - val_recall_1: 0.9814 - val_auc_1: 0.9991\n",
            "Epoch 11/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1603 - accuracy: 0.9576 - true_positives_1: 33499.0000 - false_positives_1: 1205.0000 - true_negatives_1: 1480891.0000 - false_negatives_1: 1789.0000 - sensitivity_at_specificity_1: 0.9945 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9653 - recall_1: 0.9493 - auc_1: 0.9970\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1603 - accuracy: 0.9576 - true_positives_1: 33499.0000 - false_positives_1: 1205.0000 - true_negatives_1: 1480891.0000 - false_negatives_1: 1789.0000 - sensitivity_at_specificity_1: 0.9945 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9653 - recall_1: 0.9493 - auc_1: 0.9970 - val_loss: 0.0711 - val_accuracy: 0.9788 - val_true_positives_1: 3823.0000 - val_false_positives_1: 71.0000 - val_true_negatives_1: 164611.0000 - val_false_negatives_1: 98.0000 - val_sensitivity_at_specificity_1: 0.9977 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9818 - val_recall_1: 0.9750 - val_auc_1: 0.9988\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1480 - accuracy: 0.9607 - true_positives_1: 33657.0000 - false_positives_1: 1128.0000 - true_negatives_1: 1480968.0000 - false_negatives_1: 1631.0000 - sensitivity_at_specificity_1: 0.9943 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9676 - recall_1: 0.9538 - auc_1: 0.9969\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1480 - accuracy: 0.9607 - true_positives_1: 33657.0000 - false_positives_1: 1128.0000 - true_negatives_1: 1480968.0000 - false_negatives_1: 1631.0000 - sensitivity_at_specificity_1: 0.9943 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9676 - recall_1: 0.9538 - auc_1: 0.9969 - val_loss: 0.0627 - val_accuracy: 0.9855 - val_true_positives_1: 3848.0000 - val_false_positives_1: 40.0000 - val_true_negatives_1: 164642.0000 - val_false_negatives_1: 73.0000 - val_sensitivity_at_specificity_1: 0.9987 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9897 - val_recall_1: 0.9814 - val_auc_1: 0.9993\n",
            "Epoch 13/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1354 - accuracy: 0.9655 - true_positives_1: 33844.0000 - false_positives_1: 1018.0000 - true_negatives_1: 1481078.0000 - false_negatives_1: 1444.0000 - sensitivity_at_specificity_1: 0.9948 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9708 - recall_1: 0.9591 - auc_1: 0.9972\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1354 - accuracy: 0.9655 - true_positives_1: 33844.0000 - false_positives_1: 1018.0000 - true_negatives_1: 1481078.0000 - false_negatives_1: 1444.0000 - sensitivity_at_specificity_1: 0.9948 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9708 - recall_1: 0.9591 - auc_1: 0.9972 - val_loss: 0.0617 - val_accuracy: 0.9870 - val_true_positives_1: 3864.0000 - val_false_positives_1: 42.0000 - val_true_negatives_1: 164640.0000 - val_false_negatives_1: 57.0000 - val_sensitivity_at_specificity_1: 0.9974 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9892 - val_recall_1: 0.9855 - val_auc_1: 0.9987\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1506 - accuracy: 0.9620 - true_positives_1: 33611.0000 - false_positives_1: 1106.0000 - true_negatives_1: 1475614.0000 - false_negatives_1: 1549.0000 - sensitivity_at_specificity_1: 0.9936 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9681 - recall_1: 0.9559 - auc_1: 0.9966\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1509 - accuracy: 0.9620 - true_positives_1: 33731.0000 - false_positives_1: 1111.0000 - true_negatives_1: 1480985.0000 - false_negatives_1: 1557.0000 - sensitivity_at_specificity_1: 0.9936 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9681 - recall_1: 0.9559 - auc_1: 0.9966 - val_loss: 0.0663 - val_accuracy: 0.9816 - val_true_positives_1: 3832.0000 - val_false_positives_1: 57.0000 - val_true_negatives_1: 164625.0000 - val_false_negatives_1: 89.0000 - val_sensitivity_at_specificity_1: 0.9982 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9853 - val_recall_1: 0.9773 - val_auc_1: 0.9990\n",
            "Epoch 15/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1359 - accuracy: 0.9662 - true_positives_1: 33886.0000 - false_positives_1: 948.0000 - true_negatives_1: 1481148.0000 - false_negatives_1: 1402.0000 - sensitivity_at_specificity_1: 0.9946 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9728 - recall_1: 0.9603 - auc_1: 0.9971\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1359 - accuracy: 0.9662 - true_positives_1: 33886.0000 - false_positives_1: 948.0000 - true_negatives_1: 1481148.0000 - false_negatives_1: 1402.0000 - sensitivity_at_specificity_1: 0.9946 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9728 - recall_1: 0.9603 - auc_1: 0.9971 - val_loss: 0.0780 - val_accuracy: 0.9832 - val_true_positives_1: 3844.0000 - val_false_positives_1: 54.0000 - val_true_negatives_1: 164628.0000 - val_false_negatives_1: 77.0000 - val_sensitivity_at_specificity_1: 0.9974 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9861 - val_recall_1: 0.9804 - val_auc_1: 0.9987\n",
            "Epoch 16/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1436 - accuracy: 0.9649 - true_positives_1: 33738.0000 - false_positives_1: 1018.0000 - true_negatives_1: 1475702.0000 - false_negatives_1: 1422.0000 - sensitivity_at_specificity_1: 0.9938 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9707 - recall_1: 0.9596 - auc_1: 0.9967\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1432 - accuracy: 0.9650 - true_positives_1: 33863.0000 - false_positives_1: 1020.0000 - true_negatives_1: 1481076.0000 - false_negatives_1: 1425.0000 - sensitivity_at_specificity_1: 0.9938 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9708 - recall_1: 0.9596 - auc_1: 0.9967 - val_loss: 0.0643 - val_accuracy: 0.9857 - val_true_positives_1: 3861.0000 - val_false_positives_1: 48.0000 - val_true_negatives_1: 164634.0000 - val_false_negatives_1: 60.0000 - val_sensitivity_at_specificity_1: 0.9972 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9877 - val_recall_1: 0.9847 - val_auc_1: 0.9986\n",
            "Epoch 17/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1367 - accuracy: 0.9667 - true_positives_1: 33814.0000 - false_positives_1: 974.0000 - true_negatives_1: 1475746.0000 - false_negatives_1: 1346.0000 - sensitivity_at_specificity_1: 0.9943 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9720 - recall_1: 0.9617 - auc_1: 0.9970\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1364 - accuracy: 0.9668 - true_positives_1: 33940.0000 - false_positives_1: 975.0000 - true_negatives_1: 1481121.0000 - false_negatives_1: 1348.0000 - sensitivity_at_specificity_1: 0.9944 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9721 - recall_1: 0.9618 - auc_1: 0.9970 - val_loss: 0.0507 - val_accuracy: 0.9857 - val_true_positives_1: 3857.0000 - val_false_positives_1: 42.0000 - val_true_negatives_1: 164640.0000 - val_false_negatives_1: 64.0000 - val_sensitivity_at_specificity_1: 0.9982 - val_specificity_at_sensitivity_1: 1.0000 - val_precision_1: 0.9892 - val_recall_1: 0.9837 - val_auc_1: 0.9991\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1339 - accuracy: 0.9657 - true_positives_1: 33924.0000 - false_positives_1: 1022.0000 - true_negatives_1: 1481074.0000 - false_negatives_1: 1364.0000 - sensitivity_at_specificity_1: 0.9942 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9708 - recall_1: 0.9613 - auc_1: 0.9969\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1339 - accuracy: 0.9657 - true_positives_1: 33924.0000 - false_positives_1: 1022.0000 - true_negatives_1: 1481074.0000 - false_negatives_1: 1364.0000 - sensitivity_at_specificity_1: 0.9942 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9708 - recall_1: 0.9613 - auc_1: 0.9969 - val_loss: 0.1116 - val_accuracy: 0.9748 - val_true_positives_1: 3810.0000 - val_false_positives_1: 82.0000 - val_true_negatives_1: 164600.0000 - val_false_negatives_1: 111.0000 - val_sensitivity_at_specificity_1: 0.9952 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9789 - val_recall_1: 0.9717 - val_auc_1: 0.9975\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1390 - accuracy: 0.9656 - true_positives_1: 33793.0000 - false_positives_1: 1008.0000 - true_negatives_1: 1475712.0000 - false_negatives_1: 1367.0000 - sensitivity_at_specificity_1: 0.9942 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9710 - recall_1: 0.9611 - auc_1: 0.9969\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1385 - accuracy: 0.9657 - true_positives_1: 33920.0000 - false_positives_1: 1008.0000 - true_negatives_1: 1481088.0000 - false_negatives_1: 1368.0000 - sensitivity_at_specificity_1: 0.9942 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9711 - recall_1: 0.9612 - auc_1: 0.9969 - val_loss: 0.0728 - val_accuracy: 0.9870 - val_true_positives_1: 3862.0000 - val_false_positives_1: 48.0000 - val_true_negatives_1: 164634.0000 - val_false_negatives_1: 59.0000 - val_sensitivity_at_specificity_1: 0.9957 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9877 - val_recall_1: 0.9850 - val_auc_1: 0.9978\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1554 - accuracy: 0.9631 - true_positives_1: 33822.0000 - false_positives_1: 1097.0000 - true_negatives_1: 1480999.0000 - false_negatives_1: 1466.0000 - sensitivity_at_specificity_1: 0.9923 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9686 - recall_1: 0.9585 - auc_1: 0.9960\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1554 - accuracy: 0.9631 - true_positives_1: 33822.0000 - false_positives_1: 1097.0000 - true_negatives_1: 1480999.0000 - false_negatives_1: 1466.0000 - sensitivity_at_specificity_1: 0.9923 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9686 - recall_1: 0.9585 - auc_1: 0.9960 - val_loss: 0.0939 - val_accuracy: 0.9821 - val_true_positives_1: 3844.0000 - val_false_positives_1: 62.0000 - val_true_negatives_1: 164620.0000 - val_false_negatives_1: 77.0000 - val_sensitivity_at_specificity_1: 0.9959 - val_specificity_at_sensitivity_1: 0.9999 - val_precision_1: 0.9841 - val_recall_1: 0.9804 - val_auc_1: 0.9979\n",
            "123/123 [==============================] - 1s 8ms/step - loss: 0.0939 - accuracy: 0.9821 - true_positives_1: 3844.0000 - false_positives_1: 62.0000 - true_negatives_1: 164620.0000 - false_negatives_1: 77.0000 - sensitivity_at_specificity_1: 0.9959 - specificity_at_sensitivity_1: 0.9999 - precision_1: 0.9841 - recall_1: 0.9804 - auc_1: 0.9979\n",
            "3\n",
            "Epoch 1/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 3.3752 - accuracy: 0.5123 - true_positives_2: 12139.0000 - false_positives_2: 3931.0000 - true_negatives_2: 1478165.0000 - false_negatives_2: 23149.0000 - sensitivity_at_specificity_2: 0.9491 - specificity_at_sensitivity_2: 0.9899 - precision_2: 0.7554 - recall_2: 0.3440 - auc_2: 0.9273\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 43ms/step - loss: 3.3752 - accuracy: 0.5123 - true_positives_2: 12139.0000 - false_positives_2: 3931.0000 - true_negatives_2: 1478165.0000 - false_negatives_2: 23149.0000 - sensitivity_at_specificity_2: 0.9491 - specificity_at_sensitivity_2: 0.9899 - precision_2: 0.7554 - recall_2: 0.3440 - auc_2: 0.9273 - val_loss: 0.6310 - val_accuracy: 0.8085 - val_true_positives_2: 2746.0000 - val_false_positives_2: 312.0000 - val_true_negatives_2: 164370.0000 - val_false_negatives_2: 1175.0000 - val_sensitivity_at_specificity_2: 0.9931 - val_specificity_at_sensitivity_2: 0.9998 - val_precision_2: 0.8980 - val_recall_2: 0.7003 - val_auc_2: 0.9922\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.6786 - accuracy: 0.8041 - true_positives_2: 25126.0000 - false_positives_2: 3280.0000 - true_negatives_2: 1478816.0000 - false_negatives_2: 10162.0000 - sensitivity_at_specificity_2: 0.9882 - specificity_at_sensitivity_2: 0.9996 - precision_2: 0.8845 - recall_2: 0.7120 - auc_2: 0.9891\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.6786 - accuracy: 0.8041 - true_positives_2: 25126.0000 - false_positives_2: 3280.0000 - true_negatives_2: 1478816.0000 - false_negatives_2: 10162.0000 - sensitivity_at_specificity_2: 0.9882 - specificity_at_sensitivity_2: 0.9996 - precision_2: 0.8845 - recall_2: 0.7120 - auc_2: 0.9891 - val_loss: 0.3351 - val_accuracy: 0.8967 - val_true_positives_2: 3353.0000 - val_false_positives_2: 235.0000 - val_true_negatives_2: 164447.0000 - val_false_negatives_2: 568.0000 - val_sensitivity_at_specificity_2: 0.9934 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9345 - val_recall_2: 0.8551 - val_auc_2: 0.9955\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.4369 - accuracy: 0.8744 - true_positives_2: 28923.0000 - false_positives_2: 2576.0000 - true_negatives_2: 1474144.0000 - false_negatives_2: 6237.0000 - sensitivity_at_specificity_2: 0.9912 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9182 - recall_2: 0.8226 - auc_2: 0.9935\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.4362 - accuracy: 0.8746 - true_positives_2: 29038.0000 - false_positives_2: 2583.0000 - true_negatives_2: 1479513.0000 - false_negatives_2: 6250.0000 - sensitivity_at_specificity_2: 0.9912 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9183 - recall_2: 0.8229 - auc_2: 0.9935 - val_loss: 0.2139 - val_accuracy: 0.9362 - val_true_positives_2: 3579.0000 - val_false_positives_2: 168.0000 - val_true_negatives_2: 164514.0000 - val_false_negatives_2: 342.0000 - val_sensitivity_at_specificity_2: 0.9967 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9552 - val_recall_2: 0.9128 - val_auc_2: 0.9978\n",
            "Epoch 4/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3326 - accuracy: 0.9054 - true_positives_2: 30749.0000 - false_positives_2: 2131.0000 - true_negatives_2: 1479965.0000 - false_negatives_2: 4539.0000 - sensitivity_at_specificity_2: 0.9926 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9352 - recall_2: 0.8714 - auc_2: 0.9951\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.3326 - accuracy: 0.9054 - true_positives_2: 30749.0000 - false_positives_2: 2131.0000 - true_negatives_2: 1479965.0000 - false_negatives_2: 4539.0000 - sensitivity_at_specificity_2: 0.9926 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9352 - recall_2: 0.8714 - auc_2: 0.9951 - val_loss: 0.1726 - val_accuracy: 0.9475 - val_true_positives_2: 3656.0000 - val_false_positives_2: 150.0000 - val_true_negatives_2: 164532.0000 - val_false_negatives_2: 265.0000 - val_sensitivity_at_specificity_2: 0.9969 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9606 - val_recall_2: 0.9324 - val_auc_2: 0.9981\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2635 - accuracy: 0.9267 - true_positives_2: 31698.0000 - false_positives_2: 1800.0000 - true_negatives_2: 1474920.0000 - false_negatives_2: 3462.0000 - sensitivity_at_specificity_2: 0.9933 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9463 - recall_2: 0.9015 - auc_2: 0.9959\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2637 - accuracy: 0.9267 - true_positives_2: 31816.0000 - false_positives_2: 1807.0000 - true_negatives_2: 1480289.0000 - false_negatives_2: 3472.0000 - sensitivity_at_specificity_2: 0.9933 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9463 - recall_2: 0.9016 - auc_2: 0.9959 - val_loss: 0.1410 - val_accuracy: 0.9577 - val_true_positives_2: 3714.0000 - val_false_positives_2: 131.0000 - val_true_negatives_2: 164551.0000 - val_false_negatives_2: 207.0000 - val_sensitivity_at_specificity_2: 0.9962 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9659 - val_recall_2: 0.9472 - val_auc_2: 0.9979\n",
            "Epoch 6/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2084 - accuracy: 0.9416 - true_positives_2: 32538.0000 - false_positives_2: 1481.0000 - true_negatives_2: 1475239.0000 - false_negatives_2: 2622.0000 - sensitivity_at_specificity_2: 0.9947 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9565 - recall_2: 0.9254 - auc_2: 0.9969\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2084 - accuracy: 0.9416 - true_positives_2: 32657.0000 - false_positives_2: 1488.0000 - true_negatives_2: 1480608.0000 - false_negatives_2: 2631.0000 - sensitivity_at_specificity_2: 0.9947 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9564 - recall_2: 0.9254 - auc_2: 0.9969 - val_loss: 0.1742 - val_accuracy: 0.9518 - val_true_positives_2: 3708.0000 - val_false_positives_2: 165.0000 - val_true_negatives_2: 164517.0000 - val_false_negatives_2: 213.0000 - val_sensitivity_at_specificity_2: 0.9921 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9574 - val_recall_2: 0.9457 - val_auc_2: 0.9958\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1929 - accuracy: 0.9469 - true_positives_2: 32969.0000 - false_positives_2: 1381.0000 - true_negatives_2: 1480715.0000 - false_negatives_2: 2319.0000 - sensitivity_at_specificity_2: 0.9943 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9598 - recall_2: 0.9343 - auc_2: 0.9968\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1929 - accuracy: 0.9469 - true_positives_2: 32969.0000 - false_positives_2: 1381.0000 - true_negatives_2: 1480715.0000 - false_negatives_2: 2319.0000 - sensitivity_at_specificity_2: 0.9943 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9598 - recall_2: 0.9343 - auc_2: 0.9968 - val_loss: 0.1143 - val_accuracy: 0.9674 - val_true_positives_2: 3772.0000 - val_false_positives_2: 113.0000 - val_true_negatives_2: 164569.0000 - val_false_negatives_2: 149.0000 - val_sensitivity_at_specificity_2: 0.9962 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9709 - val_recall_2: 0.9620 - val_auc_2: 0.9979\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1761 - accuracy: 0.9520 - true_positives_2: 33180.0000 - false_positives_2: 1261.0000 - true_negatives_2: 1480835.0000 - false_negatives_2: 2108.0000 - sensitivity_at_specificity_2: 0.9942 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9634 - recall_2: 0.9403 - auc_2: 0.9967\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1761 - accuracy: 0.9520 - true_positives_2: 33180.0000 - false_positives_2: 1261.0000 - true_negatives_2: 1480835.0000 - false_negatives_2: 2108.0000 - sensitivity_at_specificity_2: 0.9942 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9634 - recall_2: 0.9403 - auc_2: 0.9967 - val_loss: 0.1192 - val_accuracy: 0.9694 - val_true_positives_2: 3778.0000 - val_false_positives_2: 100.0000 - val_true_negatives_2: 164582.0000 - val_false_negatives_2: 143.0000 - val_sensitivity_at_specificity_2: 0.9962 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9742 - val_recall_2: 0.9635 - val_auc_2: 0.9979\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1637 - accuracy: 0.9551 - true_positives_2: 33356.0000 - false_positives_2: 1229.0000 - true_negatives_2: 1480867.0000 - false_negatives_2: 1932.0000 - sensitivity_at_specificity_2: 0.9948 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9645 - recall_2: 0.9453 - auc_2: 0.9971\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1637 - accuracy: 0.9551 - true_positives_2: 33356.0000 - false_positives_2: 1229.0000 - true_negatives_2: 1480867.0000 - false_negatives_2: 1932.0000 - sensitivity_at_specificity_2: 0.9948 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9645 - recall_2: 0.9453 - auc_2: 0.9971 - val_loss: 0.1361 - val_accuracy: 0.9663 - val_true_positives_2: 3776.0000 - val_false_positives_2: 108.0000 - val_true_negatives_2: 164574.0000 - val_false_negatives_2: 145.0000 - val_sensitivity_at_specificity_2: 0.9936 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9722 - val_recall_2: 0.9630 - val_auc_2: 0.9966\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1435 - accuracy: 0.9618 - true_positives_2: 33662.0000 - false_positives_2: 1073.0000 - true_negatives_2: 1481023.0000 - false_negatives_2: 1626.0000 - sensitivity_at_specificity_2: 0.9951 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9691 - recall_2: 0.9539 - auc_2: 0.9973\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1435 - accuracy: 0.9618 - true_positives_2: 33662.0000 - false_positives_2: 1073.0000 - true_negatives_2: 1481023.0000 - false_negatives_2: 1626.0000 - sensitivity_at_specificity_2: 0.9951 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9691 - recall_2: 0.9539 - auc_2: 0.9973 - val_loss: 0.0604 - val_accuracy: 0.9824 - val_true_positives_2: 3842.0000 - val_false_positives_2: 58.0000 - val_true_negatives_2: 164624.0000 - val_false_negatives_2: 79.0000 - val_sensitivity_at_specificity_2: 0.9982 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9851 - val_recall_2: 0.9799 - val_auc_2: 0.9991\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1631 - accuracy: 0.9566 - true_positives_2: 33352.0000 - false_positives_2: 1207.0000 - true_negatives_2: 1475513.0000 - false_negatives_2: 1808.0000 - sensitivity_at_specificity_2: 0.9938 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9651 - recall_2: 0.9486 - auc_2: 0.9966\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1628 - accuracy: 0.9566 - true_positives_2: 33474.0000 - false_positives_2: 1212.0000 - true_negatives_2: 1480884.0000 - false_negatives_2: 1814.0000 - sensitivity_at_specificity_2: 0.9938 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9651 - recall_2: 0.9486 - auc_2: 0.9966 - val_loss: 0.1164 - val_accuracy: 0.9681 - val_true_positives_2: 3785.0000 - val_false_positives_2: 108.0000 - val_true_negatives_2: 164574.0000 - val_false_negatives_2: 136.0000 - val_sensitivity_at_specificity_2: 0.9962 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9723 - val_recall_2: 0.9653 - val_auc_2: 0.9979\n",
            "Epoch 12/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1472 - accuracy: 0.9606 - true_positives_2: 33524.0000 - false_positives_2: 1124.0000 - true_negatives_2: 1475596.0000 - false_negatives_2: 1636.0000 - sensitivity_at_specificity_2: 0.9945 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9676 - recall_2: 0.9535 - auc_2: 0.9970\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1472 - accuracy: 0.9606 - true_positives_2: 33646.0000 - false_positives_2: 1129.0000 - true_negatives_2: 1480967.0000 - false_negatives_2: 1642.0000 - sensitivity_at_specificity_2: 0.9944 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9675 - recall_2: 0.9535 - auc_2: 0.9970 - val_loss: 0.0648 - val_accuracy: 0.9816 - val_true_positives_2: 3841.0000 - val_false_positives_2: 69.0000 - val_true_negatives_2: 164613.0000 - val_false_negatives_2: 80.0000 - val_sensitivity_at_specificity_2: 0.9982 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9824 - val_recall_2: 0.9796 - val_auc_2: 0.9991\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1440 - accuracy: 0.9604 - true_positives_2: 33539.0000 - false_positives_2: 1131.0000 - true_negatives_2: 1475589.0000 - false_negatives_2: 1621.0000 - sensitivity_at_specificity_2: 0.9953 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9674 - recall_2: 0.9539 - auc_2: 0.9974\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1439 - accuracy: 0.9605 - true_positives_2: 33664.0000 - false_positives_2: 1133.0000 - true_negatives_2: 1480963.0000 - false_negatives_2: 1624.0000 - sensitivity_at_specificity_2: 0.9952 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9674 - recall_2: 0.9540 - auc_2: 0.9974 - val_loss: 0.0984 - val_accuracy: 0.9725 - val_true_positives_2: 3797.0000 - val_false_positives_2: 94.0000 - val_true_negatives_2: 164588.0000 - val_false_negatives_2: 124.0000 - val_sensitivity_at_specificity_2: 0.9967 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9758 - val_recall_2: 0.9684 - val_auc_2: 0.9982\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1534 - accuracy: 0.9593 - true_positives_2: 33508.0000 - false_positives_2: 1175.0000 - true_negatives_2: 1475545.0000 - false_negatives_2: 1652.0000 - sensitivity_at_specificity_2: 0.9942 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9661 - recall_2: 0.9530 - auc_2: 0.9969\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1531 - accuracy: 0.9593 - true_positives_2: 33633.0000 - false_positives_2: 1177.0000 - true_negatives_2: 1480919.0000 - false_negatives_2: 1655.0000 - sensitivity_at_specificity_2: 0.9942 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9662 - recall_2: 0.9531 - auc_2: 0.9969 - val_loss: 0.0718 - val_accuracy: 0.9821 - val_true_positives_2: 3843.0000 - val_false_positives_2: 57.0000 - val_true_negatives_2: 164625.0000 - val_false_negatives_2: 78.0000 - val_sensitivity_at_specificity_2: 0.9980 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9854 - val_recall_2: 0.9801 - val_auc_2: 0.9989\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1451 - accuracy: 0.9634 - true_positives_2: 33685.0000 - false_positives_2: 1079.0000 - true_negatives_2: 1475641.0000 - false_negatives_2: 1475.0000 - sensitivity_at_specificity_2: 0.9940 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9690 - recall_2: 0.9580 - auc_2: 0.9968\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1452 - accuracy: 0.9634 - true_positives_2: 33807.0000 - false_positives_2: 1084.0000 - true_negatives_2: 1481012.0000 - false_negatives_2: 1481.0000 - sensitivity_at_specificity_2: 0.9940 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9689 - recall_2: 0.9580 - auc_2: 0.9968 - val_loss: 0.0829 - val_accuracy: 0.9791 - val_true_positives_2: 3835.0000 - val_false_positives_2: 73.0000 - val_true_negatives_2: 164609.0000 - val_false_negatives_2: 86.0000 - val_sensitivity_at_specificity_2: 0.9964 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9813 - val_recall_2: 0.9781 - val_auc_2: 0.9982\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1442 - accuracy: 0.9644 - true_positives_2: 33849.0000 - false_positives_2: 1050.0000 - true_negatives_2: 1481046.0000 - false_negatives_2: 1439.0000 - sensitivity_at_specificity_2: 0.9941 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9699 - recall_2: 0.9592 - auc_2: 0.9969\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1442 - accuracy: 0.9644 - true_positives_2: 33849.0000 - false_positives_2: 1050.0000 - true_negatives_2: 1481046.0000 - false_negatives_2: 1439.0000 - sensitivity_at_specificity_2: 0.9941 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9699 - recall_2: 0.9592 - auc_2: 0.9969 - val_loss: 0.1272 - val_accuracy: 0.9674 - val_true_positives_2: 3777.0000 - val_false_positives_2: 112.0000 - val_true_negatives_2: 164570.0000 - val_false_negatives_2: 144.0000 - val_sensitivity_at_specificity_2: 0.9936 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9712 - val_recall_2: 0.9633 - val_auc_2: 0.9967\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1440 - accuracy: 0.9652 - true_positives_2: 33853.0000 - false_positives_2: 1007.0000 - true_negatives_2: 1481089.0000 - false_negatives_2: 1435.0000 - sensitivity_at_specificity_2: 0.9937 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9711 - recall_2: 0.9593 - auc_2: 0.9967\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1440 - accuracy: 0.9652 - true_positives_2: 33853.0000 - false_positives_2: 1007.0000 - true_negatives_2: 1481089.0000 - false_negatives_2: 1435.0000 - sensitivity_at_specificity_2: 0.9937 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9711 - recall_2: 0.9593 - auc_2: 0.9967 - val_loss: 0.0638 - val_accuracy: 0.9837 - val_true_positives_2: 3850.0000 - val_false_positives_2: 60.0000 - val_true_negatives_2: 164622.0000 - val_false_negatives_2: 71.0000 - val_sensitivity_at_specificity_2: 0.9972 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9847 - val_recall_2: 0.9819 - val_auc_2: 0.9985\n",
            "Epoch 18/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1306 - accuracy: 0.9688 - true_positives_2: 33897.0000 - false_positives_2: 905.0000 - true_negatives_2: 1475815.0000 - false_negatives_2: 1263.0000 - sensitivity_at_specificity_2: 0.9941 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9740 - recall_2: 0.9641 - auc_2: 0.9969\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1303 - accuracy: 0.9689 - true_positives_2: 34024.0000 - false_positives_2: 906.0000 - true_negatives_2: 1481190.0000 - false_negatives_2: 1264.0000 - sensitivity_at_specificity_2: 0.9942 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9741 - recall_2: 0.9642 - auc_2: 0.9969 - val_loss: 0.0346 - val_accuracy: 0.9901 - val_true_positives_2: 3876.0000 - val_false_positives_2: 37.0000 - val_true_negatives_2: 164645.0000 - val_false_negatives_2: 45.0000 - val_sensitivity_at_specificity_2: 0.9987 - val_specificity_at_sensitivity_2: 1.0000 - val_precision_2: 0.9905 - val_recall_2: 0.9885 - val_auc_2: 0.9993\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1555 - accuracy: 0.9631 - true_positives_2: 33685.0000 - false_positives_2: 1084.0000 - true_negatives_2: 1475636.0000 - false_negatives_2: 1475.0000 - sensitivity_at_specificity_2: 0.9927 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9688 - recall_2: 0.9580 - auc_2: 0.9961\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1554 - accuracy: 0.9630 - true_positives_2: 33806.0000 - false_positives_2: 1089.0000 - true_negatives_2: 1481007.0000 - false_negatives_2: 1482.0000 - sensitivity_at_specificity_2: 0.9927 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9688 - recall_2: 0.9580 - auc_2: 0.9962 - val_loss: 0.1367 - val_accuracy: 0.9702 - val_true_positives_2: 3790.0000 - val_false_positives_2: 96.0000 - val_true_negatives_2: 164586.0000 - val_false_negatives_2: 131.0000 - val_sensitivity_at_specificity_2: 0.9921 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9753 - val_recall_2: 0.9666 - val_auc_2: 0.9959\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1432 - accuracy: 0.9673 - true_positives_2: 33947.0000 - false_positives_2: 986.0000 - true_negatives_2: 1481110.0000 - false_negatives_2: 1341.0000 - sensitivity_at_specificity_2: 0.9937 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9718 - recall_2: 0.9620 - auc_2: 0.9967\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1432 - accuracy: 0.9673 - true_positives_2: 33947.0000 - false_positives_2: 986.0000 - true_negatives_2: 1481110.0000 - false_negatives_2: 1341.0000 - sensitivity_at_specificity_2: 0.9937 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9718 - recall_2: 0.9620 - auc_2: 0.9967 - val_loss: 0.0591 - val_accuracy: 0.9865 - val_true_positives_2: 3857.0000 - val_false_positives_2: 43.0000 - val_true_negatives_2: 164639.0000 - val_false_negatives_2: 64.0000 - val_sensitivity_at_specificity_2: 0.9972 - val_specificity_at_sensitivity_2: 0.9999 - val_precision_2: 0.9890 - val_recall_2: 0.9837 - val_auc_2: 0.9985\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0591 - accuracy: 0.9865 - true_positives_2: 3857.0000 - false_positives_2: 43.0000 - true_negatives_2: 164639.0000 - false_negatives_2: 64.0000 - sensitivity_at_specificity_2: 0.9972 - specificity_at_sensitivity_2: 0.9999 - precision_2: 0.9890 - recall_2: 0.9837 - auc_2: 0.9985\n",
            "4\n",
            "Epoch 1/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 3.0065 - accuracy: 0.4698 - true_positives_3: 10066.0000 - false_positives_3: 3852.0000 - true_negatives_3: 1472868.0000 - false_negatives_3: 25094.0000 - sensitivity_at_specificity_3: 0.9483 - specificity_at_sensitivity_3: 0.9843 - precision_3: 0.7232 - recall_3: 0.2863 - auc_3: 0.9198\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 2.9997 - accuracy: 0.4707 - true_positives_3: 10134.0000 - false_positives_3: 3865.0000 - true_negatives_3: 1478231.0000 - false_negatives_3: 25154.0000 - sensitivity_at_specificity_3: 0.9484 - specificity_at_sensitivity_3: 0.9850 - precision_3: 0.7239 - recall_3: 0.2872 - auc_3: 0.9200 - val_loss: 0.8137 - val_accuracy: 0.7672 - val_true_positives_3: 2399.0000 - val_false_positives_3: 339.0000 - val_true_negatives_3: 164343.0000 - val_false_negatives_3: 1522.0000 - val_sensitivity_at_specificity_3: 0.9906 - val_specificity_at_sensitivity_3: 0.9991 - val_precision_3: 0.8762 - val_recall_3: 0.6118 - val_auc_3: 0.9882\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.8724 - accuracy: 0.7528 - true_positives_3: 22238.0000 - false_positives_3: 3683.0000 - true_negatives_3: 1478413.0000 - false_negatives_3: 13050.0000 - sensitivity_at_specificity_3: 0.9845 - specificity_at_sensitivity_3: 0.9991 - precision_3: 0.8579 - recall_3: 0.6302 - auc_3: 0.9839\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.8724 - accuracy: 0.7528 - true_positives_3: 22238.0000 - false_positives_3: 3683.0000 - true_negatives_3: 1478413.0000 - false_negatives_3: 13050.0000 - sensitivity_at_specificity_3: 0.9845 - specificity_at_sensitivity_3: 0.9991 - precision_3: 0.8579 - recall_3: 0.6302 - auc_3: 0.9839 - val_loss: 0.4707 - val_accuracy: 0.8735 - val_true_positives_3: 3032.0000 - val_false_positives_3: 225.0000 - val_true_negatives_3: 164457.0000 - val_false_negatives_3: 889.0000 - val_sensitivity_at_specificity_3: 0.9962 - val_specificity_at_sensitivity_3: 0.9999 - val_precision_3: 0.9309 - val_recall_3: 0.7733 - val_auc_3: 0.9958\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.5472 - accuracy: 0.8429 - true_positives_3: 27205.0000 - false_positives_3: 3027.0000 - true_negatives_3: 1473693.0000 - false_negatives_3: 7955.0000 - sensitivity_at_specificity_3: 0.9892 - specificity_at_sensitivity_3: 0.9998 - precision_3: 0.8999 - recall_3: 0.7737 - auc_3: 0.9914\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.5468 - accuracy: 0.8431 - true_positives_3: 27309.0000 - false_positives_3: 3035.0000 - true_negatives_3: 1479061.0000 - false_negatives_3: 7979.0000 - sensitivity_at_specificity_3: 0.9892 - specificity_at_sensitivity_3: 0.9998 - precision_3: 0.9000 - recall_3: 0.7739 - auc_3: 0.9914 - val_loss: 0.2275 - val_accuracy: 0.9383 - val_true_positives_3: 3544.0000 - val_false_positives_3: 163.0000 - val_true_negatives_3: 164519.0000 - val_false_negatives_3: 377.0000 - val_sensitivity_at_specificity_3: 0.9985 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9560 - val_recall_3: 0.9039 - val_auc_3: 0.9987\n",
            "Epoch 4/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.4102 - accuracy: 0.8837 - true_positives_3: 29617.0000 - false_positives_3: 2544.0000 - true_negatives_3: 1479552.0000 - false_negatives_3: 5671.0000 - sensitivity_at_specificity_3: 0.9907 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9209 - recall_3: 0.8393 - auc_3: 0.9935\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.4102 - accuracy: 0.8837 - true_positives_3: 29617.0000 - false_positives_3: 2544.0000 - true_negatives_3: 1479552.0000 - false_negatives_3: 5671.0000 - sensitivity_at_specificity_3: 0.9907 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9209 - recall_3: 0.8393 - auc_3: 0.9935 - val_loss: 0.1855 - val_accuracy: 0.9447 - val_true_positives_3: 3653.0000 - val_false_positives_3: 152.0000 - val_true_negatives_3: 164530.0000 - val_false_negatives_3: 268.0000 - val_sensitivity_at_specificity_3: 0.9959 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9601 - val_recall_3: 0.9317 - val_auc_3: 0.9976\n",
            "Epoch 5/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3187 - accuracy: 0.9118 - true_positives_3: 31100.0000 - false_positives_3: 2095.0000 - true_negatives_3: 1480001.0000 - false_negatives_3: 4188.0000 - sensitivity_at_specificity_3: 0.9923 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9369 - recall_3: 0.8813 - auc_3: 0.9950\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 0.3187 - accuracy: 0.9118 - true_positives_3: 31100.0000 - false_positives_3: 2095.0000 - true_negatives_3: 1480001.0000 - false_negatives_3: 4188.0000 - sensitivity_at_specificity_3: 0.9923 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9369 - recall_3: 0.8813 - auc_3: 0.9950 - val_loss: 0.1621 - val_accuracy: 0.9564 - val_true_positives_3: 3705.0000 - val_false_positives_3: 121.0000 - val_true_negatives_3: 164561.0000 - val_false_negatives_3: 216.0000 - val_sensitivity_at_specificity_3: 0.9962 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9684 - val_recall_3: 0.9449 - val_auc_3: 0.9978\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2750 - accuracy: 0.9235 - true_positives_3: 31727.0000 - false_positives_3: 1899.0000 - true_negatives_3: 1480197.0000 - false_negatives_3: 3561.0000 - sensitivity_at_specificity_3: 0.9927 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9435 - recall_3: 0.8991 - auc_3: 0.9955\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2750 - accuracy: 0.9235 - true_positives_3: 31727.0000 - false_positives_3: 1899.0000 - true_negatives_3: 1480197.0000 - false_negatives_3: 3561.0000 - sensitivity_at_specificity_3: 0.9927 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9435 - recall_3: 0.8991 - auc_3: 0.9955 - val_loss: 0.1334 - val_accuracy: 0.9628 - val_true_positives_3: 3738.0000 - val_false_positives_3: 112.0000 - val_true_negatives_3: 164570.0000 - val_false_negatives_3: 183.0000 - val_sensitivity_at_specificity_3: 0.9974 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9709 - val_recall_3: 0.9533 - val_auc_3: 0.9985\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2526 - accuracy: 0.9306 - true_positives_3: 32111.0000 - false_positives_3: 1791.0000 - true_negatives_3: 1480305.0000 - false_negatives_3: 3177.0000 - sensitivity_at_specificity_3: 0.9929 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9472 - recall_3: 0.9100 - auc_3: 0.9958\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2526 - accuracy: 0.9306 - true_positives_3: 32111.0000 - false_positives_3: 1791.0000 - true_negatives_3: 1480305.0000 - false_negatives_3: 3177.0000 - sensitivity_at_specificity_3: 0.9929 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9472 - recall_3: 0.9100 - auc_3: 0.9958 - val_loss: 0.1093 - val_accuracy: 0.9725 - val_true_positives_3: 3772.0000 - val_false_positives_3: 75.0000 - val_true_negatives_3: 164607.0000 - val_false_negatives_3: 149.0000 - val_sensitivity_at_specificity_3: 0.9980 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9805 - val_recall_3: 0.9620 - val_auc_3: 0.9988\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2148 - accuracy: 0.9418 - true_positives_3: 32670.0000 - false_positives_3: 1536.0000 - true_negatives_3: 1480560.0000 - false_negatives_3: 2618.0000 - sensitivity_at_specificity_3: 0.9938 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9551 - recall_3: 0.9258 - auc_3: 0.9964\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2148 - accuracy: 0.9418 - true_positives_3: 32670.0000 - false_positives_3: 1536.0000 - true_negatives_3: 1480560.0000 - false_negatives_3: 2618.0000 - sensitivity_at_specificity_3: 0.9938 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9551 - recall_3: 0.9258 - auc_3: 0.9964 - val_loss: 0.1234 - val_accuracy: 0.9674 - val_true_positives_3: 3775.0000 - val_false_positives_3: 103.0000 - val_true_negatives_3: 164579.0000 - val_false_negatives_3: 146.0000 - val_sensitivity_at_specificity_3: 0.9962 - val_specificity_at_sensitivity_3: 0.9999 - val_precision_3: 0.9734 - val_recall_3: 0.9628 - val_auc_3: 0.9979\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1901 - accuracy: 0.9470 - true_positives_3: 32985.0000 - false_positives_3: 1427.0000 - true_negatives_3: 1480669.0000 - false_negatives_3: 2303.0000 - sensitivity_at_specificity_3: 0.9942 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9585 - recall_3: 0.9347 - auc_3: 0.9967\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1901 - accuracy: 0.9470 - true_positives_3: 32985.0000 - false_positives_3: 1427.0000 - true_negatives_3: 1480669.0000 - false_negatives_3: 2303.0000 - sensitivity_at_specificity_3: 0.9942 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9585 - recall_3: 0.9347 - auc_3: 0.9967 - val_loss: 0.0868 - val_accuracy: 0.9776 - val_true_positives_3: 3813.0000 - val_false_positives_3: 67.0000 - val_true_negatives_3: 164615.0000 - val_false_negatives_3: 108.0000 - val_sensitivity_at_specificity_3: 0.9977 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9827 - val_recall_3: 0.9725 - val_auc_3: 0.9988\n",
            "Epoch 10/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1828 - accuracy: 0.9499 - true_positives_3: 33027.0000 - false_positives_3: 1354.0000 - true_negatives_3: 1475366.0000 - false_negatives_3: 2133.0000 - sensitivity_at_specificity_3: 0.9944 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9606 - recall_3: 0.9393 - auc_3: 0.9968\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1824 - accuracy: 0.9500 - true_positives_3: 33153.0000 - false_positives_3: 1356.0000 - true_negatives_3: 1480740.0000 - false_negatives_3: 2135.0000 - sensitivity_at_specificity_3: 0.9944 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9607 - recall_3: 0.9395 - auc_3: 0.9968 - val_loss: 0.0519 - val_accuracy: 0.9852 - val_true_positives_3: 3851.0000 - val_false_positives_3: 47.0000 - val_true_negatives_3: 164635.0000 - val_false_negatives_3: 70.0000 - val_sensitivity_at_specificity_3: 0.9985 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9879 - val_recall_3: 0.9821 - val_auc_3: 0.9992\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1613 - accuracy: 0.9563 - true_positives_3: 33303.0000 - false_positives_3: 1215.0000 - true_negatives_3: 1475505.0000 - false_negatives_3: 1857.0000 - sensitivity_at_specificity_3: 0.9949 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9648 - recall_3: 0.9472 - auc_3: 0.9972\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1611 - accuracy: 0.9563 - true_positives_3: 33426.0000 - false_positives_3: 1220.0000 - true_negatives_3: 1480876.0000 - false_negatives_3: 1862.0000 - sensitivity_at_specificity_3: 0.9949 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9648 - recall_3: 0.9472 - auc_3: 0.9972 - val_loss: 0.0483 - val_accuracy: 0.9865 - val_true_positives_3: 3854.0000 - val_false_positives_3: 42.0000 - val_true_negatives_3: 164640.0000 - val_false_negatives_3: 67.0000 - val_sensitivity_at_specificity_3: 0.9992 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9892 - val_recall_3: 0.9829 - val_auc_3: 0.9996\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1752 - accuracy: 0.9532 - true_positives_3: 33331.0000 - false_positives_3: 1301.0000 - true_negatives_3: 1480795.0000 - false_negatives_3: 1957.0000 - sensitivity_at_specificity_3: 0.9937 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9624 - recall_3: 0.9445 - auc_3: 0.9965\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1752 - accuracy: 0.9532 - true_positives_3: 33331.0000 - false_positives_3: 1301.0000 - true_negatives_3: 1480795.0000 - false_negatives_3: 1957.0000 - sensitivity_at_specificity_3: 0.9937 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9624 - recall_3: 0.9445 - auc_3: 0.9965 - val_loss: 0.0486 - val_accuracy: 0.9906 - val_true_positives_3: 3876.0000 - val_false_positives_3: 32.0000 - val_true_negatives_3: 164650.0000 - val_false_negatives_3: 45.0000 - val_sensitivity_at_specificity_3: 0.9985 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9918 - val_recall_3: 0.9885 - val_auc_3: 0.9992\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1521 - accuracy: 0.9600 - true_positives_3: 33481.0000 - false_positives_3: 1135.0000 - true_negatives_3: 1475585.0000 - false_negatives_3: 1679.0000 - sensitivity_at_specificity_3: 0.9943 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9672 - recall_3: 0.9522 - auc_3: 0.9969\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1526 - accuracy: 0.9598 - true_positives_3: 33599.0000 - false_positives_3: 1144.0000 - true_negatives_3: 1480952.0000 - false_negatives_3: 1689.0000 - sensitivity_at_specificity_3: 0.9942 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9671 - recall_3: 0.9521 - auc_3: 0.9969 - val_loss: 0.0591 - val_accuracy: 0.9837 - val_true_positives_3: 3843.0000 - val_false_positives_3: 55.0000 - val_true_negatives_3: 164627.0000 - val_false_negatives_3: 78.0000 - val_sensitivity_at_specificity_3: 0.9977 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9859 - val_recall_3: 0.9801 - val_auc_3: 0.9988\n",
            "Epoch 14/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1742 - accuracy: 0.9540 - true_positives_3: 33402.0000 - false_positives_3: 1307.0000 - true_negatives_3: 1480789.0000 - false_negatives_3: 1886.0000 - sensitivity_at_specificity_3: 0.9933 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9623 - recall_3: 0.9466 - auc_3: 0.9963\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1742 - accuracy: 0.9540 - true_positives_3: 33402.0000 - false_positives_3: 1307.0000 - true_negatives_3: 1480789.0000 - false_negatives_3: 1886.0000 - sensitivity_at_specificity_3: 0.9933 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9623 - recall_3: 0.9466 - auc_3: 0.9963 - val_loss: 0.0466 - val_accuracy: 0.9895 - val_true_positives_3: 3873.0000 - val_false_positives_3: 34.0000 - val_true_negatives_3: 164648.0000 - val_false_negatives_3: 48.0000 - val_sensitivity_at_specificity_3: 0.9987 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9913 - val_recall_3: 0.9878 - val_auc_3: 0.9993\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1623 - accuracy: 0.9596 - true_positives_3: 33453.0000 - false_positives_3: 1135.0000 - true_negatives_3: 1475585.0000 - false_negatives_3: 1707.0000 - sensitivity_at_specificity_3: 0.9937 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9672 - recall_3: 0.9515 - auc_3: 0.9966\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1623 - accuracy: 0.9596 - true_positives_3: 33573.0000 - false_positives_3: 1141.0000 - true_negatives_3: 1480955.0000 - false_negatives_3: 1715.0000 - sensitivity_at_specificity_3: 0.9937 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9671 - recall_3: 0.9514 - auc_3: 0.9966 - val_loss: 0.0608 - val_accuracy: 0.9855 - val_true_positives_3: 3848.0000 - val_false_positives_3: 46.0000 - val_true_negatives_3: 164636.0000 - val_false_negatives_3: 73.0000 - val_sensitivity_at_specificity_3: 0.9977 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9882 - val_recall_3: 0.9814 - val_auc_3: 0.9988\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1547 - accuracy: 0.9620 - true_positives_3: 33707.0000 - false_positives_3: 1087.0000 - true_negatives_3: 1481009.0000 - false_negatives_3: 1581.0000 - sensitivity_at_specificity_3: 0.9939 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9688 - recall_3: 0.9552 - auc_3: 0.9967\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1547 - accuracy: 0.9620 - true_positives_3: 33707.0000 - false_positives_3: 1087.0000 - true_negatives_3: 1481009.0000 - false_negatives_3: 1581.0000 - sensitivity_at_specificity_3: 0.9939 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9688 - recall_3: 0.9552 - auc_3: 0.9967 - val_loss: 0.0696 - val_accuracy: 0.9850 - val_true_positives_3: 3855.0000 - val_false_positives_3: 54.0000 - val_true_negatives_3: 164628.0000 - val_false_negatives_3: 66.0000 - val_sensitivity_at_specificity_3: 0.9982 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9862 - val_recall_3: 0.9832 - val_auc_3: 0.9991\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1538 - accuracy: 0.9616 - true_positives_3: 33747.0000 - false_positives_3: 1121.0000 - true_negatives_3: 1480975.0000 - false_negatives_3: 1541.0000 - sensitivity_at_specificity_3: 0.9938 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9679 - recall_3: 0.9563 - auc_3: 0.9967\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1538 - accuracy: 0.9616 - true_positives_3: 33747.0000 - false_positives_3: 1121.0000 - true_negatives_3: 1480975.0000 - false_negatives_3: 1541.0000 - sensitivity_at_specificity_3: 0.9938 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9679 - recall_3: 0.9563 - auc_3: 0.9967 - val_loss: 0.0703 - val_accuracy: 0.9829 - val_true_positives_3: 3841.0000 - val_false_positives_3: 56.0000 - val_true_negatives_3: 164626.0000 - val_false_negatives_3: 80.0000 - val_sensitivity_at_specificity_3: 0.9972 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9856 - val_recall_3: 0.9796 - val_auc_3: 0.9985\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1710 - accuracy: 0.9592 - true_positives_3: 33618.0000 - false_positives_3: 1186.0000 - true_negatives_3: 1480910.0000 - false_negatives_3: 1670.0000 - sensitivity_at_specificity_3: 0.9929 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9659 - recall_3: 0.9527 - auc_3: 0.9962\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1710 - accuracy: 0.9592 - true_positives_3: 33618.0000 - false_positives_3: 1186.0000 - true_negatives_3: 1480910.0000 - false_negatives_3: 1670.0000 - sensitivity_at_specificity_3: 0.9929 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9659 - recall_3: 0.9527 - auc_3: 0.9962 - val_loss: 0.0633 - val_accuracy: 0.9811 - val_true_positives_3: 3835.0000 - val_false_positives_3: 59.0000 - val_true_negatives_3: 164623.0000 - val_false_negatives_3: 86.0000 - val_sensitivity_at_specificity_3: 0.9974 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9848 - val_recall_3: 0.9781 - val_auc_3: 0.9987\n",
            "Epoch 19/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1435 - accuracy: 0.9645 - true_positives_3: 33848.0000 - false_positives_3: 1044.0000 - true_negatives_3: 1481052.0000 - false_negatives_3: 1440.0000 - sensitivity_at_specificity_3: 0.9940 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9701 - recall_3: 0.9592 - auc_3: 0.9968\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1435 - accuracy: 0.9645 - true_positives_3: 33848.0000 - false_positives_3: 1044.0000 - true_negatives_3: 1481052.0000 - false_negatives_3: 1440.0000 - sensitivity_at_specificity_3: 0.9940 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9701 - recall_3: 0.9592 - auc_3: 0.9968 - val_loss: 0.0667 - val_accuracy: 0.9829 - val_true_positives_3: 3851.0000 - val_false_positives_3: 56.0000 - val_true_negatives_3: 164626.0000 - val_false_negatives_3: 70.0000 - val_sensitivity_at_specificity_3: 0.9972 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9857 - val_recall_3: 0.9821 - val_auc_3: 0.9985\n",
            "Epoch 20/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1463 - accuracy: 0.9657 - true_positives_3: 33771.0000 - false_positives_3: 995.0000 - true_negatives_3: 1475725.0000 - false_negatives_3: 1389.0000 - sensitivity_at_specificity_3: 0.9933 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9714 - recall_3: 0.9605 - auc_3: 0.9964\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1465 - accuracy: 0.9657 - true_positives_3: 33896.0000 - false_positives_3: 998.0000 - true_negatives_3: 1481098.0000 - false_negatives_3: 1392.0000 - sensitivity_at_specificity_3: 0.9932 - specificity_at_sensitivity_3: 0.9999 - precision_3: 0.9714 - recall_3: 0.9606 - auc_3: 0.9964 - val_loss: 0.0404 - val_accuracy: 0.9906 - val_true_positives_3: 3878.0000 - val_false_positives_3: 31.0000 - val_true_negatives_3: 164651.0000 - val_false_negatives_3: 43.0000 - val_sensitivity_at_specificity_3: 0.9985 - val_specificity_at_sensitivity_3: 1.0000 - val_precision_3: 0.9921 - val_recall_3: 0.9890 - val_auc_3: 0.9992\n",
            "123/123 [==============================] - 1s 8ms/step - loss: 0.0404 - accuracy: 0.9906 - true_positives_3: 3878.0000 - false_positives_3: 31.0000 - true_negatives_3: 164651.0000 - false_negatives_3: 43.0000 - sensitivity_at_specificity_3: 0.9985 - specificity_at_sensitivity_3: 1.0000 - precision_3: 0.9921 - recall_3: 0.9890 - auc_3: 0.9992\n",
            "5\n",
            "Epoch 1/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 3.1958 - accuracy: 0.4601 - true_positives_4: 9755.0000 - false_positives_4: 4035.0000 - true_negatives_4: 1478061.0000 - false_negatives_4: 25533.0000 - sensitivity_at_specificity_4: 0.9492 - specificity_at_sensitivity_4: 0.9838 - precision_4: 0.7074 - recall_4: 0.2764 - auc_4: 0.9180\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 3.1958 - accuracy: 0.4601 - true_positives_4: 9755.0000 - false_positives_4: 4035.0000 - true_negatives_4: 1478061.0000 - false_negatives_4: 25533.0000 - sensitivity_at_specificity_4: 0.9492 - specificity_at_sensitivity_4: 0.9838 - precision_4: 0.7074 - recall_4: 0.2764 - auc_4: 0.9180 - val_loss: 0.8390 - val_accuracy: 0.7585 - val_true_positives_4: 2275.0000 - val_false_positives_4: 341.0000 - val_true_negatives_4: 164341.0000 - val_false_negatives_4: 1646.0000 - val_sensitivity_at_specificity_4: 0.9926 - val_specificity_at_sensitivity_4: 0.9989 - val_precision_4: 0.8696 - val_recall_4: 0.5802 - val_auc_4: 0.9886\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.8675 - accuracy: 0.7508 - true_positives_4: 22192.0000 - false_positives_4: 3659.0000 - true_negatives_4: 1478437.0000 - false_negatives_4: 13096.0000 - sensitivity_at_specificity_4: 0.9861 - specificity_at_sensitivity_4: 0.9991 - precision_4: 0.8585 - recall_4: 0.6289 - auc_4: 0.9847\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.8675 - accuracy: 0.7508 - true_positives_4: 22192.0000 - false_positives_4: 3659.0000 - true_negatives_4: 1478437.0000 - false_negatives_4: 13096.0000 - sensitivity_at_specificity_4: 0.9861 - specificity_at_sensitivity_4: 0.9991 - precision_4: 0.8585 - recall_4: 0.6289 - auc_4: 0.9847 - val_loss: 0.4616 - val_accuracy: 0.8636 - val_true_positives_4: 3107.0000 - val_false_positives_4: 267.0000 - val_true_negatives_4: 164415.0000 - val_false_negatives_4: 814.0000 - val_sensitivity_at_specificity_4: 0.9929 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9209 - val_recall_4: 0.7924 - val_auc_4: 0.9941\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.5482 - accuracy: 0.8435 - true_positives_4: 27241.0000 - false_positives_4: 2973.0000 - true_negatives_4: 1473747.0000 - false_negatives_4: 7919.0000 - sensitivity_at_specificity_4: 0.9901 - specificity_at_sensitivity_4: 0.9997 - precision_4: 0.9016 - recall_4: 0.7748 - auc_4: 0.9918\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.5482 - accuracy: 0.8436 - true_positives_4: 27339.0000 - false_positives_4: 2986.0000 - true_negatives_4: 1479110.0000 - false_negatives_4: 7949.0000 - sensitivity_at_specificity_4: 0.9901 - specificity_at_sensitivity_4: 0.9997 - precision_4: 0.9015 - recall_4: 0.7747 - auc_4: 0.9918 - val_loss: 0.2441 - val_accuracy: 0.9265 - val_true_positives_4: 3536.0000 - val_false_positives_4: 195.0000 - val_true_negatives_4: 164487.0000 - val_false_negatives_4: 385.0000 - val_sensitivity_at_specificity_4: 0.9964 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9477 - val_recall_4: 0.9018 - val_auc_4: 0.9975\n",
            "Epoch 4/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.3869 - accuracy: 0.8875 - true_positives_4: 29705.0000 - false_positives_4: 2427.0000 - true_negatives_4: 1474293.0000 - false_negatives_4: 5455.0000 - sensitivity_at_specificity_4: 0.9923 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9245 - recall_4: 0.8449 - auc_4: 0.9945\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.3867 - accuracy: 0.8876 - true_positives_4: 29815.0000 - false_positives_4: 2436.0000 - true_negatives_4: 1479660.0000 - false_negatives_4: 5473.0000 - sensitivity_at_specificity_4: 0.9923 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9245 - recall_4: 0.8449 - auc_4: 0.9945 - val_loss: 0.2183 - val_accuracy: 0.9403 - val_true_positives_4: 3625.0000 - val_false_positives_4: 174.0000 - val_true_negatives_4: 164508.0000 - val_false_negatives_4: 296.0000 - val_sensitivity_at_specificity_4: 0.9957 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9542 - val_recall_4: 0.9245 - val_auc_4: 0.9973\n",
            "Epoch 5/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3106 - accuracy: 0.9114 - true_positives_4: 31088.0000 - false_positives_4: 2083.0000 - true_negatives_4: 1480013.0000 - false_negatives_4: 4200.0000 - sensitivity_at_specificity_4: 0.9929 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9372 - recall_4: 0.8810 - auc_4: 0.9954\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.3106 - accuracy: 0.9114 - true_positives_4: 31088.0000 - false_positives_4: 2083.0000 - true_negatives_4: 1480013.0000 - false_negatives_4: 4200.0000 - sensitivity_at_specificity_4: 0.9929 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9372 - recall_4: 0.8810 - auc_4: 0.9954 - val_loss: 0.1602 - val_accuracy: 0.9554 - val_true_positives_4: 3697.0000 - val_false_positives_4: 135.0000 - val_true_negatives_4: 164547.0000 - val_false_negatives_4: 224.0000 - val_sensitivity_at_specificity_4: 0.9969 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9648 - val_recall_4: 0.9429 - val_auc_4: 0.9982\n",
            "Epoch 6/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2726 - accuracy: 0.9227 - true_positives_4: 31611.0000 - false_positives_4: 1884.0000 - true_negatives_4: 1474836.0000 - false_negatives_4: 3549.0000 - sensitivity_at_specificity_4: 0.9931 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9438 - recall_4: 0.8991 - auc_4: 0.9958\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2724 - accuracy: 0.9228 - true_positives_4: 31728.0000 - false_positives_4: 1888.0000 - true_negatives_4: 1480208.0000 - false_negatives_4: 3560.0000 - sensitivity_at_specificity_4: 0.9932 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9438 - recall_4: 0.8991 - auc_4: 0.9958 - val_loss: 0.1658 - val_accuracy: 0.9584 - val_true_positives_4: 3705.0000 - val_false_positives_4: 125.0000 - val_true_negatives_4: 164557.0000 - val_false_negatives_4: 216.0000 - val_sensitivity_at_specificity_4: 0.9946 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9674 - val_recall_4: 0.9449 - val_auc_4: 0.9970\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2195 - accuracy: 0.9386 - true_positives_4: 32491.0000 - false_positives_4: 1576.0000 - true_negatives_4: 1480520.0000 - false_negatives_4: 2797.0000 - sensitivity_at_specificity_4: 0.9945 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9537 - recall_4: 0.9207 - auc_4: 0.9967\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2195 - accuracy: 0.9386 - true_positives_4: 32491.0000 - false_positives_4: 1576.0000 - true_negatives_4: 1480520.0000 - false_negatives_4: 2797.0000 - sensitivity_at_specificity_4: 0.9945 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9537 - recall_4: 0.9207 - auc_4: 0.9967 - val_loss: 0.1305 - val_accuracy: 0.9694 - val_true_positives_4: 3759.0000 - val_false_positives_4: 103.0000 - val_true_negatives_4: 164579.0000 - val_false_negatives_4: 162.0000 - val_sensitivity_at_specificity_4: 0.9962 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9733 - val_recall_4: 0.9587 - val_auc_4: 0.9979\n",
            "Epoch 8/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2264 - accuracy: 0.9372 - true_positives_4: 32371.0000 - false_positives_4: 1628.0000 - true_negatives_4: 1475092.0000 - false_negatives_4: 2789.0000 - sensitivity_at_specificity_4: 0.9937 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9521 - recall_4: 0.9207 - auc_4: 0.9963\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2262 - accuracy: 0.9373 - true_positives_4: 32496.0000 - false_positives_4: 1630.0000 - true_negatives_4: 1480466.0000 - false_negatives_4: 2792.0000 - sensitivity_at_specificity_4: 0.9937 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9522 - recall_4: 0.9209 - auc_4: 0.9963 - val_loss: 0.1236 - val_accuracy: 0.9679 - val_true_positives_4: 3768.0000 - val_false_positives_4: 97.0000 - val_true_negatives_4: 164585.0000 - val_false_negatives_4: 153.0000 - val_sensitivity_at_specificity_4: 0.9969 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9749 - val_recall_4: 0.9610 - val_auc_4: 0.9983\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1848 - accuracy: 0.9490 - true_positives_4: 33054.0000 - false_positives_4: 1392.0000 - true_negatives_4: 1480704.0000 - false_negatives_4: 2234.0000 - sensitivity_at_specificity_4: 0.9946 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9596 - recall_4: 0.9367 - auc_4: 0.9969\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1848 - accuracy: 0.9490 - true_positives_4: 33054.0000 - false_positives_4: 1392.0000 - true_negatives_4: 1480704.0000 - false_negatives_4: 2234.0000 - sensitivity_at_specificity_4: 0.9946 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9596 - recall_4: 0.9367 - auc_4: 0.9969 - val_loss: 0.0901 - val_accuracy: 0.9753 - val_true_positives_4: 3796.0000 - val_false_positives_4: 77.0000 - val_true_negatives_4: 164605.0000 - val_false_negatives_4: 125.0000 - val_sensitivity_at_specificity_4: 0.9982 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9801 - val_recall_4: 0.9681 - val_auc_4: 0.9990\n",
            "Epoch 10/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1808 - accuracy: 0.9514 - true_positives_4: 33049.0000 - false_positives_4: 1319.0000 - true_negatives_4: 1475401.0000 - false_negatives_4: 2111.0000 - sensitivity_at_specificity_4: 0.9941 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9616 - recall_4: 0.9400 - auc_4: 0.9967\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1805 - accuracy: 0.9515 - true_positives_4: 33172.0000 - false_positives_4: 1321.0000 - true_negatives_4: 1480775.0000 - false_negatives_4: 2116.0000 - sensitivity_at_specificity_4: 0.9941 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9617 - recall_4: 0.9400 - auc_4: 0.9967 - val_loss: 0.1020 - val_accuracy: 0.9760 - val_true_positives_4: 3812.0000 - val_false_positives_4: 76.0000 - val_true_negatives_4: 164606.0000 - val_false_negatives_4: 109.0000 - val_sensitivity_at_specificity_4: 0.9959 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9805 - val_recall_4: 0.9722 - val_auc_4: 0.9978\n",
            "Epoch 11/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1963 - accuracy: 0.9474 - true_positives_4: 33058.0000 - false_positives_4: 1453.0000 - true_negatives_4: 1480643.0000 - false_negatives_4: 2230.0000 - sensitivity_at_specificity_4: 0.9931 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9579 - recall_4: 0.9368 - auc_4: 0.9962\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1963 - accuracy: 0.9474 - true_positives_4: 33058.0000 - false_positives_4: 1453.0000 - true_negatives_4: 1480643.0000 - false_negatives_4: 2230.0000 - sensitivity_at_specificity_4: 0.9931 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9579 - recall_4: 0.9368 - auc_4: 0.9962 - val_loss: 0.0941 - val_accuracy: 0.9786 - val_true_positives_4: 3824.0000 - val_false_positives_4: 69.0000 - val_true_negatives_4: 164613.0000 - val_false_negatives_4: 97.0000 - val_sensitivity_at_specificity_4: 0.9964 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9823 - val_recall_4: 0.9753 - val_auc_4: 0.9981\n",
            "Epoch 12/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1592 - accuracy: 0.9575 - true_positives_4: 33353.0000 - false_positives_4: 1193.0000 - true_negatives_4: 1475527.0000 - false_negatives_4: 1807.0000 - sensitivity_at_specificity_4: 0.9948 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9655 - recall_4: 0.9486 - auc_4: 0.9971\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1592 - accuracy: 0.9575 - true_positives_4: 33474.0000 - false_positives_4: 1199.0000 - true_negatives_4: 1480897.0000 - false_negatives_4: 1814.0000 - sensitivity_at_specificity_4: 0.9948 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9654 - recall_4: 0.9486 - auc_4: 0.9971 - val_loss: 0.0747 - val_accuracy: 0.9799 - val_true_positives_4: 3829.0000 - val_false_positives_4: 71.0000 - val_true_negatives_4: 164611.0000 - val_false_negatives_4: 92.0000 - val_sensitivity_at_specificity_4: 0.9969 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9818 - val_recall_4: 0.9765 - val_auc_4: 0.9984\n",
            "Epoch 13/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1824 - accuracy: 0.9511 - true_positives_4: 33246.0000 - false_positives_4: 1384.0000 - true_negatives_4: 1480712.0000 - false_negatives_4: 2042.0000 - sensitivity_at_specificity_4: 0.9934 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9600 - recall_4: 0.9421 - auc_4: 0.9964\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1824 - accuracy: 0.9511 - true_positives_4: 33246.0000 - false_positives_4: 1384.0000 - true_negatives_4: 1480712.0000 - false_negatives_4: 2042.0000 - sensitivity_at_specificity_4: 0.9934 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9600 - recall_4: 0.9421 - auc_4: 0.9964 - val_loss: 0.0893 - val_accuracy: 0.9788 - val_true_positives_4: 3826.0000 - val_false_positives_4: 71.0000 - val_true_negatives_4: 164611.0000 - val_false_negatives_4: 95.0000 - val_sensitivity_at_specificity_4: 0.9972 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9818 - val_recall_4: 0.9758 - val_auc_4: 0.9985\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1397 - accuracy: 0.9626 - true_positives_4: 33635.0000 - false_positives_4: 1062.0000 - true_negatives_4: 1475658.0000 - false_negatives_4: 1525.0000 - sensitivity_at_specificity_4: 0.9951 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9694 - recall_4: 0.9566 - auc_4: 0.9973\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1398 - accuracy: 0.9627 - true_positives_4: 33759.0000 - false_positives_4: 1064.0000 - true_negatives_4: 1481032.0000 - false_negatives_4: 1529.0000 - sensitivity_at_specificity_4: 0.9950 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9694 - recall_4: 0.9567 - auc_4: 0.9973 - val_loss: 0.0504 - val_accuracy: 0.9870 - val_true_positives_4: 3864.0000 - val_false_positives_4: 44.0000 - val_true_negatives_4: 164638.0000 - val_false_negatives_4: 57.0000 - val_sensitivity_at_specificity_4: 0.9985 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9887 - val_recall_4: 0.9855 - val_auc_4: 0.9992\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1525 - accuracy: 0.9596 - true_positives_4: 33458.0000 - false_positives_4: 1155.0000 - true_negatives_4: 1475565.0000 - false_negatives_4: 1702.0000 - sensitivity_at_specificity_4: 0.9943 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9666 - recall_4: 0.9516 - auc_4: 0.9969\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1528 - accuracy: 0.9594 - true_positives_4: 33574.0000 - false_positives_4: 1164.0000 - true_negatives_4: 1480932.0000 - false_negatives_4: 1714.0000 - sensitivity_at_specificity_4: 0.9943 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9665 - recall_4: 0.9514 - auc_4: 0.9969 - val_loss: 0.1496 - val_accuracy: 0.9702 - val_true_positives_4: 3784.0000 - val_false_positives_4: 104.0000 - val_true_negatives_4: 164578.0000 - val_false_negatives_4: 137.0000 - val_sensitivity_at_specificity_4: 0.9923 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9733 - val_recall_4: 0.9651 - val_auc_4: 0.9960\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1647 - accuracy: 0.9569 - true_positives_4: 33534.0000 - false_positives_4: 1245.0000 - true_negatives_4: 1480851.0000 - false_negatives_4: 1754.0000 - sensitivity_at_specificity_4: 0.9933 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9642 - recall_4: 0.9503 - auc_4: 0.9964\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1647 - accuracy: 0.9569 - true_positives_4: 33534.0000 - false_positives_4: 1245.0000 - true_negatives_4: 1480851.0000 - false_negatives_4: 1754.0000 - sensitivity_at_specificity_4: 0.9933 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9642 - recall_4: 0.9503 - auc_4: 0.9964 - val_loss: 0.0625 - val_accuracy: 0.9850 - val_true_positives_4: 3860.0000 - val_false_positives_4: 50.0000 - val_true_negatives_4: 164632.0000 - val_false_negatives_4: 61.0000 - val_sensitivity_at_specificity_4: 0.9967 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9872 - val_recall_4: 0.9844 - val_auc_4: 0.9983\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1491 - accuracy: 0.9617 - true_positives_4: 33705.0000 - false_positives_4: 1111.0000 - true_negatives_4: 1480985.0000 - false_negatives_4: 1583.0000 - sensitivity_at_specificity_4: 0.9941 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9681 - recall_4: 0.9551 - auc_4: 0.9968\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1491 - accuracy: 0.9617 - true_positives_4: 33705.0000 - false_positives_4: 1111.0000 - true_negatives_4: 1480985.0000 - false_negatives_4: 1583.0000 - sensitivity_at_specificity_4: 0.9941 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9681 - recall_4: 0.9551 - auc_4: 0.9968 - val_loss: 0.0645 - val_accuracy: 0.9844 - val_true_positives_4: 3839.0000 - val_false_positives_4: 51.0000 - val_true_negatives_4: 164631.0000 - val_false_negatives_4: 82.0000 - val_sensitivity_at_specificity_4: 0.9974 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9869 - val_recall_4: 0.9791 - val_auc_4: 0.9987\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1594 - accuracy: 0.9601 - true_positives_4: 33657.0000 - false_positives_4: 1146.0000 - true_negatives_4: 1480950.0000 - false_negatives_4: 1631.0000 - sensitivity_at_specificity_4: 0.9936 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9671 - recall_4: 0.9538 - auc_4: 0.9965\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1594 - accuracy: 0.9601 - true_positives_4: 33657.0000 - false_positives_4: 1146.0000 - true_negatives_4: 1480950.0000 - false_negatives_4: 1631.0000 - sensitivity_at_specificity_4: 0.9936 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9671 - recall_4: 0.9538 - auc_4: 0.9965 - val_loss: 0.0670 - val_accuracy: 0.9850 - val_true_positives_4: 3856.0000 - val_false_positives_4: 55.0000 - val_true_negatives_4: 164627.0000 - val_false_negatives_4: 65.0000 - val_sensitivity_at_specificity_4: 0.9964 - val_specificity_at_sensitivity_4: 0.9999 - val_precision_4: 0.9859 - val_recall_4: 0.9834 - val_auc_4: 0.9982\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1899 - accuracy: 0.9530 - true_positives_4: 33266.0000 - false_positives_4: 1332.0000 - true_negatives_4: 1475388.0000 - false_negatives_4: 1894.0000 - sensitivity_at_specificity_4: 0.9918 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9615 - recall_4: 0.9461 - auc_4: 0.9956\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1899 - accuracy: 0.9530 - true_positives_4: 33389.0000 - false_positives_4: 1337.0000 - true_negatives_4: 1480759.0000 - false_negatives_4: 1899.0000 - sensitivity_at_specificity_4: 0.9918 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9615 - recall_4: 0.9462 - auc_4: 0.9956 - val_loss: 0.0582 - val_accuracy: 0.9855 - val_true_positives_4: 3854.0000 - val_false_positives_4: 51.0000 - val_true_negatives_4: 164631.0000 - val_false_negatives_4: 67.0000 - val_sensitivity_at_specificity_4: 0.9972 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9869 - val_recall_4: 0.9829 - val_auc_4: 0.9986\n",
            "Epoch 20/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1335 - accuracy: 0.9651 - true_positives_4: 33758.0000 - false_positives_4: 1018.0000 - true_negatives_4: 1475702.0000 - false_negatives_4: 1402.0000 - sensitivity_at_specificity_4: 0.9949 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9707 - recall_4: 0.9601 - auc_4: 0.9973\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1333 - accuracy: 0.9651 - true_positives_4: 33880.0000 - false_positives_4: 1021.0000 - true_negatives_4: 1481075.0000 - false_negatives_4: 1408.0000 - sensitivity_at_specificity_4: 0.9949 - specificity_at_sensitivity_4: 0.9999 - precision_4: 0.9707 - recall_4: 0.9601 - auc_4: 0.9973 - val_loss: 0.0515 - val_accuracy: 0.9855 - val_true_positives_4: 3859.0000 - val_false_positives_4: 52.0000 - val_true_negatives_4: 164630.0000 - val_false_negatives_4: 62.0000 - val_sensitivity_at_specificity_4: 0.9980 - val_specificity_at_sensitivity_4: 1.0000 - val_precision_4: 0.9867 - val_recall_4: 0.9842 - val_auc_4: 0.9989\n",
            "123/123 [==============================] - 1s 8ms/step - loss: 0.0515 - accuracy: 0.9855 - true_positives_4: 3859.0000 - false_positives_4: 52.0000 - true_negatives_4: 164630.0000 - false_negatives_4: 62.0000 - sensitivity_at_specificity_4: 0.9980 - specificity_at_sensitivity_4: 1.0000 - precision_4: 0.9867 - recall_4: 0.9842 - auc_4: 0.9989\n",
            "6\n",
            "Epoch 1/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 3.9958 - accuracy: 0.3974 - true_positives_5: 7568.0000 - false_positives_5: 3706.0000 - true_negatives_5: 1473014.0000 - false_negatives_5: 27592.0000 - sensitivity_at_specificity_5: 0.9463 - specificity_at_sensitivity_5: 0.9730 - precision_5: 0.6713 - recall_5: 0.2152 - auc_5: 0.9019\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 44ms/step - loss: 3.9872 - accuracy: 0.3978 - true_positives_5: 7613.0000 - false_positives_5: 3727.0000 - true_negatives_5: 1478369.0000 - false_negatives_5: 27675.0000 - sensitivity_at_specificity_5: 0.9465 - specificity_at_sensitivity_5: 0.9730 - precision_5: 0.6713 - recall_5: 0.2157 - auc_5: 0.9021 - val_loss: 1.2119 - val_accuracy: 0.6424 - val_true_positives_5: 1561.0000 - val_false_positives_5: 339.0000 - val_true_negatives_5: 164343.0000 - val_false_negatives_5: 2360.0000 - val_sensitivity_at_specificity_5: 0.9972 - val_specificity_at_sensitivity_5: 0.9954 - val_precision_5: 0.8216 - val_recall_5: 0.3981 - val_auc_5: 0.9820\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 1.1727 - accuracy: 0.6548 - true_positives_5: 17245.0000 - false_positives_5: 4335.0000 - true_negatives_5: 1477761.0000 - false_negatives_5: 18043.0000 - sensitivity_at_specificity_5: 0.9833 - specificity_at_sensitivity_5: 0.9969 - precision_5: 0.7991 - recall_5: 0.4887 - auc_5: 0.9762\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 1.1727 - accuracy: 0.6548 - true_positives_5: 17245.0000 - false_positives_5: 4335.0000 - true_negatives_5: 1477761.0000 - false_negatives_5: 18043.0000 - sensitivity_at_specificity_5: 0.9833 - specificity_at_sensitivity_5: 0.9969 - precision_5: 0.7991 - recall_5: 0.4887 - auc_5: 0.9762 - val_loss: 0.6494 - val_accuracy: 0.8090 - val_true_positives_5: 2668.0000 - val_false_positives_5: 333.0000 - val_true_negatives_5: 164349.0000 - val_false_negatives_5: 1253.0000 - val_sensitivity_at_specificity_5: 0.9957 - val_specificity_at_sensitivity_5: 0.9994 - val_precision_5: 0.8890 - val_recall_5: 0.6804 - val_auc_5: 0.9933\n",
            "Epoch 3/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.8229 - accuracy: 0.7619 - true_positives_5: 22716.0000 - false_positives_5: 3976.0000 - true_negatives_5: 1478120.0000 - false_negatives_5: 12572.0000 - sensitivity_at_specificity_5: 0.9862 - specificity_at_sensitivity_5: 0.9990 - precision_5: 0.8510 - recall_5: 0.6437 - auc_5: 0.9858\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.8229 - accuracy: 0.7619 - true_positives_5: 22716.0000 - false_positives_5: 3976.0000 - true_negatives_5: 1478120.0000 - false_negatives_5: 12572.0000 - sensitivity_at_specificity_5: 0.9862 - specificity_at_sensitivity_5: 0.9990 - precision_5: 0.8510 - recall_5: 0.6437 - auc_5: 0.9858 - val_loss: 0.4107 - val_accuracy: 0.8778 - val_true_positives_5: 3199.0000 - val_false_positives_5: 257.0000 - val_true_negatives_5: 164425.0000 - val_false_negatives_5: 722.0000 - val_sensitivity_at_specificity_5: 0.9959 - val_specificity_at_sensitivity_5: 0.9999 - val_precision_5: 0.9256 - val_recall_5: 0.8159 - val_auc_5: 0.9962\n",
            "Epoch 4/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.6006 - accuracy: 0.8286 - true_positives_5: 26307.0000 - false_positives_5: 3353.0000 - true_negatives_5: 1473367.0000 - false_negatives_5: 8853.0000 - sensitivity_at_specificity_5: 0.9889 - specificity_at_sensitivity_5: 0.9996 - precision_5: 0.8870 - recall_5: 0.7482 - auc_5: 0.9906\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.6005 - accuracy: 0.8288 - true_positives_5: 26408.0000 - false_positives_5: 3363.0000 - true_negatives_5: 1478733.0000 - false_negatives_5: 8880.0000 - sensitivity_at_specificity_5: 0.9889 - specificity_at_sensitivity_5: 0.9996 - precision_5: 0.8870 - recall_5: 0.7484 - auc_5: 0.9906 - val_loss: 0.3258 - val_accuracy: 0.9041 - val_true_positives_5: 3342.0000 - val_false_positives_5: 225.0000 - val_true_negatives_5: 164457.0000 - val_false_negatives_5: 579.0000 - val_sensitivity_at_specificity_5: 0.9969 - val_specificity_at_sensitivity_5: 0.9999 - val_precision_5: 0.9369 - val_recall_5: 0.8523 - val_auc_5: 0.9973\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.4919 - accuracy: 0.8608 - true_positives_5: 28210.0000 - false_positives_5: 2839.0000 - true_negatives_5: 1473881.0000 - false_negatives_5: 6950.0000 - sensitivity_at_specificity_5: 0.9888 - specificity_at_sensitivity_5: 0.9998 - precision_5: 0.9086 - recall_5: 0.8023 - auc_5: 0.9919\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.4917 - accuracy: 0.8608 - true_positives_5: 28317.0000 - false_positives_5: 2850.0000 - true_negatives_5: 1479246.0000 - false_negatives_5: 6971.0000 - sensitivity_at_specificity_5: 0.9888 - specificity_at_sensitivity_5: 0.9998 - precision_5: 0.9086 - recall_5: 0.8025 - auc_5: 0.9919 - val_loss: 0.2324 - val_accuracy: 0.9317 - val_true_positives_5: 3529.0000 - val_false_positives_5: 177.0000 - val_true_negatives_5: 164505.0000 - val_false_negatives_5: 392.0000 - val_sensitivity_at_specificity_5: 0.9967 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9522 - val_recall_5: 0.9000 - val_auc_5: 0.9977\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.4143 - accuracy: 0.8801 - true_positives_5: 29503.0000 - false_positives_5: 2675.0000 - true_negatives_5: 1479421.0000 - false_negatives_5: 5785.0000 - sensitivity_at_specificity_5: 0.9912 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9169 - recall_5: 0.8361 - auc_5: 0.9938\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.4143 - accuracy: 0.8801 - true_positives_5: 29503.0000 - false_positives_5: 2675.0000 - true_negatives_5: 1479421.0000 - false_negatives_5: 5785.0000 - sensitivity_at_specificity_5: 0.9912 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9169 - recall_5: 0.8361 - auc_5: 0.9938 - val_loss: 0.2029 - val_accuracy: 0.9416 - val_true_positives_5: 3598.0000 - val_false_positives_5: 163.0000 - val_true_negatives_5: 164519.0000 - val_false_negatives_5: 323.0000 - val_sensitivity_at_specificity_5: 0.9972 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9567 - val_recall_5: 0.9176 - val_auc_5: 0.9981\n",
            "Epoch 7/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.3528 - accuracy: 0.9020 - true_positives_5: 30495.0000 - false_positives_5: 2299.0000 - true_negatives_5: 1474421.0000 - false_negatives_5: 4665.0000 - sensitivity_at_specificity_5: 0.9913 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9299 - recall_5: 0.8673 - auc_5: 0.9943\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.3524 - accuracy: 0.9021 - true_positives_5: 30611.0000 - false_positives_5: 2304.0000 - true_negatives_5: 1479792.0000 - false_negatives_5: 4677.0000 - sensitivity_at_specificity_5: 0.9913 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9300 - recall_5: 0.8675 - auc_5: 0.9943 - val_loss: 0.1819 - val_accuracy: 0.9475 - val_true_positives_5: 3646.0000 - val_false_positives_5: 142.0000 - val_true_negatives_5: 164540.0000 - val_false_negatives_5: 275.0000 - val_sensitivity_at_specificity_5: 0.9964 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9625 - val_recall_5: 0.9299 - val_auc_5: 0.9978\n",
            "Epoch 8/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.3145 - accuracy: 0.9131 - true_positives_5: 31103.0000 - false_positives_5: 2092.0000 - true_negatives_5: 1474628.0000 - false_negatives_5: 4057.0000 - sensitivity_at_specificity_5: 0.9916 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9370 - recall_5: 0.8846 - auc_5: 0.9948\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.3151 - accuracy: 0.9131 - true_positives_5: 31218.0000 - false_positives_5: 2102.0000 - true_negatives_5: 1479994.0000 - false_negatives_5: 4070.0000 - sensitivity_at_specificity_5: 0.9916 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9369 - recall_5: 0.8847 - auc_5: 0.9948 - val_loss: 0.1553 - val_accuracy: 0.9600 - val_true_positives_5: 3694.0000 - val_false_positives_5: 107.0000 - val_true_negatives_5: 164575.0000 - val_false_negatives_5: 227.0000 - val_sensitivity_at_specificity_5: 0.9974 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9718 - val_recall_5: 0.9421 - val_auc_5: 0.9984\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2711 - accuracy: 0.9246 - true_positives_5: 31778.0000 - false_positives_5: 1876.0000 - true_negatives_5: 1480220.0000 - false_negatives_5: 3510.0000 - sensitivity_at_specificity_5: 0.9929 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9443 - recall_5: 0.9005 - auc_5: 0.9957\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2711 - accuracy: 0.9246 - true_positives_5: 31778.0000 - false_positives_5: 1876.0000 - true_negatives_5: 1480220.0000 - false_negatives_5: 3510.0000 - sensitivity_at_specificity_5: 0.9929 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9443 - recall_5: 0.9005 - auc_5: 0.9957 - val_loss: 0.1220 - val_accuracy: 0.9674 - val_true_positives_5: 3748.0000 - val_false_positives_5: 104.0000 - val_true_negatives_5: 164578.0000 - val_false_negatives_5: 173.0000 - val_sensitivity_at_specificity_5: 0.9985 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9730 - val_recall_5: 0.9559 - val_auc_5: 0.9990\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2673 - accuracy: 0.9264 - true_positives_5: 31919.0000 - false_positives_5: 1882.0000 - true_negatives_5: 1480214.0000 - false_negatives_5: 3369.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9443 - recall_5: 0.9045 - auc_5: 0.9956\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.2673 - accuracy: 0.9264 - true_positives_5: 31919.0000 - false_positives_5: 1882.0000 - true_negatives_5: 1480214.0000 - false_negatives_5: 3369.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9443 - recall_5: 0.9045 - auc_5: 0.9956 - val_loss: 0.1065 - val_accuracy: 0.9694 - val_true_positives_5: 3771.0000 - val_false_positives_5: 98.0000 - val_true_negatives_5: 164584.0000 - val_false_negatives_5: 150.0000 - val_sensitivity_at_specificity_5: 0.9980 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9747 - val_recall_5: 0.9617 - val_auc_5: 0.9988\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2610 - accuracy: 0.9294 - true_positives_5: 31960.0000 - false_positives_5: 1792.0000 - true_negatives_5: 1474928.0000 - false_negatives_5: 3200.0000 - sensitivity_at_specificity_5: 0.9926 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9469 - recall_5: 0.9090 - auc_5: 0.9956\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2611 - accuracy: 0.9295 - true_positives_5: 32080.0000 - false_positives_5: 1797.0000 - true_negatives_5: 1480299.0000 - false_negatives_5: 3208.0000 - sensitivity_at_specificity_5: 0.9926 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9470 - recall_5: 0.9091 - auc_5: 0.9956 - val_loss: 0.1206 - val_accuracy: 0.9651 - val_true_positives_5: 3745.0000 - val_false_positives_5: 114.0000 - val_true_negatives_5: 164568.0000 - val_false_negatives_5: 176.0000 - val_sensitivity_at_specificity_5: 0.9969 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9705 - val_recall_5: 0.9551 - val_auc_5: 0.9983\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2541 - accuracy: 0.9337 - true_positives_5: 32314.0000 - false_positives_5: 1726.0000 - true_negatives_5: 1480370.0000 - false_negatives_5: 2974.0000 - sensitivity_at_specificity_5: 0.9916 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9493 - recall_5: 0.9157 - auc_5: 0.9952\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 0.2541 - accuracy: 0.9337 - true_positives_5: 32314.0000 - false_positives_5: 1726.0000 - true_negatives_5: 1480370.0000 - false_negatives_5: 2974.0000 - sensitivity_at_specificity_5: 0.9916 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9493 - recall_5: 0.9157 - auc_5: 0.9952 - val_loss: 0.1438 - val_accuracy: 0.9630 - val_true_positives_5: 3730.0000 - val_false_positives_5: 116.0000 - val_true_negatives_5: 164566.0000 - val_false_negatives_5: 191.0000 - val_sensitivity_at_specificity_5: 0.9952 - val_specificity_at_sensitivity_5: 0.9999 - val_precision_5: 0.9698 - val_recall_5: 0.9513 - val_auc_5: 0.9973\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2445 - accuracy: 0.9348 - true_positives_5: 32320.0000 - false_positives_5: 1720.0000 - true_negatives_5: 1475000.0000 - false_negatives_5: 2840.0000 - sensitivity_at_specificity_5: 0.9923 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9495 - recall_5: 0.9192 - auc_5: 0.9956\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2444 - accuracy: 0.9348 - true_positives_5: 32439.0000 - false_positives_5: 1724.0000 - true_negatives_5: 1480372.0000 - false_negatives_5: 2849.0000 - sensitivity_at_specificity_5: 0.9923 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9495 - recall_5: 0.9193 - auc_5: 0.9956 - val_loss: 0.1154 - val_accuracy: 0.9676 - val_true_positives_5: 3774.0000 - val_false_positives_5: 100.0000 - val_true_negatives_5: 164582.0000 - val_false_negatives_5: 147.0000 - val_sensitivity_at_specificity_5: 0.9957 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9742 - val_recall_5: 0.9625 - val_auc_5: 0.9977\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2220 - accuracy: 0.9408 - true_positives_5: 32562.0000 - false_positives_5: 1546.0000 - true_negatives_5: 1475174.0000 - false_negatives_5: 2598.0000 - sensitivity_at_specificity_5: 0.9928 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9547 - recall_5: 0.9261 - auc_5: 0.9959\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 0.2217 - accuracy: 0.9409 - true_positives_5: 32686.0000 - false_positives_5: 1549.0000 - true_negatives_5: 1480547.0000 - false_negatives_5: 2602.0000 - sensitivity_at_specificity_5: 0.9929 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9548 - recall_5: 0.9263 - auc_5: 0.9959 - val_loss: 0.1203 - val_accuracy: 0.9702 - val_true_positives_5: 3766.0000 - val_false_positives_5: 86.0000 - val_true_negatives_5: 164596.0000 - val_false_negatives_5: 155.0000 - val_sensitivity_at_specificity_5: 0.9967 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9777 - val_recall_5: 0.9605 - val_auc_5: 0.9982\n",
            "Epoch 15/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2057 - accuracy: 0.9452 - true_positives_5: 32944.0000 - false_positives_5: 1467.0000 - true_negatives_5: 1480629.0000 - false_negatives_5: 2344.0000 - sensitivity_at_specificity_5: 0.9928 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9574 - recall_5: 0.9336 - auc_5: 0.9960\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2057 - accuracy: 0.9452 - true_positives_5: 32944.0000 - false_positives_5: 1467.0000 - true_negatives_5: 1480629.0000 - false_negatives_5: 2344.0000 - sensitivity_at_specificity_5: 0.9928 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9574 - recall_5: 0.9336 - auc_5: 0.9960 - val_loss: 0.0741 - val_accuracy: 0.9788 - val_true_positives_5: 3805.0000 - val_false_positives_5: 58.0000 - val_true_negatives_5: 164624.0000 - val_false_negatives_5: 116.0000 - val_sensitivity_at_specificity_5: 0.9985 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9850 - val_recall_5: 0.9704 - val_auc_5: 0.9992\n",
            "Epoch 16/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2023 - accuracy: 0.9458 - true_positives_5: 32831.0000 - false_positives_5: 1481.0000 - true_negatives_5: 1475239.0000 - false_negatives_5: 2329.0000 - sensitivity_at_specificity_5: 0.9931 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9568 - recall_5: 0.9338 - auc_5: 0.9962\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2020 - accuracy: 0.9460 - true_positives_5: 32952.0000 - false_positives_5: 1483.0000 - true_negatives_5: 1480613.0000 - false_negatives_5: 2336.0000 - sensitivity_at_specificity_5: 0.9931 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9569 - recall_5: 0.9338 - auc_5: 0.9962 - val_loss: 0.1328 - val_accuracy: 0.9640 - val_true_positives_5: 3724.0000 - val_false_positives_5: 97.0000 - val_true_negatives_5: 164585.0000 - val_false_negatives_5: 197.0000 - val_sensitivity_at_specificity_5: 0.9967 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9746 - val_recall_5: 0.9498 - val_auc_5: 0.9981\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2079 - accuracy: 0.9451 - true_positives_5: 32940.0000 - false_positives_5: 1524.0000 - true_negatives_5: 1480572.0000 - false_negatives_5: 2348.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9558 - recall_5: 0.9335 - auc_5: 0.9959\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 70s 253ms/step - loss: 0.2079 - accuracy: 0.9451 - true_positives_5: 32940.0000 - false_positives_5: 1524.0000 - true_negatives_5: 1480572.0000 - false_negatives_5: 2348.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9558 - recall_5: 0.9335 - auc_5: 0.9959 - val_loss: 0.0731 - val_accuracy: 0.9806 - val_true_positives_5: 3831.0000 - val_false_positives_5: 65.0000 - val_true_negatives_5: 164617.0000 - val_false_negatives_5: 90.0000 - val_sensitivity_at_specificity_5: 0.9982 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9833 - val_recall_5: 0.9770 - val_auc_5: 0.9990\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2000 - accuracy: 0.9492 - true_positives_5: 33163.0000 - false_positives_5: 1434.0000 - true_negatives_5: 1480662.0000 - false_negatives_5: 2125.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9586 - recall_5: 0.9398 - auc_5: 0.9960\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 14s 49ms/step - loss: 0.2000 - accuracy: 0.9492 - true_positives_5: 33163.0000 - false_positives_5: 1434.0000 - true_negatives_5: 1480662.0000 - false_negatives_5: 2125.0000 - sensitivity_at_specificity_5: 0.9927 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9586 - recall_5: 0.9398 - auc_5: 0.9960 - val_loss: 0.0982 - val_accuracy: 0.9727 - val_true_positives_5: 3793.0000 - val_false_positives_5: 87.0000 - val_true_negatives_5: 164595.0000 - val_false_negatives_5: 128.0000 - val_sensitivity_at_specificity_5: 0.9967 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9776 - val_recall_5: 0.9674 - val_auc_5: 0.9982\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2154 - accuracy: 0.9435 - true_positives_5: 32776.0000 - false_positives_5: 1569.0000 - true_negatives_5: 1475151.0000 - false_negatives_5: 2384.0000 - sensitivity_at_specificity_5: 0.9919 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9543 - recall_5: 0.9322 - auc_5: 0.9955\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2153 - accuracy: 0.9435 - true_positives_5: 32895.0000 - false_positives_5: 1575.0000 - true_negatives_5: 1480521.0000 - false_negatives_5: 2393.0000 - sensitivity_at_specificity_5: 0.9919 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9543 - recall_5: 0.9322 - auc_5: 0.9955 - val_loss: 0.1521 - val_accuracy: 0.9702 - val_true_positives_5: 3789.0000 - val_false_positives_5: 99.0000 - val_true_negatives_5: 164583.0000 - val_false_negatives_5: 132.0000 - val_sensitivity_at_specificity_5: 0.9903 - val_specificity_at_sensitivity_5: 0.9998 - val_precision_5: 0.9745 - val_recall_5: 0.9663 - val_auc_5: 0.9949\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1798 - accuracy: 0.9548 - true_positives_5: 33396.0000 - false_positives_5: 1266.0000 - true_negatives_5: 1480830.0000 - false_negatives_5: 1892.0000 - sensitivity_at_specificity_5: 0.9933 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9635 - recall_5: 0.9464 - auc_5: 0.9964\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1798 - accuracy: 0.9548 - true_positives_5: 33396.0000 - false_positives_5: 1266.0000 - true_negatives_5: 1480830.0000 - false_negatives_5: 1892.0000 - sensitivity_at_specificity_5: 0.9933 - specificity_at_sensitivity_5: 0.9999 - precision_5: 0.9635 - recall_5: 0.9464 - auc_5: 0.9964 - val_loss: 0.0694 - val_accuracy: 0.9819 - val_true_positives_5: 3843.0000 - val_false_positives_5: 61.0000 - val_true_negatives_5: 164621.0000 - val_false_negatives_5: 78.0000 - val_sensitivity_at_specificity_5: 0.9974 - val_specificity_at_sensitivity_5: 1.0000 - val_precision_5: 0.9844 - val_recall_5: 0.9801 - val_auc_5: 0.9987\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0694 - accuracy: 0.9819 - true_positives_5: 3843.0000 - false_positives_5: 61.0000 - true_negatives_5: 164621.0000 - false_negatives_5: 78.0000 - sensitivity_at_specificity_5: 0.9974 - specificity_at_sensitivity_5: 1.0000 - precision_5: 0.9844 - recall_5: 0.9801 - auc_5: 0.9987\n",
            "7\n",
            "Epoch 1/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 2.9104 - accuracy: 0.5531 - true_positives_6: 13954.0000 - false_positives_6: 3734.0000 - true_negatives_6: 1478362.0000 - false_negatives_6: 21334.0000 - sensitivity_at_specificity_6: 0.9526 - specificity_at_sensitivity_6: 0.9937 - precision_6: 0.7889 - recall_6: 0.3954 - auc_6: 0.9378\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 2.9104 - accuracy: 0.5531 - true_positives_6: 13954.0000 - false_positives_6: 3734.0000 - true_negatives_6: 1478362.0000 - false_negatives_6: 21334.0000 - sensitivity_at_specificity_6: 0.9526 - specificity_at_sensitivity_6: 0.9937 - precision_6: 0.7889 - recall_6: 0.3954 - auc_6: 0.9378 - val_loss: 0.5691 - val_accuracy: 0.8268 - val_true_positives_6: 2863.0000 - val_false_positives_6: 271.0000 - val_true_negatives_6: 164411.0000 - val_false_negatives_6: 1058.0000 - val_sensitivity_at_specificity_6: 0.9977 - val_specificity_at_sensitivity_6: 0.9998 - val_precision_6: 0.9135 - val_recall_6: 0.7302 - val_auc_6: 0.9953\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.5975 - accuracy: 0.8289 - true_positives_6: 26528.0000 - false_positives_6: 3087.0000 - true_negatives_6: 1479009.0000 - false_negatives_6: 8760.0000 - sensitivity_at_specificity_6: 0.9883 - specificity_at_sensitivity_6: 0.9997 - precision_6: 0.8958 - recall_6: 0.7518 - auc_6: 0.9903\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.5975 - accuracy: 0.8289 - true_positives_6: 26528.0000 - false_positives_6: 3087.0000 - true_negatives_6: 1479009.0000 - false_negatives_6: 8760.0000 - sensitivity_at_specificity_6: 0.9883 - specificity_at_sensitivity_6: 0.9997 - precision_6: 0.8958 - recall_6: 0.7518 - auc_6: 0.9903 - val_loss: 0.2325 - val_accuracy: 0.9339 - val_true_positives_6: 3547.0000 - val_false_positives_6: 172.0000 - val_true_negatives_6: 164510.0000 - val_false_negatives_6: 374.0000 - val_sensitivity_at_specificity_6: 0.9977 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9538 - val_recall_6: 0.9046 - val_auc_6: 0.9982\n",
            "Epoch 3/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3566 - accuracy: 0.8987 - true_positives_6: 30324.0000 - false_positives_6: 2254.0000 - true_negatives_6: 1479842.0000 - false_negatives_6: 4964.0000 - sensitivity_at_specificity_6: 0.9925 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9308 - recall_6: 0.8593 - auc_6: 0.9949\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.3566 - accuracy: 0.8987 - true_positives_6: 30324.0000 - false_positives_6: 2254.0000 - true_negatives_6: 1479842.0000 - false_negatives_6: 4964.0000 - sensitivity_at_specificity_6: 0.9925 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9308 - recall_6: 0.8593 - auc_6: 0.9949 - val_loss: 0.1757 - val_accuracy: 0.9543 - val_true_positives_6: 3681.0000 - val_false_positives_6: 139.0000 - val_true_negatives_6: 164543.0000 - val_false_negatives_6: 240.0000 - val_sensitivity_at_specificity_6: 0.9954 - val_specificity_at_sensitivity_6: 0.9999 - val_precision_6: 0.9636 - val_recall_6: 0.9388 - val_auc_6: 0.9973\n",
            "Epoch 4/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2709 - accuracy: 0.9208 - true_positives_6: 31506.0000 - false_positives_6: 1894.0000 - true_negatives_6: 1474826.0000 - false_negatives_6: 3654.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 1.0000 - precision_6: 0.9433 - recall_6: 0.8961 - auc_6: 0.9961\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2704 - accuracy: 0.9210 - true_positives_6: 31629.0000 - false_positives_6: 1897.0000 - true_negatives_6: 1480199.0000 - false_negatives_6: 3659.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 1.0000 - precision_6: 0.9434 - recall_6: 0.8963 - auc_6: 0.9961 - val_loss: 0.0981 - val_accuracy: 0.9748 - val_true_positives_6: 3796.0000 - val_false_positives_6: 80.0000 - val_true_negatives_6: 164602.0000 - val_false_negatives_6: 125.0000 - val_sensitivity_at_specificity_6: 0.9972 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9794 - val_recall_6: 0.9681 - val_auc_6: 0.9985\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2175 - accuracy: 0.9393 - true_positives_6: 32395.0000 - false_positives_6: 1549.0000 - true_negatives_6: 1475171.0000 - false_negatives_6: 2765.0000 - sensitivity_at_specificity_6: 0.9940 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9544 - recall_6: 0.9214 - auc_6: 0.9965\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2173 - accuracy: 0.9394 - true_positives_6: 32517.0000 - false_positives_6: 1552.0000 - true_negatives_6: 1480544.0000 - false_negatives_6: 2771.0000 - sensitivity_at_specificity_6: 0.9940 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9544 - recall_6: 0.9215 - auc_6: 0.9965 - val_loss: 0.0786 - val_accuracy: 0.9801 - val_true_positives_6: 3820.0000 - val_false_positives_6: 61.0000 - val_true_negatives_6: 164621.0000 - val_false_negatives_6: 101.0000 - val_sensitivity_at_specificity_6: 0.9990 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9843 - val_recall_6: 0.9742 - val_auc_6: 0.9994\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1902 - accuracy: 0.9466 - true_positives_6: 32951.0000 - false_positives_6: 1439.0000 - true_negatives_6: 1480657.0000 - false_negatives_6: 2337.0000 - sensitivity_at_specificity_6: 0.9944 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9582 - recall_6: 0.9338 - auc_6: 0.9968\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1902 - accuracy: 0.9466 - true_positives_6: 32951.0000 - false_positives_6: 1439.0000 - true_negatives_6: 1480657.0000 - false_negatives_6: 2337.0000 - sensitivity_at_specificity_6: 0.9944 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9582 - recall_6: 0.9338 - auc_6: 0.9968 - val_loss: 0.0733 - val_accuracy: 0.9799 - val_true_positives_6: 3830.0000 - val_false_positives_6: 64.0000 - val_true_negatives_6: 164618.0000 - val_false_negatives_6: 91.0000 - val_sensitivity_at_specificity_6: 0.9972 - val_specificity_at_sensitivity_6: 0.9999 - val_precision_6: 0.9836 - val_recall_6: 0.9768 - val_auc_6: 0.9985\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1522 - accuracy: 0.9591 - true_positives_6: 33471.0000 - false_positives_6: 1108.0000 - true_negatives_6: 1480988.0000 - false_negatives_6: 1817.0000 - sensitivity_at_specificity_6: 0.9954 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9680 - recall_6: 0.9485 - auc_6: 0.9974\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1522 - accuracy: 0.9591 - true_positives_6: 33471.0000 - false_positives_6: 1108.0000 - true_negatives_6: 1480988.0000 - false_negatives_6: 1817.0000 - sensitivity_at_specificity_6: 0.9954 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9680 - recall_6: 0.9485 - auc_6: 0.9974 - val_loss: 0.0843 - val_accuracy: 0.9748 - val_true_positives_6: 3811.0000 - val_false_positives_6: 92.0000 - val_true_negatives_6: 164590.0000 - val_false_negatives_6: 110.0000 - val_sensitivity_at_specificity_6: 0.9969 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9764 - val_recall_6: 0.9719 - val_auc_6: 0.9984\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1417 - accuracy: 0.9615 - true_positives_6: 33646.0000 - false_positives_6: 1090.0000 - true_negatives_6: 1481006.0000 - false_negatives_6: 1642.0000 - sensitivity_at_specificity_6: 0.9954 - specificity_at_sensitivity_6: 1.0000 - precision_6: 0.9686 - recall_6: 0.9535 - auc_6: 0.9975\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1417 - accuracy: 0.9615 - true_positives_6: 33646.0000 - false_positives_6: 1090.0000 - true_negatives_6: 1481006.0000 - false_negatives_6: 1642.0000 - sensitivity_at_specificity_6: 0.9954 - specificity_at_sensitivity_6: 1.0000 - precision_6: 0.9686 - recall_6: 0.9535 - auc_6: 0.9975 - val_loss: 0.0829 - val_accuracy: 0.9801 - val_true_positives_6: 3829.0000 - val_false_positives_6: 64.0000 - val_true_negatives_6: 164618.0000 - val_false_negatives_6: 92.0000 - val_sensitivity_at_specificity_6: 0.9969 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9836 - val_recall_6: 0.9765 - val_auc_6: 0.9984\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1735 - accuracy: 0.9541 - true_positives_6: 33321.0000 - false_positives_6: 1285.0000 - true_negatives_6: 1480811.0000 - false_negatives_6: 1967.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9629 - recall_6: 0.9443 - auc_6: 0.9966\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1735 - accuracy: 0.9541 - true_positives_6: 33321.0000 - false_positives_6: 1285.0000 - true_negatives_6: 1480811.0000 - false_negatives_6: 1967.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9629 - recall_6: 0.9443 - auc_6: 0.9966 - val_loss: 0.0640 - val_accuracy: 0.9819 - val_true_positives_6: 3842.0000 - val_false_positives_6: 63.0000 - val_true_negatives_6: 164619.0000 - val_false_negatives_6: 79.0000 - val_sensitivity_at_specificity_6: 0.9980 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9839 - val_recall_6: 0.9799 - val_auc_6: 0.9989\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1499 - accuracy: 0.9610 - true_positives_6: 33662.0000 - false_positives_6: 1114.0000 - true_negatives_6: 1480982.0000 - false_negatives_6: 1626.0000 - sensitivity_at_specificity_6: 0.9944 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9680 - recall_6: 0.9539 - auc_6: 0.9970\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1499 - accuracy: 0.9610 - true_positives_6: 33662.0000 - false_positives_6: 1114.0000 - true_negatives_6: 1480982.0000 - false_negatives_6: 1626.0000 - sensitivity_at_specificity_6: 0.9944 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9680 - recall_6: 0.9539 - auc_6: 0.9970 - val_loss: 0.0967 - val_accuracy: 0.9750 - val_true_positives_6: 3803.0000 - val_false_positives_6: 80.0000 - val_true_negatives_6: 164602.0000 - val_false_negatives_6: 118.0000 - val_sensitivity_at_specificity_6: 0.9969 - val_specificity_at_sensitivity_6: 0.9999 - val_precision_6: 0.9794 - val_recall_6: 0.9699 - val_auc_6: 0.9983\n",
            "Epoch 11/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1480 - accuracy: 0.9611 - true_positives_6: 33660.0000 - false_positives_6: 1110.0000 - true_negatives_6: 1480986.0000 - false_negatives_6: 1628.0000 - sensitivity_at_specificity_6: 0.9949 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9681 - recall_6: 0.9539 - auc_6: 0.9972\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1480 - accuracy: 0.9611 - true_positives_6: 33660.0000 - false_positives_6: 1110.0000 - true_negatives_6: 1480986.0000 - false_negatives_6: 1628.0000 - sensitivity_at_specificity_6: 0.9949 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9681 - recall_6: 0.9539 - auc_6: 0.9972 - val_loss: 0.0542 - val_accuracy: 0.9878 - val_true_positives_6: 3867.0000 - val_false_positives_6: 40.0000 - val_true_negatives_6: 164642.0000 - val_false_negatives_6: 54.0000 - val_sensitivity_at_specificity_6: 0.9967 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9898 - val_recall_6: 0.9862 - val_auc_6: 0.9983\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1316 - accuracy: 0.9657 - true_positives_6: 33880.0000 - false_positives_6: 972.0000 - true_negatives_6: 1481124.0000 - false_negatives_6: 1408.0000 - sensitivity_at_specificity_6: 0.9950 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9721 - recall_6: 0.9601 - auc_6: 0.9973\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1316 - accuracy: 0.9657 - true_positives_6: 33880.0000 - false_positives_6: 972.0000 - true_negatives_6: 1481124.0000 - false_negatives_6: 1408.0000 - sensitivity_at_specificity_6: 0.9950 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9721 - recall_6: 0.9601 - auc_6: 0.9973 - val_loss: 0.0661 - val_accuracy: 0.9834 - val_true_positives_6: 3849.0000 - val_false_positives_6: 56.0000 - val_true_negatives_6: 164626.0000 - val_false_negatives_6: 72.0000 - val_sensitivity_at_specificity_6: 0.9972 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9857 - val_recall_6: 0.9816 - val_auc_6: 0.9986\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1579 - accuracy: 0.9590 - true_positives_6: 33494.0000 - false_positives_6: 1174.0000 - true_negatives_6: 1475546.0000 - false_negatives_6: 1666.0000 - sensitivity_at_specificity_6: 0.9936 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9661 - recall_6: 0.9526 - auc_6: 0.9966\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 0.1577 - accuracy: 0.9590 - true_positives_6: 33618.0000 - false_positives_6: 1176.0000 - true_negatives_6: 1480920.0000 - false_negatives_6: 1670.0000 - sensitivity_at_specificity_6: 0.9936 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9662 - recall_6: 0.9527 - auc_6: 0.9966 - val_loss: 0.0823 - val_accuracy: 0.9819 - val_true_positives_6: 3837.0000 - val_false_positives_6: 57.0000 - val_true_negatives_6: 164625.0000 - val_false_negatives_6: 84.0000 - val_sensitivity_at_specificity_6: 0.9967 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9854 - val_recall_6: 0.9786 - val_auc_6: 0.9983\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1440 - accuracy: 0.9641 - true_positives_6: 33698.0000 - false_positives_6: 1050.0000 - true_negatives_6: 1475670.0000 - false_negatives_6: 1462.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9698 - recall_6: 0.9584 - auc_6: 0.9968\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1438 - accuracy: 0.9641 - true_positives_6: 33822.0000 - false_positives_6: 1054.0000 - true_negatives_6: 1481042.0000 - false_negatives_6: 1466.0000 - sensitivity_at_specificity_6: 0.9940 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9698 - recall_6: 0.9585 - auc_6: 0.9968 - val_loss: 0.0523 - val_accuracy: 0.9870 - val_true_positives_6: 3864.0000 - val_false_positives_6: 42.0000 - val_true_negatives_6: 164640.0000 - val_false_negatives_6: 57.0000 - val_sensitivity_at_specificity_6: 0.9980 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9892 - val_recall_6: 0.9855 - val_auc_6: 0.9989\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1417 - accuracy: 0.9630 - true_positives_6: 33683.0000 - false_positives_6: 1082.0000 - true_negatives_6: 1475638.0000 - false_negatives_6: 1477.0000 - sensitivity_at_specificity_6: 0.9940 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9689 - recall_6: 0.9580 - auc_6: 0.9968\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1417 - accuracy: 0.9630 - true_positives_6: 33805.0000 - false_positives_6: 1088.0000 - true_negatives_6: 1481008.0000 - false_negatives_6: 1483.0000 - sensitivity_at_specificity_6: 0.9940 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9688 - recall_6: 0.9580 - auc_6: 0.9968 - val_loss: 0.0522 - val_accuracy: 0.9865 - val_true_positives_6: 3862.0000 - val_false_positives_6: 48.0000 - val_true_negatives_6: 164634.0000 - val_false_negatives_6: 59.0000 - val_sensitivity_at_specificity_6: 0.9977 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9877 - val_recall_6: 0.9850 - val_auc_6: 0.9988\n",
            "Epoch 16/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1383 - accuracy: 0.9663 - true_positives_6: 33810.0000 - false_positives_6: 978.0000 - true_negatives_6: 1475742.0000 - false_negatives_6: 1350.0000 - sensitivity_at_specificity_6: 0.9937 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9719 - recall_6: 0.9616 - auc_6: 0.9967\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1383 - accuracy: 0.9663 - true_positives_6: 33934.0000 - false_positives_6: 980.0000 - true_negatives_6: 1481116.0000 - false_negatives_6: 1354.0000 - sensitivity_at_specificity_6: 0.9937 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9719 - recall_6: 0.9616 - auc_6: 0.9967 - val_loss: 0.0468 - val_accuracy: 0.9875 - val_true_positives_6: 3871.0000 - val_false_positives_6: 45.0000 - val_true_negatives_6: 164637.0000 - val_false_negatives_6: 50.0000 - val_sensitivity_at_specificity_6: 0.9987 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9885 - val_recall_6: 0.9872 - val_auc_6: 0.9993\n",
            "Epoch 17/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1485 - accuracy: 0.9656 - true_positives_6: 33761.0000 - false_positives_6: 1015.0000 - true_negatives_6: 1475705.0000 - false_negatives_6: 1399.0000 - sensitivity_at_specificity_6: 0.9930 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9708 - recall_6: 0.9602 - auc_6: 0.9963\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1490 - accuracy: 0.9655 - true_positives_6: 33880.0000 - false_positives_6: 1019.0000 - true_negatives_6: 1481077.0000 - false_negatives_6: 1408.0000 - sensitivity_at_specificity_6: 0.9929 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9708 - recall_6: 0.9601 - auc_6: 0.9963 - val_loss: 0.0745 - val_accuracy: 0.9791 - val_true_positives_6: 3828.0000 - val_false_positives_6: 71.0000 - val_true_negatives_6: 164611.0000 - val_false_negatives_6: 93.0000 - val_sensitivity_at_specificity_6: 0.9980 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9818 - val_recall_6: 0.9763 - val_auc_6: 0.9989\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1346 - accuracy: 0.9690 - true_positives_6: 34026.0000 - false_positives_6: 961.0000 - true_negatives_6: 1481135.0000 - false_negatives_6: 1262.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9725 - recall_6: 0.9642 - auc_6: 0.9968\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1346 - accuracy: 0.9690 - true_positives_6: 34026.0000 - false_positives_6: 961.0000 - true_negatives_6: 1481135.0000 - false_negatives_6: 1262.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9725 - recall_6: 0.9642 - auc_6: 0.9968 - val_loss: 0.0446 - val_accuracy: 0.9895 - val_true_positives_6: 3876.0000 - val_false_positives_6: 38.0000 - val_true_negatives_6: 164644.0000 - val_false_negatives_6: 45.0000 - val_sensitivity_at_specificity_6: 0.9977 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9903 - val_recall_6: 0.9885 - val_auc_6: 0.9988\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1281 - accuracy: 0.9696 - true_positives_6: 33956.0000 - false_positives_6: 906.0000 - true_negatives_6: 1475814.0000 - false_negatives_6: 1204.0000 - sensitivity_at_specificity_6: 0.9942 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9740 - recall_6: 0.9658 - auc_6: 0.9970\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1279 - accuracy: 0.9696 - true_positives_6: 34083.0000 - false_positives_6: 907.0000 - true_negatives_6: 1481189.0000 - false_negatives_6: 1205.0000 - sensitivity_at_specificity_6: 0.9942 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9741 - recall_6: 0.9659 - auc_6: 0.9970 - val_loss: 0.0352 - val_accuracy: 0.9964 - val_true_positives_6: 3906.0000 - val_false_positives_6: 11.0000 - val_true_negatives_6: 164671.0000 - val_false_negatives_6: 15.0000 - val_sensitivity_at_specificity_6: 0.9987 - val_specificity_at_sensitivity_6: 1.0000 - val_precision_6: 0.9972 - val_recall_6: 0.9962 - val_auc_6: 0.9993\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1336 - accuracy: 0.9683 - true_positives_6: 34059.0000 - false_positives_6: 943.0000 - true_negatives_6: 1481153.0000 - false_negatives_6: 1229.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9731 - recall_6: 0.9652 - auc_6: 0.9968\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1336 - accuracy: 0.9683 - true_positives_6: 34059.0000 - false_positives_6: 943.0000 - true_negatives_6: 1481153.0000 - false_negatives_6: 1229.0000 - sensitivity_at_specificity_6: 0.9939 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9731 - recall_6: 0.9652 - auc_6: 0.9968 - val_loss: 0.0760 - val_accuracy: 0.9804 - val_true_positives_6: 3835.0000 - val_false_positives_6: 73.0000 - val_true_negatives_6: 164609.0000 - val_false_negatives_6: 86.0000 - val_sensitivity_at_specificity_6: 0.9972 - val_specificity_at_sensitivity_6: 0.9999 - val_precision_6: 0.9813 - val_recall_6: 0.9781 - val_auc_6: 0.9985\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0760 - accuracy: 0.9804 - true_positives_6: 3835.0000 - false_positives_6: 73.0000 - true_negatives_6: 164609.0000 - false_negatives_6: 86.0000 - sensitivity_at_specificity_6: 0.9972 - specificity_at_sensitivity_6: 0.9999 - precision_6: 0.9813 - recall_6: 0.9781 - auc_6: 0.9985\n",
            "8\n",
            "Epoch 1/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 3.2344 - accuracy: 0.4483 - true_positives_7: 9656.0000 - false_positives_7: 3628.0000 - true_negatives_7: 1473092.0000 - false_negatives_7: 25504.0000 - sensitivity_at_specificity_7: 0.9520 - specificity_at_sensitivity_7: 0.9819 - precision_7: 0.7269 - recall_7: 0.2746 - auc_7: 0.9158\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 15s 49ms/step - loss: 3.2266 - accuracy: 0.4493 - true_positives_7: 9716.0000 - false_positives_7: 3638.0000 - true_negatives_7: 1478458.0000 - false_negatives_7: 25572.0000 - sensitivity_at_specificity_7: 0.9522 - specificity_at_sensitivity_7: 0.9828 - precision_7: 0.7276 - recall_7: 0.2753 - auc_7: 0.9161 - val_loss: 0.9974 - val_accuracy: 0.6955 - val_true_positives_7: 2167.0000 - val_false_positives_7: 411.0000 - val_true_negatives_7: 164271.0000 - val_false_negatives_7: 1754.0000 - val_sensitivity_at_specificity_7: 0.9939 - val_specificity_at_sensitivity_7: 0.9983 - val_precision_7: 0.8406 - val_recall_7: 0.5527 - val_auc_7: 0.9851\n",
            "Epoch 2/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.9223 - accuracy: 0.7319 - true_positives_7: 21163.0000 - false_positives_7: 3702.0000 - true_negatives_7: 1473018.0000 - false_negatives_7: 13997.0000 - sensitivity_at_specificity_7: 0.9852 - specificity_at_sensitivity_7: 0.9989 - precision_7: 0.8511 - recall_7: 0.6019 - auc_7: 0.9831\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.9224 - accuracy: 0.7319 - true_positives_7: 21251.0000 - false_positives_7: 3719.0000 - true_negatives_7: 1478377.0000 - false_negatives_7: 14037.0000 - sensitivity_at_specificity_7: 0.9852 - specificity_at_sensitivity_7: 0.9989 - precision_7: 0.8511 - recall_7: 0.6022 - auc_7: 0.9831 - val_loss: 0.5752 - val_accuracy: 0.8327 - val_true_positives_7: 2942.0000 - val_false_positives_7: 403.0000 - val_true_negatives_7: 164279.0000 - val_false_negatives_7: 979.0000 - val_sensitivity_at_specificity_7: 0.9911 - val_specificity_at_sensitivity_7: 0.9995 - val_precision_7: 0.8795 - val_recall_7: 0.7503 - val_auc_7: 0.9920\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.6129 - accuracy: 0.8251 - true_positives_7: 26169.0000 - false_positives_7: 3203.0000 - true_negatives_7: 1473517.0000 - false_negatives_7: 8991.0000 - sensitivity_at_specificity_7: 0.9888 - specificity_at_sensitivity_7: 0.9997 - precision_7: 0.8910 - recall_7: 0.7443 - auc_7: 0.9904\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.6126 - accuracy: 0.8251 - true_positives_7: 26269.0000 - false_positives_7: 3217.0000 - true_negatives_7: 1478879.0000 - false_negatives_7: 9019.0000 - sensitivity_at_specificity_7: 0.9888 - specificity_at_sensitivity_7: 0.9997 - precision_7: 0.8909 - recall_7: 0.7444 - auc_7: 0.9904 - val_loss: 0.2958 - val_accuracy: 0.9125 - val_true_positives_7: 3445.0000 - val_false_positives_7: 234.0000 - val_true_negatives_7: 164448.0000 - val_false_negatives_7: 476.0000 - val_sensitivity_at_specificity_7: 0.9964 - val_specificity_at_sensitivity_7: 0.9999 - val_precision_7: 0.9364 - val_recall_7: 0.8786 - val_auc_7: 0.9972\n",
            "Epoch 4/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.4399 - accuracy: 0.8756 - true_positives_7: 28966.0000 - false_positives_7: 2661.0000 - true_negatives_7: 1474059.0000 - false_negatives_7: 6194.0000 - sensitivity_at_specificity_7: 0.9906 - specificity_at_sensitivity_7: 0.9998 - precision_7: 0.9159 - recall_7: 0.8238 - auc_7: 0.9932\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.4398 - accuracy: 0.8756 - true_positives_7: 29073.0000 - false_positives_7: 2670.0000 - true_negatives_7: 1479426.0000 - false_negatives_7: 6215.0000 - sensitivity_at_specificity_7: 0.9906 - specificity_at_sensitivity_7: 0.9998 - precision_7: 0.9159 - recall_7: 0.8239 - auc_7: 0.9932 - val_loss: 0.2404 - val_accuracy: 0.9378 - val_true_positives_7: 3552.0000 - val_false_positives_7: 150.0000 - val_true_negatives_7: 164532.0000 - val_false_negatives_7: 369.0000 - val_sensitivity_at_specificity_7: 0.9957 - val_specificity_at_sensitivity_7: 0.9999 - val_precision_7: 0.9595 - val_recall_7: 0.9059 - val_auc_7: 0.9972\n",
            "Epoch 5/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3558 - accuracy: 0.9012 - true_positives_7: 30559.0000 - false_positives_7: 2330.0000 - true_negatives_7: 1479766.0000 - false_negatives_7: 4729.0000 - sensitivity_at_specificity_7: 0.9911 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9292 - recall_7: 0.8660 - auc_7: 0.9942\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.3558 - accuracy: 0.9012 - true_positives_7: 30559.0000 - false_positives_7: 2330.0000 - true_negatives_7: 1479766.0000 - false_negatives_7: 4729.0000 - sensitivity_at_specificity_7: 0.9911 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9292 - recall_7: 0.8660 - auc_7: 0.9942 - val_loss: 0.2358 - val_accuracy: 0.9365 - val_true_positives_7: 3579.0000 - val_false_positives_7: 181.0000 - val_true_negatives_7: 164501.0000 - val_false_negatives_7: 342.0000 - val_sensitivity_at_specificity_7: 0.9949 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9519 - val_recall_7: 0.9128 - val_auc_7: 0.9968\n",
            "Epoch 6/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3072 - accuracy: 0.9158 - true_positives_7: 31258.0000 - false_positives_7: 2021.0000 - true_negatives_7: 1480075.0000 - false_negatives_7: 4030.0000 - sensitivity_at_specificity_7: 0.9927 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9393 - recall_7: 0.8858 - auc_7: 0.9953\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.3072 - accuracy: 0.9158 - true_positives_7: 31258.0000 - false_positives_7: 2021.0000 - true_negatives_7: 1480075.0000 - false_negatives_7: 4030.0000 - sensitivity_at_specificity_7: 0.9927 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9393 - recall_7: 0.8858 - auc_7: 0.9953 - val_loss: 0.1741 - val_accuracy: 0.9538 - val_true_positives_7: 3677.0000 - val_false_positives_7: 128.0000 - val_true_negatives_7: 164554.0000 - val_false_negatives_7: 244.0000 - val_sensitivity_at_specificity_7: 0.9954 - val_specificity_at_sensitivity_7: 0.9999 - val_precision_7: 0.9664 - val_recall_7: 0.9378 - val_auc_7: 0.9974\n",
            "Epoch 7/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2386 - accuracy: 0.9346 - true_positives_7: 32109.0000 - false_positives_7: 1647.0000 - true_negatives_7: 1475073.0000 - false_negatives_7: 3051.0000 - sensitivity_at_specificity_7: 0.9935 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9512 - recall_7: 0.9132 - auc_7: 0.9961\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2387 - accuracy: 0.9346 - true_positives_7: 32226.0000 - false_positives_7: 1656.0000 - true_negatives_7: 1480440.0000 - false_negatives_7: 3062.0000 - sensitivity_at_specificity_7: 0.9935 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9511 - recall_7: 0.9132 - auc_7: 0.9961 - val_loss: 0.1316 - val_accuracy: 0.9653 - val_true_positives_7: 3753.0000 - val_false_positives_7: 108.0000 - val_true_negatives_7: 164574.0000 - val_false_negatives_7: 168.0000 - val_sensitivity_at_specificity_7: 0.9967 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9720 - val_recall_7: 0.9572 - val_auc_7: 0.9981\n",
            "Epoch 8/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2313 - accuracy: 0.9365 - true_positives_7: 32298.0000 - false_positives_7: 1640.0000 - true_negatives_7: 1475080.0000 - false_negatives_7: 2862.0000 - sensitivity_at_specificity_7: 0.9940 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9517 - recall_7: 0.9186 - auc_7: 0.9964\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2313 - accuracy: 0.9365 - true_positives_7: 32413.0000 - false_positives_7: 1647.0000 - true_negatives_7: 1480449.0000 - false_negatives_7: 2875.0000 - sensitivity_at_specificity_7: 0.9940 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9516 - recall_7: 0.9185 - auc_7: 0.9964 - val_loss: 0.1359 - val_accuracy: 0.9651 - val_true_positives_7: 3750.0000 - val_false_positives_7: 100.0000 - val_true_negatives_7: 164582.0000 - val_false_negatives_7: 171.0000 - val_sensitivity_at_specificity_7: 0.9977 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9740 - val_recall_7: 0.9564 - val_auc_7: 0.9987\n",
            "Epoch 9/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2003 - accuracy: 0.9464 - true_positives_7: 32924.0000 - false_positives_7: 1405.0000 - true_negatives_7: 1480691.0000 - false_negatives_7: 2364.0000 - sensitivity_at_specificity_7: 0.9940 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9591 - recall_7: 0.9330 - auc_7: 0.9966\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2003 - accuracy: 0.9464 - true_positives_7: 32924.0000 - false_positives_7: 1405.0000 - true_negatives_7: 1480691.0000 - false_negatives_7: 2364.0000 - sensitivity_at_specificity_7: 0.9940 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9591 - recall_7: 0.9330 - auc_7: 0.9966 - val_loss: 0.1117 - val_accuracy: 0.9732 - val_true_positives_7: 3788.0000 - val_false_positives_7: 82.0000 - val_true_negatives_7: 164600.0000 - val_false_negatives_7: 133.0000 - val_sensitivity_at_specificity_7: 0.9962 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9788 - val_recall_7: 0.9661 - val_auc_7: 0.9979\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1911 - accuracy: 0.9479 - true_positives_7: 33011.0000 - false_positives_7: 1390.0000 - true_negatives_7: 1480706.0000 - false_negatives_7: 2277.0000 - sensitivity_at_specificity_7: 0.9937 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9596 - recall_7: 0.9355 - auc_7: 0.9964\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1911 - accuracy: 0.9479 - true_positives_7: 33011.0000 - false_positives_7: 1390.0000 - true_negatives_7: 1480706.0000 - false_negatives_7: 2277.0000 - sensitivity_at_specificity_7: 0.9937 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9596 - recall_7: 0.9355 - auc_7: 0.9964 - val_loss: 0.1385 - val_accuracy: 0.9630 - val_true_positives_7: 3751.0000 - val_false_positives_7: 122.0000 - val_true_negatives_7: 164560.0000 - val_false_negatives_7: 170.0000 - val_sensitivity_at_specificity_7: 0.9959 - val_specificity_at_sensitivity_7: 0.9999 - val_precision_7: 0.9685 - val_recall_7: 0.9566 - val_auc_7: 0.9977\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1916 - accuracy: 0.9470 - true_positives_7: 32912.0000 - false_positives_7: 1427.0000 - true_negatives_7: 1475293.0000 - false_negatives_7: 2248.0000 - sensitivity_at_specificity_7: 0.9938 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9584 - recall_7: 0.9361 - auc_7: 0.9965\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1914 - accuracy: 0.9470 - true_positives_7: 33032.0000 - false_positives_7: 1433.0000 - true_negatives_7: 1480663.0000 - false_negatives_7: 2256.0000 - sensitivity_at_specificity_7: 0.9938 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9584 - recall_7: 0.9361 - auc_7: 0.9965 - val_loss: 0.1228 - val_accuracy: 0.9694 - val_true_positives_7: 3778.0000 - val_false_positives_7: 109.0000 - val_true_negatives_7: 164573.0000 - val_false_negatives_7: 143.0000 - val_sensitivity_at_specificity_7: 0.9954 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9720 - val_recall_7: 0.9635 - val_auc_7: 0.9976\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1951 - accuracy: 0.9490 - true_positives_7: 33129.0000 - false_positives_7: 1382.0000 - true_negatives_7: 1480714.0000 - false_negatives_7: 2159.0000 - sensitivity_at_specificity_7: 0.9931 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9600 - recall_7: 0.9388 - auc_7: 0.9962\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1951 - accuracy: 0.9490 - true_positives_7: 33129.0000 - false_positives_7: 1382.0000 - true_negatives_7: 1480714.0000 - false_negatives_7: 2159.0000 - sensitivity_at_specificity_7: 0.9931 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9600 - recall_7: 0.9388 - auc_7: 0.9962 - val_loss: 0.1135 - val_accuracy: 0.9707 - val_true_positives_7: 3782.0000 - val_false_positives_7: 95.0000 - val_true_negatives_7: 164587.0000 - val_false_negatives_7: 139.0000 - val_sensitivity_at_specificity_7: 0.9964 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9755 - val_recall_7: 0.9645 - val_auc_7: 0.9981\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1507 - accuracy: 0.9588 - true_positives_7: 33417.0000 - false_positives_7: 1175.0000 - true_negatives_7: 1475545.0000 - false_negatives_7: 1743.0000 - sensitivity_at_specificity_7: 0.9953 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9660 - recall_7: 0.9504 - auc_7: 0.9974\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1509 - accuracy: 0.9588 - true_positives_7: 33534.0000 - false_positives_7: 1181.0000 - true_negatives_7: 1480915.0000 - false_negatives_7: 1754.0000 - sensitivity_at_specificity_7: 0.9952 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9660 - recall_7: 0.9503 - auc_7: 0.9974 - val_loss: 0.0606 - val_accuracy: 0.9829 - val_true_positives_7: 3844.0000 - val_false_positives_7: 56.0000 - val_true_negatives_7: 164626.0000 - val_false_negatives_7: 77.0000 - val_sensitivity_at_specificity_7: 0.9985 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9856 - val_recall_7: 0.9804 - val_auc_7: 0.9992\n",
            "Epoch 14/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1380 - accuracy: 0.9639 - true_positives_7: 33767.0000 - false_positives_7: 1005.0000 - true_negatives_7: 1481091.0000 - false_negatives_7: 1521.0000 - sensitivity_at_specificity_7: 0.9953 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9711 - recall_7: 0.9569 - auc_7: 0.9974\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1380 - accuracy: 0.9639 - true_positives_7: 33767.0000 - false_positives_7: 1005.0000 - true_negatives_7: 1481091.0000 - false_negatives_7: 1521.0000 - sensitivity_at_specificity_7: 0.9953 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9711 - recall_7: 0.9569 - auc_7: 0.9974 - val_loss: 0.1010 - val_accuracy: 0.9740 - val_true_positives_7: 3809.0000 - val_false_positives_7: 80.0000 - val_true_negatives_7: 164602.0000 - val_false_negatives_7: 112.0000 - val_sensitivity_at_specificity_7: 0.9959 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9794 - val_recall_7: 0.9714 - val_auc_7: 0.9978\n",
            "Epoch 15/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1916 - accuracy: 0.9517 - true_positives_7: 33292.0000 - false_positives_7: 1348.0000 - true_negatives_7: 1480748.0000 - false_negatives_7: 1996.0000 - sensitivity_at_specificity_7: 0.9919 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9611 - recall_7: 0.9434 - auc_7: 0.9956\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1916 - accuracy: 0.9517 - true_positives_7: 33292.0000 - false_positives_7: 1348.0000 - true_negatives_7: 1480748.0000 - false_negatives_7: 1996.0000 - sensitivity_at_specificity_7: 0.9919 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9611 - recall_7: 0.9434 - auc_7: 0.9956 - val_loss: 0.0735 - val_accuracy: 0.9786 - val_true_positives_7: 3831.0000 - val_false_positives_7: 81.0000 - val_true_negatives_7: 164601.0000 - val_false_negatives_7: 90.0000 - val_sensitivity_at_specificity_7: 0.9972 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9793 - val_recall_7: 0.9770 - val_auc_7: 0.9985\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1783 - accuracy: 0.9547 - true_positives_7: 33395.0000 - false_positives_7: 1281.0000 - true_negatives_7: 1480815.0000 - false_negatives_7: 1893.0000 - sensitivity_at_specificity_7: 0.9929 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9631 - recall_7: 0.9464 - auc_7: 0.9962\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1783 - accuracy: 0.9547 - true_positives_7: 33395.0000 - false_positives_7: 1281.0000 - true_negatives_7: 1480815.0000 - false_negatives_7: 1893.0000 - sensitivity_at_specificity_7: 0.9929 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9631 - recall_7: 0.9464 - auc_7: 0.9962 - val_loss: 0.0590 - val_accuracy: 0.9819 - val_true_positives_7: 3843.0000 - val_false_positives_7: 58.0000 - val_true_negatives_7: 164624.0000 - val_false_negatives_7: 78.0000 - val_sensitivity_at_specificity_7: 0.9982 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9851 - val_recall_7: 0.9801 - val_auc_7: 0.9991\n",
            "Epoch 17/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1684 - accuracy: 0.9565 - true_positives_7: 33521.0000 - false_positives_7: 1269.0000 - true_negatives_7: 1480827.0000 - false_negatives_7: 1767.0000 - sensitivity_at_specificity_7: 0.9935 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9635 - recall_7: 0.9499 - auc_7: 0.9965\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1684 - accuracy: 0.9565 - true_positives_7: 33521.0000 - false_positives_7: 1269.0000 - true_negatives_7: 1480827.0000 - false_negatives_7: 1767.0000 - sensitivity_at_specificity_7: 0.9935 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9635 - recall_7: 0.9499 - auc_7: 0.9965 - val_loss: 0.0694 - val_accuracy: 0.9832 - val_true_positives_7: 3843.0000 - val_false_positives_7: 54.0000 - val_true_negatives_7: 164628.0000 - val_false_negatives_7: 78.0000 - val_sensitivity_at_specificity_7: 0.9985 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9861 - val_recall_7: 0.9801 - val_auc_7: 0.9992\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1645 - accuracy: 0.9590 - true_positives_7: 33586.0000 - false_positives_7: 1203.0000 - true_negatives_7: 1480893.0000 - false_negatives_7: 1702.0000 - sensitivity_at_specificity_7: 0.9932 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9654 - recall_7: 0.9518 - auc_7: 0.9963\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1645 - accuracy: 0.9590 - true_positives_7: 33586.0000 - false_positives_7: 1203.0000 - true_negatives_7: 1480893.0000 - false_negatives_7: 1702.0000 - sensitivity_at_specificity_7: 0.9932 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9654 - recall_7: 0.9518 - auc_7: 0.9963 - val_loss: 0.0658 - val_accuracy: 0.9839 - val_true_positives_7: 3856.0000 - val_false_positives_7: 53.0000 - val_true_negatives_7: 164629.0000 - val_false_negatives_7: 65.0000 - val_sensitivity_at_specificity_7: 0.9977 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9864 - val_recall_7: 0.9834 - val_auc_7: 0.9988\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1575 - accuracy: 0.9613 - true_positives_7: 33563.0000 - false_positives_7: 1111.0000 - true_negatives_7: 1475609.0000 - false_negatives_7: 1597.0000 - sensitivity_at_specificity_7: 0.9933 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9680 - recall_7: 0.9546 - auc_7: 0.9964\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1573 - accuracy: 0.9614 - true_positives_7: 33688.0000 - false_positives_7: 1113.0000 - true_negatives_7: 1480983.0000 - false_negatives_7: 1600.0000 - sensitivity_at_specificity_7: 0.9933 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9680 - recall_7: 0.9547 - auc_7: 0.9964 - val_loss: 0.0759 - val_accuracy: 0.9791 - val_true_positives_7: 3830.0000 - val_false_positives_7: 73.0000 - val_true_negatives_7: 164609.0000 - val_false_negatives_7: 91.0000 - val_sensitivity_at_specificity_7: 0.9974 - val_specificity_at_sensitivity_7: 1.0000 - val_precision_7: 0.9813 - val_recall_7: 0.9768 - val_auc_7: 0.9987\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1594 - accuracy: 0.9605 - true_positives_7: 33699.0000 - false_positives_7: 1165.0000 - true_negatives_7: 1480931.0000 - false_negatives_7: 1589.0000 - sensitivity_at_specificity_7: 0.9927 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9666 - recall_7: 0.9550 - auc_7: 0.9961\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1594 - accuracy: 0.9605 - true_positives_7: 33699.0000 - false_positives_7: 1165.0000 - true_negatives_7: 1480931.0000 - false_negatives_7: 1589.0000 - sensitivity_at_specificity_7: 0.9927 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9666 - recall_7: 0.9550 - auc_7: 0.9961 - val_loss: 0.0978 - val_accuracy: 0.9852 - val_true_positives_7: 3860.0000 - val_false_positives_7: 57.0000 - val_true_negatives_7: 164625.0000 - val_false_negatives_7: 61.0000 - val_sensitivity_at_specificity_7: 0.9949 - val_specificity_at_sensitivity_7: 0.9999 - val_precision_7: 0.9854 - val_recall_7: 0.9844 - val_auc_7: 0.9974\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0978 - accuracy: 0.9852 - true_positives_7: 3860.0000 - false_positives_7: 57.0000 - true_negatives_7: 164625.0000 - false_negatives_7: 61.0000 - sensitivity_at_specificity_7: 0.9949 - specificity_at_sensitivity_7: 0.9999 - precision_7: 0.9854 - recall_7: 0.9844 - auc_7: 0.9974\n",
            "9\n",
            "Epoch 1/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 3.3477 - accuracy: 0.5185 - true_positives_8: 12150.0000 - false_positives_8: 3969.0000 - true_negatives_8: 1478127.0000 - false_negatives_8: 23138.0000 - sensitivity_at_specificity_8: 0.9490 - specificity_at_sensitivity_8: 0.9902 - precision_8: 0.7538 - recall_8: 0.3443 - auc_8: 0.9303\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 44ms/step - loss: 3.3477 - accuracy: 0.5185 - true_positives_8: 12150.0000 - false_positives_8: 3969.0000 - true_negatives_8: 1478127.0000 - false_negatives_8: 23138.0000 - sensitivity_at_specificity_8: 0.9490 - specificity_at_sensitivity_8: 0.9902 - precision_8: 0.7538 - recall_8: 0.3443 - auc_8: 0.9303 - val_loss: 0.6827 - val_accuracy: 0.8161 - val_true_positives_8: 2654.0000 - val_false_positives_8: 313.0000 - val_true_negatives_8: 164369.0000 - val_false_negatives_8: 1267.0000 - val_sensitivity_at_specificity_8: 0.9923 - val_specificity_at_sensitivity_8: 0.9994 - val_precision_8: 0.8945 - val_recall_8: 0.6769 - val_auc_8: 0.9909\n",
            "Epoch 2/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.7241 - accuracy: 0.7974 - true_positives_8: 24380.0000 - false_positives_8: 3287.0000 - true_negatives_8: 1473433.0000 - false_negatives_8: 10780.0000 - sensitivity_at_specificity_8: 0.9870 - specificity_at_sensitivity_8: 0.9995 - precision_8: 0.8812 - recall_8: 0.6934 - auc_8: 0.9877\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.7241 - accuracy: 0.7974 - true_positives_8: 24476.0000 - false_positives_8: 3297.0000 - true_negatives_8: 1478799.0000 - false_negatives_8: 10812.0000 - sensitivity_at_specificity_8: 0.9870 - specificity_at_sensitivity_8: 0.9995 - precision_8: 0.8813 - recall_8: 0.6936 - auc_8: 0.9877 - val_loss: 0.3358 - val_accuracy: 0.8967 - val_true_positives_8: 3388.0000 - val_false_positives_8: 246.0000 - val_true_negatives_8: 164436.0000 - val_false_negatives_8: 533.0000 - val_sensitivity_at_specificity_8: 0.9921 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9323 - val_recall_8: 0.8641 - val_auc_8: 0.9948\n",
            "Epoch 3/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.4589 - accuracy: 0.8739 - true_positives_8: 28706.0000 - false_positives_8: 2541.0000 - true_negatives_8: 1474179.0000 - false_negatives_8: 6454.0000 - sensitivity_at_specificity_8: 0.9893 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9187 - recall_8: 0.8164 - auc_8: 0.9924\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.4583 - accuracy: 0.8741 - true_positives_8: 28820.0000 - false_positives_8: 2547.0000 - true_negatives_8: 1479549.0000 - false_negatives_8: 6468.0000 - sensitivity_at_specificity_8: 0.9893 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9188 - recall_8: 0.8167 - auc_8: 0.9924 - val_loss: 0.2991 - val_accuracy: 0.9163 - val_true_positives_8: 3474.0000 - val_false_positives_8: 217.0000 - val_true_negatives_8: 164465.0000 - val_false_negatives_8: 447.0000 - val_sensitivity_at_specificity_8: 0.9923 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9412 - val_recall_8: 0.8860 - val_auc_8: 0.9952\n",
            "Epoch 4/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3473 - accuracy: 0.9071 - true_positives_8: 30711.0000 - false_positives_8: 2128.0000 - true_negatives_8: 1479968.0000 - false_negatives_8: 4577.0000 - sensitivity_at_specificity_8: 0.9914 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9352 - recall_8: 0.8703 - auc_8: 0.9945\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.3473 - accuracy: 0.9071 - true_positives_8: 30711.0000 - false_positives_8: 2128.0000 - true_negatives_8: 1479968.0000 - false_negatives_8: 4577.0000 - sensitivity_at_specificity_8: 0.9914 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9352 - recall_8: 0.8703 - auc_8: 0.9945 - val_loss: 0.1319 - val_accuracy: 0.9689 - val_true_positives_8: 3759.0000 - val_false_positives_8: 105.0000 - val_true_negatives_8: 164577.0000 - val_false_negatives_8: 162.0000 - val_sensitivity_at_specificity_8: 0.9962 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9728 - val_recall_8: 0.9587 - val_auc_8: 0.9979\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2762 - accuracy: 0.9247 - true_positives_8: 31634.0000 - false_positives_8: 1849.0000 - true_negatives_8: 1474871.0000 - false_negatives_8: 3526.0000 - sensitivity_at_specificity_8: 0.9931 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9448 - recall_8: 0.8997 - auc_8: 0.9957\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2769 - accuracy: 0.9245 - true_positives_8: 31745.0000 - false_positives_8: 1862.0000 - true_negatives_8: 1480234.0000 - false_negatives_8: 3543.0000 - sensitivity_at_specificity_8: 0.9931 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9446 - recall_8: 0.8996 - auc_8: 0.9957 - val_loss: 0.1024 - val_accuracy: 0.9730 - val_true_positives_8: 3787.0000 - val_false_positives_8: 89.0000 - val_true_negatives_8: 164593.0000 - val_false_negatives_8: 134.0000 - val_sensitivity_at_specificity_8: 0.9972 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9770 - val_recall_8: 0.9658 - val_auc_8: 0.9985\n",
            "Epoch 6/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2246 - accuracy: 0.9388 - true_positives_8: 32352.0000 - false_positives_8: 1501.0000 - true_negatives_8: 1475219.0000 - false_negatives_8: 2808.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9557 - recall_8: 0.9201 - auc_8: 0.9963\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.2245 - accuracy: 0.9387 - true_positives_8: 32469.0000 - false_positives_8: 1509.0000 - true_negatives_8: 1480587.0000 - false_negatives_8: 2819.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9556 - recall_8: 0.9201 - auc_8: 0.9963 - val_loss: 0.1090 - val_accuracy: 0.9745 - val_true_positives_8: 3778.0000 - val_false_positives_8: 76.0000 - val_true_negatives_8: 164606.0000 - val_false_negatives_8: 143.0000 - val_sensitivity_at_specificity_8: 0.9964 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9803 - val_recall_8: 0.9635 - val_auc_8: 0.9981\n",
            "Epoch 7/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2078 - accuracy: 0.9446 - true_positives_8: 32800.0000 - false_positives_8: 1465.0000 - true_negatives_8: 1480631.0000 - false_negatives_8: 2488.0000 - sensitivity_at_specificity_8: 0.9939 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9572 - recall_8: 0.9295 - auc_8: 0.9965\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.2078 - accuracy: 0.9446 - true_positives_8: 32800.0000 - false_positives_8: 1465.0000 - true_negatives_8: 1480631.0000 - false_negatives_8: 2488.0000 - sensitivity_at_specificity_8: 0.9939 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9572 - recall_8: 0.9295 - auc_8: 0.9965 - val_loss: 0.0861 - val_accuracy: 0.9778 - val_true_positives_8: 3815.0000 - val_false_positives_8: 72.0000 - val_true_negatives_8: 164610.0000 - val_false_negatives_8: 106.0000 - val_sensitivity_at_specificity_8: 0.9972 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9815 - val_recall_8: 0.9730 - val_auc_8: 0.9985\n",
            "Epoch 8/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1985 - accuracy: 0.9480 - true_positives_8: 32987.0000 - false_positives_8: 1375.0000 - true_negatives_8: 1480721.0000 - false_negatives_8: 2301.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9600 - recall_8: 0.9348 - auc_8: 0.9964\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1985 - accuracy: 0.9480 - true_positives_8: 32987.0000 - false_positives_8: 1375.0000 - true_negatives_8: 1480721.0000 - false_negatives_8: 2301.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9600 - recall_8: 0.9348 - auc_8: 0.9964 - val_loss: 0.1243 - val_accuracy: 0.9658 - val_true_positives_8: 3762.0000 - val_false_positives_8: 118.0000 - val_true_negatives_8: 164564.0000 - val_false_negatives_8: 159.0000 - val_sensitivity_at_specificity_8: 0.9957 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9696 - val_recall_8: 0.9594 - val_auc_8: 0.9977\n",
            "Epoch 9/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1778 - accuracy: 0.9535 - true_positives_8: 33162.0000 - false_positives_8: 1218.0000 - true_negatives_8: 1475502.0000 - false_negatives_8: 1998.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9646 - recall_8: 0.9432 - auc_8: 0.9967\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1777 - accuracy: 0.9535 - true_positives_8: 33282.0000 - false_positives_8: 1223.0000 - true_negatives_8: 1480873.0000 - false_negatives_8: 2006.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9646 - recall_8: 0.9432 - auc_8: 0.9966 - val_loss: 0.0983 - val_accuracy: 0.9740 - val_true_positives_8: 3805.0000 - val_false_positives_8: 86.0000 - val_true_negatives_8: 164596.0000 - val_false_negatives_8: 116.0000 - val_sensitivity_at_specificity_8: 0.9952 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9779 - val_recall_8: 0.9704 - val_auc_8: 0.9975\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1508 - accuracy: 0.9601 - true_positives_8: 33539.0000 - false_positives_8: 1066.0000 - true_negatives_8: 1481030.0000 - false_negatives_8: 1749.0000 - sensitivity_at_specificity_8: 0.9946 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9692 - recall_8: 0.9504 - auc_8: 0.9971\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1508 - accuracy: 0.9601 - true_positives_8: 33539.0000 - false_positives_8: 1066.0000 - true_negatives_8: 1481030.0000 - false_negatives_8: 1749.0000 - sensitivity_at_specificity_8: 0.9946 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9692 - recall_8: 0.9504 - auc_8: 0.9971 - val_loss: 0.0652 - val_accuracy: 0.9852 - val_true_positives_8: 3852.0000 - val_false_positives_8: 47.0000 - val_true_negatives_8: 164635.0000 - val_false_negatives_8: 69.0000 - val_sensitivity_at_specificity_8: 0.9974 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9879 - val_recall_8: 0.9824 - val_auc_8: 0.9987\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1651 - accuracy: 0.9561 - true_positives_8: 33301.0000 - false_positives_8: 1231.0000 - true_negatives_8: 1475489.0000 - false_negatives_8: 1859.0000 - sensitivity_at_specificity_8: 0.9941 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9644 - recall_8: 0.9471 - auc_8: 0.9968\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1648 - accuracy: 0.9562 - true_positives_8: 33426.0000 - false_positives_8: 1234.0000 - true_negatives_8: 1480862.0000 - false_negatives_8: 1862.0000 - sensitivity_at_specificity_8: 0.9941 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9644 - recall_8: 0.9472 - auc_8: 0.9968 - val_loss: 0.1118 - val_accuracy: 0.9735 - val_true_positives_8: 3793.0000 - val_false_positives_8: 87.0000 - val_true_negatives_8: 164595.0000 - val_false_negatives_8: 128.0000 - val_sensitivity_at_specificity_8: 0.9957 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9776 - val_recall_8: 0.9674 - val_auc_8: 0.9977\n",
            "Epoch 12/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1772 - accuracy: 0.9544 - true_positives_8: 33238.0000 - false_positives_8: 1282.0000 - true_negatives_8: 1475438.0000 - false_negatives_8: 1922.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9629 - recall_8: 0.9453 - auc_8: 0.9965\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1772 - accuracy: 0.9543 - true_positives_8: 33357.0000 - false_positives_8: 1289.0000 - true_negatives_8: 1480807.0000 - false_negatives_8: 1931.0000 - sensitivity_at_specificity_8: 0.9936 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9628 - recall_8: 0.9453 - auc_8: 0.9965 - val_loss: 0.0536 - val_accuracy: 0.9834 - val_true_positives_8: 3847.0000 - val_false_positives_8: 52.0000 - val_true_negatives_8: 164630.0000 - val_false_negatives_8: 74.0000 - val_sensitivity_at_specificity_8: 0.9987 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9867 - val_recall_8: 0.9811 - val_auc_8: 0.9993\n",
            "Epoch 13/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1521 - accuracy: 0.9609 - true_positives_8: 33535.0000 - false_positives_8: 1098.0000 - true_negatives_8: 1475622.0000 - false_negatives_8: 1625.0000 - sensitivity_at_specificity_8: 0.9943 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9683 - recall_8: 0.9538 - auc_8: 0.9969\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1519 - accuracy: 0.9610 - true_positives_8: 33659.0000 - false_positives_8: 1101.0000 - true_negatives_8: 1480995.0000 - false_negatives_8: 1629.0000 - sensitivity_at_specificity_8: 0.9943 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9683 - recall_8: 0.9538 - auc_8: 0.9969 - val_loss: 0.0597 - val_accuracy: 0.9857 - val_true_positives_8: 3857.0000 - val_false_positives_8: 50.0000 - val_true_negatives_8: 164632.0000 - val_false_negatives_8: 64.0000 - val_sensitivity_at_specificity_8: 0.9974 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9872 - val_recall_8: 0.9837 - val_auc_8: 0.9987\n",
            "Epoch 14/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1673 - accuracy: 0.9574 - true_positives_8: 33403.0000 - false_positives_8: 1203.0000 - true_negatives_8: 1475517.0000 - false_negatives_8: 1757.0000 - sensitivity_at_specificity_8: 0.9935 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9652 - recall_8: 0.9500 - auc_8: 0.9965\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 24s 86ms/step - loss: 0.1676 - accuracy: 0.9574 - true_positives_8: 33523.0000 - false_positives_8: 1207.0000 - true_negatives_8: 1480889.0000 - false_negatives_8: 1765.0000 - sensitivity_at_specificity_8: 0.9935 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9652 - recall_8: 0.9500 - auc_8: 0.9965 - val_loss: 0.1772 - val_accuracy: 0.9554 - val_true_positives_8: 3736.0000 - val_false_positives_8: 161.0000 - val_true_negatives_8: 164521.0000 - val_false_negatives_8: 185.0000 - val_sensitivity_at_specificity_8: 0.9911 - val_specificity_at_sensitivity_8: 0.9998 - val_precision_8: 0.9587 - val_recall_8: 0.9528 - val_auc_8: 0.9953\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1551 - accuracy: 0.9602 - true_positives_8: 33561.0000 - false_positives_8: 1151.0000 - true_negatives_8: 1477249.0000 - false_negatives_8: 1639.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9668 - recall_8: 0.9534 - auc_8: 0.9968\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 15s 54ms/step - loss: 0.1554 - accuracy: 0.9601 - true_positives_8: 33641.0000 - false_positives_8: 1156.0000 - true_negatives_8: 1480940.0000 - false_negatives_8: 1647.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9668 - recall_8: 0.9533 - auc_8: 0.9968 - val_loss: 0.0400 - val_accuracy: 0.9888 - val_true_positives_8: 3873.0000 - val_false_positives_8: 36.0000 - val_true_negatives_8: 164646.0000 - val_false_negatives_8: 48.0000 - val_sensitivity_at_specificity_8: 0.9982 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9908 - val_recall_8: 0.9878 - val_auc_8: 0.9991\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1698 - accuracy: 0.9584 - true_positives_8: 33586.0000 - false_positives_8: 1203.0000 - true_negatives_8: 1480893.0000 - false_negatives_8: 1702.0000 - sensitivity_at_specificity_8: 0.9928 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9654 - recall_8: 0.9518 - auc_8: 0.9961\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 15s 54ms/step - loss: 0.1698 - accuracy: 0.9584 - true_positives_8: 33586.0000 - false_positives_8: 1203.0000 - true_negatives_8: 1480893.0000 - false_negatives_8: 1702.0000 - sensitivity_at_specificity_8: 0.9928 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9654 - recall_8: 0.9518 - auc_8: 0.9961 - val_loss: 0.0669 - val_accuracy: 0.9852 - val_true_positives_8: 3858.0000 - val_false_positives_8: 50.0000 - val_true_negatives_8: 164632.0000 - val_false_negatives_8: 63.0000 - val_sensitivity_at_specificity_8: 0.9962 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9872 - val_recall_8: 0.9839 - val_auc_8: 0.9980\n",
            "Epoch 17/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1671 - accuracy: 0.9576 - true_positives_8: 33421.0000 - false_positives_8: 1207.0000 - true_negatives_8: 1475513.0000 - false_negatives_8: 1739.0000 - sensitivity_at_specificity_8: 0.9932 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9651 - recall_8: 0.9505 - auc_8: 0.9964\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1668 - accuracy: 0.9576 - true_positives_8: 33544.0000 - false_positives_8: 1211.0000 - true_negatives_8: 1480885.0000 - false_negatives_8: 1744.0000 - sensitivity_at_specificity_8: 0.9933 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9652 - recall_8: 0.9506 - auc_8: 0.9964 - val_loss: 0.0438 - val_accuracy: 0.9885 - val_true_positives_8: 3869.0000 - val_false_positives_8: 38.0000 - val_true_negatives_8: 164644.0000 - val_false_negatives_8: 52.0000 - val_sensitivity_at_specificity_8: 0.9982 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9903 - val_recall_8: 0.9867 - val_auc_8: 0.9991\n",
            "Epoch 18/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1426 - accuracy: 0.9658 - true_positives_8: 33883.0000 - false_positives_8: 986.0000 - true_negatives_8: 1481110.0000 - false_negatives_8: 1405.0000 - sensitivity_at_specificity_8: 0.9938 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9717 - recall_8: 0.9602 - auc_8: 0.9967\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1426 - accuracy: 0.9658 - true_positives_8: 33883.0000 - false_positives_8: 986.0000 - true_negatives_8: 1481110.0000 - false_negatives_8: 1405.0000 - sensitivity_at_specificity_8: 0.9938 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9717 - recall_8: 0.9602 - auc_8: 0.9967 - val_loss: 0.0477 - val_accuracy: 0.9885 - val_true_positives_8: 3874.0000 - val_false_positives_8: 39.0000 - val_true_negatives_8: 164643.0000 - val_false_negatives_8: 47.0000 - val_sensitivity_at_specificity_8: 0.9962 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9900 - val_recall_8: 0.9880 - val_auc_8: 0.9980\n",
            "Epoch 19/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1362 - accuracy: 0.9652 - true_positives_8: 33882.0000 - false_positives_8: 1024.0000 - true_negatives_8: 1481072.0000 - false_negatives_8: 1406.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9707 - recall_8: 0.9602 - auc_8: 0.9968\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1362 - accuracy: 0.9652 - true_positives_8: 33882.0000 - false_positives_8: 1024.0000 - true_negatives_8: 1481072.0000 - false_negatives_8: 1406.0000 - sensitivity_at_specificity_8: 0.9940 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9707 - recall_8: 0.9602 - auc_8: 0.9968 - val_loss: 0.0597 - val_accuracy: 0.9827 - val_true_positives_8: 3850.0000 - val_false_positives_8: 59.0000 - val_true_negatives_8: 164623.0000 - val_false_negatives_8: 71.0000 - val_sensitivity_at_specificity_8: 0.9985 - val_specificity_at_sensitivity_8: 1.0000 - val_precision_8: 0.9849 - val_recall_8: 0.9819 - val_auc_8: 0.9992\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1321 - accuracy: 0.9683 - true_positives_8: 34030.0000 - false_positives_8: 938.0000 - true_negatives_8: 1481158.0000 - false_negatives_8: 1258.0000 - sensitivity_at_specificity_8: 0.9941 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9732 - recall_8: 0.9644 - auc_8: 0.9969\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 45ms/step - loss: 0.1321 - accuracy: 0.9683 - true_positives_8: 34030.0000 - false_positives_8: 938.0000 - true_negatives_8: 1481158.0000 - false_negatives_8: 1258.0000 - sensitivity_at_specificity_8: 0.9941 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9732 - recall_8: 0.9644 - auc_8: 0.9969 - val_loss: 0.0750 - val_accuracy: 0.9821 - val_true_positives_8: 3839.0000 - val_false_positives_8: 63.0000 - val_true_negatives_8: 164619.0000 - val_false_negatives_8: 82.0000 - val_sensitivity_at_specificity_8: 0.9962 - val_specificity_at_sensitivity_8: 0.9999 - val_precision_8: 0.9839 - val_recall_8: 0.9791 - val_auc_8: 0.9980\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0750 - accuracy: 0.9821 - true_positives_8: 3839.0000 - false_positives_8: 63.0000 - true_negatives_8: 164619.0000 - false_negatives_8: 82.0000 - sensitivity_at_specificity_8: 0.9962 - specificity_at_sensitivity_8: 0.9999 - precision_8: 0.9839 - recall_8: 0.9791 - auc_8: 0.9980\n",
            "10\n",
            "Epoch 1/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 2.6844 - accuracy: 0.5571 - true_positives_9: 13886.0000 - false_positives_9: 3859.0000 - true_negatives_9: 1472903.0000 - false_negatives_9: 21275.0000 - sensitivity_at_specificity_9: 0.9518 - specificity_at_sensitivity_9: 0.9936 - precision_9: 0.7825 - recall_9: 0.3949 - auc_9: 0.9384\n",
            "Epoch 1: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 14s 46ms/step - loss: 2.6772 - accuracy: 0.5581 - true_positives_9: 13970.0000 - false_positives_9: 3863.0000 - true_negatives_9: 1478275.0000 - false_negatives_9: 21319.0000 - sensitivity_at_specificity_9: 0.9520 - specificity_at_sensitivity_9: 0.9936 - precision_9: 0.7834 - recall_9: 0.3959 - auc_9: 0.9386 - val_loss: 0.5955 - val_accuracy: 0.8283 - val_true_positives_9: 2882.0000 - val_false_positives_9: 299.0000 - val_true_negatives_9: 164341.0000 - val_false_negatives_9: 1038.0000 - val_sensitivity_at_specificity_9: 0.9941 - val_specificity_at_sensitivity_9: 0.9996 - val_precision_9: 0.9060 - val_recall_9: 0.7352 - val_auc_9: 0.9932\n",
            "Epoch 2/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.5915 - accuracy: 0.8319 - true_positives_9: 26730.0000 - false_positives_9: 3061.0000 - true_negatives_9: 1479077.0000 - false_negatives_9: 8559.0000 - sensitivity_at_specificity_9: 0.9880 - specificity_at_sensitivity_9: 0.9997 - precision_9: 0.8973 - recall_9: 0.7575 - auc_9: 0.9902\n",
            "Epoch 2: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 48ms/step - loss: 0.5915 - accuracy: 0.8319 - true_positives_9: 26730.0000 - false_positives_9: 3061.0000 - true_negatives_9: 1479077.0000 - false_negatives_9: 8559.0000 - sensitivity_at_specificity_9: 0.9880 - specificity_at_sensitivity_9: 0.9997 - precision_9: 0.8973 - recall_9: 0.7575 - auc_9: 0.9902 - val_loss: 0.2303 - val_accuracy: 0.9327 - val_true_positives_9: 3558.0000 - val_false_positives_9: 169.0000 - val_true_negatives_9: 164471.0000 - val_false_negatives_9: 362.0000 - val_sensitivity_at_specificity_9: 0.9949 - val_specificity_at_sensitivity_9: 0.9999 - val_precision_9: 0.9547 - val_recall_9: 0.9077 - val_auc_9: 0.9969\n",
            "Epoch 3/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.3715 - accuracy: 0.8976 - true_positives_9: 30195.0000 - false_positives_9: 2260.0000 - true_negatives_9: 1479878.0000 - false_negatives_9: 5094.0000 - sensitivity_at_specificity_9: 0.9913 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9304 - recall_9: 0.8556 - auc_9: 0.9942\n",
            "Epoch 3: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.3715 - accuracy: 0.8976 - true_positives_9: 30195.0000 - false_positives_9: 2260.0000 - true_negatives_9: 1479878.0000 - false_negatives_9: 5094.0000 - sensitivity_at_specificity_9: 0.9913 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9304 - recall_9: 0.8556 - auc_9: 0.9942 - val_loss: 0.1393 - val_accuracy: 0.9610 - val_true_positives_9: 3730.0000 - val_false_positives_9: 115.0000 - val_true_negatives_9: 164525.0000 - val_false_negatives_9: 190.0000 - val_sensitivity_at_specificity_9: 0.9964 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9701 - val_recall_9: 0.9515 - val_auc_9: 0.9980\n",
            "Epoch 4/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.2489 - accuracy: 0.9299 - true_positives_9: 31969.0000 - false_positives_9: 1722.0000 - true_negatives_9: 1480416.0000 - false_negatives_9: 3320.0000 - sensitivity_at_specificity_9: 0.9948 - specificity_at_sensitivity_9: 1.0000 - precision_9: 0.9489 - recall_9: 0.9059 - auc_9: 0.9967\n",
            "Epoch 4: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.2489 - accuracy: 0.9299 - true_positives_9: 31969.0000 - false_positives_9: 1722.0000 - true_negatives_9: 1480416.0000 - false_negatives_9: 3320.0000 - sensitivity_at_specificity_9: 0.9948 - specificity_at_sensitivity_9: 1.0000 - precision_9: 0.9489 - recall_9: 0.9059 - auc_9: 0.9967 - val_loss: 0.1110 - val_accuracy: 0.9673 - val_true_positives_9: 3770.0000 - val_false_positives_9: 97.0000 - val_true_negatives_9: 164543.0000 - val_false_negatives_9: 150.0000 - val_sensitivity_at_specificity_9: 0.9974 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9749 - val_recall_9: 0.9617 - val_auc_9: 0.9986\n",
            "Epoch 5/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2188 - accuracy: 0.9393 - true_positives_9: 32412.0000 - false_positives_9: 1560.0000 - true_negatives_9: 1475202.0000 - false_negatives_9: 2749.0000 - sensitivity_at_specificity_9: 0.9942 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9541 - recall_9: 0.9218 - auc_9: 0.9966\n",
            "Epoch 5: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.2193 - accuracy: 0.9392 - true_positives_9: 32525.0000 - false_positives_9: 1571.0000 - true_negatives_9: 1480567.0000 - false_negatives_9: 2764.0000 - sensitivity_at_specificity_9: 0.9942 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9539 - recall_9: 0.9217 - auc_9: 0.9966 - val_loss: 0.1038 - val_accuracy: 0.9712 - val_true_positives_9: 3778.0000 - val_false_positives_9: 85.0000 - val_true_negatives_9: 164555.0000 - val_false_negatives_9: 142.0000 - val_sensitivity_at_specificity_9: 0.9987 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9780 - val_recall_9: 0.9638 - val_auc_9: 0.9992\n",
            "Epoch 6/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.2056 - accuracy: 0.9426 - true_positives_9: 32622.0000 - false_positives_9: 1522.0000 - true_negatives_9: 1475240.0000 - false_negatives_9: 2539.0000 - sensitivity_at_specificity_9: 0.9941 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9554 - recall_9: 0.9278 - auc_9: 0.9966\n",
            "Epoch 6: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.2055 - accuracy: 0.9425 - true_positives_9: 32740.0000 - false_positives_9: 1527.0000 - true_negatives_9: 1480611.0000 - false_negatives_9: 2549.0000 - sensitivity_at_specificity_9: 0.9941 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9554 - recall_9: 0.9278 - auc_9: 0.9966 - val_loss: 0.0766 - val_accuracy: 0.9786 - val_true_positives_9: 3820.0000 - val_false_positives_9: 64.0000 - val_true_negatives_9: 164576.0000 - val_false_negatives_9: 100.0000 - val_sensitivity_at_specificity_9: 0.9977 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9835 - val_recall_9: 0.9745 - val_auc_9: 0.9988\n",
            "Epoch 7/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1671 - accuracy: 0.9555 - true_positives_9: 33232.0000 - false_positives_9: 1190.0000 - true_negatives_9: 1475572.0000 - false_negatives_9: 1929.0000 - sensitivity_at_specificity_9: 0.9946 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9654 - recall_9: 0.9451 - auc_9: 0.9970\n",
            "Epoch 7: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1674 - accuracy: 0.9555 - true_positives_9: 33353.0000 - false_positives_9: 1194.0000 - true_negatives_9: 1480944.0000 - false_negatives_9: 1936.0000 - sensitivity_at_specificity_9: 0.9946 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9654 - recall_9: 0.9451 - auc_9: 0.9970 - val_loss: 0.0848 - val_accuracy: 0.9768 - val_true_positives_9: 3819.0000 - val_false_positives_9: 81.0000 - val_true_negatives_9: 164559.0000 - val_false_negatives_9: 101.0000 - val_sensitivity_at_specificity_9: 0.9969 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9792 - val_recall_9: 0.9742 - val_auc_9: 0.9984\n",
            "Epoch 8/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1590 - accuracy: 0.9584 - true_positives_9: 33370.0000 - false_positives_9: 1162.0000 - true_negatives_9: 1475600.0000 - false_negatives_9: 1791.0000 - sensitivity_at_specificity_9: 0.9948 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9664 - recall_9: 0.9491 - auc_9: 0.9971\n",
            "Epoch 8: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1587 - accuracy: 0.9585 - true_positives_9: 33494.0000 - false_positives_9: 1164.0000 - true_negatives_9: 1480974.0000 - false_negatives_9: 1795.0000 - sensitivity_at_specificity_9: 0.9948 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9664 - recall_9: 0.9491 - auc_9: 0.9972 - val_loss: 0.0723 - val_accuracy: 0.9834 - val_true_positives_9: 3844.0000 - val_false_positives_9: 56.0000 - val_true_negatives_9: 164584.0000 - val_false_negatives_9: 76.0000 - val_sensitivity_at_specificity_9: 0.9967 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9856 - val_recall_9: 0.9806 - val_auc_9: 0.9983\n",
            "Epoch 9/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1415 - accuracy: 0.9634 - true_positives_9: 33620.0000 - false_positives_9: 1023.0000 - true_negatives_9: 1475739.0000 - false_negatives_9: 1541.0000 - sensitivity_at_specificity_9: 0.9951 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9705 - recall_9: 0.9562 - auc_9: 0.9973\n",
            "Epoch 9: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1413 - accuracy: 0.9634 - true_positives_9: 33745.0000 - false_positives_9: 1026.0000 - true_negatives_9: 1481112.0000 - false_negatives_9: 1544.0000 - sensitivity_at_specificity_9: 0.9951 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9705 - recall_9: 0.9562 - auc_9: 0.9973 - val_loss: 0.0730 - val_accuracy: 0.9793 - val_true_positives_9: 3826.0000 - val_false_positives_9: 69.0000 - val_true_negatives_9: 164571.0000 - val_false_negatives_9: 94.0000 - val_sensitivity_at_specificity_9: 0.9977 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9823 - val_recall_9: 0.9760 - val_auc_9: 0.9988\n",
            "Epoch 10/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1409 - accuracy: 0.9619 - true_positives_9: 33724.0000 - false_positives_9: 1081.0000 - true_negatives_9: 1481057.0000 - false_negatives_9: 1565.0000 - sensitivity_at_specificity_9: 0.9947 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9689 - recall_9: 0.9557 - auc_9: 0.9972\n",
            "Epoch 10: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 13s 46ms/step - loss: 0.1409 - accuracy: 0.9619 - true_positives_9: 33724.0000 - false_positives_9: 1081.0000 - true_negatives_9: 1481057.0000 - false_negatives_9: 1565.0000 - sensitivity_at_specificity_9: 0.9947 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9689 - recall_9: 0.9557 - auc_9: 0.9972 - val_loss: 0.0584 - val_accuracy: 0.9844 - val_true_positives_9: 3853.0000 - val_false_positives_9: 49.0000 - val_true_negatives_9: 164591.0000 - val_false_negatives_9: 67.0000 - val_sensitivity_at_specificity_9: 0.9972 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9874 - val_recall_9: 0.9829 - val_auc_9: 0.9985\n",
            "Epoch 11/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1377 - accuracy: 0.9637 - true_positives_9: 33693.0000 - false_positives_9: 1037.0000 - true_negatives_9: 1475725.0000 - false_negatives_9: 1468.0000 - sensitivity_at_specificity_9: 0.9945 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9701 - recall_9: 0.9582 - auc_9: 0.9971\n",
            "Epoch 11: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 42ms/step - loss: 0.1377 - accuracy: 0.9637 - true_positives_9: 33816.0000 - false_positives_9: 1042.0000 - true_negatives_9: 1481096.0000 - false_negatives_9: 1473.0000 - sensitivity_at_specificity_9: 0.9945 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9701 - recall_9: 0.9583 - auc_9: 0.9971 - val_loss: 0.0573 - val_accuracy: 0.9852 - val_true_positives_9: 3854.0000 - val_false_positives_9: 52.0000 - val_true_negatives_9: 164588.0000 - val_false_negatives_9: 66.0000 - val_sensitivity_at_specificity_9: 0.9977 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9867 - val_recall_9: 0.9832 - val_auc_9: 0.9988\n",
            "Epoch 12/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1362 - accuracy: 0.9651 - true_positives_9: 33835.0000 - false_positives_9: 995.0000 - true_negatives_9: 1481143.0000 - false_negatives_9: 1454.0000 - sensitivity_at_specificity_9: 0.9944 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9714 - recall_9: 0.9588 - auc_9: 0.9970\n",
            "Epoch 12: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1362 - accuracy: 0.9651 - true_positives_9: 33835.0000 - false_positives_9: 995.0000 - true_negatives_9: 1481143.0000 - false_negatives_9: 1454.0000 - sensitivity_at_specificity_9: 0.9944 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9714 - recall_9: 0.9588 - auc_9: 0.9970 - val_loss: 0.0583 - val_accuracy: 0.9832 - val_true_positives_9: 3847.0000 - val_false_positives_9: 58.0000 - val_true_negatives_9: 164582.0000 - val_false_negatives_9: 73.0000 - val_sensitivity_at_specificity_9: 0.9982 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9851 - val_recall_9: 0.9814 - val_auc_9: 0.9991\n",
            "Epoch 13/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1329 - accuracy: 0.9667 - true_positives_9: 33914.0000 - false_positives_9: 946.0000 - true_negatives_9: 1481192.0000 - false_negatives_9: 1375.0000 - sensitivity_at_specificity_9: 0.9946 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9729 - recall_9: 0.9610 - auc_9: 0.9971\n",
            "Epoch 13: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1329 - accuracy: 0.9667 - true_positives_9: 33914.0000 - false_positives_9: 946.0000 - true_negatives_9: 1481192.0000 - false_negatives_9: 1375.0000 - sensitivity_at_specificity_9: 0.9946 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9729 - recall_9: 0.9610 - auc_9: 0.9971 - val_loss: 0.0760 - val_accuracy: 0.9844 - val_true_positives_9: 3856.0000 - val_false_positives_9: 57.0000 - val_true_negatives_9: 164583.0000 - val_false_negatives_9: 64.0000 - val_sensitivity_at_specificity_9: 0.9954 - val_specificity_at_sensitivity_9: 0.9999 - val_precision_9: 0.9854 - val_recall_9: 0.9837 - val_auc_9: 0.9976\n",
            "Epoch 14/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1426 - accuracy: 0.9650 - true_positives_9: 33855.0000 - false_positives_9: 1030.0000 - true_negatives_9: 1481108.0000 - false_negatives_9: 1434.0000 - sensitivity_at_specificity_9: 0.9936 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9705 - recall_9: 0.9594 - auc_9: 0.9966\n",
            "Epoch 14: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1426 - accuracy: 0.9650 - true_positives_9: 33855.0000 - false_positives_9: 1030.0000 - true_negatives_9: 1481108.0000 - false_negatives_9: 1434.0000 - sensitivity_at_specificity_9: 0.9936 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9705 - recall_9: 0.9594 - auc_9: 0.9966 - val_loss: 0.0618 - val_accuracy: 0.9824 - val_true_positives_9: 3838.0000 - val_false_positives_9: 57.0000 - val_true_negatives_9: 164583.0000 - val_false_negatives_9: 82.0000 - val_sensitivity_at_specificity_9: 0.9982 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9854 - val_recall_9: 0.9791 - val_auc_9: 0.9990\n",
            "Epoch 15/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1522 - accuracy: 0.9623 - true_positives_9: 33614.0000 - false_positives_9: 1098.0000 - true_negatives_9: 1475664.0000 - false_negatives_9: 1547.0000 - sensitivity_at_specificity_9: 0.9934 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9684 - recall_9: 0.9560 - auc_9: 0.9965\n",
            "Epoch 15: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1526 - accuracy: 0.9622 - true_positives_9: 33734.0000 - false_positives_9: 1103.0000 - true_negatives_9: 1481035.0000 - false_negatives_9: 1555.0000 - sensitivity_at_specificity_9: 0.9933 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9683 - recall_9: 0.9559 - auc_9: 0.9965 - val_loss: 0.1424 - val_accuracy: 0.9635 - val_true_positives_9: 3752.0000 - val_false_positives_9: 116.0000 - val_true_negatives_9: 164524.0000 - val_false_negatives_9: 168.0000 - val_sensitivity_at_specificity_9: 0.9934 - val_specificity_at_sensitivity_9: 0.9999 - val_precision_9: 0.9700 - val_recall_9: 0.9571 - val_auc_9: 0.9965\n",
            "Epoch 16/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1443 - accuracy: 0.9647 - true_positives_9: 33852.0000 - false_positives_9: 1041.0000 - true_negatives_9: 1481097.0000 - false_negatives_9: 1437.0000 - sensitivity_at_specificity_9: 0.9931 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9702 - recall_9: 0.9593 - auc_9: 0.9964\n",
            "Epoch 16: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 44ms/step - loss: 0.1443 - accuracy: 0.9647 - true_positives_9: 33852.0000 - false_positives_9: 1041.0000 - true_negatives_9: 1481097.0000 - false_negatives_9: 1437.0000 - sensitivity_at_specificity_9: 0.9931 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9702 - recall_9: 0.9593 - auc_9: 0.9964 - val_loss: 0.0599 - val_accuracy: 0.9824 - val_true_positives_9: 3844.0000 - val_false_positives_9: 58.0000 - val_true_negatives_9: 164582.0000 - val_false_negatives_9: 76.0000 - val_sensitivity_at_specificity_9: 0.9980 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9851 - val_recall_9: 0.9806 - val_auc_9: 0.9989\n",
            "Epoch 17/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1373 - accuracy: 0.9666 - true_positives_9: 33796.0000 - false_positives_9: 976.0000 - true_negatives_9: 1475786.0000 - false_negatives_9: 1365.0000 - sensitivity_at_specificity_9: 0.9942 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9719 - recall_9: 0.9612 - auc_9: 0.9969\n",
            "Epoch 17: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1374 - accuracy: 0.9666 - true_positives_9: 33919.0000 - false_positives_9: 979.0000 - true_negatives_9: 1481159.0000 - false_negatives_9: 1370.0000 - sensitivity_at_specificity_9: 0.9942 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9719 - recall_9: 0.9612 - auc_9: 0.9969 - val_loss: 0.0287 - val_accuracy: 0.9921 - val_true_positives_9: 3885.0000 - val_false_positives_9: 26.0000 - val_true_negatives_9: 164614.0000 - val_false_negatives_9: 35.0000 - val_sensitivity_at_specificity_9: 0.9992 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9934 - val_recall_9: 0.9911 - val_auc_9: 0.9996\n",
            "Epoch 18/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1220 - accuracy: 0.9699 - true_positives_9: 33986.0000 - false_positives_9: 899.0000 - true_negatives_9: 1475863.0000 - false_negatives_9: 1175.0000 - sensitivity_at_specificity_9: 0.9943 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9742 - recall_9: 0.9666 - auc_9: 0.9970\n",
            "Epoch 18: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 45ms/step - loss: 0.1218 - accuracy: 0.9699 - true_positives_9: 34111.0000 - false_positives_9: 901.0000 - true_negatives_9: 1481237.0000 - false_negatives_9: 1178.0000 - sensitivity_at_specificity_9: 0.9944 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9743 - recall_9: 0.9666 - auc_9: 0.9970 - val_loss: 0.0386 - val_accuracy: 0.9878 - val_true_positives_9: 3869.0000 - val_false_positives_9: 44.0000 - val_true_negatives_9: 164596.0000 - val_false_negatives_9: 51.0000 - val_sensitivity_at_specificity_9: 0.9995 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9888 - val_recall_9: 0.9870 - val_auc_9: 0.9997\n",
            "Epoch 19/20\n",
            "275/276 [============================>.] - ETA: 0s - loss: 0.1360 - accuracy: 0.9688 - true_positives_9: 33921.0000 - false_positives_9: 938.0000 - true_negatives_9: 1475824.0000 - false_negatives_9: 1240.0000 - sensitivity_at_specificity_9: 0.9938 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9731 - recall_9: 0.9647 - auc_9: 0.9967\n",
            "Epoch 19: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1356 - accuracy: 0.9688 - true_positives_9: 34046.0000 - false_positives_9: 940.0000 - true_negatives_9: 1481198.0000 - false_negatives_9: 1243.0000 - sensitivity_at_specificity_9: 0.9938 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9731 - recall_9: 0.9648 - auc_9: 0.9968 - val_loss: 0.0761 - val_accuracy: 0.9814 - val_true_positives_9: 3839.0000 - val_false_positives_9: 62.0000 - val_true_negatives_9: 164578.0000 - val_false_negatives_9: 81.0000 - val_sensitivity_at_specificity_9: 0.9959 - val_specificity_at_sensitivity_9: 0.9999 - val_precision_9: 0.9841 - val_recall_9: 0.9793 - val_auc_9: 0.9979\n",
            "Epoch 20/20\n",
            "276/276 [==============================] - ETA: 0s - loss: 0.1407 - accuracy: 0.9676 - true_positives_9: 34008.0000 - false_positives_9: 957.0000 - true_negatives_9: 1481181.0000 - false_negatives_9: 1281.0000 - sensitivity_at_specificity_9: 0.9934 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9726 - recall_9: 0.9637 - auc_9: 0.9965\n",
            "Epoch 20: saving model to AlexNet\\cp.ckpt\n",
            "276/276 [==============================] - 12s 43ms/step - loss: 0.1407 - accuracy: 0.9676 - true_positives_9: 34008.0000 - false_positives_9: 957.0000 - true_negatives_9: 1481181.0000 - false_negatives_9: 1281.0000 - sensitivity_at_specificity_9: 0.9934 - specificity_at_sensitivity_9: 0.9999 - precision_9: 0.9726 - recall_9: 0.9637 - auc_9: 0.9965 - val_loss: 0.0406 - val_accuracy: 0.9911 - val_true_positives_9: 3883.0000 - val_false_positives_9: 32.0000 - val_true_negatives_9: 164608.0000 - val_false_negatives_9: 37.0000 - val_sensitivity_at_specificity_9: 0.9974 - val_specificity_at_sensitivity_9: 1.0000 - val_precision_9: 0.9918 - val_recall_9: 0.9906 - val_auc_9: 0.9987\n",
            "123/123 [==============================] - 1s 7ms/step - loss: 0.0406 - accuracy: 0.9911 - true_positives_9: 3883.0000 - false_positives_9: 32.0000 - true_negatives_9: 164608.0000 - false_negatives_9: 37.0000 - sensitivity_at_specificity_9: 0.9974 - specificity_at_sensitivity_9: 1.0000 - precision_9: 0.9918 - recall_9: 0.9906 - auc_9: 0.9987\n",
            "Loss                0.067881\n",
            "Accuracy            0.984443\n",
            "TP               3852.900000\n",
            "FP                 54.700000\n",
            "TN             164623.100000\n",
            "FN                 68.000000\n",
            "Sensitivity         0.997016\n",
            "Specificity         0.999951\n",
            "Precision           0.986000\n",
            "Recall              0.982657\n",
            "AUC                 0.998454\n",
            "dtype: float64\n"
          ]
        }
      ],
      "source": [
        "checkpoint_path = \"AlexNet/cp.ckpt\"\n",
        "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "\n",
        "# Create a callback that saves the model's weights\n",
        "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n",
        "                                                 save_weights_only=True,\n",
        "                                                 verbose=1)\n",
        "metric_history = pd.DataFrame()\n",
        "i=1\n",
        "for train_index, test_index in kf.split(X):\n",
        "    X_kf, X_cv = X[train_index], X[test_index]\n",
        "    y_kf, y_cv = y[train_index], y[test_index]\n",
        "\n",
        "    print(i)\n",
        "    i=i+1\n",
        "\n",
        "    model = keras.models.Sequential([\n",
        "        Conv2D(filters=64, kernel_size=(5,5), padding='same', activation='relu', input_shape=(30, 30, 3)),\n",
        "        MaxPool2D(pool_size=(3,3), strides=(2,2)),\n",
        "        Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'),\n",
        "        MaxPool2D(pool_size=(3,3), strides=(2,2)),\n",
        "        Flatten(),\n",
        "        Dense(1024, activation='relu'),\n",
        "        Dense(NUMBER_OF_CLASSES, activation='softmax'),\n",
        "    ])\n",
        "    \n",
        "    model.compile(loss=\"categorical_crossentropy\",\n",
        "              optimizer=\"Adam\",\n",
        "              metrics=[\"accuracy\", TruePositives(), FalsePositives(), TrueNegatives(), FalseNegatives(), SensitivityAtSpecificity(0.5), SpecificityAtSensitivity(0.5), Precision(), Recall(), AUC()])\n",
        "\n",
        "\n",
        "    history = model.fit(img_aug.flow(X_kf, y_kf, batch_size=128), validation_data=[X_cv, y_cv], epochs=20, verbose=1, callbacks=[cp_callback],)\n",
        "\n",
        "    loss, accuracy, tp, fp, tn, fn, sensitivity, specificity, precision, recall, auc = model.evaluate(X_cv, y_cv)\n",
        "    metrics = pd.DataFrame([[loss, accuracy, tp, fp, tn, fn, sensitivity, specificity, precision, recall, auc]], columns=[\"Loss\", \"Accuracy\", \"TP\", \"FP\", \"TN\", \"FN\", \"Sensitivity\",\"Specificity\", \"Precision\", \"Recall\", \"AUC\"])\n",
        "    metric_history = pd.concat([metric_history, metrics], ignore_index=True)\n",
        "print(metric_history.mean())\n",
        "info = pd.DataFrame([[\"AlexNet\"]], columns=[\"Topology\", ])\n",
        "info = pd.concat([info, pd.DataFrame(metric_history.mean()).T], axis=1,)\n",
        "table = pd.concat([table, info], ignore_index=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "zZ4t6TShv6W9"
      },
      "outputs": [],
      "source": [
        "table.to_csv(\"./log.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "_61w6-yOCxl1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Topology</th>\n",
              "      <th>Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>TP</th>\n",
              "      <th>FP</th>\n",
              "      <th>TN</th>\n",
              "      <th>FN</th>\n",
              "      <th>Sensitivity</th>\n",
              "      <th>Specificity</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>AUC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>LeNet 5</td>\n",
              "      <td>0.100975</td>\n",
              "      <td>0.969038</td>\n",
              "      <td>3783.7</td>\n",
              "      <td>100.4</td>\n",
              "      <td>164577.4</td>\n",
              "      <td>137.2</td>\n",
              "      <td>0.998215</td>\n",
              "      <td>0.999993</td>\n",
              "      <td>0.974144</td>\n",
              "      <td>0.965008</td>\n",
              "      <td>0.998989</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>AlexNet</td>\n",
              "      <td>0.067881</td>\n",
              "      <td>0.984443</td>\n",
              "      <td>3852.9</td>\n",
              "      <td>54.7</td>\n",
              "      <td>164623.1</td>\n",
              "      <td>68.0</td>\n",
              "      <td>0.997016</td>\n",
              "      <td>0.999951</td>\n",
              "      <td>0.986000</td>\n",
              "      <td>0.982657</td>\n",
              "      <td>0.998454</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0 Topology      Loss  Accuracy      TP     FP        TN     FN  \\\n",
              "0         0.0  LeNet 5  0.100975  0.969038  3783.7  100.4  164577.4  137.2   \n",
              "1         NaN  AlexNet  0.067881  0.984443  3852.9   54.7  164623.1   68.0   \n",
              "\n",
              "   Sensitivity  Specificity  Precision    Recall       AUC  \n",
              "0     0.998215     0.999993   0.974144  0.965008  0.998989  \n",
              "1     0.997016     0.999951   0.986000  0.982657  0.998454  "
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XSaQzBw1SiME"
      },
      "source": [
        "## Visualize features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "h8tCOCTKSmEP"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1226/1226 - 3s - loss: 0.6788 - accuracy: 0.7937 - 3s/epoch - 3ms/step\n",
            "Restored model, accuracy: 79.37%\n",
            "conv2d_20 (5, 5, 3, 6)\n",
            "conv2d_21 (5, 5, 6, 16)\n",
            "conv2d_22 (1, 1, 16, 120)\n"
          ]
        }
      ],
      "source": [
        "checkpoint_path = \"LeNet/cp.ckpt\"\n",
        "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "\n",
        "\n",
        "# summarize filters in each convolutional layer\n",
        "from matplotlib import pyplot\n",
        "\n",
        "model = keras.models.Sequential([\n",
        "        Conv2D(filters=6, kernel_size=(5,5), padding='same', activation='tanh', input_shape=(30, 30, 3)),\n",
        "        AveragePooling2D(strides=(2,2)),\n",
        "        Conv2D(filters=16, kernel_size=(5,5), padding='valid', activation='tanh'),\n",
        "        AveragePooling2D(strides=(2,2)),\n",
        "        Conv2D(filters=120, kernel_size=(1,1), padding='valid', activation='tanh'),\n",
        "        Flatten(),\n",
        "        Dense(84, activation='relu'),\n",
        "        Dense(NUMBER_OF_CLASSES, activation='softmax'),\n",
        "    ])\n",
        "model.load_weights(checkpoint_path)\n",
        "\n",
        "model.compile(loss=\"categorical_crossentropy\",\n",
        "              optimizer=\"Adam\",\n",
        "              metrics=[\"accuracy\",])\n",
        "\n",
        "loss, acc = model.evaluate(X, y, verbose=2)\n",
        "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))\n",
        "\n",
        "\n",
        "# summarize filter shapes\n",
        "for layer in model.layers:\n",
        "\t# check for convolutional layer\n",
        "\tif 'conv' not in layer.name:\n",
        "\t\tcontinue\n",
        "\t# get filter weights\n",
        "\tfilters, biases = layer.get_weights()\n",
        "\tprint(layer.name, filters.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "fcIvmXI7YYYM"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAGKCAYAAACy4W7+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAaTUlEQVR4nO3de4xUdxnG8Xf2MrO3GYppwCy7UYtGEGzTlEAkBW/F6NZKiq0N67WkkqYRqcS2ErWIFWkKJm4bINgSYiK1rUYMcrGUgLT2ktLGGBKsKanB2Y4RKsLMbGdnZ5jxr1NJrDnPnoXynpnv5+9nf7+zOe/Ow9lwfpuo1+t1AwDAsZZLfQEAAIShrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC41xb1C2u1muVyOUun05ZIJC7kNV0y9XrdCoWC9fb2WksLPd6omF3EVTPPbuSyyuVy1t/fH/XLXctms9bX13epLwMXCbOLuGrm2Y1cVul02szMdu/ebd3d3aH50dFRad2jR4/K1zBt2jQp98Ybb0i50dFRu+eee9763tCYgvv729/+Vprdf/zjH9K6f/rTn+RrmDdvnpRbsGCBlCsUCjZjxgxmt8EF9/fo0aPSvVbnJ5vNytfwta99Tcp95jOfkXJvvvmm3XrrraHfT+SyCh5Bu7u7raenJzTf2toqrdvZ2SlfQ1dX1wVf08wa5vEab+/82VXKSp2zVColX4O6ZiaTkdc0Y3YbXXB/0+m0NBsX41fCyWRSyqkzHgibXX65DQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMC9yO9ZBZYuXSr9X/7Pfvaz0nrbtm2b6CX9j4ULF0q5arV6wfeGX/v27ZPejfrlL38prTeekyPUd1CuvfZaKVepVOS9EX9r1qyR3nd66aWXLvjed911l5S78847pVytVpNyPFkBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyb8AkWQ0ND0tv4U6ZMkdY7ceKEvPfvf/97Kbdy5UopVy6X7ZlnnpH3R7wdOnRI+gvW3/rWt6T1xvOXghcvXizl7rvvPilXLpflvRF/hw4dkk4O+vvf/y6t981vflPee/v27VJu1qxZUu7cuXNSjicrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwL0JH7d06623WiKRCM0VCgVpPeXopsCqVauk3MGDB6WceuwHGkO9Xrd6vR6aGxoaktb70Ic+JO+9ZMmSC7pmqVSS90b8Pfvss5bJZEJzSsbMbP/+/fLeH/jAB6TcihUrpNzo6Kjdc889oTmerAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7k34BIunnnrKenp6QnPr1q2T1tu4caO89/Tp06VcrVaTcvl83i677DJ5f8Tb4cOHpTf81ZkcGBiQ916zZo2U27Jli7wmmkdbW5u1tYV/fB8+fFhab8OGDfLeJ0+elHLt7e1SrlqtSjmerAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7kU+waJer5uZ2cjIiJQfHR2VcoVCIeol/V/5fH5cueB7Q2MK7q86F+rsFotF+RrGxsbk7Hgwu40tuL/q56T6+ayeInH+NYQplUpSLvj5Cls3UY843cPDw9bf3x/lS93LZrPW19d3qS8DFwmzi7hq5tmNXFa1Ws1yuZyl02lLJBKRL9CTer1uhULBent7raWF35A2KmYXcdXMsxu5rAAAeKfwTzAAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7kU+bqmZX05DvDG7iKtmnt3IZZXL5Zr22A/EG7OLuGrm2Y1cVul02szMduzYYV1dXaF59aCMqVOnytfQ09Mj5Y4cOSLlSqWSrVix4q3vDY0puL/ZbNYymUxofs+ePdK6mzZtkq9h8uTJUu7KK6+UcuVy2X7yk58wuw0uuL833nijtbe3h+Yffvhhad2dO3fK1/DBD35Qyqmf+SMjI7Zo0aLQ2Y1cVsEjaFdXl3V3d4fm1QtXC8jM5B9MpUzP1yiP13h7wf3NZDJSWanz09am/zgpHzRmZh0dHfKaZsxuowvub3t7uzRDynybje8zUv2MHu9JfmGzyy+3AQDuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3Iv8UnCgWq1apVIJzc2ePVtab+HChfLet99+u5Rbvny5lMvn8/LeiL+7777bUqlUaO7OO++U1lu9erW89969e6Xcd77zHSmXz+ftxz/+sbw/4m358uXSYQy/+c1vpPXUk1LMzG666SYpt3nzZik3MjIi5XiyAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4N+ETLA4ePCidAnD8+HFpvSNHjsh7b926VcotWrRIyp07d07eG/F3zTXXWGdnZ2hOPX1F+TkIbNy4UcotWLBAylWrVXlvxN/jjz9uyWQyNKd+Rk6ePHmil/Q/Zs6cKeUKhYKU48kKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcC9Rr9frUb4wn8/bpEmTrLe311pawjtPyZiZnThxQr6GGTNmSLlPfOITUm5sbMy2bdtmZ8+etUwmI18H4iWY3a6uLkskEqH5n//859K6xWJRvoZ9+/ZJOfUYpUqlYrt27WJ2G1wwu/v27bPu7u7Q/MDAgLTu888/L1/Diy++KOV27dol5SqViu3duzd0dnmyAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC41zbRBa677jpLJpOhub1790rrrV69Wt77r3/9q5R74YUXpFw+n7dt27bJ+yPehoeHpdMe2tq0H5NNmzbJe587d07K/etf/5Jy6kkXaAz79++3VCoVmps1a5a03qOPPirvvX79eim3fft2KVcqlaR+4MkKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOBe5BMs6vW6mZmNjY1J+VqtJuXK5XLUS/q/8vm8lCsUCmb23+8NjSm4v+pcqEqlkpytVCpSTj2ZIsgxu40tuL/q56Q6Pxfjc1f9eQhyYbObqEec7uHhYevv74/ype5ls1nr6+u71JeBi4TZRVw18+xGLqtarWa5XM7S6bQlEonIF+hJvV63QqFgvb291tLCb0gbFbOLuGrm2Y1cVgAAvFP4JxgAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5FPsGimf+/P+KN2UVcNfPsRi6rXC7XtG9SI96YXcRVM89u5LJKp9NmZrZixQpLpVKh+cWLF0vrjucd5auuukrKHT9+XMoVi0X76Ec/+tb3hsYU3N9sNmuZTCY0f+zYMWndX/3qV/I1jIyMSLnLL79cypXLZdu4cSOz2+CC+3vFFVdYa2traH7nzp3Surfddpt8Da+++qqUu+yyy6RcrVazv/3tb6GzG7msgkfQVCollVVPT4+07njKSvmgGc/egUZ5vMbbC+5vJpORZkidH+XnIKAeMNrR0SGvacbsNrrg/ra2tkplpf7jpa1NrwL118zK9Z0vbHb55TYAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7kV8KDuzZs0d6+WvOnDnSev/85z/lvZ9//nkp9+1vf1vKjeeFZMTfe97zHukl2gcffFBab9WqVfLeZ86ckXLr1q2TcmNjY/LeiL8vf/nL0gvjd9xxh7Red3e3vPfg4KCUGxoaktdU8GQFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHBvwidYfO5zn5P+nPfVV18trXfgwAF571/84hdSrqurS8rVajUbHR2V90e8DQwMWDKZDM3t3btXXk/1hz/8Qcq9/vrrUq5arcp7I/527dol/Sn6XC4nrfe73/1O3nvBggVS7pVXXpFyxWJROuGIJysAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAvQkft5RMJqXjloaGhqT11q5dK++tHm9z8uRJKVcsFu3aa6+V90e8zZo1yzo6OkJzPT090nqDg4Py3k888YSUu+6666RcoVCwK6+8Ut4f8bZjxw5Lp9OhuYULF0rrbd68Wd575cqVUm7y5MlSTjk2yownKwBADFBWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7k34BIv77rtPyt11111S7qGHHpL3LhaLUq5er0u5crks7434e+qpp6S356+66ippvXvvvVfee926dVLugQcekNdE83j3u99tmUwmNDd16lRpvQcffFDe+80335RyZ8+elXKlUknK8WQFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHAv8gkW6qkQAfV0iNHR0Qu+5nhPsBjv94Z4Ce5vtVqV8uqcjYyMyNdwsU5LYXYbW3B/8/m8lFdnXF3PTD/BolAoSLngJKKw2U3UI0738PCw9ff3R/lS97LZrPX19V3qy8BFwuwirpp5diOXVa1Ws1wuZ+l02hKJROQL9KRer1uhULDe3l5raeE3pI2K2UVcNfPsRi4rAADeKfwTDADgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7kY9bauaX0xBvzC7iqplnN3JZ5XK5pj32A/HG7CKumnl2I5dVOp02M7PPf/7z1t7eHprfunWrtO62bdvka3j00Uel3J49e6RcoVCw97///W99b2hMwf29/vrrpdm9/fbbpXVfe+01+Rr2798v5U6ePCnlqtWqvfjii8xugwvu79DQkHV2dobmX3/9dWndK664Qr6G7u5uKVcqleTc8uXLQ2c3clkFj6Dt7e2WTCZD85lMRlpXuQGBtjbt8tW9A43yeI23d/7sKmWl/nCOZ3aVfc30GQ8wu40tuL+dnZ3SvHV0dEjrdnV1ydeg/jyMdxbD8vxyGwDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7kd+zCiSTSek9q6VLl0rrbd++Xd77ueeek3LLli2TcpVKRd4b8bd27VrpJdo5c+ZI6505c2Zceyt++MMfSrl6vS7vjfjL5XLSO1Tr16+X1lM+wwPZbFbKqe945fN5KceTFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAvQmfYLF48WLpL0d+73vfk9YbHByU9549e7aUW7dunZTjFIDmsnr1aukv9h49elRa7xvf+Ia898jIiJR79dVXpVyhULAPf/jD8v6It4cffthaWsKfNbZu3Sqt97Of/UzeW52zm266ScqVy2Upx5MVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4N6Ej1t6/PHHpSNrXnnlFWm9p59+Wt572rRpUu6WW26RcpVKxX7961/L+yPebr75Zuvq6grNZTIZab2DBw/Ke2ezWSnX2dkp5UZHR+W9EX9f//rXraOjIzR3+vRpab3xHDU3c+ZMKZdKpeQ1FTxZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDcS9TH8+ryefL5vE2aNMm++93vSm9SFwoFad1kMilfw4033ijlrrnmGnlNM7OzZ8/KpxYgfoLZVe+zegrA9OnT5Ws4c+aMlDt27JiUKxaLNnfuXGa3wQWz29vbay0t4c8ap06dktY9dOiQfA3z58+XcpMmTZJy9Xrd8vl86OzyZAUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcK8t6hcGB1+Uy2Upr+bGc6BGsViUs+MR8VAPxERwf/P5vJRXcxdjbtQZD3LMbmML7m+tVhtXPszIyEjka5ro3kEuLB/5uKXh4WHr7++P8qXuZbNZ6+vru9SXgYuE2UVcNfPsRi6rWq1muVzO0um0JRKJyBfoSb1et0KhIJ+7hXhidhFXzTy7kcsKAIB3Cv8EAwC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOBe5OOWmvnlNMQbs4u4aubZjVxWuVyuaY/9QLwxu4irZp7dyGWVTqfNzOxjH/uYtbWFL1OtVqV1N2zYIF/D7t27pdzatWvlNc3++72hMY33/i5ZskTKHT16VF5z8+bNUm7RokXymmbMbqML7u+9995rHR0dofmxsTFp3ZMnT8rXoM7ueIXNbuSyCh5B29rarL29Peoy/6Onp0fOKjcrikZ5vMbbG+/9Vee7tbVVXrO7u1vKqdcanJrG7Da24P52dHRIn3/qr4RTqdSErutCCJtdfrkNAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgXuSXggOf+tSnrLOzMzQ3MDAgrffII4/Ie+fzeSn3hS98QcpVKhXbuXOnvD/ibfny5ZZMJkNzX/3qV6X19u/fL+/93HPPSbnBwUEpV6lU7IknnpD3R7ydPn1aepH3Rz/6kbTe97//fXnvj3/841Ju7ty5Uq5cLttPf/rT0BxPVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9yZ8gsWBAwekP/t94sQJaT31jX0zs6uvvlrKzZs3T8qdO3dO3hvxt3r1astkMqG5Wq0mrTdlyhR572q1KuW+9KUvSbmRkRFOsGgic+bMsa6urtBca2urtN5tt90m7z116lQpt2PHDimn/nzxZAUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4N+HjlorForW1hS/T0dEhrffYY4/Je6dSKSmnHiVSKpXs5ZdflvdHvP3lL3+x7u7u0NxXvvIVab2bb75Z3nvDhg1SbtWqVVKuXC7LeyP+Zs6cael0OjT3xz/+UVpP+TkIKMc8mZldf/31Uk495o4nKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAexM+wWJgYEA6nWL27NnSep/+9KflvV977TUpt2XLFilXKBRs5cqV8v6It6efflqa3fnz50vrDQ0NyXu3tGj/Tly2bJmUKxaLtnnzZnl/xNvJkydtZGQkNPeDH/xAWk896cLM7MSJE1LulltukXLlctkeeOCB0BxPVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9yKfYFGv183MbHR0VMorb1uPV6VSkXKFQmFcueB7Q2MK7m+5XJby6pyNZ27UvYvFopQLfr6Y3cYW3F/187RarY5rXYX6earOeJALu4ZEPeJ0Dw8PW39/f5QvdS+bzVpfX9+lvgxcJMwu4qqZZzdyWdVqNcvlcpZOpy2RSES+QE/q9boVCgXr7e2Vz25D/DC7iKtmnt3IZQUAwDuFf4IBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOBe5BMsmvn/+yPemF3EVTPPbuSyyuVyTfsmNeKN2UVcNfPsRi6rdDptZmbLli2zZDIZmv/IRz4irfvYY4/J17B06VIp98lPflLKFQoFmzFjxlvfGxpTcH+z2axlMpnQ/EMPPSStOzg4KF/Dv//9bym3Zs0aKVepVOzJJ59kdhvceO/v5ZdfLuW2b98urzlt2jQp99JLL0m5UqlkK1euDP3eIpdV8AiaTCYtlUqF5ru6uqR129vb5WtQ11Q+kM7XKI/XeHvB/c1kMtJsdHR0SOuO54NEPRx3PD8PZsxuoxvv/VV/Jdzd3S2vqc55Z2envKZZ+PfGL7cBAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDci/xScKC9vV16cfHEiRPSevfff7+8t/om9bFjx6RcsViU90b83XHHHdLpK4888oi0nvrGvpnZu971Lim3bt06KVcsFm337t3y/oi3P//5z9LLudOnT5fWmz17trz3kiVLpNwzzzwj5fL5vC1fvjw0x5MVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMC9CZ9gccMNN0h/Enn+/PnSepMmTZL3Vt+Qnjdvnrwmmsfg4KA0u+opALt27ZL3vvvuu6Xc6dOnpVy1WpX3Rvxt2bLFUqlUaO7UqVPSenPnzpX3fvnll6Xck08+KeVGRkakHE9WAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHsTPm5p06ZN1t7eHpp74YUXpPWSyaS89xtvvCHlDh8+LOVGRkZsYGBA3h/xVi6Xra0t/Efg/vvvl9YbGhqS91aPClu/fr2UK5VK9uyzz8r7I97e9773WWdnZ2iur69PWu/gwYPy3kuWLJFyX/ziF6VcqVSScjxZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDcm/AJFu9973stlUqF5latWiWtd/z4cXnvKVOmSLkbbrhBytVqNXlvxF8+n7dqtRqaO3XqlLTetm3b5L3Pnj0r5VasWCHlxsbG5L0Rf9u3b7fW1tbQ3JEjR6T1Dhw4IO9dKBSk3MKFC6VcsViUcjxZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDci3yCRb1eNzOzcrks5fP5vJQbz5v46pvP6skUQS743tCYgvtbKpWkvJobjwv981CpVMyM2W10wf09d+6clFc/I0dHR+VruNB7B7mw2U3UI0738PCw9ff3R/lS97LZrPX19V3qy8BFwuwirpp5diOXVa1Ws1wuZ+l02hKJROQL9KRer1uhULDe3l5raeE3pI2K2UVcNfPsRi4rAADeKfwTDADgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDc+w/DCYEwF2pNwQAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 18 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAGKCAYAAACy4W7+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAaPElEQVR4nO3dbYxcZRnG8Xt2d/atO0NoSE03u2xJgGIllmJsiy/RWoqhNi19UfANiS8hFJHYNkGCxRABqUZTiVFaqqTF1qoxLXYBSYwUlEpLRQttUwsa6G6nVNdKd7Y7uzM7c/xgDtGkybnmbEvvM/P/fdEPV5/nrOfeufZsPM+mgiAIDAAAxxrO9QUAABCFsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgXlPcf1ipVCyXy1kmk7FUKnUmr+mcCYLA8vm8dXZ2WkMDPV6rmF0kVT3PbuyyyuVy1t3dHfefu9bX12ddXV3n+jJwljC7SKp6nt3YZZXJZMzMbNGiRZZOpyPze/bskdZ9+eWX5WtYunSplDtw4ICUq1Qqdvz48be+NtSm8P7eeuut1tLSEpmfM2fOGb+G7du3S7nm5mYpVywW7cc//jGzW+PC+9vX12fZbDYy//rrr0vr9vT0yNfw8Y9/XMoVCgUpNzY2Zn/84x8jZzd2WYWPoOl0Wior9VcTyg0INTVpl1/tr0Vq5fEapxfe35aWFqmsJkyYcMavQS0h5fr+F7Nb28L7m81mpc9K9YeXaj53lc97M7NSqSSvaRY9u/xyGwDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7sd+zCk2bNs1aW1sjc4cPH5bW27Fjh7z3vHnzpFwul5Ny5XLZjh07Ju+PZOvs7LS2trbI3HPPPSett3z5cnnvJ554QsqtXLlSyuXzefvRj34k749k+9KXviS97zRjxgxpPeX7IPT4449LufPPP1/KBUEg5XiyAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4N+4TLI4cOSL91dPJkydL66k5M7OLLrpIynV2dkq54eFh+8IXviDvj2T72c9+Jv21afVPg4+Ojsp7T58+Xcpdf/31Um5sbEzeG8k3MDAgza76l6Zfeuklee+PfvSjUu7pp5+WcpxgAQCoGZQVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMC9cR+3tHv3bmtsbIzM7d+/X1ovlUrJew8PD0u5dDot5TiyBqcza9YsKfeBD3xAXnPbtm1STj3qqVKpyHsj+e666y7r6OiIzN13333SekuWLJH3Vo9bOnnypJQbGxuzvXv3RuZ4sgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuDfuEyyOHj0qnTrxzDPPaBfUpF/SV77yFSk3MjIi5crlsrw3ku93v/udZbPZyJw6u+3t7fLen/nMZ6TcunXr5DVRP5YtWyZ97q5Zs0Za75///Ke8989//nMpVyqVpJz6ucuTFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAvdgnWARB8H//GeXUqVPaBVVxgoX65nO1OfVrQjKF93dwcFDKq7NbqVTkaygWi3K2Gsxubav2c7dQKEg59ZQfszN/MoX6uZsKYk53f3+/dXd3x/mn7vX19VlXV9e5vgycJcwukqqeZzd2WVUqFcvlcpbJZKQzqpIgCALL5/PW2dlpDQ38hrRWMbtIqnqe3dhlBQDA24UfwQAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4F/u4pXp+OQ3Jxuwiqep5dmOXVS6Xq9tjP5BszC6Sqp5nN3ZZZTKZt/670vDTpk2T1r3kkkvka9i4caOUO//886VcEAQWBMH/fW2oPeH9feihh6ytrS0y//jjj0vrjo2Nydcwffp0KfeNb3xDXtPMmN0aF97fD3/4w9Kh3+phsu95z3vka2hpaZFy+/btk3KlUsmeeuqpyNmNXVZhQaVSKamsGhsbpXXT6bR8DdlsVspV87gcBEHNPF7j9ML729bWZu3t7ZH55ubmqtZVtLa2ytlqMLu1Lby/TU1N0melOg9qAZnps1vNZ7lZ9LXyy20AgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3Yr8UHApPfYiyfft2ab37779f3nvXrl1SbtWqVVJudHTU1q5dK++PZJsxY4Z04sOFF14orffkk0/Ke4+MjEi5hQsXSrlSqVTV/ki2w4cPS2dAXn755dJ6f/jDH+S9ly1bJuXy+byUU09+4ckKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAODeuE+w+NznPif92e9vfvOb0npPPPGEvLfyBreZ2aJFi6Tc8PAwJ1jUkcOHD9uECRMic319fdJ6b775prz3+vXrpVxnZ6eUq1Qq8t5IvvXr10uzq54cNHPmTHlv9c/a33777VJueHjYdu7cGZnjyQoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwb9zHLX3iE5+Qjv341re+Ja33wAMPyHt/9atflXJ33nmnlMvn8/LeSL7R0VFrbGyMzBUKBWm9vXv3ynsvXbpUyinXZ2ZWLBbt73//u7w/km3fvn3SsUeTJ0+W1jt+/Li892233SblfvnLX0o59fuLJysAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHupIAiCOP9wcHDQzjvvPNu3b59lMpnI/MqVK6V1r7zySvkalH3NzB5++GEpVy6X7dChQ3by5EnLZrPydSBZwtk9ePCgNENNTdpBLwcPHpSv4SMf+YiUU09VGRwctK6uLma3xoWzq+rq6pJy7e3t8prqiRPpdFrKVSoVe+211yJnlycrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB72qv5pxEefDE0NCTlS6WSlBsZGZGvQX1DulwuV5WLeagHEqLa2VVPsDh16pR8DYODg1JOPcEizDG7ta3a+1upVKSc+hlZzZrV5qK+tthlFX5zvP/974+7xGn19vae0fXiyOfzVR1pgmQJZ3fmzJnn+ErOPGa3tqk/vIRyudxZupIzL2p2Y58NWKlULJfLWSaTsVQqFfsCPQmCwPL5vHV2dlpDA78hrVXMLpKqnmc3dlkBAPB24UcwAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDci/1ScD3///2RbMwukqqeZzd2WeVyOevu7o77z13r6+uzrq6uc30ZOEuYXSRVPc9u7LLKZDJmZnbppZdaY2NjZF49AmbLli3yNaxevVrOKkqlkv3iF79462tDbQrv7+LFi6XzJV955RVp3dmzZ8vXoHzPmJlNmDBByo2OjtratWuZ3RoX3t9bb73VWlpaIvPqmQ/Hjh2Tr0E9v7W5uVnKlUol27ZtW+Tsxi6r8BG0sbFR+sZTDwOt5ptN/R+jWrXyeI3TC+9vOp2WZkgtFuXD42yuacbs1rrw/ra0tJzRsqrms1Q99Lbaz+eo2eWX2wAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO7Ffik4dMMNN1hra2tkbv78+dJ6999/v7z3Aw88IOUGBwelXD6ft82bN8v7I9nuvvtu6SX0PXv2SOstW7ZM3juXy0m5xx57TMoVCgV5byTfT37yE+kF8KlTp0rrPf/88/Le8+bNk3IzZsyQcqOjo1KOJysAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHvjPsHi4osvtvb29sjcFVdcIa23YMECee+enh4pNzIyIq+J+nHTTTdZU1P0t8DVV18trXfixAl578WLF0u566+/XsoNDg7aypUr5f2RbMuXL5dODnrHO94hrTcwMCDvvXDhQin36KOPSrlyuSzleLICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3Bv3cUs//elPLZ1OR+aee+45ab358+fLe6tHOL3zne+UcsVi0TZv3izvj/qwdetWKffqq6/Ka7a1tUm5oaEhKVcoFOS9kXx33HGHZbPZyNwjjzwirferX/1K3ru3t1fKKddnZjY2NibleLICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALg37hMsPvWpT1l7e3tkbtasWdJ6//73v+W99+/fL+Vmzpwp5UZHR+W9kXxHjx61hobon9fK5bK03ve+9z157/7+fimXyWSknHqNqA07d+60CRMmROauvvpqab0XXnhB3vuVV16RclOmTJFyxWJRyvFkBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwL/YJFkEQmJlZoVCQ8oODg1Wteyaz6skU4ZvU1VwDkie8v5VKRcqruZGREfka1GxTk/YtGq7H7Na28P4ODw9L+Xw+L+XU9cz0EyfU7xv1czcVxJzu/v5+6+7ujvNP3evr67Ourq5zfRk4S5hdJFU9z27ssqpUKpbL5SyTyVgqlYp9gZ4EQWD5fN46OzulM+OQTMwukqqeZzd2WQEA8HbhRzAAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7sU+bqmeX05DsjG7SKp6nt3YZZXL5er22A8kG7OLpKrn2Y1dVplMpqr83/72Nyl3wQUXyGvOnj1byt15551SrlAo2M0331z114ZkCe/vHXfcYS0tLZH5uXPnSutWM7v33HPPGV2zWCzahg0bmN0aF97fF1980To6OiLzL774orSu8n0QOnTokJRTP3dDUbMbu6yqfQRVv4my2ay8ZmNjo5Rrb2+X1zSr/mtDsoT3t6WlxVpbWyPzyoeCWXU/wKXTaSlXzYeIGbNb68L729HRIc2b+tmnfB/EyVYjanb55TYAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA92K/Z1Wt48ePS7mbb75ZXvPaa6+VckeOHJFyIyMj8t5Ivr6+Pmtubo7M7dmzR1pv48aN8t5/+ctfpNzEiROlXKVSkfdG/ZgyZYqUe+aZZ+Q1//rXv0q5IAik3ODgoJ133nmROZ6sAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADujfsEi0suuUT6i7133323tN5nP/tZee+jR49KOfUvuBYKBXlvJN/XvvY1aTYmT54srbd9+3Z5b/Xt/lwuJ6+J+jEwMCCduHPvvfdK661atUreWz19Rf1+GB4elnI8WQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADujfu4pU2bNllHR0dk7qmnnpLWe/DBB+W9L7/8cik3depUKcdxS/Vl2bJl0lFhR44ckdabOHGivPe6deuk3He+8x0pVy6X7dChQ/L+SLZFixZZQ0P0s8YNN9wgrdfb2yvvPWvWLCm3ePFieU0FT1YAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPfGfYLFxIkTLZPJROauueYaab2tW7fKe6dSKSk3ZcoUKTc8PCzvjeS77bbbrK2tLTJ3zz33SOsNDAzIe+fzeSl34sQJKVepVOS9kXzXXHONNTc3R+bmzp0rrffpT39a3vuhhx6Sci+88IKUGxoasjlz5kTmeLICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALgX+wSLIAjM7L9vHyvUXLlclq+hWCxKOfVkijAXfm2oTeH9LRQKUl49HaKauTnTe4c5Zre2hfe3VCpJefWzr5q5UddUP/NPnTolXUMqiDnd/f391t3dHeefutfX12ddXV3n+jJwljC7SKp6nt3YZVWpVCyXy1kmk5HP6PMuCALL5/PW2dlpDQ38hrRWMbtIqnqe3dhlBQDA24UfwQAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4F/u4pXp+OQ3Jxuwiqep5dmOXVS6Xq9tjP5BszC6Sqp5nN3ZZZTIZMzNbunSppdPpyPy73vUuad3+/n75Gr797W9LuTVr1ki50dFR++53v/vW14baFN7fr3/969ba2hqZnzx5srTu+vXr5Wu46aabpNzAwICUGxkZsfvuu4/ZrXHh/X300Uetvb09Mn/llVdK695yyy3yNQwODkq5Xbt2yWuaWeTsxi6r8BE0nU5bc3NzZF75UDAza2lpka8hm81KOXXvUK08XuP0wvvb2toqzYbyoWBm1tSkfzu1tbVJOWYX/yu8v+3t7TZhwoTIvPoZqTxwhKqZ82pEzS6/3AYAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHBv3G93rVixwjo6OiJz6pvUjz32mLz3jTfeKOUuvfRSKccLlTidSZMmSbl//etf8pq33367lDt58qS8JurH4cOHpRfGjx07Jq03f/58ee9t27ZJOeWlZbP/ng04PDwcmePJCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDg3rhPsHj22WelP9E9depU7YKq+JPJpVJJyq1evVpeE/Xjqquukk5f2b59u7TenDlz5L3VEwN+/etfS7lisWgbN26U90eyLV++XPqT9bt375bWe/LJJ+W9165dK+UWLFgg5SqVir322muROZ6sAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPfGfdzS73//e0un05G5Cy+8UFpv69at8t7XXXedlPvtb38r5SqVip04cULeH8lWKpWsWCxG5pYsWSKtd+TIEXlv9SiaK664Qsrl83mOW6ojmzZtko65mzRpkrTel7/8ZXnvPXv2SLk//elPUm5wcNB6enoiczxZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDcG/cJFs8++6ylUqnInPK2tZnZ5z//eXnvWbNmSbne3l4pd+rUKZs7d668P5KtoaHBGhqif17bsmWLtN4bb7wh7/2hD31Iyu3cuVPKFQoFeW8k344dO6STg6ZMmSKt9/rrr8t7q5/lq1atknLKKTJmPFkBABKAsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHAv9gkWQRD8339GqVQqUm50dFS+hnw+L+VOnTpVVU79mpBM4f1V50KdyVKpJF+DOrvqyRRhjtmtbeH9HRsbk/Lq6RBn4wQUde8wFzW7qSDmdPf391t3d3ecf+peX1+fdXV1nevLwFnC7CKp6nl2Y5dVpVKxXC5nmUxGOhswCYIgsHw+b52dndKZcUgmZhdJVc+zG7usAAB4u/AjGADAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7sU+waKe///+SDZmF0lVz7Mbu6xyuVzdvkmNZGN2kVT1PLuxyyqTyby1QTabjcyfPHlSWre3t1e+hoULF0q5a6+9VsqVy2U7ePDgW18balN4fxcvXmzpdDoy//zzz0vrrlixQr6GAwcOSLmenh4pNzIyYvfeey+zW+PC+3vXXXdZa2trZH7dunXSuv39/fI13HjjjVLuN7/5jZSrVCo2MDAQObuxyyp8BM1ms1JZqQdltLW1ydeg7Gtm1tjYKK9pZjXzeI3TC+9vOp2Wykr9tVo1s9vc3CzllA+k/8Xs1rbw/ra2tkqzcTZ+JazObrV7R80uv9wGALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwL/ZLwaEFCxZYU1P0MuopEqOjo/LeP/zhD6Xce9/7XilXLBbt5ZdflvdH8ikv0X7yk5+U1po0aZK877vf/W4pN3v2bCk3NDQk743k+/73vy+9dLthwwZpvSVLlsh7b9myRcpddtllUq5cLts//vGPyBxPVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA98Z9gsXFF18s/ZnjCy64QFpv6tSp8t4XXXSRlNu9e7eUGx4eto0bN8r7I9kefvhhy2azkbldu3ZJ6+3du1fe+4033pBy73vf+6Tc2fjz5fDrwQcftPb29sjc/PnzpfXU0ybMzP785z9LuUOHDkm5oaEhu+qqqyJzTDgAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDg3riPW3r11VetqSl6mUceeURaTz0ayczsgx/8oJRbs2aNlCuXy/LeSL79+/dbR0dHZO6WW26R1kulUvLeX/ziF6Xcxz72MSlXqVTkvZF8Tz/9tHTM3aJFi6T15s2bJ++9evVqKbdjxw4pp37u8mQFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHBv3CdYbNiwwTKZTGTuzTfflNabNm2avPemTZuknPp2P6cA1JcDBw5YW1tbZO6ll16S1psyZYq89+bNm6Xc9OnTpVypVLL+/n55fyTb6OioBUEQmevp6ZHW+8EPfiDvXSwWpZy6d6FQsBUrVkTmeLICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALgX+wSL8O3poaEhKa/mqlEoFKScejJFuJ7yZjiSK7y/6vyoqjkBZWxsTMqVSqWq1mN2a1t4f9VTJNSZHB4elq9B3Vv9/hoZGTGz6NlNBTGnu7+/37q7u+P8U/f6+vqsq6vrXF8GzhJmF0lVz7Mbu6wqlYrlcjnLZDKWSqViX6AnQRBYPp+3zs5Oa2jgN6S1itlFUtXz7MYuKwAA3i78CAYAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7v0HeSaBzN2pa2gAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 18 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "for l in range(0, 4, 2):\n",
        "    # retrieve weights from the second hidden layer\n",
        "    filters, biases = model.layers[l].get_weights()\n",
        "    # normalize filter values to 0-1 so we can visualize them\n",
        "    f_min, f_max = filters.min(), filters.max()\n",
        "    filters = (filters - f_min) / (f_max - f_min)\n",
        "    # plot first few filters\n",
        "    n_filters, ix = 6, 1\n",
        "    for i in range(n_filters):\n",
        "        # get the filter\n",
        "        f = filters[:, :, :, i]\n",
        "        # plot each channel separately\n",
        "        for j in range(3):\n",
        "            # specify subplot and turn of axis\n",
        "            ax = pyplot.subplot(n_filters, 3, ix)\n",
        "            ax.set_xticks([])\n",
        "            ax.set_yticks([])\n",
        "            # plot filter channel in grayscale\n",
        "            pyplot.imshow(f[:, :, j], cmap='gray')\n",
        "            ix += 1\n",
        "    # show the figure\n",
        "    pyplot.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1226/1226 - 5s - loss: 0.0278 - accuracy: 0.9932 - 5s/epoch - 4ms/step\n",
            "Restored model, accuracy: 99.32%\n",
            "conv2d_25 (5, 5, 3, 64)\n",
            "conv2d_26 (3, 3, 64, 128)\n"
          ]
        }
      ],
      "source": [
        "checkpoint_path = \"AlexNet/cp.ckpt\"\n",
        "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "\n",
        "\n",
        "# summarize filters in each convolutional layer\n",
        "from matplotlib import pyplot\n",
        "\n",
        "model = keras.models.Sequential([\n",
        "        Conv2D(filters=64, kernel_size=(5,5), padding='same', activation='relu', input_shape=(30, 30, 3)),\n",
        "        MaxPool2D(pool_size=(3,3), strides=(2,2)),\n",
        "        Conv2D(filters=128, kernel_size=(3,3), padding='valid', activation='relu'),\n",
        "        MaxPool2D(pool_size=(3,3), strides=(2,2)),\n",
        "        Flatten(),\n",
        "        Dense(1024, activation='relu'),\n",
        "        Dense(NUMBER_OF_CLASSES, activation='softmax'),\n",
        "    ])\n",
        "\n",
        "model.load_weights(checkpoint_path)\n",
        "\n",
        "model.compile(loss=\"categorical_crossentropy\",\n",
        "              optimizer=\"Adam\",\n",
        "              metrics=[\"accuracy\",])\n",
        "\n",
        "loss, acc = model.evaluate(X, y, verbose=2)\n",
        "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))\n",
        "\n",
        "\n",
        "# summarize filter shapes\n",
        "for layer in model.layers:\n",
        "\t# check for convolutional layer\n",
        "\tif 'conv' not in layer.name:\n",
        "\t\tcontinue\n",
        "\t# get filter weights\n",
        "\tfilters, biases = layer.get_weights()\n",
        "\tprint(layer.name, filters.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Detecting that an object or model or tf.train.Checkpoint is being deleted with unrestored values. See the following logs for the specific values in question. To silence these warnings, use `status.expect_partial()`. See https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint#restorefor details about the status object returned by the restore function.\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer.iter\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer.beta_1\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer.beta_2\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer.decay\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer.learning_rate\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-0.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-0.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-1.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-1.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-3.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-3.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-4.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'm' for (root).layer_with_weights-4.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-0.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-0.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-1.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-1.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-3.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-3.bias\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-4.kernel\n",
            "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer's state 'v' for (root).layer_with_weights-4.bias\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAGKCAYAAACy4W7+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAacklEQVR4nO3de4xcdf3/8ffs7uzs7O4sxQuGYbeVNjZF24itKa4UlLKYKmlMFSlGjBeqoZIqtCQ1XigrpWJTAwqISlGhpdC0RtRKQwHrBWm5uK1RGy5NQ7rrgCA2ndndmb3N+f7xyyH+wS/ntWcpfZ+Z5+PvVz+fs5z3zmvPhvPZVBAEgQEA4FjDyb4AAACiUFYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3GuK+w+r1aoVCgXL5XKWSqXeyGs6aYIgsFKpZPl83hoa6PFaxewiqep5dmOXVaFQsK6urrj/3LX+/n7r7Ow82ZeBE4TZRVLV8+zGLqtcLmdmZj09PdbUFL3MvffeK627c+dO+RouuugiKdfT0yPlqtWqvfDCC699bahN4f1tbW2Vfjq94447pHUfe+wx+RpmzJgh5fbu3SvlxsfH7Q9/+AOzW+PC+7tt2zZrbW2NzK9fv15ad8WKFfI1PPHEE1LuzDPPlHKVSsU2bNgQObuxyyr8Jm9qarJ0Oh2Z7+jokNbNZrPyNajfmJP9tUitPF7j9YX3N5VKSfda+VAwM2tubpavoaWlRcop31v/i9mtbeH9bW1ttba2tsi88iBhNrnPXXXO1RkPRc0uv9wGALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4F7s96xC27Ztk96h2rhxo7TeN77xDXnvuXPnSrnnnntOyhWLRZs2bZq8P5Itk8lI7+Bt3rxZWu/pp5+W9/7b3/4m5a666iopVywW7bTTTpP3R7L96le/kt532r9/v7ReX1+fvPdnPvMZKTc4OCjlRkZGpBxPVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA96Z8gsXzzz9v7e3tkblFixZJ6wVBIO+t/nXL888/X8qNj4/LeyP53vKWt1hjY2Nk7q677pLW27Ztm7z3wYMHpZx60kWlUpH3RvL19fVJsztjxgxpvXe/+93y3h/84Ael3MKFC6Xc4OCgfe9734vM8WQFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuDfl45a+9rWvWVNT9DKnn366tN7NN98s7/3qq69KuRtuuEHKjYyM2P79++X9kWxXXXWVdGRXPp+X1vvEJz4h7z1r1iwpl06npdxkjilD8n3oQx+yTCYTmdu0aZO03q5du+S9P/axj0m5FStWSLnR0VEpx5MVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMC9KZ9gkc1mpRMsNm/eLK03mTep58+fL+W6urqk3Pj4uLw3ku/YsWNWLpcjcxs2bJDW27Fjh7y3+v2gntJSqVRs3bp18v5Itn379kmfuz09PdJ6fX198t7PPPOMlLvkkkuk3PDwsN17772ROZ6sAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuxT7BIggCM9NPfSgWi1JueHhYvoaJiQkpp15jmAu/NtSm8P6OjIxMKh9FnUczk07OMPt/J1NMJsfs1rbw/r7Rn32TmZtSqSTl1M/yMBd1Dakg5nQPDAzIxxglTX9/v3V2dp7sy8AJwuwiqep5dmOXVbVatUKhYLlczlKpVOwL9CQIAiuVSpbP562hgd+Q1ipmF0lVz7Mbu6wAAHiz8CMYAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPdiH7dUzy+nIdmYXSRVPc9u7LIqFAp1e+wHko3ZRVLV8+zGLqtcLmdmZh/+8IetqSl6md7eXmndJ598Ur6GefPmSblHHnlEyo2MjNjNN9/82teG2hTe340bN1o2m43Mp9Npad1du3bJ13Ds2DEpd/DgQSkXBIENDw8zuzUuvL/z5s2zxsbGyLx6WPOWLVvka/jPf/4j5fbs2SPlRkZG7Pbbb4+c3dhlFT6CNjU1Sd/M7e3t0rrKh0eora1NymUyGXlNM6uZx2u8vvD+ZrPZN7Ss1JyZST/gmU1+Fpnd2hbe38bGRqmslIyZ/vlspv/FgDf6c5dfbgMA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALgX+6Xg0DnnnGMtLS2ROfW8ssm81PjnP/9Zyv3gBz+QckEQyHsj+Y4cOSK9uPje975XWi+fz8t7qy9MlkolKVcsFu2UU06R90eyXX755dIL7bNnz5bWe/jhh+W9V61aJeW++c1vSjm1G3iyAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4N+UTLB588EHpTyffeOON0nrqn0w2M1u+fLmU+/e//y3lisWivf3tb5f3R7I988wz0p+iv+mmm6T1tmzZIu/d09Mj5davXy/lKpWKvDeSr7u7W/pT9OpMbt++Xd67tbVVyj311FNSbnx8XMrxZAUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4N+Xjls4991zLZDKRuYceekhab/Xq1fLeTz75pJTbunWrlJvMUU9Ivve9733W0tISmTvnnHOk9SYmJuS98/m8lLvlllukXHNzs7w3km/x4sWWSqUic3PnzpXWu//+++W9r7vuOimnHuFULBZtxowZkTmerAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7k35BItXX31Vent+165d0novvfSSvPfx48el3BVXXCGvifrxxz/+0Zqaor8Fpk2bJq23aNEiee/77rtPyi1fvlzKFYtFW7t2rbw/ku3ll1+2jo6OyFylUpHWGx0dlffev3+/lPvoRz8q5cbHx6UcT1YAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPdin2ARBIGZmY2NjUn5crks5dT1/vca3mgnal34EN5f9c15dSbV0wLMzIaHh6VcsVicVI7ZrW3h/VXn4kScYKHOmPr9NTExIa2bCmJO98DAgHV1dcX5p+719/dbZ2fnyb4MnCDMLpKqnmc3dllVq1UrFAqWy+UslUrFvkBPgiCwUqlk+XzeGhr4DWmtYnaRVPU8u7HLCgCANws/ggEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4F7sEyzq+f/3R7Ixu0iqep7d2GVVKBTq9k1qJBuzi6Sq59mNXVa5XM7MzDo7O6Wf5I4ePSqt29PTI1/DhRdeKOXuvPNOKVetVu2FF1547WtDbQrv79///nfpXu/bt09a99Of/rR8DfPmzZNya9askXLlctlWrlzJ7Na48P4eOnRIutdnnXWWtO7g4KB8DRs3bpRy119/vZQLgsDK5XLk1xO7rMJH0IaGhjf01w7pdFrOtrS0SLnJXl+tPF7j9YX3N5fLWUdHR2S+tbX1Db+GxsZGKTfZvZnd2jbZ2T0R85DNZqXcZPeOyvPLbQCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPdivxQcGh0dlV66DYJAWu+ee+6R954/f76Ue/zxx6Xc2NiYHTlyRN4f9WHt2rVS7qtf/aq85tKlS6Wc+uL70NCQvDeSr6+vz9ra2iJzlUpFWu/iiy+W9z527JiUu++++6Tc8PCwXXbZZZE5nqwAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5N+QSLXC4n/Ynu8847T1qvu7tb3vv++++XchdccIGUq1Qq9sADD8j7I9mef/556RSAvr4+ab0NGzbIe/f09Ei52267TcqVy2V5byTfGWecYe3t7ZG5V155RVpPPSnFzCyTyUi5q6++WsqNjIxIOZ6sAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPemfNxStVq1VCoVmZs5c6a03qc+9Sl57wMHDki54eFhKVepVOS9kXyHDx+2bDYbmRsbG5PW2717t7z3xMSElJszZ46US6fT8t5Ivh07dkjHHm3evFlab/HixfLev/3tb6Xc+vXrpZx6VBhPVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA96Z8gsX5559vzc3Nkbne3l5pvTvvvFPe++c//7mU27Bhg5QbHR2V90byLVmyxHK5XGTun//8p7TeL37xC3nvffv2SbnDhw9LuZGREXlvJN/s2bOttbU1MnfttddK66knXZiZXXLJJVLulltukXLValXK8WQFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHAv9gkWQRCYmX7qQ6lUknKVSkW+BnVN9RrDXPi1oTaF91edn6GhISmnnOQSKpfLUk49mSLMMbu1Lby/6vyon6cTExPyNaifp+rJFGEuanZTQczpHhgYsK6urjj/1L3+/n7r7Ow82ZeBE4TZRVLV8+zGLqtqtWqFQsFyuZylUqnYF+hJEARWKpUsn89bQwO/Ia1VzC6Sqp5nN3ZZAQDwZuFHMACAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuxT5uqZ5fTkOyMbtIqnqe3dhlVSgU6vbYDyQbs4ukqufZjV1WuVzOzMz27NljbW1tkfk1a9ZI615++eXyNcydO1fKPfzww1JuZGTENm3a9NrXhtoU3t9rr73WMplMZF49oHbv3r3yNaxatUrKjY2NSbnh4WH74he/yOzWuPD+XnrppZZOpyPzGzZskNb9whe+IF9DR0eHlJs5c6aUGxkZsdtvvz1ydmOXVfgI2tbWZu3t7ZH5piZtq2w2K1+Dsq+ZWUtLi7ymmdXM4zVeX3h/M5mMNBtqWakzbmbSD3hm+gnXIWa3toX3N51OS3OpFstkZlcpSTOTfhD8X1Gzyy+3AQDuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALgX+z2r0J/+9CfpXZUf/ehH0nqTea9kwYIFUu7AgQNSrlwuy3sj+Y4fP26VSiUyt337dmm9z3/+8/LeZ599tpRbuXKllFNfHkZteOSRR6RjtRYvXiyt193dLe+9fPlyKXfllVdKufHxcSnHkxUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwL0pn2Cxd+9e6S9HHj16VFpvMn9e+QMf+ICUe/bZZ6VcEATy3ki+2267Tco9+uijUu6nP/2pvPfFF18s5bZt2yblBgcHbdeuXfL+SLZ3vvOd0l/3ffHFF6X1/vrXv8p79/b2Srl77rlHypXLZXv88ccjczxZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5N+bilrVu3WkdHR2Ru7dq10nrTpk2T937iiSek3O9//3spNzQ0ZEuXLpX3R7J95StfsUwmE5k7dOiQtN71118v762uefjwYSk3PDws743k6+3ttba2tsjcmjVrpPWuvPJKee9cLifl9u7dK+VGR0elHE9WAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3pnyCxcqVKy2dTkfmxsfHpfVWr14t793UpF3+8ePHpRynANSXfD5vLS0tkbn//ve/0npz5syR916/fr2UW7ZsmZQrl8vy3ki+7u5u6eSg7373u9J6kzk56F3vepeU+8tf/iLlKpWKlOPJCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7lBUAwD3KCgDgXuwTLIIgMDOzsbExKa+eYBHnGqKoJ1OEOXVdJFN4f9U359VcsViUr0H9vlFnNzzBgtmtbeH9VWdtaGhIyqmnAZnpp6VM9vsranZTQczpHhgYsK6urjj/1L3+/n7r7Ow82ZeBE4TZRVLV8+zGLqtqtWqFQsFyuZylUqnYF+hJEARWKpUsn89bQwO/Ia1VzC6Sqp5nN3ZZAQDwZuFHMACAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuxT5uqZ5fTkOyMbtIqnqe3dhlVSgU6vbYDyQbs4ukqufZjV1WuVzOzMxWr15tmUwmMn/33XdL6x46dEi+hiVLlki5W2+9VcoNDg7aBRdc8NrXhtoU3t+dO3daa2trZP66666T1t25c6d8DeoBtd///vel3OjoqG3ZsoXZrXHh/Z09e7Y1NjZG5n/5y19K677jHe+Qr2H37t1SrlQqSblyuWyrV6+OnN3YZRU+gmYyGWtpaYnMq7+a6OjokK9BPSm4vb1dXtPMaubxGq8vvL+tra3W1tYWmVfnbDKzq3zQmJk1NzfLa5oxu7UuvL+NjY3SDKk/vExmdpUf8Mwm/5c2omaXX24DANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4F/ul4NCSJUukl25ffPFFab3TTz9d3vuBBx6Qcps2bZJyo6Oj8t5Ivi9/+cvSy+of//jHpfVOPfVUee/FixdLub6+PilXLBbtrrvukvdHsv3sZz+TPnfVU36eeuopee8LL7xQys2ZM0fKVatVKceTFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAvSmfYLFu3Trpz36fccYZ8nqq7u5uKVepVKRcsVi0LVu2yPsj2davXy/9ie5sNiut99BDD8l7Hzx4UMoNDQ29oTnUhqamJulzd9q0adJ6P/zhD+W91VMxMpmMlOMECwBAzaCsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuTfm4pZUrV0pH1vz617+W1jv77LPlvfP5vJRTj/1Qc6gNCxYssFwuF5m76aabpPX+8Y9/yHt/5CMfkXI33nijlBsZGZH3RvItWrTIUqlUZO68886T1vvkJz8p73311VdLuUcffVTKlUol6XOfJysAgHuUFQDAPcoKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHtTPsHi/e9/v3QKwK233iqtp7yVHfrNb34j5V555RUpVyqV5L2RfHfccYd0aol6UsqKFSvkvadNmyblenp6pFy1WpX3RvJt2rTJstlsZO6KK66Q1tu1a5e89+9+9zspd+mll0q5crks5XiyAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOAeZQUAcI+yAgC4F/sEiyAIzEw/9WF8fFzKDQ0NydfQ1KRdvnqNYS782lCbwvs7Ojoq5SuVipQbHByUr6GxsVHKqSdThDlmt7aF91c99aFYLEq54eHh2Nf0/6NeY5iLmt1UEHO6BwYGrKurK84/da+/v986OztP9mXgBGF2kVT1PLuxy6parVqhULBcLjep8/w8C4LASqWS5fN5a2jgN6S1itlFUtXz7MYuKwAA3iz8CAYAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuxT7Co5//fH8nG7CKp6nl2Y5dVoVCo2zepkWzMLpKqnmc3dlnlcjkzM1u2bJml0+nI/IEDB6R1L7zwQvkaXnrpJSnX29sr5QYHB+3cc8997WtDbQrv78yZM6Uz+o4ePSqte9FFF8nXMH36dCl3ww03SLlisWhnnnkms1vjwvvb3t4uPVk99thj0rqf+9zn5GtQ53zWrFlSrlwu2zXXXBM5u7HLKvwPlU6nrbm5OTKvHtyZyWTka1BK0swm/Q1cK4/XeH3h/W1sbJTmUp0HdR7N9Dnv6OiQ1zRjdmtdeH9TqZR0r9XPPvXz2Uyf3Ww2K69pFj27/HIbAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAvdgvBYe+/vWvSy+e/eQnP5HWe+tb3yrvvXDhQim3c+dOKVepVOS9kXxbt2619vb2yFyhUJDW+/a3vy3v/Z73vEfKnXXWWVKuWq3KeyP5Dh8+LL0w3tLSIq13/Phxee8dO3ZIuW9961tSTj3LkicrAIB7lBUAwD3KCgDgHmUFAHCPsgIAuEdZAQDco6wAAO5RVgAA9ygrAIB7Uz7BYtasWdKb1OVyWVpvz5498t67d++WcvPnz5dyExMT8t5Ivh//+MfW3Nwcmbv77rul9dQ5MzM75ZRTpNzhw4flNVE/Fi9eLP0p+t7eXmk9NWdmtm7dOimnfuarOZ6sAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPemfNySemzMgw8+KOUuu+wyee+FCxdKuXw+L+XGx8ft2WeflfdHsu3fv186smbVqlXSesuWLZP37u7ulnK5XE7Klctlu+aaa+T9kWxf+tKXLJvNRuamT58urbdgwQJ573/9619S7umnn5Zyw8PDUo4nKwCAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe1M+weK5556T3rLfvn27tN6xY8fkvZcuXSrljhw5IuXGxsbkvZF806dPt3Q6HZk77bTTpPXmzp0r7/22t71Nyp166qlSrlqtynsj+V5++WVraWmJzM2bN09a77Of/ay8d1OTVhvf+c53pNzExISU48kKAOAeZQUAcI+yAgC4R1kBANyjrAAA7lFWAAD3KCsAgHuUFQDAPcoKAOBe7BMsgiAwM7PBwUEpX6lUpNzIyIh8DePj41JOPZkizIVfG2pTeH/V+VFnt1gsytegnjgx2RyzW9vC+6t+TqozOTo6Kl9DqVSScurJFGEuanZTQczpHhgYsK6urjj/1L3+/n7r7Ow82ZeBE4TZRVLV8+zGLqtqtWqFQsFyuZylUqnYF+hJEARWKpUsn89bQwO/Ia1VzC6Sqp5nN3ZZAQDwZuFHMACAe5QVAMA9ygoA4B5lBQBwj7ICALhHWQEA3KOsAADuUVYAAPcoKwCAe5QVAMA9ygoA4B5lBQBw7/8AyNJRzyy1TbEAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 18 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAAGKCAYAAACy4W7+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAR0klEQVR4nO3dT4itdf3A8c8Z74wzc++ZSxGFw4zXRQpCUYiWizZiIIEZEVFKhhVtWkir3IQkBbUIbKWChf2hTKFoF2RQizbVIpFaaS2a8Ugh5T1nHOdPc54W8vQLkt9cns9zvZ9zzuu1nvPhe3w+d97zHJx5Bk3TNAEAhS1d6QMAwGnECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyznR94XQ6jdFoFMPhMAaDQZ9numKaponJZBKbm5uxtKTj88ruMqsWeXc7x2o0GsX29nbXl5e2s7MTW1tbV/oYXCZ2l1m1yLvbOVbD4TAiIj74wQ/GmTOdx0RExFNPPZV6fcTrP3FkjcfjuHDhwn/eG/Opvb6PPPJIrK2tpWZ95jOfSZ/n4sWL6Rnj8Ti2t7ft7pxrr+8dd9wRy8vLqVnPPPNM+jyf+tSn0jOOjo7iBz/4wam727ky7S3omTNn0v/RNjY2Uq+P6CdWrXm5veaNtdd3bW0t1tfXr/Bp+tn/lt2db+31XV5eTn/f7WNXVlZW0jNap53Hh9sAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQ3aJqm6fLC8Xgc58+fj69+9auxurqaOsSf/vSn1Osj+nk2y3Q6jZdeeikuXrzY62MbqKXd3be//e3pp+reeeed6fOcPXs2PePw8DAee+wxuzvn2t3t4zr38UTp/f399IzxeBzveMc7Tn1P7qwAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKO9MdsDy8nIsLy+nZrzvfe/LHiO++93vpmewWF555ZUYDAapGYeHh+lz3HXXXekZ+/v78dhjj6XnMBs++9nPpr/vPvfcc+lzrK2tpWdcKndWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJTX+eGLTdNERMTBwUH6ECcnJ+kZfWrfG/Opvb59XOejo6P0jP39/d5m2N351l7f4+Pj9Ky9vb30jD6dtruDpuN27+7uxvb2dqdDVbezsxNbW1tX+hhcJnaXWbXIu9s5VtPpNEajUQyHw/SjwatomiYmk0lsbm7G0pJPSOeV3WVWLfLudo4VALxZ/AgGQHliBUB5YgVAeWIFQHliBUB5YgVAeWIFQHmd/9zSIv9yGrPN7jKrFnl3O8dqNBot7J/9YLbZXWbVIu9u51gNh8OuL/0fv//979Mzfvazn6VnHB4exje/+c1e3xv19Hl9X3zxxfSMf/3rX+kZk8kk3vWud9ndOdde352dndjY2EjNevTRR9Pneeihh9IzmqaJg4ODU3e3c6z6vAU9d+5cesbq6moPJ3ndvNxe88b6vL7ZbxgR/cSqZXfnW3t9NzY20rvXx/fMPvfttFk+3AagPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqC8zs+zal28eDH9XJW//vWv2WP08vDFk5OT9Axmx/PPP59+WOGHP/zh9Dn+8Y9/pGfY3cXyve99L9bW1lIzfvnLX6bPsb+/n55xqdxZAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVBe+uGLDz74YFx99dWpGTfffHP2GPGRj3wkPePg4CCeffbZ9Bxmw49+9KNYXV1NzfjVr36VPsf111+fnjGdTtMzmB3Hx8dx5kzu2/e73/3u9Dmef/759IyTk5N44YUXTv06d1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlCdWAJQnVgCUJ1YAlNf56V1N00RExOHhYfoQ+/v76RkHBwfpGe17ad8b86m9vn3szHg8Ts/o48GJ7Qy7O9/63N0+vnefnJykZ1zq7g6ajtu9u7sb29vbXV5a3s7OTmxtbV3pY3CZ2F1m1SLvbudYTafTGI1GMRwOYzAYdD5gJU3TxGQyic3NzVha8gnpvLK7zKpF3t3OsQKAN4sfwQAoT6wAKE+sAChPrAAoT6wAKE+sACiv81+wWOT/35/ZZneZVYu8u51jNRqNFvY3qZltdpdZtci72zlWw+EwIiKuu+669E9yd911V+r1ERHf+ta30jNa7XtjPrXX989//nP6Wt9zzz3p8/zkJz9JzxiPx7G9vW1351x7fR9//PFYX19Pzfr617+ePs+nP/3p9IyDg4N48MEHT93dzrFqb0GXlpbSsbr66qtTr+/bvNxe88ba6zscDmNjYyM168yZzv+E/iN7hv9md+dbe33X19fTsbrqqqvS51lbW0vPaJ22uz7cBqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoLz0w3juvvvuWF1dTc348pe/nD1GfO5zn0vP2Nvbi5tuuik9h9kwnU5jOp2mZ2Tdcsst6RknJyfpGcyOO++8M/0ctKeffjp9jnvvvTc9Yzwex5e+9KVTv86dFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDlpR+++MlPfjKGw2Fqxq9//evsMeKJJ55Izzg6OkrPYHZcuHAhBoNBasYNN9yQPsdzzz2XnsFi+fa3vx1ra2upGU8++WT6HNdee216xuHh4SV9nTsrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMrr/PDFpmkiImJvby99iFdffTU9o48HJx4fH0fE/7035lN7ffu4zicnJ+kZfbK78629vgcHB1f4JK+71AcnXsqM03Z30HTc7t3d3dje3u7y0vJ2dnZia2vrSh+Dy8TuMqsWeXc7x2o6ncZoNIrhcJh+NHgVTdPEZDKJzc3NWFryCem8srvMqkXe3c6xAoA3ix/BAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sACiv859bWuRfTmO22V1m1SLvbudYjUajhf2zH8w2u8usWuTd7Ryr4XAYEREf/ehHY3l5ueuYiIh4/PHHU6+PiDh//nx6Rqt9b8yn9vrec889sbKykpr13ve+N32eL37xi+kZLbs739rr+4tf/CLOnj2bmnXfffelz3PTTTelZxwfH8dPf/rTU3e3c6zaW9Dl5eX0P/iNjY3U6/s2L7fXvLH2+q6srKR3d21trY8j9cbuzrf2+p49ezbOnTuXmnXVVVelz5P99/PfTttdH24DUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVBe50eEtG699db0YxKeffbZ7DHirW99a3pG0zTxz3/+Mz2H2fD3v/89/Sy2Rx99NH2O97znPekZJycn8cc//jE9h9nw8MMPp3f31ltvTZ/jiSeeSM8Yj8fx1FNPnfp17qwAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKE+sAChPrAAoT6wAKC/98MWHH344lpZyzXv55Zezx4hrrrkmPWM6nXr44gK5+eabY3V1NTWjjwcevvDCC+kZ4/E4zp8/n57DbPjQhz4U6+vrqRl33313+hw33nhjesbBwcElfZ07KwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDK6/zwxaZpIuL1BxZmtbMy+jhHO6OP81BXe30PDw/Ts/rYu/F43NsMuzvf2uv72muvXeGTvO5SH5x4KTNO291B03G7d3d3Y3t7u8tLy9vZ2Ymtra0rfQwuE7vLrFrk3e0cq+l0GqPRKIbDYQwGg84HrKRpmphMJrG5uRlLSz4hnVd2l1m1yLvbOVYA8GbxIxgA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5XX+c0uL/MtpzDa7y6xa5N3tHKvRaLSwf/aD2WZ3mVWLvLudYzUcDiMi4itf+Uqsrq52HRMRERcuXEi9PiLine98Z3rG3t5e3Hbbbf95b8yn9vped9116buQP/zhD+nz9PFHSSeTSVx//fV2d86113dnZyc2NjZSsz7/+c+nz/P000+nZ7RO293OsWpvQVdXV9OxWl9fT70+IuLcuXPpGa15ub3mjbXXd2lpKR2r7DeMiIjl5eX0jJbdnW/t9d3Y2Ejv3srKSh9H6s1pu+vDbQDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMrr/Dyr1m9+85v083gmk0n2GPHkk0+mZxwfH6dnMDu+853vpJ+DduONN6bP8f73vz894+joKD2D2fH9738/1tbWUjP62N0+nlo8nU7jxRdfPPXr3FkBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUF764Ys//OEPY2NjIzXjC1/4QvYY8eMf/zg9g8Vy2223pWf85S9/Sc/4xje+kZ4xGAzSM5gdDzzwQPqaf/zjH0+f45FHHknP2N/fj0984hOnfp07KwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDKEysAyhMrAMoTKwDK6/zwxaZpIiJiPB6nD3F0dJSe0af2vTGf+ry+k8kkPaOP/W9n2N351l7fPq5zH3u3v7/f24zT3tOg6fiud3d3Y3t7u8tLy9vZ2Ymtra0rfQwuE7vLrFrk3e0cq+l0GqPRKIbD4dw8UrtpmphMJrG5uRlLSz4hnVd2l1m1yLvbOVYA8GbxIxgA5YkVAOWJFQDliRUA5YkVAOWJFQDldf4LFov8//sz2+wus2qRd7dzrEaj0cL+JjWzze4yqxZ5dzvHajgcRkTEQw89FKurq13HRETEM888k3p9RMTvfve79Iy28O17Yz6113drayt9F3LDDTekz7O+vp6ecXx8HD//+c/t7pzr8/ree++96Rm33HJLesZrr70WDzzwwKnvrXOs2lvQ1dXVWFtb6zrm9UOc6XyM/zlPH+bl9po31l7fpaWldKyWl5fT5+ljRsvuzrc+r+/Kykp6RvZ7/3877b35cBuA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDyxAqA8sQKgPLECoDy0g+Suvbaa9MPj7vmmmuyx4hXXnklPWM8Hsf58+fTc5gNd9xxR/qZPl/72tfS53jLW96SnsFiuf3229PPAfzABz6QPsd9992XnjEej+P+++8/9evcWQFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQXvrhi/fff38sLeWa97e//S17jDg4OEjPOD4+Ts9gdnzsYx+Ls2fPpma8/PLL6XM0TZOe4cGhi+W3v/1tDAaD1Iy3ve1t6XPcfvvt6RmTyeSSvs6dFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDliRUA5YkVAOWJFQDldX74YvvAuOl02tthMvp4cGI7o4+H4VFXe31fffXV9KxLfXDc/2c8Hvc2w+7Ot/b69nGd+/ie2cf+7+3tRcTp72nQdHzXu7u7sb293eWl5e3s7MTW1taVPgaXid1lVi3y7naO1XQ6jdFoFMPhMP145SqaponJZBKbm5uxtOQT0nlld5lVi7y7nWMFAG8WP4IBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQnlgBUJ5YAVCeWAFQ3r8Bf3TATheS0Y8AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 18 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "for l in range(0, 4, 2):\n",
        "    # retrieve weights from the second hidden layer\n",
        "    filters, biases = model.layers[l].get_weights()\n",
        "    # normalize filter values to 0-1 so we can visualize them\n",
        "    f_min, f_max = filters.min(), filters.max()\n",
        "    filters = (filters - f_min) / (f_max - f_min)\n",
        "    # plot first few filters\n",
        "    n_filters, ix = 6, 1\n",
        "    for i in range(n_filters):\n",
        "        # get the filter\n",
        "        f = filters[:, :, :, i]\n",
        "        # plot each channel separately\n",
        "        for j in range(3):\n",
        "            # specify subplot and turn of axis\n",
        "            ax = pyplot.subplot(n_filters, 3, ix)\n",
        "            ax.set_xticks([])\n",
        "            ax.set_yticks([])\n",
        "            # plot filter channel in grayscale\n",
        "            pyplot.imshow(f[:, :, j], cmap='gray')\n",
        "            ix += 1\n",
        "    # show the figure\n",
        "    pyplot.show()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3.10.8 ('tf')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.15"
    },
    "vscode": {
      "interpreter": {
        "hash": "7f3991beda84519ddbebd03ee67fce523f4a815cc55d3e3530775fe1c7f2ada7"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}