exp_resnet_age.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-27 10:47:03.290225: I tensorflow/core/platform/cpu_feature_guard.cc:183] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "import numpy as np\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path(\"/scratch/ajb5d/ecg/tfrecords/\")\n",
    "TRAIN_RECS = list(DATA_PATH.glob(\"train*.tfrecords\"))\n",
    "VAL_RECS = list(DATA_PATH.glob(\"val*.tfrecords\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "\n",
    "record_format = {\n",
    "    'ecg/data': tf.io.FixedLenSequenceFeature([], tf.float32, allow_missing=True),\n",
    "    'age': tf.io.FixedLenFeature([], tf.float32),\n",
    "    'gender': tf.io.FixedLenFeature([], tf.int64),\n",
    "}\n",
    "\n",
    "def _parse_record(record):\n",
    "    example = tf.io.parse_single_example(record, record_format)\n",
    "    ecg_data = tf.reshape(example['ecg/data'], [5000,12])\n",
    "    label = example['age']\n",
    "    return ecg_data, label\n",
    "\n",
    "def drop_na_ages(x,y):\n",
    "    return not tf.math.reduce_any(tf.math.is_nan(y))\n",
    "\n",
    "def age_lt_90(x,y):\n",
    "    return tf.math.reduce_all(tf.math.less_equal(y, tf.constant([90.0])))\n",
    "\n",
    "def load_dataset(filenames):\n",
    "    ignore_order = tf.data.Options()\n",
    "    ignore_order.experimental_deterministic = False\n",
    "    dataset = tf.data.TFRecordDataset(filenames)\n",
    "    dataset = dataset.with_options(ignore_order)\n",
    "    dataset = dataset.map(_parse_record, num_parallel_calls=tf.data.AUTOTUNE)\n",
    "    dataset = dataset.filter(drop_na_ages)\n",
    "    dataset = dataset.filter(age_lt_90)\n",
    "    return dataset\n",
    "\n",
    "def get_dataset(filenames, labeled=True):\n",
    "    dataset = load_dataset(filenames)\n",
    "    dataset = dataset.shuffle(2048)\n",
    "    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-27 10:47:07.850207: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1636] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 78717 MB memory:  -> device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:07:00.0, compute capability: 8.0\n"
     ]
    }
   ],
   "source": [
    "train_dataset = get_dataset(TRAIN_RECS)\n",
    "val_dataset = get_dataset(VAL_RECS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def residual_unit(x, y, n_samples_out, n_filters_out, prefix, kernel_size = 16):\n",
    "    n_samples_in = y.shape[1]\n",
    "    downsample = n_samples_in // n_samples_out\n",
    "    n_filters_in = y.shape[2]\n",
    "    \n",
    "    if downsample == 1:\n",
    "        y = y\n",
    "    else:\n",
    "        y = layers.MaxPooling1D(downsample, strides=downsample, padding='same', name = f\"{prefix}_mp_opt\")(y)\n",
    "        \n",
    "    if n_filters_in != n_filters_out:\n",
    "        y = layers.Conv1D(n_filters_out, 1, padding='same', use_bias=False, name = f\"{prefix}_conv_opt\")(y)\n",
    "        \n",
    "    x = layers.Conv1D(n_filters_out, kernel_size, padding='same', use_bias=False, name = f\"{prefix}_conv1\")(x)\n",
    "    x = layers.BatchNormalization(name = f\"{prefix}_bn1\")(x)\n",
    "    x = layers.Activation(\"relu\", name = f\"{prefix}_act1\")(x)\n",
    "    x = layers.Dropout(0.2, name = f\"{prefix}_dropout1\")(x)\n",
    "    x = layers.Conv1D(n_filters_out, kernel_size, strides=downsample, padding='same', use_bias=False, name = f\"{prefix}_conv2\")(x)\n",
    "\n",
    "    x = layers.Add(name = f\"{prefix}_add\")([x,y])\n",
    "    y = x\n",
    "    x = layers.BatchNormalization(name = f\"{prefix}_bn2\")(x)\n",
    "    x = layers.Activation(\"relu\", name = f\"{prefix}_act2\")(x)\n",
    "    x = layers.Dropout(0.2, name = f\"{prefix}_dropout2\")(x)\n",
    "    return (x,y)\n",
    "    \n",
    "input_layer = tf.keras.layers.Input(shape=(5000,12), name=\"input\")\n",
    "x = input_layer\n",
    "x = layers.Conv1D(64, 16, padding='same', use_bias=False, name = \"conv_1\")(x)\n",
    "x = layers.BatchNormalization(name=\"bn\")(x)\n",
    "x = layers.Activation(\"relu\", name=\"relu\")(x)\n",
    "\n",
    "x, y = residual_unit(x,x,1024,128, \"res1\")\n",
    "x, y = residual_unit(x,y,256,196, \"res2\")\n",
    "x, y = residual_unit(x,y,64,256, \"res3\")\n",
    "x, _ = residual_unit(x,y,16,320, \"res4\")\n",
    "\n",
    "x = tf.keras.layers.Flatten(name=\"flatten\")(x)\n",
    "x = tf.keras.layers.Dense(1, name=\"output\")(x)\n",
    "\n",
    "model = tf.keras.models.Model(input_layer, x)\n",
    "\n",
    "model.compile(\n",
    "    optimizer='adam',\n",
    "    loss=tf.keras.losses.MeanSquaredError(),\n",
    "    metrics=['mse', 'mae']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input (InputLayer)          [(None, 5000, 12)]           0         []                            \n",
      "                                                                                                  \n",
      " conv_1 (Conv1D)             (None, 5000, 64)             12288     ['input[0][0]']               \n",
      "                                                                                                  \n",
      " bn (BatchNormalization)     (None, 5000, 64)             256       ['conv_1[0][0]']              \n",
      "                                                                                                  \n",
      " relu (Activation)           (None, 5000, 64)             0         ['bn[0][0]']                  \n",
      "                                                                                                  \n",
      " res1_conv1 (Conv1D)         (None, 5000, 128)            131072    ['relu[0][0]']                \n",
      "                                                                                                  \n",
      " res1_bn1 (BatchNormalizati  (None, 5000, 128)            512       ['res1_conv1[0][0]']          \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res1_act1 (Activation)      (None, 5000, 128)            0         ['res1_bn1[0][0]']            \n",
      "                                                                                                  \n",
      " res1_dropout1 (Dropout)     (None, 5000, 128)            0         ['res1_act1[0][0]']           \n",
      "                                                                                                  \n",
      " res1_mp_opt (MaxPooling1D)  (None, 1250, 64)             0         ['relu[0][0]']                \n",
      "                                                                                                  \n",
      " res1_conv2 (Conv1D)         (None, 1250, 128)            262144    ['res1_dropout1[0][0]']       \n",
      "                                                                                                  \n",
      " res1_conv_opt (Conv1D)      (None, 1250, 128)            8192      ['res1_mp_opt[0][0]']         \n",
      "                                                                                                  \n",
      " res1_add (Add)              (None, 1250, 128)            0         ['res1_conv2[0][0]',          \n",
      "                                                                     'res1_conv_opt[0][0]']       \n",
      "                                                                                                  \n",
      " res1_bn2 (BatchNormalizati  (None, 1250, 128)            512       ['res1_add[0][0]']            \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res1_act2 (Activation)      (None, 1250, 128)            0         ['res1_bn2[0][0]']            \n",
      "                                                                                                  \n",
      " res1_dropout2 (Dropout)     (None, 1250, 128)            0         ['res1_act2[0][0]']           \n",
      "                                                                                                  \n",
      " res2_conv1 (Conv1D)         (None, 1250, 196)            401408    ['res1_dropout2[0][0]']       \n",
      "                                                                                                  \n",
      " res2_bn1 (BatchNormalizati  (None, 1250, 196)            784       ['res2_conv1[0][0]']          \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res2_act1 (Activation)      (None, 1250, 196)            0         ['res2_bn1[0][0]']            \n",
      "                                                                                                  \n",
      " res2_dropout1 (Dropout)     (None, 1250, 196)            0         ['res2_act1[0][0]']           \n",
      "                                                                                                  \n",
      " res2_mp_opt (MaxPooling1D)  (None, 313, 128)             0         ['res1_add[0][0]']            \n",
      "                                                                                                  \n",
      " res2_conv2 (Conv1D)         (None, 313, 196)             614656    ['res2_dropout1[0][0]']       \n",
      "                                                                                                  \n",
      " res2_conv_opt (Conv1D)      (None, 313, 196)             25088     ['res2_mp_opt[0][0]']         \n",
      "                                                                                                  \n",
      " res2_add (Add)              (None, 313, 196)             0         ['res2_conv2[0][0]',          \n",
      "                                                                     'res2_conv_opt[0][0]']       \n",
      "                                                                                                  \n",
      " res2_bn2 (BatchNormalizati  (None, 313, 196)             784       ['res2_add[0][0]']            \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res2_act2 (Activation)      (None, 313, 196)             0         ['res2_bn2[0][0]']            \n",
      "                                                                                                  \n",
      " res2_dropout2 (Dropout)     (None, 313, 196)             0         ['res2_act2[0][0]']           \n",
      "                                                                                                  \n",
      " res3_conv1 (Conv1D)         (None, 313, 256)             802816    ['res2_dropout2[0][0]']       \n",
      "                                                                                                  \n",
      " res3_bn1 (BatchNormalizati  (None, 313, 256)             1024      ['res3_conv1[0][0]']          \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res3_act1 (Activation)      (None, 313, 256)             0         ['res3_bn1[0][0]']            \n",
      "                                                                                                  \n",
      " res3_dropout1 (Dropout)     (None, 313, 256)             0         ['res3_act1[0][0]']           \n",
      "                                                                                                  \n",
      " res3_mp_opt (MaxPooling1D)  (None, 79, 196)              0         ['res2_add[0][0]']            \n",
      "                                                                                                  \n",
      " res3_conv2 (Conv1D)         (None, 79, 256)              1048576   ['res3_dropout1[0][0]']       \n",
      "                                                                                                  \n",
      " res3_conv_opt (Conv1D)      (None, 79, 256)              50176     ['res3_mp_opt[0][0]']         \n",
      "                                                                                                  \n",
      " res3_add (Add)              (None, 79, 256)              0         ['res3_conv2[0][0]',          \n",
      "                                                                     'res3_conv_opt[0][0]']       \n",
      "                                                                                                  \n",
      " res3_bn2 (BatchNormalizati  (None, 79, 256)              1024      ['res3_add[0][0]']            \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res3_act2 (Activation)      (None, 79, 256)              0         ['res3_bn2[0][0]']            \n",
      "                                                                                                  \n",
      " res3_dropout2 (Dropout)     (None, 79, 256)              0         ['res3_act2[0][0]']           \n",
      "                                                                                                  \n",
      " res4_conv1 (Conv1D)         (None, 79, 320)              1310720   ['res3_dropout2[0][0]']       \n",
      "                                                                                                  \n",
      " res4_bn1 (BatchNormalizati  (None, 79, 320)              1280      ['res4_conv1[0][0]']          \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res4_act1 (Activation)      (None, 79, 320)              0         ['res4_bn1[0][0]']            \n",
      "                                                                                                  \n",
      " res4_dropout1 (Dropout)     (None, 79, 320)              0         ['res4_act1[0][0]']           \n",
      "                                                                                                  \n",
      " res4_mp_opt (MaxPooling1D)  (None, 20, 256)              0         ['res3_add[0][0]']            \n",
      "                                                                                                  \n",
      " res4_conv2 (Conv1D)         (None, 20, 320)              1638400   ['res4_dropout1[0][0]']       \n",
      "                                                                                                  \n",
      " res4_conv_opt (Conv1D)      (None, 20, 320)              81920     ['res4_mp_opt[0][0]']         \n",
      "                                                                                                  \n",
      " res4_add (Add)              (None, 20, 320)              0         ['res4_conv2[0][0]',          \n",
      "                                                                     'res4_conv_opt[0][0]']       \n",
      "                                                                                                  \n",
      " res4_bn2 (BatchNormalizati  (None, 20, 320)              1280      ['res4_add[0][0]']            \n",
      " on)                                                                                              \n",
      "                                                                                                  \n",
      " res4_act2 (Activation)      (None, 20, 320)              0         ['res4_bn2[0][0]']            \n",
      "                                                                                                  \n",
      " res4_dropout2 (Dropout)     (None, 20, 320)              0         ['res4_act2[0][0]']           \n",
      "                                                                                                  \n",
      " flatten (Flatten)           (None, 6400)                 0         ['res4_dropout2[0][0]']       \n",
      "                                                                                                  \n",
      " output (Dense)              (None, 1)                    6401      ['flatten[0][0]']             \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 6401313 (24.42 MB)\n",
      "Trainable params: 6397585 (24.40 MB)\n",
      "Non-trainable params: 3728 (14.56 KB)\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import os\n",
    "\n",
    "def make_checkpoint_dir(data_path, label):\n",
    "    current_datetime = datetime.now()\n",
    "    formatted_datetime = current_datetime.strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
    "    output_dir = f\"{label}-{formatted_datetime}\"\n",
    "    output_path = f\"{data_path}/{output_dir}\"\n",
    "    \n",
    "    if not os.path.exists(output_path):\n",
    "        os.makedirs(output_path)\n",
    "    \n",
    "    return output_path\n",
    "\n",
    "model_name = \"resnet-age\"\n",
    "output_path = make_checkpoint_dir(\"data/models\", model_name)\n",
    "\n",
    "print(f\"Model: {model_name} Run Path: {output_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-27 10:47:11.666549: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:432] Loaded cuDNN version 8904\n",
      "2023-11-27 10:47:13.018117: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:625] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
      "2023-11-27 10:47:13.044564: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x557d1b22bb00 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2023-11-27 10:47:13.044607: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA A100-SXM4-80GB, Compute Capability 8.0\n",
      "2023-11-27 10:47:13.116809: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:255] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
      "2023-11-27 10:47:13.587052: I ./tensorflow/compiler/jit/device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   7479/Unknown - 311s 40ms/step - loss: 129.7336 - mse: 129.7336 - mae: 8.9518"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-27 10:52:44.311833: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous recv item cancelled. Key hash: 8031426800233044737\n",
      "2023-11-27 10:52:44.311926: I tensorflow/core/framework/local_rendezvous.cc:405] Local rendezvous recv item cancelled. Key hash: 17256161008876952570\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 339s 44ms/step - loss: 129.7336 - mse: 129.7336 - mae: 8.9518 - val_loss: 146.3850 - val_mse: 146.3850 - val_mae: 9.5457 - lr: 0.0010\n",
      "Epoch 2/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 109.7593 - mse: 109.7593 - mae: 8.2473INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 321s 43ms/step - loss: 109.7593 - mse: 109.7593 - mae: 8.2473 - val_loss: 129.2401 - val_mse: 129.2401 - val_mae: 9.0000 - lr: 0.0010\n",
      "Epoch 3/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 102.7204 - mse: 102.7204 - mae: 7.9732INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 321s 43ms/step - loss: 102.7204 - mse: 102.7204 - mae: 7.9732 - val_loss: 124.7040 - val_mse: 124.7040 - val_mae: 8.8191 - lr: 0.0010\n",
      "Epoch 4/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 97.3381 - mse: 97.3381 - mae: 7.7602INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 322s 43ms/step - loss: 97.3381 - mse: 97.3381 - mae: 7.7602 - val_loss: 124.1345 - val_mse: 124.1345 - val_mae: 8.7878 - lr: 0.0010\n",
      "Epoch 5/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 92.8989 - mse: 92.8989 - mae: 7.5809 - val_loss: 129.5034 - val_mse: 129.5034 - val_mae: 9.1177 - lr: 0.0010\n",
      "Epoch 6/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 88.6910 - mse: 88.6910 - mae: 7.4046 - val_loss: 131.1395 - val_mse: 131.1395 - val_mae: 9.1128 - lr: 0.0010\n",
      "Epoch 7/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 84.8381 - mse: 84.8381 - mae: 7.2482INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 322s 43ms/step - loss: 84.8381 - mse: 84.8381 - mae: 7.2482 - val_loss: 113.9278 - val_mse: 113.9278 - val_mae: 8.4290 - lr: 0.0010\n",
      "Epoch 8/50\n",
      "7479/7479 [==============================] - 317s 42ms/step - loss: 81.0523 - mse: 81.0523 - mae: 7.0832 - val_loss: 116.8566 - val_mse: 116.8566 - val_mae: 8.5343 - lr: 0.0010\n",
      "Epoch 9/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 77.2457 - mse: 77.2457 - mae: 6.9188 - val_loss: 123.2538 - val_mse: 123.2538 - val_mae: 8.7076 - lr: 0.0010\n",
      "Epoch 10/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 73.9339 - mse: 73.9339 - mae: 6.7744 - val_loss: 115.1247 - val_mse: 115.1247 - val_mae: 8.4746 - lr: 0.0010\n",
      "Epoch 11/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 70.7421 - mse: 70.7421 - mae: 6.6323 - val_loss: 115.2487 - val_mse: 115.2487 - val_mae: 8.4464 - lr: 0.0010\n",
      "Epoch 12/50\n",
      "7479/7479 [==============================] - 318s 43ms/step - loss: 67.7304 - mse: 67.7304 - mae: 6.4944 - val_loss: 120.0597 - val_mse: 120.0597 - val_mae: 8.5774 - lr: 0.0010\n",
      "Epoch 13/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 65.0930 - mse: 65.0930 - mae: 6.3703 - val_loss: 114.0742 - val_mse: 114.0742 - val_mae: 8.4253 - lr: 0.0010\n",
      "Epoch 14/50\n",
      "7479/7479 [==============================] - 317s 42ms/step - loss: 62.7091 - mse: 62.7091 - mae: 6.2552 - val_loss: 114.8139 - val_mse: 114.8139 - val_mae: 8.4528 - lr: 0.0010\n",
      "Epoch 15/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 60.3091 - mse: 60.3091 - mae: 6.1385 - val_loss: 115.5121 - val_mse: 115.5121 - val_mae: 8.4405 - lr: 0.0010\n",
      "Epoch 16/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 58.2626 - mse: 58.2626 - mae: 6.0349 - val_loss: 117.3271 - val_mse: 117.3271 - val_mae: 8.5018 - lr: 0.0010\n",
      "Epoch 17/50\n",
      "7479/7479 [==============================] - 317s 42ms/step - loss: 56.2254 - mse: 56.2254 - mae: 5.9291 - val_loss: 121.3017 - val_mse: 121.3017 - val_mae: 8.6339 - lr: 0.0010\n",
      "Epoch 18/50\n",
      "7478/7479 [============================>.] - ETA: 0s - loss: 57.9805 - mse: 57.9805 - mae: 6.0313INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 322s 43ms/step - loss: 57.9779 - mse: 57.9779 - mae: 6.0312 - val_loss: 112.1971 - val_mse: 112.1971 - val_mae: 8.3170 - lr: 1.0000e-04\n",
      "Epoch 19/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 54.3352 - mse: 54.3352 - mae: 5.8432 - val_loss: 113.2653 - val_mse: 113.2653 - val_mae: 8.3455 - lr: 1.0000e-04\n",
      "Epoch 20/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 52.7165 - mse: 52.7165 - mae: 5.7625 - val_loss: 113.6931 - val_mse: 113.6931 - val_mae: 8.3679 - lr: 1.0000e-04\n",
      "Epoch 21/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 51.4877 - mse: 51.4877 - mae: 5.6933 - val_loss: 112.3213 - val_mse: 112.3213 - val_mae: 8.3234 - lr: 1.0000e-04\n",
      "Epoch 22/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 50.4970 - mse: 50.4970 - mae: 5.6406 - val_loss: 113.8818 - val_mse: 113.8818 - val_mae: 8.3770 - lr: 1.0000e-04\n",
      "Epoch 23/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 49.7123 - mse: 49.7123 - mae: 5.5937 - val_loss: 114.0713 - val_mse: 114.0713 - val_mae: 8.3789 - lr: 1.0000e-04\n",
      "Epoch 24/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 48.8447 - mse: 48.8447 - mae: 5.5491 - val_loss: 115.9292 - val_mse: 115.9292 - val_mae: 8.4397 - lr: 1.0000e-04\n",
      "Epoch 25/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 48.2274 - mse: 48.2274 - mae: 5.5115 - val_loss: 113.5741 - val_mse: 113.5741 - val_mae: 8.3570 - lr: 1.0000e-04\n",
      "Epoch 26/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 47.4612 - mse: 47.4612 - mae: 5.4670 - val_loss: 114.6217 - val_mse: 114.6217 - val_mae: 8.3932 - lr: 1.0000e-04\n",
      "Epoch 27/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 46.9575 - mse: 46.9575 - mae: 5.4401 - val_loss: 114.7703 - val_mse: 114.7703 - val_mae: 8.4061 - lr: 1.0000e-04\n",
      "Epoch 28/50\n",
      "7479/7479 [==============================] - 318s 42ms/step - loss: 46.3298 - mse: 46.3298 - mae: 5.4032 - val_loss: 114.8688 - val_mse: 114.8688 - val_mae: 8.4102 - lr: 1.0000e-04\n",
      "Epoch 29/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 48.9400 - mse: 48.9400 - mae: 5.5532INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 322s 43ms/step - loss: 48.9400 - mse: 48.9400 - mae: 5.5532 - val_loss: 109.4445 - val_mse: 109.4445 - val_mae: 8.1897 - lr: 1.0000e-05\n",
      "Epoch 30/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 47.8904 - mse: 47.8904 - mae: 5.4922INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 322s 43ms/step - loss: 47.8904 - mse: 47.8904 - mae: 5.4922 - val_loss: 109.3084 - val_mse: 109.3084 - val_mae: 8.1859 - lr: 1.0000e-05\n",
      "Epoch 31/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 47.6895 - mse: 47.6895 - mae: 5.4819INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 324s 43ms/step - loss: 47.6895 - mse: 47.6895 - mae: 5.4819 - val_loss: 109.1913 - val_mse: 109.1913 - val_mae: 8.1818 - lr: 1.0000e-05\n",
      "Epoch 32/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 47.4646 - mse: 47.4646 - mae: 5.4682 - val_loss: 109.1996 - val_mse: 109.1996 - val_mae: 8.1834 - lr: 1.0000e-05\n",
      "Epoch 33/50\n",
      "7479/7479 [==============================] - 318s 43ms/step - loss: 47.2837 - mse: 47.2837 - mae: 5.4535 - val_loss: 109.2777 - val_mse: 109.2777 - val_mae: 8.1855 - lr: 1.0000e-05\n",
      "Epoch 34/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 47.2148 - mse: 47.2148 - mae: 5.4567 - val_loss: 109.2027 - val_mse: 109.2027 - val_mae: 8.1835 - lr: 1.0000e-05\n",
      "Epoch 35/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 47.1129 - mse: 47.1129 - mae: 5.4494 - val_loss: 109.3114 - val_mse: 109.3114 - val_mae: 8.1866 - lr: 1.0000e-05\n",
      "Epoch 36/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 47.0263 - mse: 47.0263 - mae: 5.4435INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 324s 43ms/step - loss: 47.0263 - mse: 47.0263 - mae: 5.4435 - val_loss: 109.1736 - val_mse: 109.1736 - val_mae: 8.1812 - lr: 1.0000e-05\n",
      "Epoch 37/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 46.8220 - mse: 46.8220 - mae: 5.4319INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 324s 43ms/step - loss: 46.8220 - mse: 46.8220 - mae: 5.4319 - val_loss: 109.1105 - val_mse: 109.1105 - val_mae: 8.1799 - lr: 1.0000e-05\n",
      "Epoch 38/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.7881 - mse: 46.7881 - mae: 5.4326 - val_loss: 109.2898 - val_mse: 109.2898 - val_mae: 8.1866 - lr: 1.0000e-05\n",
      "Epoch 39/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.7658 - mse: 46.7658 - mae: 5.4300 - val_loss: 109.2142 - val_mse: 109.2142 - val_mae: 8.1842 - lr: 1.0000e-05\n",
      "Epoch 40/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 46.7279 - mse: 46.7279 - mae: 5.4322 - val_loss: 109.1718 - val_mse: 109.1718 - val_mae: 8.1830 - lr: 1.0000e-05\n",
      "Epoch 41/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 46.6019 - mse: 46.6019 - mae: 5.4201 - val_loss: 109.3761 - val_mse: 109.3761 - val_mae: 8.1897 - lr: 1.0000e-05\n",
      "Epoch 42/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.5502 - mse: 46.5502 - mae: 5.4170 - val_loss: 109.3586 - val_mse: 109.3586 - val_mae: 8.1894 - lr: 1.0000e-05\n",
      "Epoch 43/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.3303 - mse: 46.3303 - mae: 5.4028 - val_loss: 109.2410 - val_mse: 109.2410 - val_mae: 8.1846 - lr: 1.0000e-05\n",
      "Epoch 44/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.2168 - mse: 46.2168 - mae: 5.3999 - val_loss: 109.2585 - val_mse: 109.2585 - val_mae: 8.1853 - lr: 1.0000e-05\n",
      "Epoch 45/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 46.2587 - mse: 46.2587 - mae: 5.4017 - val_loss: 109.2833 - val_mse: 109.2833 - val_mae: 8.1853 - lr: 1.0000e-05\n",
      "Epoch 46/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 46.1452 - mse: 46.1452 - mae: 5.3914 - val_loss: 109.4691 - val_mse: 109.4691 - val_mae: 8.1918 - lr: 1.0000e-05\n",
      "Epoch 47/50\n",
      "7479/7479 [==============================] - 319s 43ms/step - loss: 46.2209 - mse: 46.2209 - mae: 5.3991 - val_loss: 109.4941 - val_mse: 109.4941 - val_mae: 8.1921 - lr: 1.0000e-05\n",
      "Epoch 48/50\n",
      "7479/7479 [==============================] - 320s 43ms/step - loss: 46.4060 - mse: 46.4060 - mae: 5.4103 - val_loss: 109.1736 - val_mse: 109.1736 - val_mae: 8.1682 - lr: 1.0000e-06\n",
      "Epoch 49/50\n",
      "7479/7479 [==============================] - ETA: 0s - loss: 46.0779 - mse: 46.0779 - mae: 5.3867INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 323s 43ms/step - loss: 46.0779 - mse: 46.0779 - mae: 5.3867 - val_loss: 109.0694 - val_mse: 109.0694 - val_mae: 8.1631 - lr: 1.0000e-06\n",
      "Epoch 50/50\n",
      "7478/7479 [============================>.] - ETA: 0s - loss: 46.1204 - mse: 46.1204 - mae: 5.3980INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: data/models/resnet-age-2023-11-27_10-47-08/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7479/7479 [==============================] - 323s 43ms/step - loss: 46.1211 - mse: 46.1211 - mae: 5.3979 - val_loss: 108.9814 - val_mse: 108.9814 - val_mae: 8.1601 - lr: 1.0000e-06\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7f99e7fbec80>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "callbacks = [\n",
    "    tf.keras.callbacks.TerminateOnNaN(),\n",
    "    tf.keras.callbacks.ReduceLROnPlateau(),\n",
    "    tf.keras.callbacks.ModelCheckpoint(output_path, save_best=True),\n",
    "    tf.keras.callbacks.CSVLogger(f\"data/models/{model_name}-history.csv\")\n",
    "]\n",
    "\n",
    "history = model.fit(train_dataset, epochs=50, validation_data=val_dataset, callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(f\"data/models/{model_name}.keras\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Tensorflow 2.13.0",
   "language": "python",
   "name": "tensorflow-2.13.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}