MNIST_widgets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import ipywidgets\n",
    "from ipywidgets import widgets\n",
    "from IPython.display import clear_output\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters tuning dashboard: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fb5e71909fb432eb64d6d501f018095",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=20, description='Batch size:', min=10)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fdfd4d893b14e4d8b79bfea48a08532",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=512, description='Size layer 1:', max=1000, min=1)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3060c3723f9a49f2b425f829be45a3ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=256, description='Size layer 2:', max=1000, min=1)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d989fef152194ce2a7afcd5b9c38280b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=64, description='Size layer 3:', max=1000, min=1)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "094d85e43ea741489b94d40eccaed1ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatSlider(value=0.2, description='Dropout prob:', max=1.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00901c6bb7a142be99f14388fac3b202",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatLogSlider(value=0.01, description='Learning rate:', max=0.0, min=-4.0, step=0.0001)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f70f5b0e7f2c446d964b0b7bbc350680",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=2, description='No of epochs:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_size_slider = widgets.IntSlider(value=20, min=10, max=100, description='Batch size:')\n",
    "hl_slider1 = widgets.IntSlider(value=512, min=1, max=1000, description='Size layer 1:')\n",
    "hl_slider2 = widgets.IntSlider(value=256, min=1, max=1000, description='Size layer 2:')\n",
    "hl_slider3 = widgets.IntSlider(value=64, min=1, max=1000, description='Size layer 3:')\n",
    "dropout_slider = widgets.FloatSlider(value=0.2, min=0.0, max=1.0, description='Dropout prob:')\n",
    "lr_slider = widgets.FloatLogSlider(value=0.01, base=10, min=-4, max=0, step=0.0001,description='Learning rate:')\n",
    "epoch_slider = widgets.IntSlider(value=2, min=0, max=100, description='No of epochs:')\n",
    "\n",
    "def change_batch_size_slider(change):\n",
    "    batch_size_slider.value = change.new\n",
    "    init_data()\n",
    "    \n",
    "\n",
    "def change_slider_1(change):\n",
    "    hl_slider1.value = change.new\n",
    "    init_model()\n",
    "    \n",
    "def change_slider_2(change):\n",
    "    hl_slider2.value = change.new\n",
    "    init_model()\n",
    "    \n",
    "def change_slider_3(change):\n",
    "    hl_slider3.value = change.new\n",
    "    init_model()\n",
    "\n",
    "def change_dropout_slider(change):\n",
    "    dropout_slider.value = change.new\n",
    "    init_model()\n",
    "\n",
    "def change_lr_slider(change):\n",
    "    lr_slider.value = change.new\n",
    "    init_optimizer()\n",
    "\n",
    "def change_epoch_slider(change):\n",
    "    epoch_slider.value = change.new\n",
    "    \n",
    "\n",
    "batch_size_slider.observe(change_batch_size_slider, names='value')\n",
    "hl_slider1.observe(change_slider_1, names='value')\n",
    "hl_slider2.observe(change_slider_2, names='value')\n",
    "hl_slider3.observe(change_slider_3, names='value')\n",
    "dropout_slider.observe(change_dropout_slider, names='value')\n",
    "lr_slider.observe(change_lr_slider, names='value')\n",
    "epoch_slider.observe(change_epoch_slider, names='value')\n",
    "\n",
    "# Add all the sliders to output\n",
    "display(batch_size_slider)\n",
    "display(hl_slider1)\n",
    "display(hl_slider2)\n",
    "display(hl_slider3)\n",
    "display(dropout_slider)\n",
    "display(lr_slider)\n",
    "display(epoch_slider)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model structure:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c40b60e57e4e4846ab93a5058dc5e6b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "out_model_widget = ipywidgets.Output()\n",
    "display(out_model_widget)\n",
    "\n",
    "#Define model\n",
    "class Model(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        # Network will have input of 784 nodes and 3 hidden layers of 512, 256 and 128, \n",
    "        # with an output layer for class scores of size 10\n",
    "        hidden_1_size = hl_slider1.value\n",
    "        hidden_2_size = hl_slider2.value\n",
    "        hidden_3_size = hl_slider3.value\n",
    "        output_size = 10\n",
    "        DROPOUT_PROB = dropout_slider.value\n",
    "        \n",
    "        self.fc1 = torch.nn.Linear(784, hidden_1_size)\n",
    "        self.fc2 = torch.nn.Linear(hidden_1_size, hidden_2_size)\n",
    "        self.fc3 = torch.nn.Linear(hidden_2_size, hidden_3_size)\n",
    "        self.out = torch.nn.Linear(hidden_3_size, output_size)\n",
    "        self.dropout = torch.nn.Dropout(p=DROPOUT_PROB)\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        #Flatten\n",
    "        x = x.view(-1, 784)\n",
    "        # First hidden layer output\n",
    "        x = torch.nn.functional.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        # Second hidden layer output\n",
    "        x = torch.nn.functional.relu(self.fc2(x))\n",
    "        x = self.dropout(x)\n",
    "        # Third hidden layer output\n",
    "        x = torch.nn.functional.relu(self.fc3(x))\n",
    "        x = self.dropout(x)\n",
    "        # Last layer, i.e., the output\n",
    "        x = self.out(x)\n",
    "        \n",
    "        return x\n",
    "        \n",
    "model = Model()\n",
    "\n",
    "def init_model():\n",
    "    model = Model()\n",
    "    with out_model_widget:\n",
    "        print(model)\n",
    "        out_model_widget.clear_output(wait=True)\n",
    "\n",
    "init_model()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizer function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00d3c17cf9b643cc9d6f1901c19c91e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "out_optim_widget = ipywidgets.Output()\n",
    "display(out_optim_widget)\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = lr_slider.value)\n",
    "\n",
    "def init_optimizer():\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr = lr_slider.value)\n",
    "    with out_optim_widget:\n",
    "        print(optimizer)\n",
    "        out_optim_widget.clear_output(wait=True)\n",
    "\n",
    "init_optimizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch details:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bbf1e707106482ca36dc7e952621eb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "out_data_loaders_widget = ipywidgets.Output()\n",
    "display(out_data_loaders_widget)\n",
    "\n",
    "def init_data():\n",
    "    transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
    "\n",
    "    train_data = torchvision.datasets.MNIST(root='data', download=True, train=True, transform = transform)\n",
    "    test_data = torchvision.datasets.MNIST(root='data', download=True, train=False, transform = transform)\n",
    "\n",
    "    #Obtain train-validation split\n",
    "    indices = list(range(len(train_data)))\n",
    "    np.random.shuffle(indices)\n",
    "\n",
    "    BATCH_SIZE = batch_size_slider.value\n",
    "    validation_ratio = 0.2\n",
    "    split_point = int(np.floor(len(indices)*validation_ratio))\n",
    "    validation_indices = indices[:split_point]\n",
    "    train_indices = indices[split_point:]\n",
    "    \n",
    "    train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indices)\n",
    "    validation_sampler = torch.utils.data.sampler.SubsetRandomSampler(validation_indices)\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
    "    validation_loader = torch.utils.data.DataLoader(train_data, sampler=validation_sampler, batch_size=BATCH_SIZE)\n",
    "    test_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE)\n",
    "    with out_data_loaders_widget:\n",
    "        print(\"Train-batches: \",len(train_loader))\n",
    "        print(\"Valid-batches: \",len(validation_loader))\n",
    "        print(\"Test-batches: \",len(test_loader))\n",
    "        out_data_loaders_widget.clear_output(wait=True)\n",
    "    return train_loader, validation_loader, test_loader\n",
    "\n",
    "train_loader, validation_loader, test_loader = init_data()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6257634e382040e7a6161487cab3591a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(description='TRAIN', style=ButtonStyle())"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ee2129dda0c455180f4cf9408bace7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_button = widgets.Button(description='TRAIN')\n",
    "display(train_button)\n",
    "out_training_widget = ipywidgets.Output()\n",
    "display(out_training_widget)\n",
    "\n",
    "def train(b):\n",
    "    with out_training_widget:\n",
    "\n",
    "        print(\"Training now...\")\n",
    "        EPOCHS = epoch_slider.value\n",
    "        valid_loss_min = np.Inf\n",
    "        total_time = 0\n",
    "        \n",
    "        for epoch in range(EPOCHS):\n",
    "            start_time = time.time()\n",
    "\n",
    "            train_loss = 0.0\n",
    "            validation_loss = 0.0\n",
    "\n",
    "            model.train()\n",
    "            for data, label in train_loader:\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                loss = criterion(output, label)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "                train_loss += loss.item()*data.size(0)\n",
    "\n",
    "            model.eval()\n",
    "            for data, label in validation_loader:\n",
    "                output = model(data)\n",
    "                loss = criterion(output, label)\n",
    "\n",
    "                validation_loss += loss.item()*data.size(0)\n",
    "\n",
    "            train_loss = train_loss / len(train_loader)\n",
    "            validation_loss = validation_loss / len(validation_loader)\n",
    "            epoch_time = time.time() - start_time\n",
    "            total_time += epoch_time\n",
    "\n",
    "            print(\"Epoch: {} / Training Loss: {:.4f} / Validation Loss: {:.4f}\".format(\n",
    "                epoch+1,\n",
    "                train_loss,\n",
    "                validation_loss\n",
    "                ))\n",
    "            print(\"Time taken: {:.2f} seconds.\".format(epoch_time))\n",
    "\n",
    "            if validation_loss < valid_loss_min:\n",
    "                print(\"Validation loss decreased from {:.4f} to {:.4f}. Saving the model.\".format(\n",
    "                    valid_loss_min,\n",
    "                    validation_loss))\n",
    "                torch.save(model.state_dict(), 'mnist_model.pt')\n",
    "                valid_loss_min = validation_loss\n",
    "        print(\"Training complete!\")\n",
    "        print(\"Total time taken: {:.2f} seconds.\".format(total_time))\n",
    "        \n",
    "        out_training_widget.clear_output(wait=True)\n",
    "\n",
    "train_button.on_click(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24081768d2dd4a9ab9d3a6c500972c29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(description='TEST', style=ButtonStyle())"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1a9e372c4f645f6ac0ce94774543425",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_button = widgets.Button(description='TEST')\n",
    "display(test_button)\n",
    "out_testing_widget = ipywidgets.Output()\n",
    "display(out_testing_widget)\n",
    "\n",
    "def test(b):\n",
    "    with out_testing_widget:\n",
    "        print(\"Start testing...\")\n",
    "        test_loss = 0.0\n",
    "        class_correct = list(0. for i in range(10))\n",
    "        class_total = list(0. for i in range(10))\n",
    "        start_time = time.time()\n",
    "        \n",
    "        test_loss = 0.0\n",
    "        correct_guesses_for_class = [0 for i in range(10)]\n",
    "        total_samples_per_class = [0 for i in range(10)]\n",
    "        # Setup the model for testing\n",
    "        model.eval()\n",
    "        count = 0\n",
    "        correct_pred = 0\n",
    "        total_pred = 0\n",
    "        for data, labels in test_loader:\n",
    "            count = 1\n",
    "            output = model(data)\n",
    "            max_score, argmax_class_index = torch.max(output, 1)\n",
    "\n",
    "            is_same_bool_array = argmax_class_index.eq(labels)\n",
    "            correct_pred += int(is_same_bool_array.sum())\n",
    "            total_pred += len(is_same_bool_array)\n",
    "\n",
    "\n",
    "        print(\"The accuracy for overall test-set is: {:.4f}\".format(100*(correct_pred/total_pred)))\n",
    "        print(\"Time taken: {:.2f} seconds.\".format(time.time() - start_time))\n",
    "\n",
    "        out_testing_widget.clear_output(wait=True)\n",
    "        \n",
    "test_button.on_click(test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}