Lab07_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IeYtZStL1_UC"
   },
   "source": [
    "# Lab 7-1: Tips"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mDt8Ekbp1_UI"
   },
   "source": [
    "Author: Seungjae Lee (이승재)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "4NyWlgO-1_UJ"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7pK3ojFD1_UM",
    "outputId": "1345dfe9-d0ca-4314-efd6-1d9cac50ab66"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x1edd7269230>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For reproducibility\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ndAeQykM1_UO"
   },
   "source": [
    "## Training and Test Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "5MUleULn1_UO"
   },
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[1, 2, 1],\n",
    "                             [1, 3, 2],\n",
    "                             [1, 3, 4],\n",
    "                             [1, 5, 5],\n",
    "                             [1, 7, 5],\n",
    "                             [1, 2, 5],\n",
    "                             [1, 6, 6],\n",
    "                             [1, 7, 7]\n",
    "                            ])\n",
    "y_train = torch.LongTensor([2, 2, 2, 1, 1, 1, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "enCJPkKE1_UP"
   },
   "outputs": [],
   "source": [
    "x_test = torch.FloatTensor([[2, 1, 1], [3, 1, 2], [3, 3, 4]])\n",
    "y_test = torch.LongTensor([2, 2, 2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LLFJ3F3R1_UQ"
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "XgQaGBOm1_UR"
   },
   "outputs": [],
   "source": [
    "class SoftmaxClassifierModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(3, 8) #x_train의 shape과 y_train의 class의 갯수를 고려해보세요.\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "k504gsZv1_US"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()# #SoftmaxClassifierModel()로 모델을 초기화하세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "tDSegsws1_US"
   },
   "outputs": [],
   "source": [
    "# optimizer 설정\n",
    "optimizer = optim.SGD(model.parameters(),lr=0.1)# #SGD optimizer를 사용하고 learning rate는 0.1로 적용하세요."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "FpJEADyk1_UT"
   },
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train) #모델을 생성해서 예측값을 구해보세요.\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.cross_entropy(prediction,y_train) #PyTorch에서 제공하는 F.cross_entropy함수를 사용해 cost를 구현해보세요.\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()  # gradient를 0으로 초기화\n",
    "        cost.backward()  # 비용 함수를 미분하여 gradient 계산\n",
    "        optimizer.step()  # W와 b를 업데이트\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "MzwtgZPU1_UU"
   },
   "outputs": [],
   "source": [
    "def test(model, optimizer, x_test, y_test):\n",
    "    prediction = model(x_test)\n",
    "    predicted_classes = prediction.max(1)[1] # 각각의 데이터 마다 가장 큰 값의 인덱스를 취함 => class를 예측하는 것\n",
    "    correct_count = (predicted_classes == y_test).sum().item()\n",
    "    cost = F.cross_entropy(prediction, y_test)\n",
    "\n",
    "    print('Accuracy: {}% Cost: {:.6f}'.format(\n",
    "         correct_count / len(y_test) * 100, cost.item()\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "AOjN96fz1_UU",
    "outputId": "3df05551-23c2-4f11-eb95-2fa51cc04417"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 1.991479\n",
      "Epoch    1/20 Cost: 1.663707\n",
      "Epoch    2/20 Cost: 1.484954\n",
      "Epoch    3/20 Cost: 1.387691\n",
      "Epoch    4/20 Cost: 1.340652\n",
      "Epoch    5/20 Cost: 1.308806\n",
      "Epoch    6/20 Cost: 1.284747\n",
      "Epoch    7/20 Cost: 1.264902\n",
      "Epoch    8/20 Cost: 1.247684\n",
      "Epoch    9/20 Cost: 1.232249\n",
      "Epoch   10/20 Cost: 1.218112\n",
      "Epoch   11/20 Cost: 1.204964\n",
      "Epoch   12/20 Cost: 1.192604\n",
      "Epoch   13/20 Cost: 1.180891\n",
      "Epoch   14/20 Cost: 1.169724\n",
      "Epoch   15/20 Cost: 1.159025\n",
      "Epoch   16/20 Cost: 1.148738\n",
      "Epoch   17/20 Cost: 1.138815\n",
      "Epoch   18/20 Cost: 1.129221\n",
      "Epoch   19/20 Cost: 1.119926\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "NfzfnSRc1_UV",
    "outputId": "aa1c602b-65e3-4a26-ac25-feb38e414d0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.0% Cost: 2.241292\n"
     ]
    }
   ],
   "source": [
    "test(model, optimizer, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9qLNy-E41_UV"
   },
   "source": [
    "## Learning Rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "genGgcdk1_UW"
   },
   "source": [
    "Gradient Descent 에서의 $\\alpha$ 값"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xGzjnTkM1_UW"
   },
   "source": [
    "`optimizer = optim.SGD(model.parameters(), lr=0.1)` 에서 `lr=0.1` 이다"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QVpaEQFNBLNj"
   },
   "source": [
    "Q1. Learning Rate가 다음과 같을 때 어떤 일이 발생할까요?\n",
    "\n",
    "Learing rate가 클 경우 : 변화값이 커 가장 작은 gradient를 지나칠 수 있음\n",
    "\n",
    "Learing rate가 작을 경우 : gradient값을 작게 하는데 시간이 너무 오래 걸림"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C5z1kNJy1_UW"
   },
   "source": [
    "Large learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "CHsBXQFT1_UW"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "hhqLUXTL1_UX"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "EbzgO5nq1_UX",
    "outputId": "5a600459-81c1-49b3-dfc5-dbb7a6c0b964"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 2.610557\n",
      "Epoch    1/20 Cost: 859732.187500\n",
      "Epoch    2/20 Cost: 2240830.000000\n",
      "Epoch    3/20 Cost: 103330.054688\n",
      "Epoch    4/20 Cost: 1747232.250000\n",
      "Epoch    5/20 Cost: 1649857.625000\n",
      "Epoch    6/20 Cost: 792392.500000\n",
      "Epoch    7/20 Cost: 1869107.250000\n",
      "Epoch    8/20 Cost: 388137.812500\n",
      "Epoch    9/20 Cost: 1000217.750000\n",
      "Epoch   10/20 Cost: 982670.125000\n",
      "Epoch   11/20 Cost: 1387705.000000\n",
      "Epoch   12/20 Cost: 1519107.250000\n",
      "Epoch   13/20 Cost: 951767.500000\n",
      "Epoch   14/20 Cost: 786155.250000\n",
      "Epoch   15/20 Cost: 887357.687500\n",
      "Epoch   16/20 Cost: 1579892.500000\n",
      "Epoch   17/20 Cost: 1253482.250000\n",
      "Epoch   18/20 Cost: 1143955.000000\n",
      "Epoch   19/20 Cost: 584492.812500\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Wc12FsyN1_UX"
   },
   "source": [
    "Small learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "URf6c-0i1_UY"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "y6nPVEGV1_UY"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "ARubAn4x1_UY",
    "outputId": "980be48a-ad72-43cb-b027-4ce370d9fdba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 3.777051\n",
      "Epoch    1/20 Cost: 3.777051\n",
      "Epoch    2/20 Cost: 3.777051\n",
      "Epoch    3/20 Cost: 3.777051\n",
      "Epoch    4/20 Cost: 3.777051\n",
      "Epoch    5/20 Cost: 3.777051\n",
      "Epoch    6/20 Cost: 3.777051\n",
      "Epoch    7/20 Cost: 3.777051\n",
      "Epoch    8/20 Cost: 3.777051\n",
      "Epoch    9/20 Cost: 3.777051\n",
      "Epoch   10/20 Cost: 3.777051\n",
      "Epoch   11/20 Cost: 3.777051\n",
      "Epoch   12/20 Cost: 3.777051\n",
      "Epoch   13/20 Cost: 3.777051\n",
      "Epoch   14/20 Cost: 3.777051\n",
      "Epoch   15/20 Cost: 3.777051\n",
      "Epoch   16/20 Cost: 3.777051\n",
      "Epoch   17/20 Cost: 3.777051\n",
      "Epoch   18/20 Cost: 3.777051\n",
      "Epoch   19/20 Cost: 3.777051\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AAfx42ne1_UZ"
   },
   "source": [
    "적절한 숫자로 시작해 발산하면 작게, cost가 줄어들지 않으면 크게 조정하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "dAb7ZOKw1_UZ"
   },
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "V069kTVp1_UZ"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "Aq9HM-Z_1_UZ",
    "outputId": "a6df5cf4-9acd-4d4e-a0a0-6488a2b18a26"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 4.354907\n",
      "Epoch    1/20 Cost: 2.100595\n",
      "Epoch    2/20 Cost: 1.474521\n",
      "Epoch    3/20 Cost: 1.095100\n",
      "Epoch    4/20 Cost: 1.039693\n",
      "Epoch    5/20 Cost: 1.008386\n",
      "Epoch    6/20 Cost: 0.985914\n",
      "Epoch    7/20 Cost: 0.968357\n",
      "Epoch    8/20 Cost: 0.953931\n",
      "Epoch    9/20 Cost: 0.941626\n",
      "Epoch   10/20 Cost: 0.930854\n",
      "Epoch   11/20 Cost: 0.921229\n",
      "Epoch   12/20 Cost: 0.912500\n",
      "Epoch   13/20 Cost: 0.904484\n",
      "Epoch   14/20 Cost: 0.897053\n",
      "Epoch   15/20 Cost: 0.890110\n",
      "Epoch   16/20 Cost: 0.883581\n",
      "Epoch   17/20 Cost: 0.877408\n",
      "Epoch   18/20 Cost: 0.871546\n",
      "Epoch   19/20 Cost: 0.865958\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FPSYQ-lH1_Ua"
   },
   "source": [
    "## Data Preprocessing (데이터 전처리)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "551p9eWA1_Ua"
   },
   "source": [
    "데이터를 zero-center하고 normalize하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "eVBCPsDD1_Ua"
   },
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[73, 80, 75],\n",
    "                             [93, 88, 93],\n",
    "                             [89, 91, 90],\n",
    "                             [96, 98, 100],\n",
    "                             [73, 66, 70]])\n",
    "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "17vDm7i01_Ub"
   },
   "source": [
    "$$ x'_j = \\frac{x_j - \\mu_j}{\\sigma_j} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y7shP1Jt1_Uc"
   },
   "source": [
    "여기서 $\\sigma$ 는 standard deviation, $\\mu$ 는 평균값 이다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "r0pu6TcI1_Uc"
   },
   "outputs": [],
   "source": [
    "mu = x_train.mean(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "opFSudRu1_Ue"
   },
   "outputs": [],
   "source": [
    "sigma = x_train.std(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "2OEH3LeZ1_Ue"
   },
   "outputs": [],
   "source": [
    "norm_x_train = (x_train - mu) / sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AOuJijWj1_Ue",
    "outputId": "eec2d117-d97b-485e-d3b2-e19d5a45dd05"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0674, -0.3758, -0.8398],\n",
      "        [ 0.7418,  0.2778,  0.5863],\n",
      "        [ 0.3799,  0.5229,  0.3486],\n",
      "        [ 1.0132,  1.0948,  1.1409],\n",
      "        [-1.0674, -1.5197, -1.2360]])\n"
     ]
    }
   ],
   "source": [
    "print(norm_x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ig-JfKxp1_Uf"
   },
   "source": [
    "Normalize와 zero center한 X로 학습해서 성능을 보자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "UJwCzulq1_Uf"
   },
   "outputs": [],
   "source": [
    "class MultivariateLinearRegressionModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(3, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "eRBrSASh1_Uf"
   },
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "hs0e8drb1_Uf"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "pWZUOQgB1_Ug"
   },
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "7eAnbGrj1_Ug",
    "outputId": "255440df-c35b-4035-8a88-5e8b71a85df0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 6.655849\n",
      "Epoch    1/20 Cost: 4.381054\n",
      "Epoch    2/20 Cost: 2.947375\n",
      "Epoch    3/20 Cost: 2.033244\n",
      "Epoch    4/20 Cost: 1.446264\n",
      "Epoch    5/20 Cost: 1.067212\n",
      "Epoch    6/20 Cost: 0.820901\n",
      "Epoch    7/20 Cost: 0.659572\n",
      "Epoch    8/20 Cost: 0.552737\n",
      "Epoch    9/20 Cost: 0.480899\n",
      "Epoch   10/20 Cost: 0.431577\n",
      "Epoch   11/20 Cost: 0.396801\n",
      "Epoch   12/20 Cost: 0.371456\n",
      "Epoch   13/20 Cost: 0.352255\n",
      "Epoch   14/20 Cost: 0.337107\n",
      "Epoch   15/20 Cost: 0.324669\n",
      "Epoch   16/20 Cost: 0.314054\n",
      "Epoch   17/20 Cost: 0.304717\n",
      "Epoch   18/20 Cost: 0.296290\n",
      "Epoch   19/20 Cost: 0.288540\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer,norm_x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lMVmpArvAXnz"
   },
   "source": [
    "Q2. 이렇게 Data Preprocessing을 하는 이유가 무엇일까요?\n",
    "\n",
    "=> 데이터에서 누락된 부분, 오차, 가공할 부분있는지 확인하기 위해서"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UyQBI9z61_Ug"
   },
   "source": [
    "## Overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CaKMCCEZCBSD"
   },
   "source": [
    "Q3. Overfitting의 개념은?\n",
    "\n",
    "=> 훈련 데이터에 지나치게 맞춰져서 다른 데이터로 테스트할 때 오류가 발생하는 것"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bcJLcLhSCBVh"
   },
   "source": [
    "Q4. 그렇다면 Overfitting을 방지하는 방법에는 무엇이 있을까요?\n",
    "\n",
    "=> train data, test data, validation data를 구분하고 훈련시킨다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RwbeaBBG1_Uh"
   },
   "source": [
    "Regularization: Let's not have too big numbers in the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "XCdG_T_l1_Uh"
   },
   "outputs": [],
   "source": [
    "def train_with_regularization(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "        \n",
    "        # l2 norm 계산\n",
    "        l2_reg = 0\n",
    "        for param in model.parameters():\n",
    "            l2_reg += torch.norm(param)\n",
    "            \n",
    "        cost += l2_reg\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch+1, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "uTw_T6D11_Uh"
   },
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "YzVc7CUb1_Uh"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "ZafKSN8N1_Ui",
    "outputId": "d53951cb-5c8d-4c0f-a8a2-e80db172d312"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    1/20 Cost: 29657.029297\n",
      "Epoch    2/20 Cost: 18852.949219\n",
      "Epoch    3/20 Cost: 12092.840820\n",
      "Epoch    4/20 Cost: 7794.446777\n",
      "Epoch    5/20 Cost: 5051.640625\n",
      "Epoch    6/20 Cost: 3298.616455\n",
      "Epoch    7/20 Cost: 2177.366211\n",
      "Epoch    8/20 Cost: 1459.961182\n",
      "Epoch    9/20 Cost: 1000.875244\n",
      "Epoch   10/20 Cost: 707.071472\n",
      "Epoch   11/20 Cost: 519.036621\n",
      "Epoch   12/20 Cost: 398.690338\n",
      "Epoch   13/20 Cost: 321.664398\n",
      "Epoch   14/20 Cost: 272.363312\n",
      "Epoch   15/20 Cost: 240.806305\n",
      "Epoch   16/20 Cost: 220.605591\n",
      "Epoch   17/20 Cost: 207.673233\n",
      "Epoch   18/20 Cost: 199.392700\n",
      "Epoch   19/20 Cost: 194.089783\n",
      "Epoch   20/20 Cost: 190.692474\n"
     ]
    }
   ],
   "source": [
    "train_with_regularization(model, optimizer, norm_x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Lab07_1.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}