sw_00_regressions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use PyTorch to train linear and logistic regressions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simulate linear and logistic regressions. Use PyTorch to recover the true values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import util\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression example 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000; D = 4; K = 3; # know N, D, and K\n",
    "w = np.random.random((D, K-1))*10 # use K-1 because it is identifiable in estimation\n",
    "w = np.concatenate((np.zeros((D, 1)), w), axis = 1) \n",
    "y, X = util.gen_multiple_choice_data(N, D, K, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         3.87636046 8.06844608]\n",
      " [0.         9.47307596 2.92279439]\n",
      " [0.         9.84289957 1.98111079]\n",
      " [0.         0.1366462  0.74166592]]\n"
     ]
    }
   ],
   "source": [
    "# print(y)\n",
    "# print(X)\n",
    "print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "cuda:0\n",
      "cuda:1\n"
     ]
    }
   ],
   "source": [
    "# cuda device\n",
    "device_id = torch.cuda.device_count()\n",
    "print(device_id) # 0 and 1 are both available\n",
    "# if torch.cuda.is_available():\n",
    "device_0 = torch.device(\"cuda:0\")\n",
    "device_1 = torch.device(\"cuda:1\")\n",
    "print(device_0)\n",
    "print(device_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_torch = torch.from_numpy(X)\n",
    "y_torch = torch.from_numpy(y)\n",
    "w_torch = torch.randn(4, 3, requires_grad = True, dtype = torch.double)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.9491, -0.2354, -0.2943],\n",
      "        [ 2.0769,  0.4156,  0.1282],\n",
      "        [ 0.9985,  2.2843,  1.0736],\n",
      "        ...,\n",
      "        [ 0.4647,  1.3650,  0.8925],\n",
      "        [-0.9256, -2.0626, -1.4511],\n",
      "        [-0.1128,  0.3966,  0.1262]], dtype=torch.float64,\n",
      "       grad_fn=<MmBackward>)\n"
     ]
    }
   ],
   "source": [
    "def model(inputs):\n",
    "    U = torch.matmul(inputs, w_torch)\n",
    "    return U\n",
    "\n",
    "print(model(X_torch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True w is:  [[0.         3.87636046 8.06844608]\n",
      " [0.         9.47307596 2.92279439]\n",
      " [0.         9.84289957 1.98111079]\n",
      " [0.         0.1366462  0.74166592]]\n",
      "Learnt w is:  tensor([[ 1.1622,  1.5789,  0.8825],\n",
      "        [-0.7149,  0.8155,  0.2295],\n",
      "        [-0.0621,  0.5774,  0.2981],\n",
      "        [ 0.0531,  0.7800,  0.0961]], dtype=torch.float64, requires_grad=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f46600340d0>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhV1dn+8e+TAcI8BgTCPAjIzJEwBlRkVrBSFFCoqIiCCOj7tnay1vrr21aZRRBESisKKiiTIlAgYSYgMwIREAJoIvM8rt8fObTRJhDICTs5uT/Xda7k7LWT8+xsuLOzztprmXMOEREJXiFeFyAiIllLQS8iEuQU9CIiQU5BLyIS5BT0IiJBLszrAtJSsmRJV6lSJa/LEBHJMdavX/+Dcy4yrbZsGfSVKlUiPj7e6zJERHIMM/s2vTZ13YiIBDkFvYhIkFPQi4gEuRsGvZlNNrMkM9uaTntXM9tsZhvNLN7MWqZqu+LfvtHMZgeycBERyZiMvBk7BRgLTE2nfTEw2znnzKweMAOo6W8755xrkOkqRUTklt3wit45FwscvU77afefmdEKAJolTUQkGwlIH72ZPWRmXwPzgH6pmiL83TmrzazbDb5Hf/++8cnJyYEoS0RECFDQO+dmOedqAt2A11I1VXDO+YBewEgzq3qd7/GOc87nnPNFRqY55v+GxizezdaDJ27pa0VEglVAR934u3mqmllJ//ND/o97gKVAw0C+XmrHzlxk2tr9dB+/ks82HsyqlxERyXEyHfRmVs3MzP95IyAPcMTMiplZXv/2kkALYHtmXy89xQrkYfagltSLKsoLH27ktbnbuXzlala9nIhIjnHDUTdm9gHQBihpZonAK0A4gHNuPPAw0MfMLgHngEf8I3BqARPM7Copv1D+zzmXZUEPEFkoL+8/Fc3r83bw7vK9bD90krG9GlKiYN6sfFkRkWzNsuNSgj6fz2V2rpuP1yfy61lbiCyYlwmPN6ZOuSIBqk5EJPsxs/X+90T/S9DeGdu9cRSfDGiOc46H317JzA2JXpckIuKJoA16gLpRRZj9fEsalC/KsBmbeHXONi6p315EcpmgDnqAkgXz8s+nounXojLvrdjH4++u4YfTF7wuS0Tktgn6oAcIDw3h9w/UZniP+ny1/zgPjlnO5sTjXpclInJb5Iqgv+ZnjaL45NnmmBndx6/ik/XqtxeR4Jergh6gTrkizB7UAl/FYrz40Sb+MFv99iIS3HJd0AOUKJiXqf2a8GTLykxZuY/ek9RvLyLBK1cGPUBYaAi/61KbkY80YHPicR4Ys5xNB9RvLyLBJ9cG/TXdGpbj4wHNCTHj5xNWMSP+gNcliYgEVK4Pekjpt5/zfEvurlSM//14My/P3MKFy1e8LktEJCAU9H7FC+Th70804dk2Vflg7X56jF/FwePnvC5LRCTTFPSphIWG8MsONZnweGP2JJ+hy+g4lu/+weuyREQyRUGfhvZ33cFng1oQWSgvfSav4a0lCVy9mv0mfxMRyQgFfTqqRBbk04Et6FyvLH9bsJNn/rmek+cveV2WiMhNU9BfR/48YYx+tAGvPFCbJV8n8eCY5Xz93UmvyxIRuSkK+hswM55oUZkP+jflzMUrPPSWlioUkZxFQZ9Bd1cqzrznW1K3XBFe+HAjf5i9jYuXNXWCiGR/CvqbUKpwBO8/Hf3vqRN6TlzN9yfPe12WiMh1ZSjozWyymSWZ2dZ02rua2WYz22hm8WbWMlVbXzPb7X/0DVThXgn3T50wtldDdhw+SefRy1m954jXZYmIpCujV/RTgA7XaV8M1HfONQD6AZMAzKw4KYuJRwNNgFfMrNgtV5uNdKlXls8GtqBwRBi9J61hYuwesuP6uyIiGQp651wscPQ67afdf1KuAHDt8/bAQufcUefcMWAh1/+FkaNUL12Izwa1oG2tUrw+fwcDp23g9IXLXpclIvIjAeujN7OHzOxrYB4pV/UA5YDUs4Ql+rel9fX9/d0+8cnJyYEqK8sVighn/GONebljTb7Y+h1dxy5n1/envC5LROTfAhb0zrlZzrmaQDfgNf9mS2vXdL7+Heeczznni4yMDFRZt4WZ8UzrqvzzqWhOnLtE17ErmPWVVq8Skewh4KNu/N08Vc2sJClX8OVTNUcBhwL9mtlF86olmTe4FXXLFWHo9E38etYWzl/SLJgi4q2ABL2ZVTMz83/eCMgDHAEWAO3MrJj/Tdh2/m1Bq3ThCKY9Hc2A1lWZtmY/3cevZP+Rs16XJSK5WEaHV34ArALuNLNEM3vSzAaY2QD/Lg8DW81sI/AW8IhLcZSUbpx1/scf/duCWlhoCL/qWJNJfXzsP3KWzmPi+HLbd16XJSK5lGXHIYE+n8/Fx8d7XUZAHDh6lufe38CWgyd4JqYKL7W/k/BQ3acmIoFlZuudc7602pQ4Wax88fx8NKAZjzWtwITYPfTS3bQicpsp6G+DiPBQ/tStLqMebcC2QyfpPDqOFQla0EREbg8F/W3UtUE5Zg9qQbH8eXjs3TWMWbxbC5qISJZT0N9m1Uql3E3btX5Z3ly4iyemrOPomYtelyUiQUxB74H8ecIY8UgDXn+oDqu+OUKX0XFs2H/M67JEJEgp6D1iZvSOrsgnzzYnNNR4ZMIqJi/fq4nRRCTgFPQeqxtVhLmDWtG6Rin+OHc7A6dt0Nq0IhJQCvpsoEj+cCb2SZkYbcG27+kyejlbEk94XZaIBAkFfTZxbWK0Gc805fKVqzz89kqmrFBXjohknoI+m2lcsTjzBrcipkZJ/jBnOwP+uZ4TZ9WVIyK3TkGfDRUrkIeJfXz8tnMtFu9IovOYODYeOO51WSKSQynosykz46lWVfhoQDOcg+5vr2RSnJYrFJGbp6DP5hpWKMb8wa24t2Yp/jRvB09Pjef4Wd1gJSIZp6DPAYrkD2fC44155YHaLNuVTKdRcaz/NuhnexaRAFHQ5xBmxhMtKvPJs80JCw2hx4TVjF/2jebKEZEbUtDnMPWiijJ3cEva31Wa//v8a/r9XXPliMj1KehzoMIR4bzVqxGvdb2LlQlH6DQqjrV71ZUjImlT0OdQZsbjzSox87nmRISH0HPiat5akqCuHBH5Lwr6HK5OuSLMeb4lneqW4W8LdtL3vbUkn7rgdVkiko3cMOjNbLKZJZnZ1nTae5vZZv9jpZnVT9W2z8y2mNlGMwuORWCzoUIR4Yx+tAF//lld1u49SsdRscTuSva6LBHJJjJyRT8F6HCd9r1Aa+dcPeA14J2ftN/jnGuQ3qK1EhhmRs8mFZg9qCXFC+Shz+S1/Hn+Di5evup1aSLisRsGvXMuFkj3nT7n3Ern3LVVM1YDUQGqTW7BnXcU4rOBLekdnbIY+c/Hr+TbI2e8LktEPBToPvongc9TPXfAl2a23sz6X+8Lzay/mcWbWXxysrodMiNfnlBef6gub/duxN4fztB59HI+/eqg12WJiEcCFvRmdg8pQf/LVJtbOOcaAR2BgWYWk97XO+fecc75nHO+yMjIQJWVq3WsW4bPh8RQq0whhkzfyIszNnHmwmWvyxKR2ywgQW9m9YBJQFfn3JFr251zh/wfk4BZQJNAvJ5kXLmi+fjg6aYMvq86s75KpMuY5Ww9qEVNRHKTTAe9mVUAZgKPO+d2pdpewMwKXfscaAekOXJHslZYaAjD7q/BtKebcu7iFR4at4J3tT6tSK6RkeGVHwCrgDvNLNHMnjSzAWY2wL/L74ESwLifDKMsDSw3s03AWmCec+6LLDgGyaCmVUrw+QutaHNnKV6bu51+U9Zx5LTG3IsEO8uOV3U+n8/Fx2vYfVZxzvGP1d/yp3k7KJovnJGPNKB5tZJelyUimWBm69Mbxq47Y3MhM6NPs0p8NrAFhfOF0/vdNfz1i6+5dEVj7kWCkYI+F6tVpjCzB7Xg0bvLM27pN/SYsIoDR896XZaIBJiCPpfLnyeMP/+sHmN7NSQh6TSdRsXx2UaNuRcJJgp6AaBLvbLMH9yKGncU4oUPNzJ0+kZOnb/kdVkiEgAKevm38sXzM71/U4a2rcHsTYfoNFpLFooEAwW9/EhYaAgvtK3OjGeaAfDz8asYsXAXl/VGrUiOpaCXNDWuWIz5g1vRrWE5Ri3eTY8Jq9h/RG/UiuRECnpJV6GIcIb3aMDong3ZnXSaTqPjmLkhUXfUiuQwCnq5oQfrl+WLITHULluYYTM2MfjDjZw4pzdqRXIKBb1kyLXJ0f6n/Z18vuUwnUbFsWbPkRt/oYh4TkEvGRYaYgy8pxofP9uc8FDj0Ymr+dsC3VErkt0p6OWmNShflHmDW9GjcXneWvIN3d9eyd4ftIqVSHaloJdbUiBvGH/pXo+3ezdi35GzdB4dx4x1B/RGrUg2pKCXTOlYtwxfDGlF/aii/O8nm3nu/Q0cO3PR67JEJBUFvWRamSL5eP+paF7uWJNFO76n/chYlu3Sur8i2YWCXgIiJMR4pnVVZj3XgiL5wuk7eS2//2wr5y5e8bo0kVxPQS8BVadcEeY835InW1Zm6qpv6Tw6jk0HjntdlkiupqCXgIsID+V3XWoz7alozl26ws/eXsnIRbs0DFPEIwp6yTLNq5XkiyExPFi/LCMX7ab7+FXsST7tdVkiuU5GFgefbGZJZrY1nfbeZrbZ/1hpZvVTtXUws51mlmBmvwpk4ZIzFMkXzohHGjC2V0P2/XCGTqPj+MfqbzUMU+Q2ysgV/RSgw3Xa9wKtnXP1gNeAdwDMLBR4C+gI1AZ6mlntTFUrOVaXemX5cmgMd1cqzu8+3cov3ltH0snzXpclkivcMOidc7FAuqtPOOdWOueO+Z+uBqL8nzcBEpxze5xzF4EPga6ZrFdysNKFI5jarwl/7HoXa/Yeod3IWOZvOex1WSJBL9B99E8Cn/s/LwccSNWW6N+WJjPrb2bxZhafnKwx2MHKzOjTrBLzBreiYvH8PPf+BoZN38hJLVsokmUCFvRmdg8pQf/La5vS2C3djlnn3DvOOZ9zzhcZGRmosiSbqhpZkI+fbc4L91Xns02H6DgyjlXfaDZMkawQkKA3s3rAJKCrc+7a/9ZEoHyq3aKAQ4F4PQkO4aEhDL2/Bh8PaEaesBB6TVrNn+Zu5/wl3WQlEkiZDnozqwDMBB53zu1K1bQOqG5mlc0sD/AoMDuzryfBp2GFYswb3JLe0RWYtHwvD4xZzpbEE16XJRI0MjK88gNgFXCnmSWa2ZNmNsDMBvh3+T1QAhhnZhvNLB7AOXcZGAQsAHYAM5xz27LkKCTHy58njD91q8uUJ+7m5PlLdBu3guELd3Hxsm6yEsksy47jmX0+n4uPj/e6DPHIibOXeHXONmZ+dZC7yhbmzR71qXlHYa/LEsnWzGy9c86XVpvujJVsp0j+cIY/0oAJjzfm+5PneWDMcsYtTeCyplAQuSUKesm22t91BwuGxNC2Vmn++sVOuo9fxTeaQkHkpinoJVsrUTAv43o3YtSjDdj7wxk6jYrj3eV7uXo1+3U5imRXCnrJ9syMrg3KsXBoDC2rleS1udvpOXE1B46e9bo0kRxBQS85RqnCEUzq6+Ov3eux7dBJ2o+M5f01miBN5EYU9JKjmBk9fOVZMDSGRhWK8ZtZW+kzeS2Hjp/zujSRbEtBLzlSuaL5+MeTTXitWx3i9x2j/chYPlmfqKt7kTQo6CXHMjMeb1qRL4a0ouYdhXjxo008PXU9Sac0/bFIagp6yfEqlijAh/2b8dvOtYjdnUy7EbF8+tVBXd2L+CnoJSiEhhhPtarC/MGtqFyyAEOmb0y5utfiJiIKegku1UoV5OMBzflt51rE7U6m7fBlfKy+e8nlFPQSdK5d3X/+QivuvKMQL320iX5T1nH4hEbmSO6koJegVSWyINP7N+OVB2qzas8R2g2PZfq6/bq6l1xHQS9BLSTEeKJFZb54IYZaZQvzy0+20GfyWg5q3L3kIgp6yRUqlSzAh0835Y9d72L9t8doP0J31UruoaCXXCMkJGVh8gVDYqgXVYTfzNrKY++u0Zw5EvQU9JLrlC+en/efiub1h+qwcf9x2o+MZeqqfZoRU4KWgl5yJTOjd3RFFgyNoXHFYvz+s230nLiab4+c8bo0kYBT0EuuFlUsP1P7NeEvD9dl+6GTdBgZx+Tle7miq3sJIhlZHHyymSWZ2dZ02mua2Sozu2BmL/2kbZ+ZbUm9aLhIdmNmPHJ3Bb4cFkPTKsX549ztdB+/kl3fn/K6NJGAyMgV/RSgw3XajwKDgTfSab/HOdcgvUVrRbKLMkXyMfkXdzPikfrs++EMnUfHMXLRLi5e1lq1krPdMOidc7GkhHl67UnOuXXApUAWJuIFM+OhhlEsGtaajnXKMHLRbrqMiWPD/mNelyZyy7K6j94BX5rZejPrf70dzay/mcWbWXxycnIWlyVyfSUK5mV0z4ZM/oWPU+cv8/DbK3l1zjbOXLjsdWkiNy2rg76Fc64R0BEYaGYx6e3onHvHOedzzvkiIyOzuCyRjLm3Zmm+HBrDY9EVeW/FPtqNiCV2ly5EJGfJ0qB3zh3yf0wCZgFNsvL1RLJCoYhwXutWh48GNCNveAh9Jq9l2IyNHDtz0evSRDIky4LezAqYWaFrnwPtgDRH7ojkBHdXKs78wa0YdE81Zm88xP0jljFn0yFNoyDZnt3oH6mZfQC0AUoC3wOvAOEAzrnxZnYHEA8UBq4Cp4Ha/v1n+b9NGDDNOfd6Rory+XwuPl6jMSX72n7oJL/8ZDNbDp6gba1SvNatDmWK5PO6LMnFzGx9eqMbbxj0XlDQS05w+cpVJq/Yy/CFuwgPCeFXnWrS8+4KhISY16VJLnS9oNedsSK3KCw0hP4xVVkwJIY65VImSXt04mr2JJ/2ujSRH1HQi2RSxRIFmPZ0NH95uC47Dp+kw6g4xv5rt260kmxDQS8SANemUVg8rDVta5XijS930WVMHPH70r3XUOS2UdCLBFCpwhGM692Yd/v6OHPhCt3Hr+LXs7Zw4pxuHBfvKOhFssB9tVJutHqyZWU+XLuftsOXMXezhmKKNxT0IlmkQN4wftelNp8NbEnpwnkZNO0r+k1ZR+IxrWglt5eCXiSL1Y0qwqfPteC3nWuxZu9R7h8ey8TYPVy+ojdr5fZQ0IvcBmGhITzVqgoLh7WmedUSvD5/B13fWsHmxONelya5gIJe5DYqVzQfk/r6GNe7EcmnLtDtrRW8OmcbpzUrpmQhBb3IbWZmdKpbhkUvtqZXdAWmrNxHu+HLWLj9e69LkyCloBfxSOGIcP7UrS4fD2hOoYhwnp4az4B/rOe7E+e9Lk2CjIJexGONKxZj7uCW/E/7O1myM4n7hy9jygotUC6Bo6AXyQbCQ0MYeE81vhwaQ4MKRfnDnO10fWs5mw7ozVrJPAW9SDZSsUQBpvZrwtheDUk6eYFu41bw20+3cOKs7qyVW6egF8lmzIwu9cqy+MXWPNG8MtPW7Oe+4UuZuSFRd9bKLVHQi2RThSLC+f0DtZnzfEuiiuVn2IxN9Jy4moSkU16XJjmMgl4km7urbBFmPtuc//dQXXYcPkXHUXH89YuvOXfxitelSQ6hoBfJAUJCjF7RFVj8YmserF+OcUu/4f4Ry1i8Q2Pv5cZuGPRmNtnMkswszYW9zaymma0yswtm9tJP2jqY2U4zSzCzXwWqaJHcqmTBvLzZoz7T+zclX3goT/49nv5T4zl4/JzXpUk2lpEr+ilAh+u0HwUGA2+k3mhmocBbQEdSFgvvaWa1b61MEUktukoJ5g1uxS871CRu9w+0fXMZ45d9wyVNlCZpuGHQO+diSQnz9NqTnHPrgJ+O/2oCJDjn9jjnLgIfAl0zU6yI/EeesBCebVOVhcNiaFm9JP/3+dd0Hh3H2r1a1Up+LCv76MsBB1I9T/RvS5OZ9TezeDOLT05OzsKyRIJLVLH8TOzjY1KflFWtekxYxYszNpF86oLXpUk2kZVBb2lsS3cQsHPuHeeczznni4yMzMKyRIJT29qlWTgshmfbVGX2poPc+8ZS3luxV/PeS5YGfSJQPtXzKOBQFr6eSK6XP08Yv+xQky+GpEyl8Oqc7XQZs5w1e454XZp4KCuDfh1Q3cwqm1ke4FFgdha+noj4VY0syNR+TRj/WCNOnb/MI++sZsiHX5F0UjNj5kZhN9rBzD4A2gAlzSwReAUIB3DOjTezO4B4oDBw1cyGALWdcyfNbBCwAAgFJjvntmXNYYjIT5kZHeqUoXWNUoxbmsCEZXtYtCOJIW2r07d5JcJDdRtNbmHZce4Mn8/n4uPjvS5DJKjs++EMr87ZxpKdyVQvVZBXu95F86olvS5LAsTM1jvnfGm16Ve6SC5RqWQBJv/ibib28XHu0hV6TVzDoGkbOHxCN1sFOwW9SC5iZtxfuzSLhrXmhfuq8+X277nPf7PVxcsanROsFPQiuVBEeChD76/BoqGtaV415WarDqNiidute1iCkYJeJBerUCI/k/r6eO8Xd3PlquPxd9fy7D/Xa+6cIHPDUTciEvzuqVmKZlVLMCluD2OXJLBkZxLPtq7GM62rEBEe6nV5kkm6ohcRIKU7Z9C91Vk0rDX31izFiEW7uO/NZczfclgrW+VwCnoR+ZGoYvkZ17sx056OplBEGM+9v4GeE1ez4/BJr0uTW6SgF5E0Na9akrnPt+S1bnX4+rtTdB4dx+8+3cqxMxe9Lk1ukoJeRNIVFhrC400rsvSlNjzetCLT1u6nzRtL+fvKfZosLQdR0IvIDRXNn4dXu9Zh/uBW3FW2MK/M3kbn0ctZmfCD16VJBijoRSTD7ryjEO8/Fc34xxpz9tJlek1awzP/iOfA0bNelybXoaAXkZuSMlnaHSwc2pqX2tUgdtcP3Dd8GW8s2MnZi5e9Lk/SoKAXkVtybTjmkpfa0KnOHYxdksC9byzjs40HNRwzm1HQi0im3FEkgpGPNuSTZ5sRWSgvL3y4ke7jV7E58bjXpYmfgl5EAqJxxeJ8NrAFf324Ht8eOcODY1cwbMZGvjuhxU68pqAXkYAJCTF63F2eJS+1YUDrqszddJh73ljKyEW71H/vIQW9iARcoYhwftWxJotfbM29tUoxctFu7n1jGZ+sT+TqVfXf324KehHJMuWL5+etXo34eEAzShfOy4sfbaLbuBWs3XvU69JyFQW9iGQ5X6XizHquBSMeqU/SyQv0mLCK595fz/4jGn9/O9ww6M1sspklmdnWdNrNzEabWYKZbTazRqnarpjZRv9jdiALF5GcJSTEeKhhFEteasPQtjVY8nUybYcv48/zd3Dy/CWvywtqGbminwJ0uE57R6C6/9EfeDtV2znnXAP/48FbrlJEgka+PKG80DZl/P0D9csyIXYP9/xtKf9c/a3mz8kiNwx651wscL0Ota7AVJdiNVDUzMoEqkARCU53FIngzR71mTOoJVUjC/LbT7fSaXQcsbu0nGGgBaKPvhxwINXzRP82gAgzizez1WbW7XrfxMz6+/eNT07WiRbJLepGFWH6M015u3cjzl+6Sp/Ja/nFe2tJSDrldWlBIxBBb2lsuzZ+qoJzzgf0AkaaWdX0volz7h3nnM8554uMjAxAWSKSU5gZHeuWYeGwGH7dqSbr9x2j/cg4fjNrC8mnLnhdXo4XiKBPBMqneh4FHAJwzl37uAdYCjQMwOuJSJDKGxZK/5iqLP2fNjwWXYHp6w7Q5m9LGL14t264yoRABP1soI9/9E1T4IRz7rCZFTOzvABmVhJoAWwPwOuJSJArUTAvr3atw5dDY2hVPZLhC3fR5m9L+XDtfq7ohqubZjeaZc7MPgDaACWB74FXgHAA59x4MzNgLCkjc84CTzjn4s2sOTABuErKL5SRzrl3M1KUz+dz8fHxt3RAIhJ84vcd5fX5O/hq/3FqlC7Iyx1r0ebOSFLiRwDMbL2/q/y/27LjdKIKehH5Keccn2/9jr988TXfHjlLi2oleLljLeqUK+J1adnC9YJed8aKSI5gZnSqW4aFQ1vz+y612XboJA+MXc6w6Rs5ePyc1+Vla7qiF5Ec6cS5S4xbmsB7K/YB0K9FZZ67pyqFI8K9Lcwj6roRkaB18Pg53lywk5lfHaRY/nCev7c6jzWtSJ6w3NVhoa4bEQla5YrmY/gjDZj7fEtqly3MH+du5/4Ry5i3+bCWNPRT0ItIUKhTrgj/fDKa9564m4iwUAZO28BD41ayes8Rr0vznIJeRIKGmXHPnaWY/0Ir/vJwXb47cZ5H31nNL95by/ZDJ70uzzPqoxeRoHX+0hWmrNzHuCUJnLpwma71y/JiuzspXzy/16UFnN6MFZFc7cTZS7y97BveW7GXq87RO7oig+6tRsmCeb0uLWAU9CIiwHcnzjNq8S5mxCcSERbCU62q8HRMFQrmDfO6tExT0IuIpJKQdJo3v9zJ51u/o0SBPAy6txq9oiuQNyzU69JumYZXioikUq1UQd5+rDGfDmxBjdKFeHXOdu57cxmffnWQq0E4aZqCXkRyrQblizLt6Wj+3q8JhSPCGTJ9I53HLGfJzqSgGoOvoBeRXM3MaF0jkrnPt2TUow04c+EyT7y3jkffWc2G/ce8Li8gFPQiIkBIiNG1QTkWDWvNqw/exTfJp/nZuJX0nxrPzu9y9rKGejNWRCQNZy5c5t3le5kYu4fTFy/zYP2yDG1bg0olC3hdWpo06kZE5BYdP3uR8cv2MGXlXi5dcfTwRfH8vdUpWzSf16X9iIJeRCSTkk6dZ9ySb5i2Zj8AvZtWYOA92eemKwW9iEiAJB47y+jFu/lkw0HyhoXwRItK9G9VlSL5vZ0HX0EvIhJge5JPM2LRbuZsOkThiDD6x1ThiRaVKeDRXbaZvmHKzCabWZKZbU2n3cxstJklmNlmM2uUqq2vme32P/re2iGIiGQvVSILMqZnQ+YPbkWTysV548tdxPx1Ce8u38v5S1e8Lu9HMnRFb2YxwGlgqnOuThrtnYDngU5ANDDKORdtZsWBeMAHOGA90Ng5d93BqbqiF5GcZsP+Y7z55U5WJByhTJEIBt9Xne6NowgPvT2j2DN9Re+ciwWOXmeXrqT8EnDOudVAUTMrA7QHFjrnjvrDfSHQ4ebKFxHJ/nn1KTAAAAZISURBVBpVKMb7TzVl2lPR3FEkgpdnbqHt8JRpFa54PK1CoH7VlAMOpHqe6N+W3vb/Ymb9zSzezOKTk5MDVJaIyO3VvFpJZj7bnHf7+sifJ4wh0zfScVQs87cc9mwenUAFvaWxzV1n+39vdO4d55zPOeeLjIwMUFkiIrefmXFfrdLMe74lY3s15KqD597fQKfRcSzY9t1tn0cnUEGfCJRP9TwKOHSd7SIiQS8kxOhSrywLhsQw8pEGXLh8lWf+sZ4Hxi7nX19/f9sCP1BBPxvo4x990xQ44Zw7DCwA2plZMTMrBrTzbxMRyTVCQ4xuDcuxcGgMb/y8PifPXabflHi6jVvJsl3JWR74GRrwaWYfAG2AkmaWCLwChAM458YD80kZcZMAnAWe8LcdNbPXgHX+b/VH59z13tQVEQlaYaEhdG8cRdcGZZm5IZHRixPoO3ktvorFGHZ/DZpVLYFZWj3emaMbpkREPHLx8lVmxB9g7L8S+O7keaIrF+fv/ZoQEX7zK11db3hlzl8oUUQkh8oTFsJjTSvSvXEU09cdYMfhk7cU8jeioBcR8VhEeCh9m1fKsu+vhUdERIKcgl5EJMgp6EVEgpyCXkQkyCnoRUSCnIJeRCTIKehFRIKcgl5EJMhlyykQzCwZ+PYWv7wk8EMAy8kJdMy5g445+GXmeCs659Kc4z1bBn1mmFl8evM9BCsdc+6gYw5+WXW86roREQlyCnoRkSAXjEH/jtcFeEDHnDvomINflhxv0PXRi4jIjwXjFb2IiKSioBcRCXJBE/Rm1sHMdppZgpn9yut6AsXMypvZEjPbYWbbzOwF//biZrbQzHb7PxbzbzczG+3/OWw2s0beHsGtM7NQM/vKzOb6n1c2szX+Y55uZnn82/P6nyf42yt5WfetMrOiZvaxmX3tP9/Ngv08m9lQ/7/rrWb2gZlFBNt5NrPJZpZkZltTbbvp82pmff377zazvjdTQ1AEvZmFAm8BHYHaQE8zq+1tVQFzGXjROVcLaAoM9B/br4DFzrnqwGL/c0j5GVT3P/oDb9/+kgPmBWBHqud/AUb4j/kY8KR/+5PAMedcNWCEf7+caBTwhXOuJlCflGMP2vNsZuWAwYDPOVcHCAUeJfjO8xSgw0+23dR5NbPiwCtANNAEeOXaL4cMcc7l+AfQDFiQ6vnLwMte15VFx/oZcD+wEyjj31YG2On/fALQM9X+/94vJz2AKP9/gHuBuYCRcsdg2E/PObAAaOb/PMy/n3l9DDd5vIWBvT+tO5jPM1AOOAAU95+3uUD7YDzPQCVg662eV6AnMCHV9h/td6NHUFzR859/MNck+rcFFf+fqg2BNUBp59xhAP/HUv7dguVnMRL4X+Cq/3kJ4Lhz7rL/eerj+vcx+9tP+PfPSaoAycB7/u6qSWZWgCA+z865g8AbwH7gMCnnbT3BfZ6vudnzmqnzHSxBb2lsC6pxo2ZWEPgEGOKcO3m9XdPYlqN+FmbWBUhyzq1PvTmNXV0G2nKKMKAR8LZzriFwhv/8OZ+WHH/M/q6HrkBloCxQgJSui58KpvN8I+kdY6aOPViCPhEon+p5FHDIo1oCzszCSQn5951zM/2bvzezMv72MkCSf3sw/CxaAA+a2T7gQ1K6b0YCRc0szL9P6uP69zH724sAR29nwQGQCCQ659b4n39MSvAH83luC+x1ziU75y4BM4HmBPd5vuZmz2umznewBP06oLr/3fo8pLyhM9vjmgLCzAx4F9jhnBueqmk2cO2d976k9N1f297H/+59U+DEtT8Rcwrn3MvOuSjnXCVSzuW/nHO9gSVAd/9uPz3maz+L7v79c9SVnnPuO+CAmd3p33QfsJ0gPs+kdNk0NbP8/n/n1445aM9zKjd7XhcA7cysmP8voXb+bRnj9ZsUAXyzoxOwC/gG+I3X9QTwuFqS8ifaZmCj/9GJlL7JxcBu/8fi/v2NlBFI3wBbSBnR4PlxZOL42wBz/Z9XAdYCCcBHQF7/9gj/8wR/exWv677FY20AxPvP9adAsWA/z8CrwNfAVuAfQN5gO8/AB6S8B3GJlCvzJ2/lvAL9/MeeADxxMzVoCgQRkSAXLF03IiKSDgW9iEiQU9CLiAQ5Bb2ISJBT0IuIBDkFvYhIkFPQi4gEuf8P1HOKGmugDeYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# training set up 1.\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "n_epoch = 1000\n",
    "loss_list = []\n",
    "\n",
    "# training\n",
    "for i in range(n_epoch):\n",
    "    U = model(X_torch)\n",
    "    loss = criterion(U, y_torch)\n",
    "    loss_list.append(loss.item())\n",
    "    loss.backward()\n",
    "    # update coefficients.\n",
    "    with torch.no_grad():\n",
    "        w_torch -= w_torch.grad * 1e-3\n",
    "        w_torch.grad.zero_()\n",
    "\n",
    "# check.\n",
    "print(\"True w is: \", w)\n",
    "print(\"Learnt w is: \", w_torch)\n",
    "plt.plot(loss_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression example 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True w is:  [[0.         3.87636046 8.06844608]\n",
      " [0.         9.47307596 2.92279439]\n",
      " [0.         9.84289957 1.98111079]\n",
      " [0.         0.1366462  0.74166592]]\n",
      "Learnt w is:  tensor([[-1.9492,  0.9414,  0.2377],\n",
      "        [-0.9784,  0.4168, -0.0909],\n",
      "        [ 0.6762, -0.6026, -0.8796],\n",
      "        [-0.5333,  0.9961, -0.5187]], dtype=torch.float64, requires_grad=True)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcYUlEQVR4nO3da3Bc9Znn8e+jvupqSZZsGdsgGxzACwESQSCXzY3MONQs1O4kU7iSSSY39kWYZDep3Ul2tsJO5sXOJTuZZDc3is2wm0pgyKUyXooMk2GYTQYCgwiXBRsTY0MsjG3Jlq271N169sU5ktuy7G5LLbfPOb9PVVefy1/dz/Fx/frf/z4Xc3dERCT6GupdgIiI1IYCXUQkJhToIiIxoUAXEYkJBbqISEyk6/XGXV1d3tvbW6+3FxGJpCeffHLI3bsXW1e3QO/t7aW/v79eby8iEklm9srp1mnIRUQkJhToIiIxoUAXEYkJBbqISEwo0EVEYkKBLiISEwp0EZGYiFyg7z44ypce3M3R8Zl6lyIicl6JXKDvGxrjfzy8h0MjU/UuRUTkvBK5QG/JZQAYmy7WuRIRkfNL9AI9H1ytYGxKgS4iUi56gZ4LAn1kqlDnSkREzi+RC/TWuR66hlxERE4SuUCf66FryEVE5GSRC/SmbAoz9dBFRBaKXKCbGS25NKPqoYuInCRygQ7Qmkurhy4iskAkA70ln9YYuojIAhUD3cy+bWaHzey5Cu2uNbOSmb2vduUtrkU9dBGRU1TTQ78b2HamBmaWAv4UeLAGNVXUms8wqkAXETlJxUB3958BRys0+33gh8DhWhRVSTDkohOLRETKLXsM3czWA/8a+GYVbW8zs34z6x8cHFzye+pHURGRU9XiR9G/BP7A3UuVGrr7ne7e5+593d3dS37Dlpx+FBURWShdg9foA+41M4Au4CYzK7r7j2vw2otqyacZnylRmnVSDbZSbyMiEinLDnR33zQ3bWZ3A/evZJhD2en/00VWNWZW8q1ERCKjYqCb2T3AO4AuMxsA7gAyAO5ecdx8JZRfoEuBLiISqBjo7r692hdz999bVjVVmr/JhcbRRUTmRfZMUYCxaR26KCIyJ5qBHo6h6wJdIiInRDLQdZMLEZFTRTLQdZMLEZFTRTPQ1UMXETlFJAO9OasxdBGRhSIZ6KkGozmbUg9dRKRMJAMddJMLEZGFohvouTSjOg5dRGReZAO9NZ/RGLqISJnIBnpbY4aRSfXQRUTmRDfQ82lG1EMXEZkX2UBfpR66iMhJIhvobY0ZRqYKuHu9SxEROS9EN9DzGQolZ6owW+9SRETOC9EN9MbgbNHjGnYREQGiHOj54CYXI1MKdBERiHKgh7ee0w+jIiKB6AZ6eMVF9dBFRALRDfT5HrqORRcRgSgHusbQRUROEt1AD49y0Ri6iEigYqCb2bfN7LCZPXea9R8ws2fDx6NmdlXtyzxVLp0in2nQ6f8iIqFqeuh3A9vOsH4f8HZ3fz3wx8CdNairKm15nf4vIjInXamBu//MzHrPsP7RstnHgA3LL6s6bY0ZnVgkIhKq9Rj6x4CfnG6lmd1mZv1m1j84OLjsNwuuuKhAFxGBGga6mb2TIND/4HRt3P1Od+9z977u7u5lv2dwTXSNoYuIQI0C3cxeD9wF3OLuR2rxmtVoy2fUQxcRCS070M3sQuBHwO+6+4vLL6l6bY1p/SgqIhKq+KOomd0DvAPoMrMB4A4gA+Du3wS+AKwGvm5mAEV371upgssFPfQi7k743iIiiVXNUS7bK6z/OPDxmlV0FlY1ZijNOhMzJZpzFTdFRCTWInumKJRdz0Xj6CIiEQ/08HouOhZdRCTqgT5316IJBbqISKQDvaMpC8CwAl1EJNqB3t40N+QyU+dKRETqL9KBrh66iMgJkQ70pmyKTMoYnlAPXUQk0oFuZrQ3ZfWjqIgIEQ90gI6mjHroIiLEINDbm7IaQxcRIQ6B3pjRkIuICDEI9I6mrIZcRESIQaC3N2c4NlHA3etdiohIXUU+0DuassyUZpkslOpdiohIXUU+0NvDKy7qh1ERSbroB/rc2aLjGkcXkWSLfKB3NOkSuiIiEINAn++h60gXEUm4yAf6XA9dY+giknSRD/S5Hvpx9dBFJOEiH+jZdAPN2ZR66CKSeJEPdJi7not66CKSbDEJ9OBsURGRJKsY6Gb2bTM7bGbPnWa9mdlXzWyPmT1rZm+ofZln1tmsHrqISDU99LuBbWdY/15gS/i4DfjG8ss6O10tOY6MKdBFJNkqBrq7/ww4eoYmtwD/2wOPAe1mtq5WBVZjdXOWI2PT5/ItRUTOO7UYQ18P7C+bHwiXncLMbjOzfjPrHxwcrMFbBzpbsozPlJic0QW6RCS5ahHotsiyRa9l6+53unufu/d1d3fX4K0DXc05AI6Mq5cuIslVi0AfADaWzW8ADtTgdau2uiU4uUjj6CKSZLUI9B3Ah8KjXa4Hjrv7azV43aqtblEPXUQkXamBmd0DvAPoMrMB4A4gA+Du3wQeAG4C9gATwEdWqtjTWd0c9NCH1EMXkQSrGOjuvr3Cegc+WbOKlkBDLiIiMTlTtCmbpimb0qGLIpJosQh0CHrpR3TXIhFJsPgEenOOIfXQRSTBYhPoXS1ZjaGLSKLFJtBXN+d02KKIJFp8Aj3soQcH3YiIJE+MAj1HcdYZmSzWuxQRkbqITaB3hceiD2nYRUQSKjaBvjq8QNfQqAJdRJIpNoHe3RoE+qAOXRSRhIpNoK9tCwL90IgCXUSSKTaBvqoxQzbdwOGRqXqXIiJSF7EJdDNjbVuOQwp0EUmo2AQ6wNrWvIZcRCSx4hXobXkOjaqHLiLJFKtAX9OW47B66CKSULEK9LVtecami4xN62xREUmemAV6cOiijnQRkSSKV6C35gEdiy4iyRSrQF/TFgT6Yf0wKiIJFKtAP3G2qAJdRJInVoHekgtuFq0hFxFJoqoC3cy2mdluM9tjZp9bZP2FZvawmT1lZs+a2U21L7WqOoNj0dVDF5EEqhjoZpYCvga8F9gKbDezrQua/WfgPne/BrgV+HqtC63Wmlad/i8iyVRND/06YI+773X3GeBe4JYFbRxoC6dXAQdqV+LZWbcqz4FjCnQRSZ5qAn09sL9sfiBcVu6/AB80swHgAeD3F3shM7vNzPrNrH9wcHAJ5Va2oaOJgyNTFEuzK/L6IiLnq2oC3RZZtvBOzNuBu919A3AT8B0zO+W13f1Od+9z977u7u6zr7YK6zsaKc06BzXsIiIJU02gDwAby+Y3cOqQyseA+wDc/RdAHuiqRYFna317IwCvDk/W4+1FROqmmkB/AthiZpvMLEvwo+eOBW1+DbwbwMwuJwj0lRlTqWB9RxDoB44r0EUkWSoGursXgduBB4FdBEezPG9mXzSzm8NmnwU+YWbPAPcAv+fuC4dlzgn10EUkqdLVNHL3Bwh+7Cxf9oWy6Z3AW2pb2tLkMym6WrK8ekyBLiLJEqszReesb29kQD10EUmYeAZ6R6N66CKSOPEM9PZGDhybpE7D+CIidRHbQJ8qzHJkfKbepYiInDPxDPSOJkBHuohIssQy0Dd2Bocu7h+eqHMlIiLnTiwD/aLOZgBeHhqvcyUiIudOLAO9MZuipy3PviH10EUkOWIZ6AC9XU28fEQ9dBFJjtgG+qauZvZpyEVEEiTWgX50fIbjk4V6lyIick7ENtB7V+uHURFJltgG+qauMNA1ji4iCRHbQN/Y2YQZ7B1UoItIMsQ20POZFBesalQPXUQSI7aBDjrSRUSSJdaBfsmaFvYcHmN2VlddFJH4i3WgX9bTysRMSTe7EJFEiHWgv66nFYAXDo7UuRIRkZUX70BfGwT67oOjda5ERGTlxTrQW3JpNnY28sIhBbqIxF+sAx3g0rVt6qGLSCLEPtAv62ll39A408VSvUsREVlRVQW6mW0zs91mtsfMPneaNr9jZjvN7Hkz+15ty1y6S3taKc06Lx3W8egiEm/pSg3MLAV8DXgPMAA8YWY73H1nWZstwOeBt7j7sJmtWamCz9Zl4ZEuu14bYesFbXWuRkRk5VTTQ78O2OPue919BrgXuGVBm08AX3P3YQB3P1zbMpduc3cLzdkUzw4cq3cpIiIrqppAXw/sL5sfCJeVex3wOjN7xMweM7Nti72Qmd1mZv1m1j84OLi0is9SqsG4csMqnh44fk7eT0SkXqoJdFtk2cJz6dPAFuAdwHbgLjNrP+WP3O909z537+vu7j7bWpfsqo3t7Dowoh9GRSTWqgn0AWBj2fwG4MAibf7G3Qvuvg/YTRDw54WrN7QzU5pl12s6fFFE4quaQH8C2GJmm8wsC9wK7FjQ5sfAOwHMrItgCGZvLQtdjqs2Bl8WntmvcXQRia+Kge7uReB24EFgF3Cfuz9vZl80s5vDZg8CR8xsJ/Aw8B/c/chKFX221q3K092aU6CLSKxVPGwRwN0fAB5YsOwLZdMOfCZ8nHfMjKs3tvOUAl1EYiz2Z4rOuba3g31D4xwamap3KSIiKyIxgX7D5i4AHtt73owEiYjUVGICfesFbbTl0zy6R4EuIvGUmEBPNRhv2ryaX6iHLiIxlZhAB7hh82p+fXSCgeGJepciIlJziQr0N1+yGkDDLiISS4kK9EvXttLTluehFw7VuxQRkZpLVKCbGTduXcPPXhxiqqDruohIvCQq0AHes7WHyUKJR/YM1bsUEZGaSlygX7+5k5Zcmp/u1LCLiMRL4gI9l07x9ku7+ftdhyiWZutdjohIzSQu0AH+1esvYGhshp//SsMuIhIfiQz0d122ho6mDD/85UC9SxERqZlEBno23cDNV13A3+08xPHJQr3LERGpiUQGOsBvv3EDM8VZdjyz8OZLIiLRlNhAv3L9Kq5Y38bdj+xjdnbhLVJFRKInsYFuZnz8rZt5aXCc//viYL3LERFZtsQGOsBNV66jpy3PXf903tz+VERkyRId6Nl0Ax95Sy+P7DnCEy8frXc5IiLLkuhAB/jQDb2sac3xJz95geDWqCIi0ZT4QG/Mpvj0jVt48pVhXQ5ARCIt8YEO8Dt9G7lkTQt/9H92Mj5drHc5IiJLUlWgm9k2M9ttZnvM7HNnaPc+M3Mz66tdiSsvk2rgv/6bK3n12CR/8dMX612OiMiSVAx0M0sBXwPeC2wFtpvZ1kXatQKfAh6vdZHnwrW9nXzgTRfy7Uf28agurSsiEVRND/06YI+773X3GeBe4JZF2v0x8GfAVA3rO6f+002Xs7mrmU/d+zSHRyO7GSKSUNUE+npgf9n8QLhsnpldA2x09/vP9EJmdpuZ9ZtZ/+Dg+XcyT3Muzdc/8EbGpgv82+88yeSM7mokItFRTaDbIsvmj+8zswbgy8BnK72Qu9/p7n3u3tfd3V19lefQpT2tfOXWa3hm/zFu/94vdc10EYmMagJ9ANhYNr8BKL+iVStwBfCPZvYycD2wI2o/jJb7zX/RwxdvuYKHXjjM7d97iumieuoicv6rJtCfALaY2SYzywK3AjvmVrr7cXfvcvded+8FHgNudvf+Fan4HPng9Rfxhd/ayt8+f5CP3d3PyJQusysi57eKge7uReB24EFgF3Cfuz9vZl80s5tXusB6+uhbN/Gl91/FY3uPcPN//yd2vTZS75JERE7L6nW6e19fn/f3R6MT/8TLR/nkd3/JyFSBz7/3cn73+otoaFjspwURkZVlZk+6+6JD2jpTtArX9nZy/6feynWbVnPHjud5/7d+od66iJx3FOhVWtOa53995Fr+2/uv4qXBMW766s/5zH1P8+qxyXqXJiICQLreBUSJmfHbb9zAjZev5ev/uIe/evRldjx9gJuvvoBPvG0zl69rq3eJIpJgGkNfhlePTXLXz/fy10/sZ2KmxJs2dfL+vo3cdGUPTVl9VopI7Z1pDF2BXgPHJwp8959f4b4n9vPykQmasym2XbGObVf08LYtXeQzqXqXKCIxoUA/R9yd/leG+X7/fn7y3EFGp4o0ZlL8y9d18e7L1/LWS7q4oL2x3mWKSIQp0OtgpjjL4/uO8HfPH+KnOw9xcCS42Ffv6iZuuLiLN1+8mmt7O+lZla9zpSISJQr0OpuddXYfGuXRl47wi5eGeHzvUUbDG2n0tOW5emM7V21s5+qN7bx+wyqacxp/F5HFKdDPM8XSLM8dGOGpXw/z9P5jPL3/GK8cmQDADC7qbOKynjYu7Wnl8nWtXNbTxoWdTTqZSUTOGOjqCtZBOtXA1WGPfM7R8Rme2X+MZweOs/vQCC+8NsqDOw8y93nbmElxyZoWNnc3s6kreFzc3UJvVzMt6tGLCAr080Znc5Z3XraGd162Zn7Z5EyJFw+NsvvgKDtfG+GlwTGefGWYHc8coPyL1ZrWHJu7m7mos5kNHY1s6GxkQ0cTGzuaWNOaU89eJCEU6OexxmyKq8Lx9XJThRKvHJlg39AYe4fG2Ts4zt7BMf5h92EGR6dPaptJGevbg4Df0NHI+vZG1q7K09OWp2dVnrVtedryacwU+iJRp0CPoHwmxaU9rVza03rKuqlCiVePTTIwPMnA8ET4PMn+oxP8/a7DDI1Nn/I3jZlUGO45etry84G/ti1Pd2uO1c1ZulpztOYU/CLnMwV6zOQzKS7ubuHi7pZF108VShwemebgyBQHR6Y4dHzqpOn+V4Y5PDLNzCJ3asqmG+hqzrK6JUdXy9xzMN3VkmN1+NzZnKW9KUMurROqRM4lBXrC5DMpLlzdxIWrm07bxt0Znihw8PgUQ2PTHBmfZmh0hqHw+cj4NINj07xwcJShsWkKpcWPlGrKpmhvzNDelKWjOUN7YxD0HU3Bc3tTlo6mTNl0llWNGVIa8xdZEgW6nMLM6GzO0tmcrdjW3RmZKgbBPzbD0Ng0R8dnODYxw/BEgWMThXB6hteOjXBsMpifPcPRsi25NG35NK35DG2N4fMp8xla82naGsPnsE1bY4ZcukFDQ5JICnRZFjNjVWOGVY0ZLq7yvt+zs87oVJFjk0HoD08EHwDHJgoMTxQYnSowMlkMnqcKHBqZ4leHC4xOFRmZLJzxwwCCH4Lb8hmac2mac2lacimasmlacmmac6lgefbkdcF02fpcmpZsmqZcikxKV5mWaFCgyznX0GCsasqwqinDRavP7m/dnYmZEiNTJwJ+dKrIyFSBkani/IfByFSB8eli+CgxPDHD/uEJxqeLTEyXGJspUu05ddl0Ay25NE3ZFM3ZNI3ZFI2ZFE3ZFPmy6cZMan7d6Z6bsmkaMyny2Yb5aQ0xSa0o0CVSzGy+B71u1dJfx92ZLJQYny4xPl1kbLrIxMyJ6fHpIuPhfDBdnG87WSgxVShxcKQQTM+UmCiUmJwpMV089cfkSrKphpOCP59Jkc80kEs3kEvPTafIpRvIZ4LnXKaBfDpFLnNym/K2uczc34TLytpkUxqWiiMFuiSSmdGUTdOUTdPdmqvZ687OBh8Uk2HAn/Q8N11h+cRMiZnSLFOFEscmC0yPBB8U04XgeSp8LlYaezrj9jP/gTH3QZFJGdl0imy6gWzKwucGMqmGYDqcn1++YH6uzXz7+deYe+0ztQ0eOglueRToIjXU0HDiG8RKK5Zmg6AvC/npYompwqnhXz5d3na6cOJvZkqzzBSDR6E0y3RhltGpYrBswbq5Zac7wmmp0g1B8KcbjEz4YZJOBdMnlhnp8HlueTr8QEinjHRD2bpU2d80lP/tiWXplJ30t9n0ieXzNTSceL2F75MK16UajHSD1fWbjwJdJKLSqQbSqQaaa/cF46y5+3zYF0o+H/ozJ4X+7KnL5z4YSqcunwm/fRTCvy2WnMKsUwznCyWnOBs8j00Xg/VzbWf95PmSU5gNnpfzjeZszAX73AdN8HziQyLdYGy/7kI+/rbNNX9vBbqILJmZhcM25/9JZO5+4sOgeCLoyz8M5j8ETvrgCKfD+bkPnGL5B8jsLKXwg6dU9gFS3qY4G/x9cdbpalmZT+GqAt3MtgFfAVLAXe7+JwvWfwb4OFAEBoGPuvsrNa5VRGTJzIxs2sjSAJVPsYikigfYmlkK+BrwXmArsN3Mti5o9hTQ5+6vB34A/FmtCxURkTOr5oyJ64A97r7X3WeAe4Fbyhu4+8PuPhHOPgZsqG2ZIiJSSTWBvh7YXzY/EC47nY8BP1lshZndZmb9ZtY/ODhYfZUiIlJRNYG+2DE4i/5cbGYfBPqAP19svbvf6e597t7X3V3leeIiIlKVan4UHQA2ls1vAA4sbGRmNwJ/CLzd3U+96LaIiKyoanroTwBbzGyTmWWBW4Ed5Q3M7BrgW8DN7n649mWKiEglFQPd3YvA7cCDwC7gPnd/3sy+aGY3h83+HGgBvm9mT5vZjtO8nIiIrJCqjkN39weABxYs+0LZ9I01rktERM6SebXXEK31G5sNAks9+agLGKphOVGgbU4GbXMyLGebL3L3RY8qqVugL4eZ9bt7X73rOJe0zcmgbU6Gldpm3YpFRCQmFOgiIjER1UC/s94F1IG2ORm0zcmwItscyTF0ERE5VVR76CIisoACXUQkJiIX6Ga2zcx2m9keM/tcveupFTPbaGYPm9kuM3vezD4dLu80s5+a2a/C545wuZnZV8N/h2fN7A313YKlMbOUmT1lZveH85vM7PFwe/86vNwEZpYL5/eE63vrWfdymFm7mf3AzF4I9/cNcd7PZvbvw//Tz5nZPWaWj+N+NrNvm9lhM3uubNlZ71cz+3DY/ldm9uGzqSFSgV7lzTaiqgh81t0vB64HPhlu2+eAh9x9C/BQOA/Bv8GW8HEb8I1zX3JNfJrgkhJz/hT4cri9wwSXYyZ8Hnb3S4Avh+2i6ivA37r7ZcBVBNsfy/1sZuuBTxHcAOcKgrue3Uo89/PdwLYFy85qv5pZJ3AH8CaCe1HcMfchUBV3j8wDuAF4sGz+88Dn613XCm3r3wDvAXYD68Jl64Dd4fS3gO1l7efbReVBcOXOh4B3AfcTXKp5CEgv3N8E1xK6IZxOh+2s3tuwhG1uA/YtrD2u+5kT91PoDPfb/cBvxnU/A73Ac0vdr8B24Ftly09qV+kRqR46Z3+zjUgKv2ZeAzwOrHX31wDC5zVhszj8W/wl8B+B2XB+NXDMgwvCwcnbNL+94frjYfuo2Uxw392/Coea7jKzZmK6n939VeBLwK+B1wj225PEfz/POdv9uqz9HbVAr/pmG1FlZi3AD4F/5+4jZ2q6yLLI/FuY2W8Bh939yfLFizT1KtZFSRp4A/ANd78GGOfE1/DFRHq7w+GCW4BNwAVAM8Fww0Jx28+VnG47l7X9UQv0qm62EVVmliEI8++6+4/CxYfMbF24fh0wd735qP9bvAW42cxeJrhP7bsIeuztZjZ3FdDybZrf3nD9KuDouSy4RgaAAXd/PJz/AUHAx3U/3wjsc/dBdy8APwLeTPz385yz3a/L2t9RC/SKN9uIKjMz4H8Cu9z9L8pW7QDmfun+MMHY+tzyD4W/ll8PHJ/7ahcF7v55d9/g7r0E+/Ef3P0DwMPA+8JmC7d37t/hfWH7yPXc3P0gsN/MLg0XvRvYSUz3M8FQy/Vm1hT+H5/b3ljv5zJnu18fBH7DzDrCbze/ES6rTr1/RFjCjw43AS8CLwF/WO96arhdbyX4avUs8HT4uIlg/PAh4Ffhc2fY3giO+HkJ+H8ERxHUfTuWuO3vAO4PpzcD/wzsAb4P5MLl+XB+T7h+c73rXsb2Xg30h/v6x0BHnPcz8EfAC8BzwHeAXBz3M3APwe8EBYKe9seWsl+Bj4bbvwf4yNnUoFP/RURiImpDLiIichoKdBGRmFCgi4jEhAJdRCQmFOgiIjGhQBcRiQkFuohITPx/Ivadr2bnVu8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# training set up 2.\n",
    "# generate data\n",
    "X_torch = torch.from_numpy(X)\n",
    "y_torch = torch.from_numpy(y)\n",
    "w_torch = torch.randn(4, 3, requires_grad = True, dtype = torch.double)\n",
    "\n",
    "# class\n",
    "class SimpleLinear(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # sw: note the way to define parameters. Use nn.Parameter() class\n",
    "        self.w = nn.Parameter(torch.randn(4, 3, requires_grad = True, dtype = torch.double))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = torch.matmul(x, self.w)\n",
    "        return x\n",
    "\n",
    "# opt\n",
    "model = SimpleLinear()\n",
    "# model.parameters()\n",
    "opt = torch.optim.SGD(model.parameters(), 1e-1)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "n_epoch = 1000\n",
    "loss_list = []\n",
    "\n",
    "# training\n",
    "for i in range(n_epoch):\n",
    "    output = model(X_torch)\n",
    "    loss = criterion(output, y_torch)\n",
    "    loss.backward()\n",
    "    loss_list.append(loss.item())\n",
    "    with torch.no_grad():\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "        \n",
    "# \n",
    "plt.plot(loss_list)\n",
    "print(\"True w is: \", w)\n",
    "print(\"Learnt w is: \", w_torch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}