lectures/lecture-4/Seminar_on_PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vccetCugONM_",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# PyTorch\n",
    "## Why do we use it?\n",
    "1. Many research works related to Deep Learning (DL) are implemented on PyTorch\n",
    "2. It's a constantly evolving and expanding package (functorch)\n",
    "3. Package provide wide variety of tools to make your code computationally efficient (GPU)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B0Nm9e5EPPWt",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# How to initialize vectors, matrices, tensors (object with number of dimensions more than 2)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "kHXT5gA5QAXy",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sCRWnnR6Upk-",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Manual definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_Jyw0YiaUoks",
    "outputId": "37d151f5-a964-47be-b4e1-3cf81bd75818",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 0],\n",
       "        [1, 2, 3],\n",
       "        [2, 4, 6]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor([[0,1,0],\n",
    "                  [1,2,3],\n",
    "                  [2,4,6]\n",
    "                  ])\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uYM4_hx7evtj",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### From numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5kqxygFceukD",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6WrZGpcde0d1",
    "outputId": "6c518dce-613f-4e37-ac3b-a2249c1e63e1",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4860, 0.0246, 0.7205],\n",
       "        [0.4484, 0.6652, 0.0402],\n",
       "        [0.9689, 0.5528, 0.7985]], dtype=torch.float64)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.random.uniform(size=(3,3))\n",
    "torch.from_numpy(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BL-1MNYgPvEc",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Zero, ones, random  vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TWnuiy_0RFl8",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bdPYXiqQOIMV",
    "outputId": "0e736bc8-a96b-4ce2-bc6e-e10d36e7bc95",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0.])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = (4)\n",
    "x = torch.zeros(dims)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FNW-I7A1eE2V",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ixmhUKMzQY5F",
    "outputId": "3a82cf17-c80c-4e04-ae24-5f90f5a2caa9",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 1., 1., 1.])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = (4)\n",
    "x = torch.ones(dims)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bxDQ-N32ZAd3",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ltouO_w9ZFMG",
    "outputId": "884912cb-a6ea-40b4-a92e-c8595cff30b3",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(10)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2u4n3unyeLv0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Linspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "73A0izxyZI3T",
    "outputId": "a4707a04-c25e-4aa9-a0b9-0b0052ad9372",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-1.0000, -0.7778, -0.5556, -0.3333, -0.1111,  0.1111,  0.3333,  0.5556,\n",
       "         0.7778,  1.0000])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.linspace(-1,1,10)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-8VUO34ieXc0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4W6A2lZZUUQK",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Identity matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mD1vOs3BUTPV",
    "outputId": "47970be3-c620-4d78-d858-a672c57212e1",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.eye(3)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0cSGhIaLUacf",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Diagonal of matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ENjyok5PYyoi",
    "outputId": "92ec2ce0-1aa7-4748-9ccd-4795955f0123",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 1., 1.])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.diag()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xA8kM0rqZcFs",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Diagonal matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ar7lqfKtZfv5",
    "outputId": "6b09b5f8-8329-4baa-c320-6118b763e812",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0],\n",
       "        [0, 1, 0],\n",
       "        [0, 0, 2]])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.diag(torch.arange(3))\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "W_SczjmFSp0L",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Random matrix (uniform distribution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qI3K-_kXQsei",
    "outputId": "60fe1637-3b74-4126-f919-11dc8b2c79ee",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1512, 0.5654, 0.4128],\n",
       "        [0.4651, 0.1956, 0.7034],\n",
       "        [0.7692, 0.8898, 0.7750]])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = (3,3)\n",
    "x = torch.rand(dims)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6mKSmCCxTFda",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Random matrix (normal distribution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EoX12gEcTFdb",
    "outputId": "dcd721b9-8aaa-4302-f97a-97cc8ff87429",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.2327, -0.0362,  0.5722],\n",
       "        [ 0.9502,  1.0798,  0.4765],\n",
       "        [-0.0577, -2.3726,  0.5873]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = (3,3)\n",
    "x = torch.randn(dims)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OsoDl82WTosR",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## How calculate statistics on tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eCa2L_2FTnDM",
    "outputId": "1c93ee27-6675-4637-9adb-84581d3a0b1d",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.1074),\n",
       " tensor(1.0340),\n",
       " tensor(0.4765),\n",
       " tensor(-2.3726),\n",
       " tensor(1.0798))"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(), x.std(), x.median(), x.min(), x.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mxgFKxg2XsZO",
    "outputId": "675998ca-5a94-491b-f6c5-d2d71a9d41b6",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 0.1011,  0.8355, -0.6144]),\n",
       " tensor([0.4197, 0.3176, 1.5565]),\n",
       " tensor([-0.0362,  0.9502, -0.0577]),\n",
       " tensor([-0.2327,  0.4765, -2.3726]),\n",
       " tensor([0.5722, 1.0798, 0.5873]))"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.mean(dim=1), x.std(dim=1), x.median(dim=1)[0], x.min(dim=1)[0], x.max(dim=1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4wopzf8DX-Fz",
    "outputId": "c27c818c-72d1-42d2-89e9-2fb6d77cb403",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1, 0, 0]), tensor([0, 2, 1]), tensor([2, 1, 2]))"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.median(dim=1)[1], x.min(dim=1)[1], x.max(dim=1)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "g4_PVlg1TN5E",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Numpy operations in PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "klFk3eKaz6n_",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5564nF9tS0fu",
    "outputId": "7825843a-1ee3-4173-e3ed-920216a278cf",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0930, -1.2068, -0.9818],\n",
       "        [ 0.2951,  1.0160,  0.2693],\n",
       "        [-1.1653, -1.0365,  1.9469]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn((3,2))\n",
    "y = torch.randn((2,3))\n",
    "x @ y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AiMRM6MMfNmZ",
    "outputId": "da9c45c9-a860-4ba6-f645-1ebaafc82218",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.8451,  1.2279, -1.0831]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn((1,3))\n",
    "y = torch.randn((3,1))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cHeJikWmxxPc",
    "outputId": "fb936bca-927c-4b7b-def4-1d129cbe8629",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3426],\n",
       "        [0.9155],\n",
       "        [0.2896]])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ABESeDDFxyly",
    "outputId": "6fac50c1-998a-413c-ce47-288e7287d2ee",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.5209]])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x @ y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SJEokM1HfTLf",
    "outputId": "4980165e-d839-4841-d737-3645942c48b0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0292,  0.2806,  0.4464],\n",
       "        [ 0.0248, -0.2386, -0.3797],\n",
       "        [-0.0213,  0.2045,  0.3254]])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y @ x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6-PS8J-yyRof",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Reshaping and dimension permutation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bUq26qfwyaX3",
    "outputId": "e45d8283-3685-4099-d819-32a92bf1442e",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1.4408,  1.0554, -0.4302,  2.4403],\n",
       "         [-1.5278, -1.2142,  0.7821,  0.2796],\n",
       "         [ 0.5764, -0.8881, -2.0423, -0.9508]],\n",
       "\n",
       "        [[ 0.9360, -0.5286, -0.2148, -0.6397],\n",
       "         [-0.5315,  0.3589, -0.6161, -1.6595],\n",
       "         [ 0.2836, -0.8500,  1.2469, -0.2715]]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn((2,3,4))\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "T1V-6JLvyfR9",
    "outputId": "2e2b706f-2a41-460a-dc59-d6b6e085878d",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.4408,  1.0554, -0.4302,  2.4403],\n",
       "        [-1.5278, -1.2142,  0.7821,  0.2796],\n",
       "        [ 0.5764, -0.8881, -2.0423, -0.9508],\n",
       "        [ 0.9360, -0.5286, -0.2148, -0.6397],\n",
       "        [-0.5315,  0.3589, -0.6161, -1.6595],\n",
       "        [ 0.2836, -0.8500,  1.2469, -0.2715]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.reshape(2*3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rLsm-foCbEVQ",
    "outputId": "9d4dcee7-a63a-4cb4-8c42-5e6c0280ca33",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 0.0159,  0.5230,  0.4219],\n",
       "         [-0.8664, -1.3224, -0.2815],\n",
       "         [ 0.3624, -0.9631, -0.1161],\n",
       "         [-0.2072, -1.8525,  0.1093]]), torch.Size([4, 3]))"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.reshape(-1,4), x.reshape(-1,4).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2KcH0MBx0ASG",
    "outputId": "03af885e-1c0b-49d1-d805-8a64ab746dc3",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 2, 4])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.permute(1,0,2).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ui48IA8n0M84",
    "outputId": "33ba068f-e86e-41de-f6a6-50ea8a49a9f2",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3])"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.permute(0,2,1).reshape(-1,3).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qyHtiHXvvdWh",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Decompositions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "n54WAprXvgd-",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "VUPsHhP7viZK",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn((100,100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "vJYqcSMpvoeJ",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "U, S, V = torch.svd(x)\n",
    "Q, R = torch.linalg.qr(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "IOYnq2CNv3u0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn((100,10)) @ torch.randn((10,100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hqrfby_Hwvi7",
    "outputId": "833f9421-4dc9-4738-a63e-fa33975b58f7",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "714 µs ± 36.4 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit torch.svd(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OI6gZUpFw0vf",
    "outputId": "ce2631e3-6832-43ba-e192-46e75fcb59f8",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "283 µs ± 8.97 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit torch.svd_lowrank(x,q=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AOEvu7N3zv_0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### How to transfer data to GPU\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CGvt5sGm0-HU",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gC7nS18c06WE",
    "outputId": "bdcbace7-c529-492b-96a7-85e40e6c5898",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "t4HC_kKQ1HKf",
    "outputId": "00ea2c11-6ea2-4217-fe51-3bfa15f830f0",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cdCSSzZobMJB",
    "outputId": "9775dca3-3b93-4b68-b1dd-35fb7c60e2ed",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "106 µs ± 2.29 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit x = torch.randn((100,100)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AWx2ZKaW1USa",
    "outputId": "b9a81610-6d68-4d59-ee3c-bf76d18a415a",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.96 µs ± 114 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit x = torch.randn((100,100), device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "P8CPF39W1a_t",
    "outputId": "018f1628-29cb-4d9b-8d69-3957607c70e3",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "-1\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn((5,5), device=device)\n",
    "print(x.get_device())\n",
    "x = x.detach().cpu()\n",
    "print(x.get_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "az7LpU7Z2ENo",
    "outputId": "47eac60e-b27c-41f5-c3e6-f2ad6f5fb885",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357 µs ± 5.17 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn((100,1000),device=\"cpu\")\n",
    "y = torch.randn((1000,100),device=\"cpu\")\n",
    "%timeit x @ y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Gli-kHfE2slg",
    "outputId": "4b545bcf-5e37-4bc5-a3f9-3d41c942bbdf",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21.8 µs ± 506 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn((100,1000),device=\"cuda\")\n",
    "y = torch.randn((1000,100),device=\"cuda\")\n",
    "%timeit x @ y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 200
    },
    "id": "xxoiDFl5A4hv",
    "outputId": "476169c8-b88e-484c-dbc0-f0983503dc11",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-22e087ccddd4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m@\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument mat2 in method wrapper_mm)"
     ]
    }
   ],
   "source": [
    "x = torch.randn((100,1000),device=\"cuda\")\n",
    "y = torch.randn((1000,100),device=\"cpu\")\n",
    "x @ y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iW3Bi9AI3g-b",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xNQjQELGBIqG",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nNe0kdfaD7id",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Simple network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cuCpvy8iBZvz",
    "outputId": "39b1d452-2d57-4d14-840a-67d645e0b338",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 100])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = torch.nn.Linear(100,10)\n",
    "x = torch.randn((5,100))\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1QQ3llcZ3nQB",
    "outputId": "569a0aff-97ce-4d5c-9ff6-f41112eb8671",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 10])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out = net(x)\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "68boZao7B0mJ",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "out_manual = []\n",
    "for x_one in x:\n",
    "    out_manual.append(net(x_one[None]))\n",
    "out_manual = torch.cat(out_manual, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JA2nBEAEC22s",
    "outputId": "39b2730d-84ad-4bba-e200-25eb862a97b5",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(7.7970e-07, grad_fn=<CopyBackwards>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.norm(out-out_manual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CMNrx27ADZ6Y",
    "outputId": "d2aa5408-a011-4919-d880-007f58418c97",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([100]), torch.Size([1, 100]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_one.shape, x_one[None].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9qc7R52PDUgQ",
    "outputId": "9bc64aa2-e1cd-4f90-c88e-9ce77066f025",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 100])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_one.unsqueeze(0).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MBHHeutGD6K_",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### More complex networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ajp3VD2QEAry",
    "outputId": "7ee53f48-8094-48ea-8832-ceb8c492e8f7",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=100, out_features=50, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(100,50),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(50,10)\n",
    ")\n",
    "net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "s5fwyLwBHVuu",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "$$\\mathrm{net} = f_2(f_1(f_0(x)))$$\n",
    "\n",
    "$$f_0() = Linear(100,50)()$$\n",
    "$$f_1() = ReLU()$$\n",
    "$$f_2() = Linear(50,10)()$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HbKANjDUERTX",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn((5,100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iWzGuEpSEimz",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "out = net(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qW6JHBoLElRb",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "out_manual = x\n",
    "for i in range(len(net)):\n",
    "    out_manual = net[i](out_manual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Dez62rEcExZQ",
    "outputId": "83039b92-b180-44e0-be46-082ceb23fb2c",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0., grad_fn=<CopyBackwards>)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.norm(out - out_manual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TM1zsnC_KiZu",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### How to calc gradients and jacobians? (functorch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 679
    },
    "id": "3a-i-esfXOxb",
    "outputId": "f454000f-2ccf-49ac-d419-69c26c97c261",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting functorch\n",
      "  Downloading functorch-1.13.0-py2.py3-none-any.whl (2.1 kB)\n",
      "Collecting torch<1.13.1,>=1.13.0\n",
      "  Downloading torch-1.13.0-cp37-cp37m-win_amd64.whl (167.3 MB)\n",
      "     -------------------------------------- 167.3/167.3 MB 4.5 MB/s eta 0:00:00\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\egor\\anaconda3\\lib\\site-packages (from torch<1.13.1,>=1.13.0->functorch) (4.4.0)\n",
      "Installing collected packages: torch, functorch\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.10.1+cu102\n",
      "    Uninstalling torch-1.10.1+cu102:\n",
      "      Successfully uninstalled torch-1.10.1+cu102\n",
      "Successfully installed functorch-1.13.0 torch-1.13.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Error parsing requirements for jupyter: [Errno 2] No such file or directory: 'c:\\\\users\\\\egor\\\\anaconda3\\\\lib\\\\site-packages\\\\jupyter-1.0.0.dist-info\\\\METADATA'\n",
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "torchvision 0.11.2+cu102 requires torch==1.10.1, but you have torch 1.13.0 which is incompatible.\n",
      "torchaudio 0.10.1+cu102 requires torch==1.10.1, but you have torch 1.13.0 which is incompatible.\n"
     ]
    }
   ],
   "source": [
    "!pip install functorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2iG-q2imXEE_",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from functorch import jacrev,jacfwd,vmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cogGf5NHYbVG",
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vTWd21QqFQp6",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(100,10),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(10,1)\n",
    ").to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fZNeTXL7Whs7",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn((1,100), device=\"cuda\")\n",
    "x_var = torch.autograd.Variable(x,requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Vd7oYyTEW-ou",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "grads = torch.autograd.grad(outputs=net(x_var), inputs=x_var, grad_outputs=torch.ones_like(net(x)))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "M62sYy6UZHBw",
    "outputId": "f1779fa7-10ec-428c-fa29-70356a70a87e",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 100])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1kk0_nTJZVmz",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Calculation of jacobian using standard tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8bi1yvo4ZOB7",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(100,50),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(50,20)\n",
    ").to(device)\n",
    "x = torch.randn((1,100), device=\"cuda\")\n",
    "x_var = torch.autograd.Variable(x,requires_grad=True)\n",
    "grads = torch.autograd.grad(outputs=net(x_var), inputs=x_var, grad_outputs=torch.ones_like(net(x)))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bSBDE-ZsZg8q",
    "outputId": "18c77f69-5371-4529-ca09-0311c16f5129",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 100]), torch.Size([1, 20]))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads.shape, net(x).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1rp8lqe7ZvKD",
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "[1,20,100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qzj9qqK1Zr6B",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "def slow_grad_calc(x):\n",
    "    grads = []\n",
    "    for i in range(20):\n",
    "        x_var = torch.autograd.Variable(x,requires_grad=True)\n",
    "        out = net(x_var)[:,i]\n",
    "        grad = torch.autograd.grad(out,x_var,torch.ones_like(out))[0][:,None]\n",
    "        grads.append(grad)\n",
    "    return torch.cat(grads,dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0ovmgxx1ZpEX",
    "outputId": "ef7f3866-4921-4d26-f30f-0813dfd3f061",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 20, 100])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads = slow_grad_calc(x)\n",
    "grads.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3M-t6HSBb38T",
    "outputId": "f06487c7-f20d-4d7d-ef8a-9166b756e035",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.51 ms ± 152 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit slow_grad_calc(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z_cb70YXb-kh",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### FuncTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-kS02FjHb7tW",
    "outputId": "f13ffb45-30da-49ac-ccb5-5d5c3bfe6ae9",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 20, 1, 100])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jac_func = jacfwd(net)\n",
    "jac_func(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_U88_u2qchRJ",
    "outputId": "d953ac70-ee3f-4b6d-b7ab-a324ea9fc9eb",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.17 ms ± 24.8 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit jac_func(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WHwgIMWMdVUq",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### FuncTorch operations over batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LC1YDfxJcyd4",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = torch.randn((10,100),device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Y5Bre5qcdwmX",
    "outputId": "4f4e8e1d-afc8-441a-cb43-787de4a311a1",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 20, 10, 100])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads = jac_func(x)\n",
    "grads.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ibGFXMLzd7pB",
    "outputId": "a8cbb491-ac97-4d68-89c4-29808a13e22a",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0077,  0.0243, -0.0019,  ...,  0.0293, -0.0044,  0.0351],\n",
       "        [-0.0030,  0.0255,  0.0129,  ...,  0.0025, -0.0403,  0.0171],\n",
       "        [ 0.0011, -0.0244,  0.0128,  ..., -0.0139,  0.0124,  0.0274],\n",
       "        ...,\n",
       "        [-0.0033, -0.0449, -0.0122,  ..., -0.0086,  0.0235, -0.0176],\n",
       "        [-0.0164, -0.0018, -0.0123,  ...,  0.0323, -0.0443, -0.0130],\n",
       "        [ 0.0219, -0.0086,  0.0348,  ..., -0.0059, -0.0297,  0.0355]],\n",
       "       device='cuda:0', grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads[0,:,0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YOu-n51xeG_v",
    "outputId": "b25dfea5-b0fb-4c44-c815-49ac809c6091",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]], device='cuda:0',\n",
       "       grad_fn=<SliceBackward0>)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grads[0,:,1,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jtoG-oi-ea8t",
    "outputId": "2139097c-d378-4180-d587-6956f12cdcb5",
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 1, 20, 1, 100])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vmap(jac_func)(x.unsqueeze(1)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vD1y5g-Eew8Q",
    "outputId": "d84a324c-f65f-4445-f3ea-bd9f1d2368de",
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 1, 100])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.unsqueeze(1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net(x,index=0)\n",
    "i = 0\n",
    "func = lambda x: net(x,index=i)\n",
    "jacfwd(func)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "celltoolbar": "Slideshow",
  "colab": {
   "collapsed_sections": [],
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}