Neural_network_architecture_modules_ani_khachatryan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Module** is an abstract class which defines fundamental methods necessary for a training a neural network. You do not need to change anything here, just read the comments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Module(object):\n",
    "    \n",
    "    def __init__ (self):\n",
    "        self.output = None\n",
    "        self.gradInput = None\n",
    "        self.train = True\n",
    "    \"\"\"\n",
    "    Basically, you can think of a module as of a something (black box) \n",
    "    which can process `input` data and produce `ouput` data.\n",
    "    This is like applying a function which is called `forward`: \n",
    "        \n",
    "        output = module.forward(input)\n",
    "    \n",
    "    The module should be able to perform a backward pass: to differentiate the `forward` function. \n",
    "    More, it should be able to differentiate it if is a part of chain (chain rule).\n",
    "    The latter implies there is a gradient from previous step of a chain rule. \n",
    "    \n",
    "        gradInput = module.backward(input, gradOutput)\n",
    "    \"\"\"\n",
    "    \n",
    "    def forward(self, inpt):\n",
    "        \"\"\"\n",
    "        Takes an input object, and computes the corresponding output of the module.\n",
    "        \"\"\"\n",
    "        return self.updateOutput(inpt)\n",
    "\n",
    "    def backward(self, inpt, gradOutput):\n",
    "        \"\"\"\n",
    "        Performs a backpropagation step through the module, with respect to the given input.\n",
    "        \n",
    "        This includes \n",
    "         - computing a gradient w.r.t. `input` (is needed for further backprop),\n",
    "         - computing a gradient w.r.t. parameters (to update parameters while optimizing).\n",
    "        \"\"\"\n",
    "        self.updateGradInput(inpt, gradOutput)\n",
    "        self.accGradParameters(inpt, gradOutput)\n",
    "        return self.gradInput\n",
    "    \n",
    "\n",
    "    def updateOutput(self, inpt):\n",
    "        \"\"\"\n",
    "        Computes the output using the current parameter set of the class and input.\n",
    "        This function returns the result which is stored in the `output` field.\n",
    "        \n",
    "        Make sure to both store the data in `output` field and return it. \n",
    "        \"\"\"\n",
    "        \n",
    "        # The easiest case:\n",
    "            \n",
    "        # self.output = input \n",
    "        # return self.output\n",
    "        \n",
    "        pass\n",
    "\n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        \"\"\"\n",
    "        Computing the gradient of the module with respect to its own input. \n",
    "        This is returned in `gradInput`. Also, the `gradInput` state variable is updated accordingly.\n",
    "        \n",
    "        The shape of `gradInput` is always the same as the shape of `input`.\n",
    "        \n",
    "        Make sure to both store the gradients in `gradInput` field and return it.\n",
    "        \"\"\"\n",
    "        \n",
    "        # The easiest case:\n",
    "        \n",
    "        # self.gradInput = gradOutput \n",
    "        # return self.gradInput\n",
    "        \n",
    "        pass   \n",
    "    \n",
    "    def accGradParameters(self, inpt, gradOutput):\n",
    "        \"\"\"\n",
    "        Computing the gradient of the module with respect to its own parameters.\n",
    "        No need to override if module has no parameters (e.g. ReLU).\n",
    "        \"\"\"\n",
    "        pass\n",
    "    \n",
    "    def zeroGradParameters(self): \n",
    "        \"\"\"\n",
    "        Zeroes `gradParams` variable if the module has params.\n",
    "        \"\"\"\n",
    "        pass\n",
    "        \n",
    "    def getParameters(self):\n",
    "        \"\"\"\n",
    "        Returns a list with its parameters. \n",
    "        If the module does not have parameters return empty list. \n",
    "        \"\"\"\n",
    "        return []\n",
    "        \n",
    "    def getGradParameters(self):\n",
    "        \"\"\"\n",
    "        Returns a list with gradients with respect to its parameters. \n",
    "        If the module does not have parameters return empty list. \n",
    "        \"\"\"\n",
    "        return []\n",
    "    \n",
    "    def training(self):\n",
    "        \"\"\"\n",
    "        Sets training mode for the module.\n",
    "        Training and testing behaviour differs for Dropout, BatchNorm.\n",
    "        \"\"\"\n",
    "        self.training = True\n",
    "    \n",
    "    def evaluate(self):\n",
    "        \"\"\"\n",
    "        Sets evaluation mode for the module.\n",
    "        Training and testing behaviour differs for Dropout, BatchNorm.\n",
    "        \"\"\"\n",
    "        self.training = False\n",
    "    \n",
    "    def __repr__(self):\n",
    "        \"\"\"\n",
    "        Pretty printing. Should be overrided in every module if you want \n",
    "        to have readable description. \n",
    "        \"\"\"\n",
    "        return \"Module\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequential container"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define** a forward and backward pass procedures."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sequential(Module):\n",
    "    \"\"\"\n",
    "         This class implements a container, which processes `input` data sequentially. \n",
    "         \n",
    "         `input` is processed by each module (layer) in self.modules consecutively.\n",
    "         The resulting array is called `output`. \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__ (self):\n",
    "        super(Sequential, self).__init__()\n",
    "        self.modules = []\n",
    "        self.y = []\n",
    "   \n",
    "    def add(self, module):\n",
    "        \"\"\"\n",
    "        Adds a module to the container.\n",
    "        \"\"\"\n",
    "        self.modules.append(module)\n",
    "\n",
    "    def updateOutput(self, inpt):\n",
    "#         self.y_list = []\n",
    "#         for i in range(len(self.modules)):\n",
    "#             if i == 0:\n",
    "#                 self.y_list.append(self.modules[i].forward(inpt))\n",
    "#             else:\n",
    "#                 self.y_list.append(self.modules[i].forward(self.y_list[i-1]))\n",
    "#                 y_ind = self.y_list[i]\n",
    "        \n",
    "#         self.output = np.array(y_ind)\n",
    "#         return self.output\n",
    "        \"\"\"\n",
    "        Basic workflow of FORWARD PASS:\n",
    "        \n",
    "            y_0    = module[0].forward(input)\n",
    "            y_1    = module[1].forward(y_0)\n",
    "            ...\n",
    "            output = module[n-1].forward(y_{n-2})   \n",
    "            \n",
    "            \n",
    "        Just write a little loop. \n",
    "        \"\"\"\n",
    "        self.y = []\n",
    "        \n",
    "        self.y.append(self.modules[0].forward(inpt))\n",
    "        for i in range (1,len(self.modules)):\n",
    "            self.y.append(self.modules[i].forward(self.y[i-1]))\n",
    "        self.output = self.y[-1]            \n",
    "            \n",
    "        return self.output\n",
    "        \n",
    "\n",
    "    def backward(self, inpt, gradOutput):\n",
    "        \"\"\"\n",
    "        Workflow of BACKWARD PASS:\n",
    "            \n",
    "            g_{n-1} = module[n-1].backward(y_{n-2}, gradOutput)\n",
    "            g_{n-2} = module[n-2].backward(y_{n-3}, g_{n-1})\n",
    "            ...\n",
    "            g_1 = module[1].backward(y_0, g_2)   \n",
    "            gradInput = module[0].backward(input, g_1)   \n",
    "             \n",
    "             \n",
    "        !!!\n",
    "                \n",
    "        To each module you need to provide the input, module saw while forward pass, \n",
    "        it is used while computing gradients. \n",
    "        Make sure that the input for `i-th` layer the output of `module[i]` (just the same input as in forward pass) \n",
    "        and NOT `input` to this Sequential module. \n",
    "        \n",
    "        !!!\n",
    "        \n",
    "       \"\"\"\n",
    "#         n = len(self.modules)\n",
    "#         g = (n-1)*[None]\n",
    "#         g.append(gradOutput)\n",
    "        \n",
    "#         for i in reversed(range(1,len(self.modules))):\n",
    "#             g[i] = (self.modules[i]).backward(self.y[i-1], g[i])\n",
    "        \n",
    "        \n",
    "            \n",
    "#         self.gradInput = self.modules[0].backward(inpt,g[0])  \n",
    "\n",
    "\n",
    "        g = []\n",
    "        g.append(gradOutput)\n",
    "        n = len(self.modules)\n",
    "        for i in range(1,n):\n",
    "            g.append(self.modules[n-i].backward(self.y[n-i-1], g[i-1]))\n",
    "            grad_inpt = g[i]\n",
    "        self.gradInput = np.array(grad_inpt) \n",
    "       \n",
    "        return self.gradInput\n",
    "      \n",
    "\n",
    "    def zeroGradParameters(self): \n",
    "        for module in self.modules:\n",
    "            module.zeroGradParameters()\n",
    "    \n",
    "    def getParameters(self):\n",
    "        \"\"\"\n",
    "        Should gather all parameters in a list.\n",
    "        \"\"\"\n",
    "        return [x.getParameters() for x in self.modules]\n",
    "    \n",
    "    def getGradParameters(self):\n",
    "        \"\"\"\n",
    "        Should gather all gradients w.r.t parameters in a list.\n",
    "        \"\"\"\n",
    "        return [x.getGradParameters() for x in self.modules]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        string = \"\".join([str(x) + '\\n' for x in self.modules])\n",
    "        return string\n",
    "    \n",
    "    def __getitem__(self,x):\n",
    "        return self.modules.__getitem__(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- input:   **`batch_size x n_features1`**\n",
    "- output: **`batch_size x n_features2`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Module):\n",
    "    \"\"\"\n",
    "    A module which applies a linear transformation \n",
    "    A common name is fully-connected layer, dense layer, InnerProductLayer in caffe. \n",
    "    \n",
    "    The module should work with 2D input of shape (n_samples, n_feature).\n",
    "    \"\"\"\n",
    "    def __init__(self, n_in, n_out):\n",
    "        super(Linear, self).__init__()\n",
    "       \n",
    "        # This is a nice initialization\n",
    "        stdv = 1./np.sqrt(n_in)\n",
    "        self.W = np.random.uniform(-stdv, stdv, size = (n_out, n_in))\n",
    "        self.b = np.random.uniform(-stdv, stdv, size = n_out)\n",
    "        \n",
    "        self.gradW = np.zeros_like(self.W)\n",
    "        self.gradb = np.zeros_like(self.b)\n",
    "        \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = inpt.dot(self.W.T) + self.b\n",
    "        \n",
    "        return self.output\n",
    "\n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput.dot(self.W)\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def accGradParameters(self, inpt, gradOutput):\n",
    "        self.gradW = gradOutput.T.dot(inpt)\n",
    "        self.gradb = gradOutput.sum(axis=0)\n",
    "    \n",
    "    def zeroGradParameters(self):\n",
    "        self.gradW.fill(0)\n",
    "        self.gradb.fill(0)\n",
    "        \n",
    "    def getParameters(self):\n",
    "        return [self.W, self.b]\n",
    "    \n",
    "    def getGradParameters(self):\n",
    "        return [self.gradW, self.gradb]\n",
    "    \n",
    "    def __repr__(self):\n",
    "        s = self.W.shape\n",
    "        q = 'Linear %d -> %d' %(s[1],s[0])\n",
    "        return q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This one is probably the hardest but as others only takes 5 lines of code in total. \n",
    "- input:   **`batch_size x n_feats`**\n",
    "- output: **`batch_size x n_feats`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftMax(Module):\n",
    "    def __init__(self):\n",
    "            super(SoftMax, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt):\n",
    "        exps = np.exp(inpt - np.max(inpt))\n",
    "        if inpt.ndim != 1:\n",
    "            self.output = exps/np.sum(exps, axis = 1, keepdims = True) \n",
    "        else: \n",
    "            self.output = exps/np.sum(exps, keepdims = True)\n",
    "        \n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = self.output*(gradOutput - np.sum(self.output*gradOutput,axis = 1, \n",
    "                                                          keepdims = True))\n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "            return \"SoftMax\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One of the most significant recent ideas that impacted NNs a lot is [**Batch normalization**](http://arxiv.org/abs/1502.03167). The idea is simple, yet effective: the features should be whitened ($mean = 0$, $std = 1$) all the way through NN. This improves the convergence for deep models letting it train them for days but not weeks. **You are** to implement a part of the layer: mean subtraction. That is, the module should calculate mean value for every feature (every column) and subtract it.\n",
    "\n",
    "Note, that you need to estimate the mean over the dataset to be able to predict on test examples. The right way is to create a variable which will hold smoothed mean over batches (exponential smoothing works good) and use it when forwarding test examples.\n",
    "\n",
    "When training calculate mean as folowing: \n",
    "```\n",
    "    mean_to_subtract = self.old_mean * alpha + batch_mean * (1 - alpha)\n",
    "```\n",
    "when evaluating (`self.training == False`) set $alpha = 1$.\n",
    "\n",
    "\n",
    "- input:   **`batch_size x n_feats`**\n",
    "- output: **`batch_size x n_feats`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchMeanSubtraction(Module):\n",
    "    def __init__(self, alpha = 0.95):\n",
    "        super(BatchMeanSubtraction, self).__init__()\n",
    "        \n",
    "        self.alpha = alpha\n",
    "        self.old_mean = np.array([None])\n",
    "        \n",
    "    def updateOutput(self, inpt):\n",
    "        batch_mean = np.mean(inpt, axis = 0)\n",
    "        if any(self.old_mean == None):\n",
    "            self.old_mean = batch_mean\n",
    "        if self.training:\n",
    "            self.old_mean = self.old_mean * self.alpha + batch_mean * (1 - self.alpha)\n",
    "            self.output = inpt - batch_mean\n",
    "        if self.training == False:\n",
    "            self.output = inpt - self.old_mean\n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        N = inpt.size\n",
    "        der = (-1/N + np.eye(N))\n",
    "        self.gradInput = der.dot(gradOutput)\n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"BatchMeanNormalization\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement [**dropout**](https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf). The idea and implementation is really simple: just multimply the input by $Bernoulli(p)$ mask. \n",
    "\n",
    "This is a very cool regularizer. In fact, when you see your net is overfitting try to add more dropout. It is hard to test, since every `forward` requires sampling a new mask, that is the only reason we need `fix_mask` parameter in there. \n",
    "\n",
    "While training (`self.training == True`) it should sample a mask on each iteration (for every batch). When testing this module should implement identity transform i.e. `self.output = input * p`.\n",
    "\n",
    "- input:   **`batch_size x n_feats`**\n",
    "- output: **`batch_size x n_feats`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import bernoulli\n",
    "\n",
    "class Dropout(Module):\n",
    "    def __init__(self, p=0.5):\n",
    "        super(Dropout, self).__init__()\n",
    "        \n",
    "        self.p = p\n",
    "        self.mask = None\n",
    "\n",
    "    def updateOutput(self, inpt):\n",
    "        if self.training == False:\n",
    "            self.output = inpt * (1-self.p)\n",
    "            return self.output\n",
    "        else:\n",
    "            self.mask = np.random.binomial(1,1 - self.p,size = inpt.shape)\n",
    "            self.output  = inpt * self.mask\n",
    "            return  self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.mask = np.random.binomial(1,1 - self.p,size = inpt.shape)\n",
    "        self.gradInput = gradOutput*self.mask\n",
    "        \n",
    "        return self.gradInput\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return \"Dropout\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement well-known **Sigmoid** non-linearity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid(Module):\n",
    "    def __init__(self):\n",
    "         super(Sigmoid, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = 1/(1 + np.exp(-inpt))\n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput*self.output*(1 - self.output)\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"Sigmoid\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement **hyperbolic tangent** non-linearity (aka **Tanh**): \n",
    "Note that Tanh is scaled version of the sigmoid function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tanh(Module):\n",
    "    def __init__(self):\n",
    "         super(Tanh, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = 2 / (1 + np.exp(-2*inpt)) - 1       \n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = 4*gradOutput * np.exp(-2*self.output)/(1 + np.exp(-2*self.output))**2\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"Tanh\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement **Rectified Linear Unit** non-linearity (aka **ReLU**): "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReLU(Module):\n",
    "    def __init__(self):\n",
    "         super(ReLU, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = np.maximum(0, inpt)\n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput*np.where(self.output > 0, 1, 0)\n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"ReLU\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement [**Leaky Rectified Linear Unit**](http://en.wikipedia.org/wiki%2FRectifier_%28neural_networks%29%23Leaky_ReLUs). Expriment with slope. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeakyReLU(Module):\n",
    "    def __init__(self, slope = 0.03):\n",
    "        super(LeakyReLU, self).__init__()\n",
    "            \n",
    "        self.slope = slope\n",
    "        \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = np.where(inpt > 0, inpt, inpt * self.slope)\n",
    "        \n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput *np.where(self.output > 0, 1, self.slope) \n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"LeakyReLU\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement [**Exponential Linear Units**](http://arxiv.org/abs/1511.07289) activations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ELU(Module):\n",
    "    def __init__(self, alpha = 1.0):\n",
    "        super(ELU, self).__init__()\n",
    "        \n",
    "        self.alpha = alpha\n",
    "        \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = np.where(inpt > 0, inpt, self.alpha*(np.exp(inpt) - 1))\n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput *np.where(self.output > 0, 1, np.exp(self.output)*self.alpha)\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"ELU\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implement [**SoftPlus**](https://en.wikipedia.org/wiki%2FRectifier_%28neural_networks%29) activations. Look, how they look a lot like ReLU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftPlus(Module):\n",
    "    def __init__(self):\n",
    "        super(SoftPlus, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt):\n",
    "        self.output = np.log(1 + np.exp(inpt))\n",
    "        return self.output\n",
    "    \n",
    "    def updateGradInput(self, inpt, gradOutput):\n",
    "        self.gradInput = gradOutput * np.exp(self.output)/(1 + np.exp(self.output))\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"SoftPlus\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Criterions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Criterions are used to score the models answers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Criterion(object):\n",
    "    def __init__ (self):\n",
    "        self.output = None\n",
    "        self.gradInput = None\n",
    "    \n",
    "    def forward(self, inpt, target):\n",
    "        \"\"\"\n",
    "            Given an input and a target, compute the loss function \n",
    "            associated to the criterion and return the result.\n",
    "            \n",
    "            For consistency this function should not be overrided,\n",
    "            all the code goes in `updateOutput`.\n",
    "        \"\"\"\n",
    "        return self.updateOutput(inpt, target)\n",
    "\n",
    "    def backward(self, inpt, target):\n",
    "        \"\"\"\n",
    "            Given an input and a target, compute the gradients of the loss function\n",
    "            associated to the criterion and return the result. \n",
    "\n",
    "            For consistency this function should not be overrided,\n",
    "            all the code goes in `updateGradInput`.\n",
    "        \"\"\"\n",
    "        return self.updateGradInput(inpt, target)\n",
    "    \n",
    "    def updateOutput(self, inpt, target):\n",
    "        \"\"\"\n",
    "        Function to override.\n",
    "        \"\"\"\n",
    "        return self.output\n",
    "\n",
    "    def updateGradInput(self, inpt, target):\n",
    "        \"\"\"\n",
    "        Function to override.\n",
    "        \"\"\"\n",
    "        return self.gradInput   \n",
    "\n",
    "    def __repr__(self):\n",
    "        \"\"\"\n",
    "        Pretty printing. Should be overrided in every module if you want \n",
    "        to have readable description. \n",
    "        \"\"\"\n",
    "        return \"Criterion\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The **MSECriterion**, which is basic L2 norm usually used for regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MSECriterion(Criterion):\n",
    "    def __init__(self):\n",
    "        super(MSECriterion, self).__init__()\n",
    "\n",
    "    def updateOutput(self, inpt, target):   \n",
    "        self.output = np.sum((inpt - target)**2)/inpt.size\n",
    "        \n",
    "        return self.output \n",
    " \n",
    "    def updateGradInput(self, inpt, target):\n",
    "        self.gradInput = 2*(inpt - target)/inpt.size\n",
    "        \n",
    "        return self.gradInput\n",
    "\n",
    "    def __repr__(self):\n",
    "        return \"MSECriterion\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You task is to implement the **CrossEntropyCriterion**. It should implement [multiclass log loss](http://ml-cheatsheet.readthedocs.io/en/latest/loss_functions.html#cross-entropy). Nevertheless there is a sum over `y` (target) in that formula, \n",
    "remember that targets are one-hot encoded. This fact simplifies the computations a lot. Note, that criterions are the only places, where you divide by batch size. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CrossEntropyCriterion(Criterion):\n",
    "    def __init__(self):\n",
    "        super(CrossEntropyCriterion, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt, target): \n",
    "        \n",
    "        # Use this trick to avoid numerical errors\n",
    "        input_clamp = np.maximum(1e-15, np.minimum(inpt, 1 - 1e-15))\n",
    "        self.output = - np.sum(target*np.log(input_clamp))/input_clamp.size\n",
    "                                \n",
    "        return self.output \n",
    "        \n",
    "        \n",
    "\n",
    "    def updateGradInput(self, inpt, target):\n",
    "        \n",
    "        # Use this trick to avoid numerical errors\n",
    "        input_clamp = np.maximum(1e-15, np.minimum(inpt, 1 - 1e-15))\n",
    "        self.gradInput = -target /(input_clamp*input_clamp.size)\n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"CrossEntropyCriterion\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**MultiLabelCriterion** for atribute classification, i.e. target is multiple-hot encoded, could be multiple ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiLabelCriterion(Criterion):\n",
    "    def __init__(self):\n",
    "        super(MultiLabelCriterion, self).__init__()\n",
    "    \n",
    "    def updateOutput(self, inpt, target): \n",
    "        \n",
    "        # Use this trick to avoid numerical errors\n",
    "        input_clamp = np.maximum(1e-15, np.minimum(inpt, 1 - 1e-15))\n",
    "        self.output = -np.sum(np.multiply(target, np.log(input_clamp)) +\n",
    "                       np.multiply(1-target, np.log(input_clamp)))/input_clamp.size\n",
    "        \n",
    "        return self.output\n",
    "\n",
    "    def updateGradInput(self, inpt, target):\n",
    "        \n",
    "        # Use this trick to avoid numerical errors\n",
    "        input_clamp = np.maximum(1e-15, np.minimum(inpt, 1 - 1e-15))\n",
    "        self.gradInput = -target /(input_clamp*input_clamp.size) + (1 - target)/(input_clamp*input_clamp.size)    \n",
    "        \n",
    "        \n",
    "        return self.gradInput\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"MultiLabelCriterion\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}