RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Qt3KzJzBPdHU"
   },
   "outputs": [],
   "source": [
    "\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "from collections import defaultdict\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import Dataset\n",
    "import numpy as np\n",
    "from scipy.io.wavfile import read\n",
    "import librosa\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "cuda = True if torch.cuda.is_available() else False\n",
    "\n",
    "Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed_value):\n",
    "    \"\"\"Set seed for reproducibility.\n",
    "    \"\"\"\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    torch.cuda.manual_seed_all(seed_value)\n",
    "\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QKSnqpAJLVwx"
   },
   "outputs": [],
   "source": [
    "class SpeechCommandsDataset(Dataset):\n",
    "    \"\"\"Google Speech Commands dataset.\"\"\"\n",
    "\n",
    "    def __init__(self, root_dir, split):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            root_dir (string): Directory with all the data files.\n",
    "            split    (string): In [\"train\", \"valid\", \"test\"].\n",
    "        \"\"\"\n",
    "        self.root_dir = root_dir\n",
    "        self.split = split\n",
    "\n",
    "        self.number_of_classes = len(self.get_classes())\n",
    "\n",
    "        self.class_to_file = defaultdict(list)\n",
    "\n",
    "        self.valid_filenames = self.get_valid_filenames()\n",
    "        self.test_filenames = self.get_test_filenames()\n",
    "\n",
    "        for c in self.get_classes():\n",
    "            file_name_list = sorted(os.listdir(self.root_dir + \"data_speech_commands_v0.02/\" + c))\n",
    "            for filename in file_name_list:\n",
    "                if split == \"train\":\n",
    "                    if (filename not in self.valid_filenames[c]) and (filename not in self.test_filenames[c]):\n",
    "                        self.class_to_file[c].append(filename)\n",
    "                elif split == \"valid\":\n",
    "                    if filename in self.valid_filenames[c]:\n",
    "                        self.class_to_file[c].append(filename)\n",
    "                elif split == \"test\":\n",
    "                    if filename in self.test_filenames[c]:\n",
    "                        self.class_to_file[c].append(filename)\n",
    "                else:\n",
    "                    raise ValueError(\"Invalid split name.\")\n",
    "\n",
    "        self.filepath_list = list()\n",
    "        self.label_list = list()\n",
    "        for cc, c in enumerate(self.get_classes()):\n",
    "            f_extension = sorted(list(self.class_to_file[c]))\n",
    "            l_extension = [cc for i in f_extension]\n",
    "            f_extension = [self.root_dir + \"data_speech_commands_v0.02/\" + c + \"/\" + filename for filename in f_extension]\n",
    "            self.filepath_list.extend(f_extension)\n",
    "            self.label_list.extend(l_extension)\n",
    "        self.number_of_samples = len(self.filepath_list)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.number_of_samples\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        sample = np.zeros((16000, ), dtype=np.float32)\n",
    "\n",
    "        sample_file = self.filepath_list[idx]\n",
    "\n",
    "        sample_from_file = read(sample_file)[1]\n",
    "        sample[:sample_from_file.size] = sample_from_file\n",
    "        sample = sample.reshape((16000, ))\n",
    "        \n",
    "        sample = librosa.feature.mfcc(y=sample, sr=16000, hop_length=512, n_fft=2048).transpose().astype(np.float32)\n",
    "\n",
    "        label = self.label_list[idx]\n",
    "\n",
    "        return sample, label\n",
    "\n",
    "    def get_classes(self):\n",
    "        return ['one', 'two', 'three']\n",
    "\n",
    "    def get_valid_filenames(self):\n",
    "        class_names = self.get_classes()\n",
    "\n",
    "        class_to_filename = defaultdict(set)\n",
    "        with open(self.root_dir + \"data_speech_commands_v0.02/validation_list.txt\", \"r\") as fp:\n",
    "            for line in fp:\n",
    "                clean_line = line.strip().split(\"/\")\n",
    "\n",
    "                if clean_line[0] in class_names:\n",
    "                    class_to_filename[clean_line[0]].add(clean_line[1])\n",
    "\n",
    "        return class_to_filename\n",
    "\n",
    "    def get_test_filenames(self):\n",
    "        class_names = self.get_classes()\n",
    "\n",
    "        class_to_filename = defaultdict(set)\n",
    "        with open(self.root_dir + \"data_speech_commands_v0.02/testing_list.txt\", \"r\") as fp:\n",
    "            for line in fp:\n",
    "                clean_line = line.strip().split(\"/\")\n",
    "\n",
    "                if clean_line[0] in class_names:\n",
    "                    class_to_filename[clean_line[0]].add(clean_line[1])\n",
    "\n",
    "        return class_to_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vx8ptirGKa9u"
   },
   "outputs": [],
   "source": [
    "\n",
    "train_dataset = SpeechCommandsDataset(dataset_folder,\n",
    "                                      \"train\")\n",
    "valid_dataset = SpeechCommandsDataset(dataset_folder,\n",
    "                                      \"valid\")\n",
    "\n",
    "test_dataset = SpeechCommandsDataset(dataset_folder,\n",
    "                                     \"test\")\n",
    "\n",
    "batch_size = 100\n",
    "\n",
    "\n",
    "num_epochs = 5\n",
    "valid_every_n_steps = 20\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=True)\n",
    "valid_loader = torch.utils.data.DataLoader(dataset=valid_dataset,\n",
    "                                           batch_size=batch_size,\n",
    "                                           shuffle=False)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eJwOesOQOSh9"
   },
   "source": [
    "## LSTM and GRU Cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LQu9Yxfy-Wqj"
   },
   "outputs": [],
   "source": [
    "class LSTMCell(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, bias=True):\n",
    "        super(LSTMCell, self).__init__()\n",
    "        self.input_size = input_size \n",
    "        self.hidden_size = hidden_size \n",
    "        self.bias = bias\n",
    "\n",
    "        self.x2h = nn.Linear(self.input_size, self.hidden_size*4, bias=self.bias)\n",
    "        self.h2h = nn.Linear(self.hidden_size, self.hidden_size*4, bias=self.bias)\n",
    "\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        std = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for w in self.parameters():\n",
    "            w.data.uniform_(-std, std)\n",
    "\n",
    "    def forward(self, input, hx=None):\n",
    "        if hx is None:\n",
    "            hx = input.new_zeros(input.size(0), self.hidden_size, requires_grad=False)\n",
    "            hx = (hx, hx)\n",
    "            \n",
    "        # We used hx to pack both the hidden and cell states\n",
    "        hx, cx = hx\n",
    "\n",
    "        # Given h_t-1, c_t-1 (hx, cx):\n",
    "        # x2h layer packs all weight matrices and biases\n",
    "\n",
    "        all = self.x2h(input) + self.h2h(hx)\n",
    "        fy, iy, oy, c_tilda_y = torch.chunk(all, 4, dim=1)\n",
    "\n",
    "        fy = nn.Sigmoid()(fy)\n",
    "        iy = nn.Sigmoid()(iy)\n",
    "        oy = nn.Sigmoid()(oy)\n",
    "        c_tilda_y = nn.Tanh()(c_tilda_y)\n",
    "        \n",
    "        cy = (fy * cx) + (iy * c_tilda_y)\n",
    "        hy = oy * nn.Tanh()(cy)\n",
    "\n",
    "        return (hy, cy)\n",
    "\n",
    "class BasicRNNCell(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, bias=True, nonlinearity=\"tanh\"):\n",
    "        super(BasicRNNCell, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.bias = bias\n",
    "        self.nonlinearity = nonlinearity\n",
    "        if self.nonlinearity not in [\"tanh\", \"relu\"]:\n",
    "            raise ValueError(\"Invalid nonlinearity selected for RNN.\")\n",
    "\n",
    "        self.x2h = nn.Linear(input_size, hidden_size, bias=bias)\n",
    "        self.h2h = nn.Linear(hidden_size, hidden_size, bias=bias)\n",
    "\n",
    "        self.reset_parameters()\n",
    "        \n",
    "\n",
    "    def reset_parameters(self):\n",
    "        std = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for w in self.parameters():\n",
    "            w.data.uniform_(-std, std)\n",
    "\n",
    "            \n",
    "    def forward(self, input, hx=None):\n",
    "        if hx is None:\n",
    "            hx = input.new_zeros(input.size(0), self.hidden_size, requires_grad=False)\n",
    "\n",
    "        activation = getattr(nn.functional, self.nonlinearity)\n",
    "        hy = activation(self.x2h(input) + self.h2h(hx))\n",
    "\n",
    "        return hy\n",
    "\n",
    "    \n",
    "    \n",
    "class GRUCell(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, bias=True):\n",
    "        super(GRUCell, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size \n",
    "        self.bias = bias\n",
    "\n",
    "        self.x2h = nn.Linear(self.input_size, self.hidden_size*2, bias=self.bias)\n",
    "        self.h2h = nn.Linear(self.hidden_size, self.hidden_size*2, bias=self.bias)\n",
    "\n",
    "        self.x2r = nn.Linear(self.input_size, hidden_size, bias=bias)       \n",
    "        self.h2r = nn.Linear(self.hidden_size, hidden_size, bias=self.bias)\n",
    "\n",
    "        self.reset_parameters()\n",
    "        \n",
    "\n",
    "    def reset_parameters(self):\n",
    "        std = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for w in self.parameters():\n",
    "            w.data.uniform_(-std, std)\n",
    "\n",
    "    def forward(self, input, hx=None):\n",
    "        if hx is None:\n",
    "            hx = input.new_zeros(input.size(0), self.hidden_size, requires_grad=False)\n",
    "        \n",
    "        all = self.x2h(input) + self.h2h(hx)\n",
    "        zy, ry = torch.chunk(all, 2, dim=1)\n",
    "        zy = nn.Sigmoid()(zy)\n",
    "        ry = nn.Sigmoid()(ry)\n",
    "        \n",
    "        input_h2r = hx * ry\n",
    "        h_tilda_y = nn.Tanh()(self.x2r(input) + self.h2r(input_h2r))\n",
    "        hy = (1 - zy) * hx + zy * h_tilda_y\n",
    "        return hy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN and Bidirectional RNN Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LQu9Yxfy-Wqj"
   },
   "outputs": [],
   "source": [
    "class RNNModel(nn.Module):\n",
    "    def __init__(self, mode, input_size, hidden_size, num_layers, bias, output_size):\n",
    "        super(RNNModel, self).__init__()\n",
    "        self.mode = mode\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.bias = bias\n",
    "        self.output_size = output_size\n",
    "        \n",
    "        self.rnn_cell_list = nn.ModuleList()\n",
    "        \n",
    "        if mode == 'LSTM':\n",
    "\n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(LSTMCell(self.input_size, self.hidden_size, bias=self.bias))      # bottom layer from input and across\n",
    "      \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(LSTMCell(self.hidden_size, self.hidden_size, bias=self.bias))  # forward layer from prev layer and across\n",
    "\n",
    "        elif mode == 'GRU':\n",
    "\n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(GRUCell(self.input_size, self.hidden_size, bias=self.bias))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(GRUCell(self.hidden_size, self.hidden_size, bias=self.bias))  # forward layer from prev layer and across    \n",
    "        \n",
    "        elif mode == 'RNN_TANH':\n",
    "            \n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))  # forward layer from prev layer and across\n",
    "                \n",
    "        elif mode == 'RNN_RELU':\n",
    "\n",
    "           # Forward\n",
    "            self.rnn_cell_list.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))  # forward layer from prev layer and across\n",
    "\n",
    "        else:\n",
    "            raise ValueError(\"Invalid RNN mode selected.\")\n",
    "\n",
    "\n",
    "        self.att_fc = nn.Linear(self.hidden_size, 1)\n",
    "        self.fc = nn.Linear(self.hidden_size, self.output_size)\n",
    " \n",
    "\n",
    "        \n",
    "    def forward(self, input, hx=None):\n",
    "\n",
    "        outs = []\n",
    "        h0 = [None] * self.num_layers if hx is None else list(hx)\n",
    "        \n",
    "        # In this forward pass we want to create our RNN from the rnn cells,\n",
    "        # ..taking the hidden states from the final RNN layer and passing these \n",
    "        # ..through our fully connected layer (fc).\n",
    "        \n",
    "        # The multi-layered RNN should be able to run when the mode is either \n",
    "        # .. LSTM, GRU, RNN_TANH or RNN_RELU.\n",
    "        \n",
    "        if self.mode == 'LSTM':\n",
    "            # Iterate over each time step\n",
    "            for j in range(input.shape[1]):\n",
    "                # Iterate over each layer step\n",
    "                for i, cell in enumerate(self.rnn_cell_list):\n",
    "                    # First on h0\n",
    "                    if i == 0:\n",
    "                        hx_new = cell.forward(input[:,j,:], hx=h0[i])\n",
    "                    # Go through other cells\n",
    "                    else:\n",
    "                        hx_new = cell.forward(hx_new[0], hx=h0[i])\n",
    "\n",
    "                    h0[i] = hx_new\n",
    "                    outs.append(hx_new[0])\n",
    "                \n",
    "        else:\n",
    "            # Iterate over each time step\n",
    "            for j in range(input.shape[1]):\n",
    "                # Iterate over each layer step\n",
    "                for i, cell in enumerate(self.rnn_cell_list):\n",
    "                    # First on h0\n",
    "                    if i == 0:\n",
    "                        hx_new = cell.forward(input[:,j,:], hx=h0[i])\n",
    "                    # Go through other cells\n",
    "                    else:\n",
    "                        hx_new = cell.forward(hx_new, hx=h0[i])\n",
    "                    \n",
    "                    h0[i] = hx_new\n",
    "                    outs.append(hx_new)\n",
    "\n",
    "        out = outs[-1].squeeze()\n",
    "\n",
    "        out = self.fc(out)\n",
    "        \n",
    "        \n",
    "        return out\n",
    "\n",
    "\n",
    "\n",
    "class BidirRecurrentModel(nn.Module):\n",
    "    def __init__(self, mode, input_size, hidden_size, num_layers, bias, output_size):\n",
    "        super(BidirRecurrentModel, self).__init__()\n",
    "        self.mode = mode\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.bias = bias\n",
    "        self.output_size = output_size\n",
    "        \n",
    "        self.rnn_cell_list = nn.ModuleList()\n",
    "        self.rnn_cell_list_rev = nn.ModuleList()\n",
    "        \n",
    "        if mode == 'LSTM':\n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(LSTMCell(self.input_size, self.hidden_size, bias=self.bias))      # bottom layer from input and across\n",
    "      \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(LSTMCell(self.hidden_size, self.hidden_size, bias=self.bias))  # forward layer from prev layer and across\n",
    "\n",
    "            # Backward\n",
    "            self.rnn_cell_list_rev.append(LSTMCell(self.input_size, self.hidden_size, bias=self.bias))      # from input to first backward and across\n",
    "\n",
    "            for i in range(self.num_layers - 1):\n",
    "                self.rnn_cell_list_rev.append(LSTMCell(self.hidden_size, self.hidden_size, bias=self.bias))  # backward layer from prev layer and across\n",
    "            \n",
    "        elif mode == 'GRU':\n",
    "\n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(GRUCell(self.input_size, self.hidden_size, bias=self.bias))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(GRUCell(self.hidden_size, self.hidden_size, bias=self.bias))  # forward layer from prev layer and across\n",
    "            \n",
    "            # Backward\n",
    "            self.rnn_cell_list_rev.append(GRUCell(self.input_size, self.hidden_size, bias=self.bias))      # from input to first backward and across\n",
    "\n",
    "            for i in range(self.num_layers - 1):\n",
    "                self.rnn_cell_list_rev.append(GRUCell(self.hidden_size, self.hidden_size, bias=self.bias))  # backward layer from prev layer and across \n",
    "\n",
    "        elif mode == 'RNN_TANH':\n",
    "            # Forward\n",
    "            self.rnn_cell_list.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))  # forward layer from prev layer and across\n",
    "\n",
    "\n",
    "            # Backward\n",
    "            self.rnn_cell_list_rev.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))      # from input to first backward and across\n",
    "\n",
    "            for i in range(self.num_layers - 1):\n",
    "                self.rnn_cell_list_rev.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"tanh\"))  # backward layer from prev layer and across\n",
    "  \n",
    "        elif mode == 'RNN_RELU':\n",
    "\n",
    "           # Forward\n",
    "            self.rnn_cell_list.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))      # bottom layer from input and across\n",
    "        \n",
    "            for i in range(self.num_layers - 1): \n",
    "                self.rnn_cell_list.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))  # forward layer from prev layer and across\n",
    "\n",
    "\n",
    "            # Backward\n",
    "            self.rnn_cell_list_rev.append(BasicRNNCell(self.input_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))      # last forward layer layer to first backward and across\n",
    "\n",
    "            for i in range(self.num_layers - 1):\n",
    "                self.rnn_cell_list_rev.append(BasicRNNCell(self.hidden_size, self.hidden_size, bias=self.bias, nonlinearity=\"relu\"))  # backward layer from prev layer and across\n",
    "\n",
    "\n",
    "        else:\n",
    "            raise ValueError(\"Invalid RNN mode selected.\")\n",
    "        \n",
    "        self.fc = nn.Linear(self.hidden_size*2, self.output_size)  # multiply by 2 since using concat of forward and back outputs \n",
    " \n",
    "        \n",
    "        \n",
    "    def forward(self, input, hx=None):\n",
    "        \n",
    "        # In this forward pass we want to create our Bidirectional RNN from the rnn cells,\n",
    "        # .. taking the hidden states from the final RNN layer with their reversed counterparts\n",
    "        # .. before concatening these and running them through the fully connected layer (fc)\n",
    "        \n",
    "        # The multi-layered RNN should be able to run when the mode is either \n",
    "        # .. LSTM, GRU, RNN_TANH or RNN_RELU.\n",
    "        \n",
    "        outs = []\n",
    "        outs_rev = []\n",
    "\n",
    "        # Keep separate lists because do not use output of forward layers as input to backward layers\n",
    "        h0 = [None] * self.num_layers if hx is None else list(hx)\n",
    "        h0_rev = [None] * self.num_layers if hx is None else list(hx)#.reverse()\n",
    "\n",
    "\n",
    "        if self.mode == 'LSTM':\n",
    "\n",
    "            # Iterate through forward direction layers\n",
    "            # Iterate over each  time step\n",
    "            for j in range(input.shape[1]):\n",
    "                # Iterate over each layer step\n",
    "                for i, cell in enumerate(self.rnn_cell_list):\n",
    "                    # Forward direction layers\n",
    "                    # First on input\n",
    "                    if i == 0:\n",
    "                        hx_new = cell.forward(input[:,j,:], hx=h0[i])\n",
    "                    # Go through other forward layers\n",
    "                    else:\n",
    "                        # Input is output h of previous cell, and h is h of previous layer\n",
    "                        hx_new = cell.forward(hx_new[0], hx=h0[i])\n",
    "  \n",
    "                    # Backward direction layers\n",
    "                    # First on input\n",
    "                    if i == 0:\n",
    "                        hx_rev = cell.forward(input[:,-(j+1),:], hx=h0_rev[i])     # Reverse order of input   \n",
    "                    # Go through other forward layers\n",
    "                    else:\n",
    "                        hx_rev = cell.forward(hx_rev[0], hx=h0_rev[i])\n",
    "\n",
    "                    # Update h0 to store previous layers outputs\n",
    "                    h0[i] = hx_new\n",
    "                    outs.append(hx_new[0])\n",
    "\n",
    "                    # Update h0_rev to store previous layers outputs\n",
    "                    h0_rev[i] = hx_rev\n",
    "                    outs_rev.append(hx_rev[0])\n",
    "\n",
    "        else:\n",
    "            # Iterate over each time step\n",
    "            for j in range(input.shape[1]):\n",
    "                # Iterate over each layer step\n",
    "                for i, cell in enumerate(self.rnn_cell_list):\n",
    "                    # Forward directio layers\n",
    "                    # First on h0\n",
    "                    if i == 0:\n",
    "                        hx_new = cell.forward(input[:,j,:], hx=h0[i])\n",
    "                    # Go through other forward layers\n",
    "                    else:\n",
    "                        hx_new = cell.forward(hx_new, hx=h0[i])\n",
    "\n",
    "                    # Backward direction layers\n",
    "                    # First on h0\n",
    "                    if i == 0:\n",
    "                        hx_rev = cell.forward(input[:,-(j+1),:], hx=h0_rev[i])   # reverse order of input\n",
    "                    # Go through other forward layers\n",
    "                    else:\n",
    "                        hx_rev = cell.forward(hx_rev, hx=h0_rev[i])\n",
    "\n",
    "\n",
    "                    h0[i] = hx_new\n",
    "                    outs.append(hx_new)\n",
    "                    h0_rev[i] = hx_rev\n",
    "                    outs_rev.append(hx_rev)\n",
    "\n",
    "        out = outs[-1].squeeze()\n",
    "        out_rev = outs_rev[0].squeeze()\n",
    "        out = torch.cat((out, out_rev), 1)\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eJwOesOQOSh9"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LQu9Yxfy-Wqj"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 20. Loss: 1.0907610654830933. V-Accuracy: 33  T-Accuracy: 35\n",
      "Iteration: 40. Loss: 0.9884630441665649. V-Accuracy: 61  T-Accuracy: 60\n",
      "Iteration: 60. Loss: 0.8351296782493591. V-Accuracy: 68  T-Accuracy: 68\n",
      "Iteration: 80. Loss: 0.7208311557769775. V-Accuracy: 67  T-Accuracy: 68\n",
      "Iteration: 100. Loss: 0.6368197798728943. V-Accuracy: 70  T-Accuracy: 68\n",
      "Iteration: 120. Loss: 0.6104040741920471. V-Accuracy: 70  T-Accuracy: 68\n",
      "Iteration: 140. Loss: 0.6348065733909607. V-Accuracy: 69  T-Accuracy: 68\n",
      "Iteration: 160. Loss: 0.6063405275344849. V-Accuracy: 73  T-Accuracy: 71\n",
      "Iteration: 180. Loss: 0.5756142735481262. V-Accuracy: 71  T-Accuracy: 71\n",
      "Iteration: 200. Loss: 0.7492965459823608. V-Accuracy: 69  T-Accuracy: 71\n",
      "Iteration: 220. Loss: 0.7609963417053223. V-Accuracy: 73  T-Accuracy: 71\n",
      "Iteration: 240. Loss: 0.6040704846382141. V-Accuracy: 76  T-Accuracy: 73\n",
      "Iteration: 260. Loss: 0.5462620854377747. V-Accuracy: 75  T-Accuracy: 73\n",
      "Iteration: 280. Loss: 0.5252678394317627. V-Accuracy: 74  T-Accuracy: 73\n",
      "Iteration: 300. Loss: 0.5464482307434082. V-Accuracy: 77  T-Accuracy: 74\n",
      "Iteration: 320. Loss: 0.6104760766029358. V-Accuracy: 75  T-Accuracy: 74\n",
      "Iteration: 340. Loss: 0.5058141350746155. V-Accuracy: 76  T-Accuracy: 74\n",
      "Iteration: 360. Loss: 0.5488533973693848. V-Accuracy: 78  T-Accuracy: 77\n",
      "Iteration: 380. Loss: 0.5991271734237671. V-Accuracy: 75  T-Accuracy: 77\n",
      "Iteration: 400. Loss: 0.4403088688850403. V-Accuracy: 76  T-Accuracy: 77\n",
      "Iteration: 420. Loss: 0.5070282816886902. V-Accuracy: 77  T-Accuracy: 77\n",
      "Iteration: 440. Loss: 0.4440942406654358. V-Accuracy: 77  T-Accuracy: 77\n",
      "Iteration: 460. Loss: 0.5029195547103882. V-Accuracy: 78  T-Accuracy: 77\n"
     ]
    }
   ],
   "source": [
    "\n",
    "seq_dim, input_dim = train_dataset[0][0].shape\n",
    "output_dim = 3\n",
    "\n",
    "hidden_dim = 32\n",
    "layer_dim = 1\n",
    "bias = True\n",
    "\n",
    "### Change the code below to try running different models:\n",
    "#model = RNNModel(\"LSTM\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = RNNModel(\"GRU\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = RNNModel(\"RNN_RELU\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = RNNModel(\"RNN_TANH\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "\n",
    "model = BidirRecurrentModel(\"LSTM\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = BidirRecurrentModel(\"GRU\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = BidirRecurrentModel(\"RNN_RELU\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "#model = BidirRecurrentModel(\"RNN_TANH\", input_dim, hidden_dim, layer_dim, bias, output_dim)\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model.cuda()\n",
    "    \n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "learning_rate = 0.01\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "loss_list = []\n",
    "iter = 0\n",
    "max_v_accuracy = 0\n",
    "reported_t_accuracy = 0\n",
    "max_t_accuracy = 0\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (audio, labels) in enumerate(train_loader):\n",
    "        if torch.cuda.is_available():\n",
    "            audio = Variable(audio.view(-1, seq_dim, input_dim).cuda())\n",
    "            labels = Variable(labels.cuda())\n",
    "        else:\n",
    "            audio = Variable(audio.view(-1, seq_dim, input_dim))\n",
    "            labels = Variable(labels)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(audio)\n",
    "\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        if torch.cuda.is_available():\n",
    "            loss.cuda()\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        loss_list.append(loss.item())\n",
    "        iter += 1\n",
    "\n",
    "        if iter % valid_every_n_steps == 0:\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            for audio, labels in valid_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    audio = Variable(audio.view(-1, seq_dim, input_dim).cuda())\n",
    "                else:\n",
    "                    audio = Variable(audio.view(-1, seq_dim, input_dim))\n",
    "\n",
    "                outputs = model(audio)\n",
    "\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "\n",
    "                total += labels.size(0)\n",
    "\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                else:\n",
    "                    correct += (predicted == labels).sum()\n",
    "\n",
    "            v_accuracy = 100 * correct // total\n",
    "            \n",
    "            is_best = False\n",
    "            if v_accuracy >= max_v_accuracy:\n",
    "                max_v_accuracy = v_accuracy\n",
    "                is_best = True\n",
    "\n",
    "            if is_best:\n",
    "                for audio, labels in test_loader:\n",
    "                    if torch.cuda.is_available():\n",
    "                        audio = Variable(audio.view(-1, seq_dim, input_dim).cuda())\n",
    "                    else:\n",
    "                        audio = Variable(audio.view(-1, seq_dim, input_dim))\n",
    "\n",
    "                    outputs = model(audio)\n",
    "\n",
    "                    _, predicted = torch.max(outputs.data, 1)\n",
    "\n",
    "                    total += labels.size(0)\n",
    "\n",
    "                    if torch.cuda.is_available():\n",
    "                        correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    else:\n",
    "                        correct += (predicted == labels).sum()\n",
    "\n",
    "                t_accuracy = 100 * correct // total\n",
    "                reported_t_accuracy = t_accuracy\n",
    "\n",
    "            print('Iteration: {}. Loss: {}. V-Accuracy: {}  T-Accuracy: {}'.format(iter, loss.item(), v_accuracy, reported_t_accuracy))\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "RNN tutorial and coursework.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}