RNN-task.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating names with recurrent neural networks\n",
    "\n",
    "This time you'll find yourself delving into the heart (and other intestines) of recurrent neural networks on a class of toy problems.\n",
    "\n",
    "Struggle to find a name for the variable? Let's see how you'll come up with a name for your son/daughter. Surely no human has expertize over what is a good child name, so let us train RNN instead;\n",
    "\n",
    "It's dangerous to go alone, take these:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.696201Z",
     "start_time": "2018-08-13T20:26:38.104103Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.2.1\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "import keras_utils\n",
    "import tqdm_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data\n",
    "The dataset contains ~8k earthling names from different cultures, all in latin transcript.\n",
    "\n",
    "This notebook has been designed so as to allow you to quickly swap names for something similar: deep learning article titles, IKEA furniture, pokemon names, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.701832Z",
     "start_time": "2018-08-13T20:26:42.697766Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_token = \" \"  # so that the network knows that we're generating a first token\n",
    "\n",
    "# this is the token for padding,\n",
    "# we will add fake pad token at the end of names \n",
    "# to make them of equal size for further batching\n",
    "pad_token =\"#\"\n",
    "\n",
    "with open(\"names\") as f:\n",
    "    names = f.read()[:-1].split('\\n')\n",
    "    names = [start_token + name for name in names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.707885Z",
     "start_time": "2018-08-13T20:26:42.703302Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of samples: 7944\n",
      " Abagael\n",
      " Claresta\n",
      " Glory\n",
      " Liliane\n",
      " Prissie\n",
      " Geeta\n",
      " Giovanne\n",
      " Piggy\n"
     ]
    }
   ],
   "source": [
    "print('number of samples:', len(names))\n",
    "for x in names[::1000]:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.857411Z",
     "start_time": "2018-08-13T20:26:42.709371Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max length: 16\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEICAYAAAC55kg0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGntJREFUeJzt3X+UXWV97/H3h/CjgPwIZgyQBCZiQIGlAaeAVRAvBcKP\nS9B7i6FeCIoGWrB6ZV0v0NtCRbpSK6WyxNAAaaBCMOVHSQWESFVKa5AJxpBAkAECmTBJBsMPC65o\n4Hv/2M/oZjhn5vyaOQnP57XWWbPP93n2s7/7THK+Zz97n9mKCMzMLE/btDsBMzNrHxcBM7OMuQiY\nmWXMRcDMLGMuAmZmGXMRMDPLmIuAva1JCknvacN2j5bU28T6l0r6dlreR9J/SRrTotyukfQXrciz\nwthHSnqiVePZyHMRyICkj0j6T0kvS9oo6T8k/X6783o7GcliExHPRcQ7IuL1YXI4S9KDNYx3bkRc\n1orcBu93RPx7RBzQirFtdGzb7gRsZEnaFfgu8CfAQmB74EhgUzvzsvaQNGa4YmJ58ZHA29/+ABGx\nICJej4hfRcR9EbF8oIOkz0h6XNKLku6VtG+p7VhJq9JRxDcl/UjSZ1Pbb6cs0vPO9Mlw2/R8N0nX\nS+qTtFbSVwemNAY+tUr6etruM5JOKI21h6R/lPR8av+XUtvJkpZJeikd4by/lhdC0g5pe89JWp+m\nRXZMbUdL6pV0gaQNKedPl9Z9p6R/lfSKpIfTvjyY2h5I3X6Wpm0+WVqv4ngVcpucXttfSloMjBvi\ndT1L0tOp7zOSPiXpfcA1wIdSDi+lvvMlzZF0t6RXgY+l2FcHbf9iSS9IWi3pU6X4Dwd+3+XfW7X9\nHjy9JOl9aYyXJK2UdEqpbb6kqyXdlfblIUn7Dfd7tNZyEXj7+znwuqQbJJ0gaWy5UdJ04GLgE0AH\n8O/AgtQ2Drgd+H8Ub0pPAR+uY9vzgc3Ae4BDgOOAz5baDweeSGN/DbheklLbPwE7AQcB7wKuTDkd\nAswDzgHeCfwDsEjSDjXkM5uiKE5NOU0A/rLUviewW4qfDVxder2uBl5NfWamBwARcVRa/ECatvlO\nDeMNdjOwNL0Wl5XHL5O0M3AVcEJE7AL8AbAsIh4HzgV+nHLYvbTaHwOXA7sAlaaL9kzbnZC2O1fS\nsFM6Q+z3QK7bAf8K3EfxO/w8cNOgsWcAfwWMBXpSnjaaIsKPt/kDeB/FG3IvxZvyImB8arsHOLvU\ndxvgNWBf4ExgSalNaYzPpueXAt8utXcCQTHNOJ5iymnHUvvpwA/S8llAT6ltp7TunsBewBvA2Ar7\nMge4bFDsCeCjVfY9KN7wRfEmvl+p7UPAM2n5aOBXwLal9g3AEcAY4DfAAaW2rwIPDt5O6XnV8Srk\nuE/6vexcit088NoOel13Bl4C/kf5tS29pg8Ois0HbqwQ+2opz8HbXgj8RVr+4cDvu9I2qux3b1o+\nElgHbFNqXwBcWsrjulLbicCqdv9/ye3hI4EMRMTjEXFWREwEDgb2Bv4+Ne8LfCMdrr8EbKR4w5yQ\n+q0pjRPl58PYF9gO6CuN/Q8UnwgHrCuN/VpafAcwCdgYES9WGfeCgTHTuJNSrkPpoCg0S0vrfS/F\nB/wiIjaXnr+W8umgeAMu73str0O18QbbG3gxIl4txZ6tNGDq80mKT/19aSrlvcPkMVyulbY93OtZ\ni72BNRHxxqCxJ5SerystV3t9bAS5CGQmIlZRfAI7OIXWAOdExO6lx44R8Z9AH8UbLABpqmZSabhX\nKd5YB+xZWl5DcSQwrjTurhFxUA1prgH2kLR7lbbLB+W7U0QsGGbMFyg+mR9UWm+3iKjlTaef4tPy\nxFJsUpW+jegDxqapngH7VOscEfdGxLEUR0yrgGsHmqqtMsz2K237+bQ81O94OM8DkySV32f2AdbW\nMYaNMBeBtzlJ700nJyem55MopmWWpC7XABdJOii17ybpj1LbXcBBkj6RTkr+GW9+E1gGHKXiOvbd\ngIsGGiKij2Iu+ApJu0raRtJ+kj46XM5p3XuAb0kaK2k7SQPzz9cC50o6XIWdJZ0kaZdhxnwjrXul\npHelfZ0g6fga8nmd4tzIpZJ2Sp+8zxzUbT3w7uHGqjL+s0A38FeStpf0EeC/V+orabyk6elNexPw\nXxRTZwM5TJS0fQNpDGz7SOBk4J9TfBnwibTf76E4t1E21H4/RPHp/svpd3h02q9bGsjPRoiLwNvf\nLylOwD6Urg5ZAqwALgCIiDuAvwFukfRKajshtb0A/BHFCdVfAFOA/xgYOCIWA98BllOc1PzuoG2f\nSXFJ6mPAi8CtFJ9ea3EGxTz8Koq59C+mbXYDnwO+mcbsoZinrsX/Tf2XpH39PlDrNe3nU5zkXUdx\n0noBb77M9lLghjTVdFqNY5b9McXvaSNwCXBjlX7bAF+i+JS9EfgoxeW/AP8GrATWSXqhjm2vo3gt\nnwduAs5NR4xQnJD/NcWb/Q2pvexSqux3RPya4k3/BIojsW8BZ5bGti2Aimles9pI+iHFCcvr2p1L\nO0n6G2DPiKh4FY/Z1sJHAmY1SNNq709TUIdRTIvc0e68zJrlbwyb1WYXiimgvSmmRq4A7mxrRmYt\n4OkgM7OMeTrIzCxjW/x00Lhx46Kzs7PdaZiZbTWWLl36QkR0DN9zKygCnZ2ddHd3tzsNM7OthqSK\n3zivxNNBZmYZcxEwM8uYi4CZWcZcBMzMMuYiYGaWMRcBM7OMuQiYmWXMRcDMLGMuAmZmGdvivzFs\nW5bOC++qq//q2SeNUCZm1go+EjAzy9iwRUDSJEk/kPSYpJWSvpDie0haLOnJ9HNsikvSVZJ6JC2X\ndGhprJmp/5OSfEcmM7M2q+VIYDNwQUQcCBwBnCfpQOBC4P6ImALcn55DcT/RKekxC5gDRdGguHfq\n4cBhwCUDhcPMzNpj2CIQEX0R8Uha/iXwODABmE5x42nSz1PT8nTgxigsAXaXtBdwPLA4IjZGxIvA\nYmBaS/fGzMzqUtc5AUmdwCHAQ8D4iOhLTeuA8Wl5ArCmtFpvilWLV9rOLEndkrr7+/vrSdHMzOpQ\ncxGQ9A7gNuCLEfFKuS2Ke1S27D6VETE3Iroioqujo6b7IpiZWQNqKgKStqMoADdFxO0pvD5N85B+\nbkjxtcCk0uoTU6xa3MzM2qSWq4MEXA88HhF/V2paBAxc4TMTuLMUPzNdJXQE8HKaNroXOE7S2HRC\n+LgUMzOzNqnly2IfBs4AHpW0LMUuBmYDCyWdDTwLnJba7gZOBHqA14BPA0TERkmXAQ+nfl+JiI0t\n2QszM2vIsEUgIh4EVKX5mAr9AzivyljzgHn1JGhmZiPH3xg2M8uYi4CZWcZcBMzMMuYiYGaWMRcB\nM7OMuQiYmWXMN5V5m/FNX8ysHj4SMDPLmIuAmVnGXATMzDLmImBmljEXATOzjLkImJllzEXAzCxj\nLgJmZhlzETAzy1gtt5ecJ2mDpBWl2HckLUuP1QN3HJPUKelXpbZrSut8UNKjknokXZVuW2lmZm1U\ny5+NmA98E7hxIBARnxxYlnQF8HKp/1MRMbXCOHOAzwEPUdyCchpwT/0pm5lZqwx7JBARDwAV7wWc\nPs2fBiwYagxJewG7RsSSdPvJG4FT60/XzMxaqdlzAkcC6yPiyVJssqSfSvqRpCNTbALQW+rTm2IV\nSZolqVtSd39/f5MpmplZNc0WgdN581FAH7BPRBwCfAm4WdKu9Q4aEXMjoisiujo6OppM0czMqmn4\nT0lL2hb4BPDBgVhEbAI2peWlkp4C9gfWAhNLq09MMTMza6NmjgT+EFgVEb+d5pHUIWlMWn43MAV4\nOiL6gFckHZHOI5wJ3NnEts3MrAVquUR0AfBj4ABJvZLOTk0zeOsJ4aOA5emS0VuBcyNi4KTynwLX\nAT3AU/jKIDOztht2OigiTq8SP6tC7Dbgtir9u4GD68zPzMxGkL8xbGaWMRcBM7OMuQiYmWXMRcDM\nLGMuAmZmGXMRMDPLmIuAmVnGXATMzDLmImBmljEXATOzjLkImJllzEXAzCxjLgJmZhlzETAzy5iL\ngJlZxlwEzMwyVsudxeZJ2iBpRSl2qaS1kpalx4mltosk9Uh6QtLxpfi0FOuRdGHrd8XMzOpVy5HA\nfGBahfiVETE1Pe4GkHQgxW0nD0rrfEvSmHTf4auBE4ADgdNTXzMza6Nabi/5gKTOGsebDtwSEZuA\nZyT1AIeltp6IeBpA0i2p72N1Z2xmZi3TzDmB8yUtT9NFY1NsArCm1Kc3xarFK5I0S1K3pO7+/v4m\nUjQzs6E0WgTmAPsBU4E+4IqWZQRExNyI6IqIro6OjlYObWZmJcNOB1USEesHliVdC3w3PV0LTCp1\nnZhiDBE3M7M2aehIQNJepacfBwauHFoEzJC0g6TJwBTgJ8DDwBRJkyVtT3HyeFHjaZuZWSsMeyQg\naQFwNDBOUi9wCXC0pKlAAKuBcwAiYqWkhRQnfDcD50XE62mc84F7gTHAvIhY2fK9MTOzutRyddDp\nFcLXD9H/cuDyCvG7gbvrys7MzEZUQ+cEzEZK54V31b3O6tknjUAmZnnwn40wM8uYi4CZWcZcBMzM\nMuYiYGaWMRcBM7OMuQiYmWXMRcDMLGMuAmZmGXMRMDPLmIuAmVnGXATMzDLmImBmljEXATOzjLkI\nmJllzEXAzCxjwxYBSfMkbZC0ohT7W0mrJC2XdIek3VO8U9KvJC1Lj2tK63xQ0qOSeiRdJUkjs0tm\nZlarWo4E5gPTBsUWAwdHxPuBnwMXldqeioip6XFuKT4H+BzFfYenVBjTzMxG2bBFICIeADYOit0X\nEZvT0yXAxKHGSDem3zUilkREADcCpzaWspmZtUorzgl8Brin9HyypJ9K+pGkI1NsAtBb6tObYhVJ\nmiWpW1J3f39/C1I0M7NKmioCkv4c2AzclEJ9wD4RcQjwJeBmSbvWO25EzI2Irojo6ujoaCZFMzMb\nQsM3mpd0FnAycEya4iEiNgGb0vJSSU8B+wNrefOU0cQUMzOzNmroSEDSNODLwCkR8Vop3iFpTFp+\nN8UJ4Kcjog94RdIR6aqgM4E7m87ezMyaMuyRgKQFwNHAOEm9wCUUVwPtACxOV3ouSVcCHQV8RdJv\ngDeAcyNi4KTyn1JcabQjxTmE8nkEMzNrg2GLQEScXiF8fZW+twG3VWnrBg6uKzszMxtR/sawmVnG\nXATMzDLmImBmljEXATOzjLkImJllzEXAzCxjLgJmZhlzETAzy5iLgJlZxlwEzMwy5iJgZpYxFwEz\ns4y5CJiZZcxFwMwsYy4CZmYZcxEwM8uYi4CZWcZqKgKS5knaIGlFKbaHpMWSnkw/x6a4JF0lqUfS\nckmHltaZmfo/KWlm63fHzMzqUeuRwHxg2qDYhcD9ETEFuD89BziB4gbzU4BZwBwoigbF/YkPBw4D\nLhkoHGZm1h41FYGIeADYOCg8HbghLd8AnFqK3xiFJcDukvYCjgcWR8TGiHgRWMxbC4uZmY2iZs4J\njI+IvrS8DhiflicAa0r9elOsWvwtJM2S1C2pu7+/v4kUzcxsKC05MRwRAUQrxkrjzY2Irojo6ujo\naNWwZmY2SDNFYH2a5iH93JDia4FJpX4TU6xa3MzM2qSZIrAIGLjCZyZwZyl+ZrpK6Ajg5TRtdC9w\nnKSx6YTwcSlmZmZtsm0tnSQtAI4GxknqpbjKZzawUNLZwLPAaan73cCJQA/wGvBpgIjYKOky4OHU\n7ysRMfhks5mZjaKaikBEnF6l6ZgKfQM4r8o484B5NWdnZmYjyt8YNjPLWE1HAtYanRfeVVf/1bNP\nGqFMzMwKPhIwM8uYi4CZWcZcBMzMMuYiYGaWMRcBM7OMuQiYmWXMRcDMLGP+noBlx9/XMPsdHwmY\nmWXMRcDMLGMuAmZmGXMRMDPLmIuAmVnGXATMzDLWcBGQdICkZaXHK5K+KOlSSWtL8RNL61wkqUfS\nE5KOb80umJlZoxr+nkBEPAFMBZA0huKm8XdQ3E7yyoj4erm/pAOBGcBBwN7A9yXtHxGvN5qDmZk1\np1XTQccAT0XEs0P0mQ7cEhGbIuIZinsQH9ai7ZuZWQNaVQRmAAtKz8+XtFzSPEljU2wCsKbUpzfF\n3kLSLEndkrr7+/tblKKZmQ3WdBGQtD1wCvDPKTQH2I9iqqgPuKLeMSNibkR0RURXR0dHsymamVkV\nrTgSOAF4JCLWA0TE+oh4PSLeAK7ld1M+a4FJpfUmppiZmbVJK4rA6ZSmgiTtVWr7OLAiLS8CZkja\nQdJkYArwkxZs38zMGtTUXxGVtDNwLHBOKfw1SVOBAFYPtEXESkkLgceAzcB5vjLIzKy9mioCEfEq\n8M5BsTOG6H85cHkz2zQzs9bxN4bNzDLmImBmljEXATOzjLkImJllzEXAzCxjLgJmZhlzETAzy5iL\ngJlZxlwEzMwy5iJgZpYxFwEzs4y5CJiZZcxFwMwsYy4CZmYZcxEwM8uYi4CZWcZacaP51ZIelbRM\nUneK7SFpsaQn08+xKS5JV0nqkbRc0qHNbt/MzBrXqiOBj0XE1IjoSs8vBO6PiCnA/ek5FDeln5Ie\ns4A5Ldq+mZk1YKSmg6YDN6TlG4BTS/Ebo7AE2H3QjenNzGwUtaIIBHCfpKWSZqXY+IjoS8vrgPFp\neQKwprRub4q9iaRZkroldff397cgRTMzq6SpG80nH4mItZLeBSyWtKrcGBEhKeoZMCLmAnMBurq6\n6lrXzMxq1/SRQESsTT83AHcAhwHrB6Z50s8NqftaYFJp9YkpZmZmbdBUEZC0s6RdBpaB44AVwCJg\nZuo2E7gzLS8CzkxXCR0BvFyaNjIzs1HW7HTQeOAOSQNj3RwR35P0MLBQ0tnAs8Bpqf/dwIlAD/Aa\n8Okmt29mZk1oqghExNPAByrEfwEcUyEewHnNbNPMzFrH3xg2M8uYi4CZWcZcBMzMMuYiYGaWMRcB\nM7OMuQiYmWXMRcDMLGMuAmZmGXMRMDPLWCv+iqiZlXReeFdd/VfPPmmEMjEbno8EzMwy5iJgZpYx\nFwEzs4y5CJiZZcxFwMwsYy4CZmYZa7gISJok6QeSHpO0UtIXUvxSSWslLUuPE0vrXCSpR9ITko5v\nxQ6YmVnjmvmewGbggoh4JN1neKmkxantyoj4ermzpAOBGcBBwN7A9yXtHxGvN5FDS/n6bjPLTcNH\nAhHRFxGPpOVfAo8DE4ZYZTpwS0RsiohnKO4zfFij2zczs+a15JyApE7gEOChFDpf0nJJ8ySNTbEJ\nwJrSar0MXTTMzGyENV0EJL0DuA34YkS8AswB9gOmAn3AFQ2MOUtSt6Tu/v7+ZlM0M7MqmioCkraj\nKAA3RcTtABGxPiJej4g3gGv53ZTPWmBSafWJKfYWETE3Iroioqujo6OZFM3MbAjNXB0k4Hrg8Yj4\nu1J8r1K3jwMr0vIiYIakHSRNBqYAP2l0+2Zm1rxmrg76MHAG8KikZSl2MXC6pKlAAKuBcwAiYqWk\nhcBjFFcWnbclXRlkZpajhotARDwIqELT3UOsczlweaPbNDOz1vI3hs3MMuYiYGaWMRcBM7OMuQiY\nmWXMRcDMLGMuAmZmGXMRMDPLmIuAmVnGmvnGsJm1Qb33vQDf+8Kq85GAmVnGXATMzDLmImBmljEX\nATOzjLkImJllzEXAzCxjLgJmZhlzETAzy9iof1lM0jTgG8AY4LqImD3aOZjZ0Or9Qpq/jLb1GtUi\nIGkMcDVwLNALPCxpUUQ8NhLba+SblWZmORntI4HDgJ6IeBpA0i3AdIqbz5tZJkb6SMN/WqN2iojR\n25j0P4FpEfHZ9PwM4PCIOH9Qv1nArPT0AOCJUUuyduOAF9qdRIOce3s499G3teYNzeW+b0R01NJx\ni/wDchExF5jb7jyGIqk7IrranUcjnHt7OPfRt7XmDaOX+2hfHbQWmFR6PjHFzMysDUa7CDwMTJE0\nWdL2wAxg0SjnYGZmyahOB0XEZknnA/dSXCI6LyJWjmYOLbRFT1cNw7m3h3MffVtr3jBKuY/qiWEz\nM9uy+BvDZmYZcxEwM8uYi0CDJI2R9FNJ3213LvWQtLukWyWtkvS4pA+1O6daSPrfklZKWiFpgaTf\na3dO1UiaJ2mDpBWl2B6SFkt6Mv0c284cq6mS+9+mfy/LJd0hafd25lhNpdxLbRdICknj2pHbcKrl\nLunz6bVfKelrI7FtF4HGfQF4vN1JNOAbwPci4r3AB9gK9kHSBODPgK6IOJjiooIZ7c1qSPOBaYNi\nFwL3R8QU4P70fEs0n7fmvhg4OCLeD/wcuGi0k6rRfN6aO5ImAccBz412QnWYz6DcJX2M4i8qfCAi\nDgK+PhIbdhFogKSJwEnAde3OpR6SdgOOAq4HiIhfR8RL7c2qZtsCO0raFtgJeL7N+VQVEQ8AGweF\npwM3pOUbgFNHNakaVco9Iu6LiM3p6RKK7/dscaq87gBXAl8GttirYKrk/ifA7IjYlPpsGIltuwg0\n5u8p/lG90e5E6jQZ6Af+MU1lXSdp53YnNZyIWEvxKeg5oA94OSLua29WdRsfEX1peR0wvp3JNOEz\nwD3tTqJWkqYDayPiZ+3OpQH7A0dKekjSjyT9/khsxEWgTpJOBjZExNJ259KAbYFDgTkRcQjwKlvu\ntMRvpfnz6RRFbG9gZ0n/q71ZNS6K67K32E+l1Uj6c2AzcFO7c6mFpJ2Ai4G/bHcuDdoW2AM4Avg/\nwEJJavVGXATq92HgFEmrgVuA/ybp2+1NqWa9QG9EPJSe30pRFLZ0fwg8ExH9EfEb4HbgD9qcU73W\nS9oLIP0ckUP7kSLpLOBk4FOx9Xy5aD+KDw4/S/9fJwKPSNqzrVnVrhe4PQo/oZh5aPmJbReBOkXE\nRRExMSI6KU5O/ltEbBWfSiNiHbBG0gEpdAxbx5/xfg44QtJO6ZPQMWwFJ7QHWQTMTMszgTvbmEtd\n0o2gvgycEhGvtTufWkXEoxHxrojoTP9fe4FD0/+DrcG/AB8DkLQ/sD0j8BdRXQTy83ngJknLganA\nX7c5n2GlI5dbgUeARyn+3W6xfw5A0gLgx8ABknolnQ3MBo6V9CTFkc0WeUe9Krl/E9gFWCxpmaRr\n2ppkFVVy3ypUyX0e8O502egtwMyROArzn40wM8uYjwTMzDLmImBmljEXATOzjLkImJllzEXAzCxj\nLgJmZhlzETAzy9j/B8WHKERRkkO/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f139a25cd30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "MAX_LENGTH = max(map(len, names))\n",
    "print(\"max length:\", MAX_LENGTH)\n",
    "\n",
    "plt.title('Sequence length distribution')\n",
    "plt.hist(list(map(len, names)), bins=25);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text processing\n",
    "\n",
    "First we need to collect a \"vocabulary\" of all unique tokens i.e. unique characters. We can then encode inputs as a sequence of character ids."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.864592Z",
     "start_time": "2018-08-13T20:26:42.858725Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_tokens: 55\n"
     ]
    }
   ],
   "source": [
    "tokens =set(chain(*map(lambda x: list(x),names))) \n",
    "### YOUR CODE HERE: all unique characters go here, padding included!\n",
    "\n",
    "tokens = list(tokens)\n",
    "n_tokens = len(tokens)\n",
    "print ('n_tokens:', n_tokens)\n",
    "\n",
    "assert 50 < n_tokens < 60"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cast everything from symbols into identifiers\n",
    "\n",
    "Tensorflow string manipulation is a bit tricky, so we'll work around it. \n",
    "We'll feed our recurrent neural network with ids of characters from our dictionary.\n",
    "\n",
    "To create such dictionary, let's assign `token_to_id`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.870330Z",
     "start_time": "2018-08-13T20:26:42.866135Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "token_to_id = {s: idx for idx, s in enumerate(tokens)}\n",
    "### YOUR CODE HERE: create a dictionary of {symbol -> its  index in tokens}\n",
    "\n",
    "assert len(tokens) == len(token_to_id), \"dictionaries must have same size\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.875943Z",
     "start_time": "2018-08-13T20:26:42.871834Z"
    }
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'#'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-106-d3c43fc7d925>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mto_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtoken_to_id\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpad_token\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;34m\"\"\"Casts a list of names into rnn-digestable padded matrix\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mmax_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_len\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mnames_ix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_len\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: '#'"
     ]
    }
   ],
   "source": [
    "def to_matrix(names, max_len=None, pad=token_to_id[pad_token], dtype=np.int32):\n",
    "    \"\"\"Casts a list of names into rnn-digestable padded matrix\"\"\"\n",
    "    \n",
    "    max_len = max_len or max(map(len, names))\n",
    "    names_ix = np.zeros([len(names), max_len], dtype) + pad\n",
    "\n",
    "    for i in range(len(names)):\n",
    "        name_ix = list(map(token_to_id.get, names[i]))\n",
    "        names_ix[i, :len(name_ix)] = name_ix\n",
    "\n",
    "    return names_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:42.883107Z",
     "start_time": "2018-08-13T20:26:42.877186Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Abagael\n",
      " Glory\n",
      " Prissie\n",
      " Giovanne\n",
      "[[23  8 30  9 41  9 47 39 23]\n",
      " [23 40 39 12  2 46 23 23 23]\n",
      " [23 17  2 11 27 27 11 47 23]\n",
      " [23 40 11 12  1  9 54 54 47]]\n"
     ]
    }
   ],
   "source": [
    "# Example: cast 4 random names to padded matrices (so that we can easily batch them)\n",
    "print('\\n'.join(names[::2000]))\n",
    "print(to_matrix(names[::2000]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Defining a recurrent neural network\n",
    "\n",
    "We can rewrite recurrent neural network as a consecutive application of dense layer to input $x_t$ and previous rnn state $h_t$. This is exactly what we're gonna do now.\n",
    "<img src=\"./rnn.png\" width=600>\n",
    "\n",
    "Since we're training a language model, there should also be:\n",
    "* An embedding layer that converts character id x_t to a vector.\n",
    "* An output layer that predicts probabilities of next phoneme based on h_t+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:44.039419Z",
     "start_time": "2018-08-13T20:26:42.884581Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remember to reset your session if you change your graph!\n",
    "s = keras_utils.reset_tf_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:44.044903Z",
     "start_time": "2018-08-13T20:26:44.041084Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.layers import concatenate, Dense, Embedding\n",
    "\n",
    "rnn_num_units = 64  # size of hidden state\n",
    "embedding_size = 16  # for characters\n",
    "\n",
    "# Let's create layers for our recurrent network\n",
    "# Note: we create layers but we don't \"apply\" them yet (this is a \"functional API\" of Keras)\n",
    "# Note: set the correct activation (from keras.activations) to Dense layers!\n",
    "\n",
    "# an embedding layer that converts character ids into embeddings\n",
    "embed_x = Embedding(n_tokens, embedding_size)\n",
    "\n",
    "# a dense layer that maps input and previous state to new hidden state, [x_t,h_t]->h_t+1\n",
    "get_h_next =Dense(rnn_num_units, activation='relu') ### YOUR CODE HERE\n",
    "\n",
    "# a dense layer that maps current hidden state to probabilities of characters [h_t+1]->P(x_t+1|h_t+1)\n",
    "get_probas =Dense(n_tokens, activation='softmax') ### YOUR CODE HERE "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will generate names character by character starting with `start_token`:\n",
    "\n",
    "<img src=\"./char-nn.png\" width=600>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:44.053212Z",
     "start_time": "2018-08-13T20:26:44.048389Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rnn_one_step(x_t, h_t):\n",
    "    \"\"\"\n",
    "    Recurrent neural network step that produces \n",
    "    probabilities for next token x_t+1 and next state h_t+1\n",
    "    given current input x_t and previous state h_t.\n",
    "    We'll call this method repeatedly to produce the whole sequence.\n",
    "    \n",
    "    You're supposed to \"apply\" above layers to produce new tensors.\n",
    "    Follow inline instructions to complete the function.\n",
    "    \"\"\"\n",
    "    # convert character id into embedding\n",
    "    x_t_emb = embed_x(tf.reshape(x_t, [-1, 1]))[:, 0]\n",
    "    \n",
    "    # concatenate x_t embedding and previous h_t state\n",
    "    x_and_h = concatenate([x_t_emb, h_t])### YOUR CODE HERE\n",
    "    \n",
    "    # compute next state given x_and_h\n",
    "    h_next =get_h_next(x_and_h) ### YOUR CODE HERE\n",
    "    \n",
    "    # get probabilities for language model P(x_next|h_next)\n",
    "    output_probas =get_probas(h_next) ### YOUR CODE HERE\n",
    "    \n",
    "    return output_probas, h_next"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN: loop\n",
    "\n",
    "Once `rnn_one_step` is ready, let's apply it in a loop over name characters to get predictions.\n",
    "\n",
    "Let's assume that all names are at most length-16 for now, so we can simply iterate over them in a for loop.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:44.342948Z",
     "start_time": "2018-08-13T20:26:44.056136Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_sequence = tf.placeholder(tf.int32, (None, MAX_LENGTH))  # batch of token ids\n",
    "batch_size = tf.shape(input_sequence)[0]\n",
    "\n",
    "predicted_probas = []\n",
    "h_prev = tf.zeros([batch_size, rnn_num_units])  # initial hidden state\n",
    "\n",
    "for t in range(MAX_LENGTH):\n",
    "    x_t = input_sequence[:, t]  # column t\n",
    "    probas_next, h_next = rnn_one_step(x_t, h_prev)\n",
    "    \n",
    "    h_prev = h_next\n",
    "    predicted_probas.append(probas_next)\n",
    "    \n",
    "# combine predicted_probas into [batch, time, n_tokens] tensor\n",
    "predicted_probas = tf.transpose(tf.stack(predicted_probas), [1, 0, 2])\n",
    "\n",
    "# next to last token prediction is not needed\n",
    "predicted_probas = predicted_probas[:, :-1, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN: loss and gradients\n",
    "\n",
    "Let's gather a matrix of predictions for $P(x_{next}|h)$ and the corresponding correct answers.\n",
    "\n",
    "We will flatten our matrices to shape [None, n_tokens] to make it easier.\n",
    "\n",
    "Our network can then be trained by minimizing crossentropy between predicted probabilities and those answers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:44.354310Z",
     "start_time": "2018-08-13T20:26:44.344648Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# flatten predictions to [batch*time, n_tokens]\n",
    "predictions_matrix = tf.reshape(predicted_probas, [-1, n_tokens])\n",
    "\n",
    "# flatten answers (next tokens) and one-hot encode them\n",
    "answers_matrix = tf.one_hot(tf.reshape(input_sequence[:, 1:], [-1]), n_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Usually it's a good idea to ignore gradients of loss for padding token predictions.\n",
    "\n",
    "Because we don't care about further prediction after the pad_token is predicted for the first time, so it doesn't make sense to punish our network after the pad_token is predicted.\n",
    "\n",
    "For simplicity you can ignore this comment, it's up to you."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:45.076642Z",
     "start_time": "2018-08-13T20:26:44.355594Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the loss as categorical cross-entropy (e.g. from keras.losses).\n",
    "# Mind that predictions are probabilities and NOT logits!\n",
    "# Remember to apply tf.reduce_mean to get a scalar loss!\n",
    "loss =tf.reduce_mean(keras.losses.categorical_crossentropy(answers_matrix, predictions_matrix)) ### YOUR CODE HERE\n",
    "\n",
    "optimize = tf.train.AdamOptimizer().minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN: training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:55.322187Z",
     "start_time": "2018-08-13T20:26:45.078296Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VFXawPHfMyUJkAQIhB4IqICIAhoQLIAgYue1rGXX\nXlh91bWtLupa110Lu+q69l3r2nDVdy0oNlBQaQHpRYr0FgKEElJmct4/5t7JzGQmmSSThLk8388n\nH2buPXPn3Fzy3HNPFWMMSimlnMXV1BlQSimVeBrclVLKgTS4K6WUA2lwV0opB9LgrpRSDqTBXSml\nHEiDu1JKOZAGd6WUciAN7kop5UCepvritm3bmtzc3Kb6eqWUSkpz5szZbozJrildkwX33Nxc8vPz\nm+rrlVIqKYnI2njSabWMUko5kAZ3pZRyIA3uSinlQE1W566UUolQXl7Ohg0bKCkpaeqsJFRaWhpd\nunTB6/XW6fMa3JVSSW3Dhg1kZGSQm5uLiDR1dhLCGENhYSEbNmyge/fudTpG3NUyIuIWkZ9E5NMo\n+1JFZIKIrBSRmSKSW6fcKKVULZWUlNCmTRvHBHYAEaFNmzb1ehqpTZ37zcDSGPuuBnYaYw4FngQe\nq3OOlFKqlpwU2G31Pae4gruIdAHOAP4VI8kY4HXr9fvASGmg3/aGncU8+Mliyv0VDXF4pZRyhHhL\n7k8BdwKxImpnYD2AMcYHFAFt6p27KJZs2s2rP6zhtR/WNMThlVKq1tLT05s6C1XUGNxF5ExgmzFm\nTn2/TETGiki+iOQXFBTU6Rij+rTnxMPa8uLU1fi09K6UUlHFU3I/HjhbRNYA7wIjROTNiDQbgRwA\nEfEALYHCyAMZY14yxuQZY/Kys2ucGiEqEeGSwd3YvreU71dur9MxlFKqIRhjuOOOO+jbty9HHnkk\nEyZMAGDz5s0MHTqU/v3707dvX6ZNm4bf7+eKK64Ipn3yyScTmpcau0IaY+4C7gIQkeHA740xl0Qk\n+xi4HJgOnA9MNsaYhOY0xLCe2XhcQv6anQzv1a6hvkYplWQe/GQxSzbtTugx+3TK5P6zjogr7Ycf\nfsi8efOYP38+27dvZ+DAgQwdOpS3336b0aNHc8899+D3+ykuLmbevHls3LiRRYsWAbBr166E5rvO\nI1RF5CEROdt6+zLQRkRWArcB4xKRuVjSvG4Oa5/Bwo1FDfk1SilVK99//z0XX3wxbreb9u3bM2zY\nMGbPns3AgQN59dVXeeCBB1i4cCEZGRn06NGD1atXc9NNNzFp0iQyMzMTmpdaDWIyxnwLfGu9vi9k\newnwq0RmrCZ9O2Uyedk2jDGO7AallKq9eEvYjW3o0KFMnTqViRMncsUVV3Dbbbdx2WWXMX/+fL74\n4gteeOEF3nvvPV555ZWEfWfSzi3Tu2MmhfvK2LGvrKmzopRSAJx44olMmDABv99PQUEBU6dOZdCg\nQaxdu5b27dtz7bXXcs011zB37ly2b99ORUUF5513Hg8//DBz585NaF6SdvqBQ7JbALCqYB9t0lOb\nODdKKQXnnHMO06dPp1+/fogIjz/+OB06dOD1119n/PjxeL1e0tPTeeONN9i4cSNXXnklFRWBXn+P\nPPJIQvOStMG9a1ZzADbuKgaymjYzSqmD2t69e4FAb77x48czfvz4sP2XX345l19+eZXPJbq0Hipp\nq2XaZaYBsHV3aRPnRCmlDjxJG9zTUz2kp3rYuttZ03wqpVQiJG1wB2iXmcq2PVpyV+pg14DDappM\nfc8pqYN7+4w0tmnJXamDWlpaGoWFhY4K8PZ87mlpaXU+RtI2qAK0z0xlzrqdTZ0NpVQT6tKlCxs2\nbKCu81UdqOyVmOoqqYN76xYp7NpX3tTZUEo1Ia/XW+fVipwsqatlMtK87C3zUVHhnMcxpZRKhKQO\n7plpHoyBfWW+ps6KUkodUJI6uKenBmqV9pRocFdKqVBJHdwz0ryABnellIqU1ME9PS1Qct9bqo2q\nSikVKqmDe4YV3HdryV0ppcIkdXDPTNM6d6WUiiapg3t6ql3nrtUySikVKqmDu10ts1dL7kopFSap\ng3vzFDcu0WoZpZSKlNTBXURI87opKfc3dVaUUuqAktTBHSDV46LUV9HU2VBKqQNK0gf3NK+bUp+W\n3JVSKlTSB/dUj4uSci25K6VUqBqDu4ikicgsEZkvIotF5MEoaa4QkQIRmWf9XNMw2a0q1aMld6WU\nihTPfO6lwAhjzF4R8QLfi8jnxpgZEekmGGNuTHwWq5fm1ZK7UkpFqjG4m8DaVXutt17r54CZQF1L\n7kopVVVcde4i4haRecA24CtjzMwoyc4TkQUi8r6I5CQ0l9VI1ZK7UkpVEVdwN8b4jTH9gS7AIBHp\nG5HkEyDXGHMU8BXwerTjiMhYEckXkfxErXcYKLlrcFdKqVC16i1jjNkFTAFOjdheaIwptd7+Czgm\nxudfMsbkGWPysrOz65LfKlK9Lq2WUUqpCPH0lskWkVbW62bAKGBZRJqOIW/PBpYmMpPVSfO4KdVq\nGaWUChNPb5mOwOsi4iZwM3jPGPOpiDwE5BtjPgZ+JyJnAz5gB3BFQ2U4kpbclVKqqnh6yywABkTZ\nfl/I67uAuxKbtfikedzaoKqUUhGSf4SqltyVUqqKpA/uaR435X6Dv+KA6XqvlFJNLumDe6o3cApa\neldKqUrJH9w9VnDXenellApK+uCe5nUDUKIld6WUCkr64J7i1pK7UkpFSvrg7rWqZXwVGtyVUsqW\n9ME9xS0AlPm0t4xSStmSPrh7rWqZcr+W3JVSyuaY4K7VMkopVckxwV2rZZRSqlLSB/cUT6DOXatl\nlFKqUtIHd61zV0qpqjS4K6WUAzkguFtdIf1a566UUjYHBHer5K7rqCqlVJBzgrtWyyilVJBzgrvO\n566UUkFJH9xTtFpGKaWqSPrg7tV+7kopVUXyB3etc1dKqSqSPrh7XNoVUimlIiV9cBcRUtwuLbkr\npVSIpA/uAB63aIOqUkqFqDG4i0iaiMwSkfkislhEHoySJlVEJojIShGZKSK5DZHZWLxacldKqTDx\nlNxLgRHGmH5Af+BUERkckeZqYKcx5lDgSeCxxGazel63S+vclVIqRI3B3QTstd56rZ/ISDoGeN16\n/T4wUkQkYbmsQYpb8GnJXSmlguKqcxcRt4jMA7YBXxljZkYk6QysBzDG+IAioE2U44wVkXwRyS8o\nKKhfzkN4PVoto5RSoeIK7sYYvzGmP9AFGCQifevyZcaYl4wxecaYvOzs7LocIqpAnbtWyyillK1W\nvWWMMbuAKcCpEbs2AjkAIuIBWgKFichgPAJ17lpyV0opWzy9ZbJFpJX1uhkwClgWkexj4HLr9fnA\nZGNMoxWlU9yi1TJKKRXCE0eajsDrIuImcDN4zxjzqYg8BOQbYz4GXgb+LSIrgR3ARQ2W4yg82hVS\nKaXC1BjcjTELgAFRtt8X8roE+FVisxY/r1u0zl0ppUI4YoSqDmJSSqlwjgjuHpfg05K7UkoFOSO4\na8ldKaXCOCK4e92CT5fZU0qpIIcEdy25K6VUKEcEd4/LpXXuSikVwhHB3auDmJRSKowjgrtH69yV\nUiqMI4K71+3SlZiUUiqEc4J7hQZ3pZSyOSK46yAmpZQK54zg7nbhqzA04kSUSil1QHNEcE9xB1b0\n08nDlFIqwBHB3eMOnIZP692VUgpwSnB3acldKaVCOSK4e+2Suw5kUkopwGHBXUvuSikV4Ijg7gk2\nqGrJXSmlwCHB3WsFd52CQCmlAhwR3D0urXNXSqlQjgjudp17mQZ3pZQCHBPcrWoZbVBVSinAIcFd\nBzEppVS4GoO7iOSIyBQRWSIii0Xk5ihphotIkYjMs37ua5jsRufVQUxKKRXGE0caH3C7MWauiGQA\nc0TkK2PMkoh004wxZyY+izXzeuwGVQ3uSikFcZTcjTGbjTFzrdd7gKVA54bOWG1UTj+g1TJKKQW1\nrHMXkVxgADAzyu4hIjJfRD4XkSNifH6siOSLSH5BQUGtMxtL5QhVDe5KKQW1CO4ikg58ANxijNkd\nsXsu0M0Y0w/4B/DfaMcwxrxkjMkzxuRlZ2fXNc9VeHQQk1JKhYkruIuIl0Bgf8sY82HkfmPMbmPM\nXuv1Z4BXRNomNKfVsAcxacldKaUC4uktI8DLwFJjzBMx0nSw0iEig6zjFiYyo9VJ0YnDlFIqTDy9\nZY4HLgUWisg8a9vdQFcAY8wLwPnA9SLiA/YDF5lGXPMuWC2jJXellALiCO7GmO8BqSHNM8AzicpU\nbQVnhdQ6d6WUAhwyQtWrE4cppVQYZwR3jzaoKqVUKEcEd11DVSmlwjkiuFeuoarBXSmlwCHB3e0S\nRHRWSKWUsjkiuEOg9K6LdSilVIBzgrtLtFpGKaUsjgnuHrdLu0IqpZTFMcHd6xYdxKSUUhYHBXcX\n5T4tuSulFDgouHvcolP+KqWUxTHB3et2UaYld6WUAhwU3Jt53ZSU+5s6G0opdUBwVnD3aXBXSilw\nUHBP87rZX6bBXSmlwGnBvVzr3JVSChwU3JulaJ27UkrZnBPcvS6tllFKKYtjgnuaNqgqpVSQY4J7\n8xQP+0p9NOK63EopdcByTHBv1dxLud9QrFUzSinloODezAvArv3lAOwv87N9b2lTZkkppZqMc4J7\n8xQAdhWXAXDBi9PJe/jrpsySUko1mRqDu4jkiMgUEVkiIotF5OYoaUREnhaRlSKyQESObpjsxtbS\nKrkXFQdK7gs3FjV2FpRS6oDhiSOND7jdGDNXRDKAOSLylTFmSUia04DDrJ9jgeetfxtNqjdwnyrV\nycOUUqrmkrsxZrMxZq71eg+wFOgckWwM8IYJmAG0EpGOCc9tNVLcGtyVUspWqzp3EckFBgAzI3Z1\nBtaHvN9A1RtAg0r1BE5FF8lWSqlaBHcRSQc+AG4xxuyuy5eJyFgRyReR/IKCgrocIqYUO7j7KnQa\nAqXUQS+u4C4iXgKB/S1jzIdRkmwEckLed7G2hTHGvGSMyTPG5GVnZ9clvzGFBveb3/0pocdWSqlk\nE09vGQFeBpYaY56Ikexj4DKr18xgoMgYszmB+ayRXede5vMzedm24HYdsaqUOhjF01vmeOBSYKGI\nzLO23Q10BTDGvAB8BpwOrASKgSsTn9Xq2SX3cr/B43JR7g9UzfgrDB63NHZ2lFKqSdUY3I0x3wPV\nRkcTKB7fkKhM1UVKSIOq1y1YA1XxGxPXHUwppZzEMSNUQ7tCet2Vp1WhnWeUUgchxwR3ESHF7aIs\nIrif+PhkFm/S0apKqYOLY4I7gK+ighe+W8WW3SXBbdv3ljFh9vpqPqWUUs7jqOBeEaNjTIXVY2bR\nxiKuem02ZTqKVSnlcI4K7n884/Co29+csY6KCsMd7y9g8rJtrNi2p5FzppRSjctRwf3co7vE3Ldk\n8+5gn3eXaNdIpZSzOSq42wt2RLOqYG+wekZju1LK6RwV3F2u2FF7V3F5sE5+0qItjZQjpZRqGo4K\n7tUp2l8erJZ56usVTZwbpZRqWAdNcH/iq5/RaWaUUgcLxwb3fjmtqmzza3RXSh0kHBvcO7dKq7Kt\nQoO7Uuog4djgLlG6xKzfsT9q2nWFxcz6ZUfMY+0uKSd33ETe05GuSqkk4dzgXou0Q8dP4YIXp1NS\n7mdt4b4q+zfuDNwUXvnhlwTlTimlGpbjgnszrxuIXnIPVVLuJ3fcRN6Yvia4bdwHCxg2/luK9pfz\nzdKtwe1+qw+lu5qulkopdSBxXHA/sktLoHLBbIAHzupTJd2FL80A4B+TVwa3zV6zE4Db35vP1a/n\nk79mB9NWFLB6e6A0Hyu47you44THdPZJpdSBw3HB/cVLjuHx84/iT2P6Brd53FVPc/76XVW2tU1P\nAWDm6kIAduwr49KXZ/G7dwJrssYK7j+sLGTDzv08O2Vl1P1KKdXYHLdIUesWKVyQlxO2LSVKcLdV\nhEwlmWlNX7Cn1BfYF9G7xhMjuBu0F45S6sDiuJJ7NNWtoVq4ryz4etqK7WH7IqcQ1gnHlFLJ4iAJ\n7nU7TV9EdK9poW2pVR8dpZRqOI4O7q2bB6pZUmoIyrGUlPvD3rtdgV9XRYWhqLi8fplTSqkG5Ojg\nPu0PI/jp3lF4XHU7zdLI4G7dI/721XL6PfRlMMDrwFel1IHG0cE9PdVD6xYpeD11O817P1oc9t7t\nEkp9fj6ZvxmAlQW6opNS6sBUY9QTkVdEZJuILIqxf7iIFInIPOvnvsRns368Ib1cRAL93ls1j72w\nRyxfL93G6Censm5HMQDnPT89eMy6KCn3c83rs1mzveqoWKWUqo94irSvAafWkGaaMaa/9fNQ/bOV\nWKENqh0z07ji+O50atmsTsdaU1hcZdvCDVUHL63Yuod+D37J5qLo89kAfL9iO18v3cafPl1Sp7wo\npVQsNfZzN8ZMFZHchs9Kw8nr1ppXrshjw879DOuZDUBOVjOWbN6dkOO/OHV14EVICf7NGWsp2l/O\npEVbWFtYzKHt0rlkcLeEfJ9SStUkUYOYhojIfGAT8HtjzOKaPtCYXC5hRO/2YdseO+8oSsrn8d3P\nBfU6du64iVG323Pb+CsMr/24BkCDu1Kq0SSiQXUu0M0Y0w/4B/DfWAlFZKyI5ItIfkFB/YJqfbVq\nnsK1J/ZokGN/v2J7MKDH6kmzadd+VhbsbZDvV0qpegd3Y8xuY8xe6/VngFdE2sZI+5IxJs8Yk5ed\nnV3fr663Y7q15rhD2iTseBMXbGbHvjIueXlmcFvk6k8+fwXPTlnJcY9O5tHPlwG1b5D959TVvDR1\nVb3zq5RyrnoHdxHpIFYdhIgMso5ZWN/jNoZmKW7evnYwVxyXC0DvDhn1PmbkzJCh89M8+dXPPPn1\nz4z/Ynm1x1hbuI+b3vmJUp8/6v4/f7aUv3y2rN55VUo5VzxdId8BpgO9RGSDiFwtIteJyHVWkvOB\nRVad+9PARcYk17Cecn8FAL8+tmtw2z8uHhCW5p7TD4/rWPbCHraP520Kvv77Nyt4dkrNJe67PlzI\nJ/M3MfuXnXF9p1JKRYqnt8zFNex/BngmYTlqAtEW42hjTf9ry2zm4e1rj+XX/wxUuZzWtwOfL9pS\n5VjjPlwY9n7ZltoPdCr1BW42KXUcfNXQ9pSUs3jTbgb3SFyVllIqsQ7M6NHIBuZmAdCnYyYAPdun\n06VVcwBG9Qn0sjm2exuOO6SyKaFZijuheXh80jKueT2fzUX7g9Ux8cx39szkFdz/UdTxZQ3m5nfn\ncdFLM9gZMqOmUurA4rj53Ovi3KM7c8JhbWmfmcb8+08h1eMizesm/48n06ZFStQl+zbtCq9+GXda\n72ADaW19vXQbXy/dBsDe0nLKrJL7ec9P59ObTqBv55YxP/vXL38G4MGQxUkADr93Eucd05mH/+fI\nOuWpOks2BcYHlMRoE2gIy7fsYfRTU3ntyoEM79Wu0b5XqWSlJXcCfdLbZ6YB0LKZlzRrHda26akx\n12Ldtrs07H23rObB178dVvculjNW7+DnrZVdJB+bFH7DCF1cJHLWylD7y/28OWNd1H0+fwVfLt5C\nXZtG7F+Jz1/5+UPu/owb355bp+PFY/aaHQB8sXhrDSmVUqDBvc52FAeqJOwFuVs2q5yrJqt5CpNv\nH8bI3vUvYfr8hu9XbGfKskDJfn9IQD/zH9/X6ZhvzljL2H/P4eP5m2pOHIW9aElJuZ/lW/awr9SH\nv8Lw6YLNdTqeUirxNLjX0o/jRvD5zScGByd1bBUo8bdIrazhymqRQo/sdC4e1DXaIQAY0LVVXN83\nfXUhl7w8kytfmw3AvjJfcN/KbfENgpqxujDYaAxQtD9wjOV1aOwNta/Mz+inpvK/bzVciV0FFmDP\nHTeR7yNWCmso/grDlqKSRvku1XA0uNdSp1bNOLxjJu+OHcy403qTnlq12eKsfp0AaFnNzJPXDzuk\nTt9fXBq9KubJr35mqTVXTmjVzbz1u7jopRn87cvKvvVZVk+gwr1llPkq2LRrP4V7K6uZKioM4z5Y\nwKKNVSdEA7Cnx99pPb1MX524YQ3+CsPUnwtiVhkdjCsdLrSuw/PfNc4C7H/9cjmDH/mGbbs1wCcz\nbVCto8M7ZnJ4x0y+WBzoDumrMMy8eySZaZV19naVTaRpd55ETlZz3C4JK1HHo7gsenD/+zcr+Ps3\nK/jzOX05olNlA+wr3/8CQP6ayj7z9kLfK7btoecfPw9u//jG4zn7mR847pA2/LiqkHdnr2fZn04N\nno/NXk7QXqzEZ40TANixr4ysFiks2bSbdpmptE1PrdX5vfz9av7y2TL+eVlesKcScFAvQd7YyzdO\nthr3C/eV0c5qi1LJR0vu9XT36YeTk9WM3h0yaJ+ZFtZFMiMt+r0zOyMQ8L6+bVhw21/OqblXy9g3\n8tlaQ2nqnv9bxP88+0PwvV2vvn1foGQ+adEW/m/uRgDmrtsV9tmznwl87sdVlSXx0GPZ7OEAdsk9\n9P509J++4qKXpnP609M49alpANz/0aKYE6xF+sWa2z7WeUaGuRVb97A/yg2vtjdNVckc1LdS59Dg\nXk8Dc7OYdueIsDp3W7c2LXj03CO5Y3QvOrUMlIAuH9ItWBLu3rZFMO0FeV04pU/7KscI9eWSrcG6\n99paXbCPxZuKuO7NOcyyep7Ewx6E9dO6neSOm8iUZduCDaq7YqwjO2N14Pjbraqe16evDe4r2FNa\nbS8de5crsv4lymdKfX5GPTmVG0J66ZT6/OSv2cEhd3/Gj6u2U7S/nHOe++GAXRBlc9H+Ovdaaih2\ndhqzCuz1H9eQO25icLR4PKavKqRgT2nNCQ9SGtwb2EWDunLDSYdyZJdAVcmoPh2ipvO4XVw6pGGn\nBL7y1brdGOav38U5z/0YOMZrs4N/9LuKazeIaW3hPgb++Wv+Ne0XSn1+zvrH9/y4KryR0J6LRyQw\nu6Yv4o+9wlR2AbXHA3y/svIYAx76ivNfCKyQNfXn7Xy1ZCs/rdvF09+sqFVeG8O89bsY8shk/pO/\noamzEsa+1TRmdZDdJrSv1FdDykoX/3MG5zxX9clSBWhwbySPnnsU1w8/hME9ssK2f33bMP72q34A\ntRrOP+3Ok2qdh211LOWMiaiasfv+79ofveQeKrSHxwyr4fWbZVvJe/hrFm4s4u6I6RrsUuO8dbu4\n5OWZwUFadsB5Z9Y6jnzgC0p9fsrtfvbWP3tLfWFtEi6prMYJLRtv31v976HMV1FlArhA3kxCS9l2\nb6dENkgngn2OjVk943JVrn8QDzuPG3bGXunsYKfBvZG0bpHCH07tHbbkH8Ch7dI575guAHjjmW8A\nuDAvh5yQQVO2gbmt65/RCNHmt7FLzB/Nq7mffOj0x3/4IBDI1xYWs6ckUEIr9VXw+KRl/OZfM4DK\n+vu9Vgnuhe9WMX/9LmaHNAiX+w0/b9nLs1NWWp8JfKi4LLzU5xIJPmXYaXLHTSTv4a+ZtGgLz05Z\nGXUZxOe+XckZT39fJcB3v+uzKt0+d+4rCz5dTFq0hTdnrCVe9uWOP6BVvi4u8/HIZ0urHchWV/bX\nNES7xfz1u3j1h1+qbLdvwmVxVsv4tE2lRhrcDzAvXXoMn/3uxOD7cwd0BuD+s/pw8uGBOnm7i+V7\nvx0S9tmHxvTlsfMSO92AHchD7a9nQNkc0od6c1EJz327ih9WBkqvdokstO51zLM/8EnEgKs1hft4\n2eoJZAfu0vLwvLqkst44ssB93ZtzGP/Fcm6dMK/KeAF7aomb3v6J139cw9x1lTeW0MniSsr9DPjT\nVzz4yZLgMf/43/jn+bHbFSLn/I/kqwg/r32lPv73rbm8OHU1E2avj/v74mZlx19h2Fvq473Z6xP2\nxDLm2R948JMlVY5nPw1G+/8WTejo6Ibwfz9tqPcqbU1Ng/sB5pQjOtCnUyYLHjiFBQ+cwrBegUVN\nBnXPCg58soNCaIMsQKvmXi4c2JXnf3N0jd9zXRz97GPNb7+npObqmLqy/2QLa5iU7KZ3fgq+tgtx\nkaU+EQnWG8cKBTNW7+DkJ77jwhenc8Nbc3l80rJg983V2/dx/8eLOddqb7B9NC/Q28iu3vlkwaYq\npdzr35wT1s00Gvs6VtRQCo0MZNe9OYdvlwcCT00zh+4r9bGqlit+2d/mqzD86ZMl3PnBAmb9UnMj\n/Hc/FzAxzlHKv2zfR+64idxgPQnZPbBqCu4Fe0op91dQXhF/w2td3DphPpe/MqtBv6OhaXA/QGWm\neclM8zKmf2cWPHAKR3RqGfwjt/upp3rDL589BcJpR3YM6+kQbbWpoT2jLpYVJrJ/O0C3Ns0pKW+Y\nP6yjHviCb5cH+ljXVC8e6aN5G6s0GItUPgF8vnBzcExCNDN/2cHEhZt57tvo8+3vCLnZ3PzuPABO\neGwKAG4Rnv+2coDRpEVb+HzRlqiBasqybdw2IfB5u0ReU/WHnW7uup1sKSrhh5AG5My06APlPpm/\niStfncUR93/ByL99V6VhOh7+ChPs7hpP+8rlr8zihrfncunLM/mH1YAdq51ixN++A2DiQvtmEPgP\nW1pNcPf5Kxj456/5wwcLGqTkvrpgL9/9XFDlZlhRYXjth1+CVYWx7Ckp5/B7Jx0wJX4dxJQE7D9g\nv/VH7nEH/hDSPJXB9+1rjqV5SuXl/Pnh0yjaX44x0DY9he53fRZ2zFgDrACG9cxmyCFtmLO26mIh\nawuLq6St7j9zTlYzhvXMjjmJWajdJZV/PLUd/m4H3FAukWCw8FUYfvvvOXEda92O4irbbgipa3cJ\nYaM3RYQVIVU7171Z+T0+f0VYO8tv/z2HMn8Fvx/di39ODVQrfblkK7e8+xNPXRRYIKaouJxNRfvZ\nXLSfnNbNgw3HJeUVjHryu7BxBXaJt9Tnp6S8gpbNvDw7ZWWV1b6Wb93D9FWFXHl897B1C6Kxg7HP\nb0i1/p9UF3QjTVuxnWkrtnPTyMN44OPFvD59LWsePaPaz9iFkerq3EusPHw4dyPz1u+Kma4uVm7b\ny8lPBG44Pdunh+37ZMEmHvhkCWsKi3ng7CNiHuPdWevZX+7nqa9/ZljPpl9GVIN7EimvCC+5e91C\nj7YtuHFL5g1FAAASSElEQVTEoRx3aHhJ3Ot2hY0OPXdAZz78aWPwvV0qFwmvjw4dkXr9mzUHw9tP\n6RkzuP/nuiH06pDB3LU74wruoUp9FaSnemosLVXHJbBgQ+2DQLQJ0EJ7tFSYypInBJ4yYnXh+2TB\nJrpmtWB/mZ+P5m2kY6s01hYWc9yjk8PS/XfeJob1ymbaiu18OHdj2L7RR1SOf9hTEv492/eVUVRc\nzrgPF/D5oi2s/svpUZdxPOPpwCRz3du2YOTh4eMpfP4KKkxlFY/936HE5w+W+KM13H73cwFbivZz\nUq92MTsD2GMcNuwsruzdFEU81TKlIXlYXVDzuIUi62nDfqLNX7MDt0vYvreMUX3a8/bMdWRnpDKq\nT/uwqTZC87mruCxYcIg2WM62YMMu/vzZUutcDow5MjS4J5HRR3Tg+W9XMaJ34I9TRJj8++FxffaJ\nC/vTsVUas9fs5IkL+gUfa9NTPOwJCUyhVTG3nNyT/LU7uWN0L+58f0HU44ZWC6z6y+nc+PZcPl+0\nhfn3nxL8oxrQtXWVm0s8UjwuCKmd6dQyjU1FJYzp3ymunjpbdpfwXpx9yM8/pgvvz4m/v3nkTcee\njz/SrRPmx33MWGmrm+b43v8u4k+fLgkGxeVb99C7Q0bMFcAen7ScjDQvHTLTOOPpabx46TE8OmkZ\nCzYUBW/s9s1+7Bv5wUBXuLeMFVv30KV1c/zGkJ7qCauTTvNWDe6Tl1Xm267CisVuGwkN7iXlfvaV\n+miTnspz364kJcYNpLjMx6OfL+OO0b3IsP4/fjBnA7f/J/D7/PzmE9m+t5RLX67M76RbTuTu/wv0\n3vrq1qFhjeahN+rQth97oOKWohJG/u1b3r/+OA63FvixR3cDzFm7k81F++nYshmLNhbRspmXnKzm\nfPdzAYNysxK+0E8sGtyTSP+cVjU+3lbnjtG9g6837AxUPTRPdXPvWX2iBu9eHTKYfc/JzLcegXu2\nT+eCvBzK/BWceWQnmqW4aZFa+R/V7RKe+83R+CpMWEmuZTMvT1zYv0pwn3vvKHbsKws+Dkdq5nUz\nonc7JlvTHbdJT2VTUQnDemaHBfffDu3Bi1NXV/l8bZ4WhvRoEzW4D+6RxaKNu+v1BNHQynwVZKZ5\n2F3iY/qqwmqXdly+dQ8XvDiddhmp7Cn18eWSrSzYECi19r53En884/Bg//bQEuxjk5bx2KRldGyZ\nxuaikir/D6O1w1z1Wn5c+V9XWMwWq5qrzFfBjn1lTFm2jbdnrWPO2p388sjpPD4p9qLyb85YyxvT\nAz9rHj2Dj+dvCgZ2gNP+Pq3KZ+ypMQBGPTk1bF/oeJDQp1qvWzjhscnBvvWv/bCG7IxUzre6Moca\nP2k5T1zYPzgt96PnHsm4DxdyzoDOPHlh/+p+HQmjwf0gldUiMDPkLSf35IK8HLpmNY/5SGzX0bZp\nkco1J1ZdiOSUPu2DPWtEBK+7+sfSib87gTXbi8lqkUJWixRm3DWSwY98gwg8dt5RZKZ5g/XWoY+4\nXbOas3BjUZUqgNF9O9Avp1W1Uw9/d8dwho3/Nub+1i3CGyZTPS4ym3l5aExfzni6anCAQFvG9r0H\nxlKDdnvFQ58uiSu9HcAih/s/PHFptZ+zu7He8Z/4n0hqMnR8Zam+YG8pY9/IJz+kvcfu8hrLXz6r\nXNDmrg8XUJjAaxJaHbV+Z3HYoKkJ+YFuqM9MqTpbZ3GZP6yh3F5befaaHRz1wBdcflwut5/SK2H5\njEaD+0GqeYonrPRV3ejYIzplcuvJPbloUE7U/S9dller7z6iU8uwmSvbZaTyq2O68OtjuzKga+vg\nMn4uV/g6suN/dRQnHNaWM47sSItUN2/NWEdOVnP6dmrJ0V0rB3BNveOksIDx4f8eR+dWzYKl22jS\nU8OD+0/3jQo2UMeqK/7mtuH0e+jLWp17fTXzuus9ziDUWzNr1xZi+08tqrBq466IEctQ8w0n1Duz\nEtvvf0ZIW8tnC2P3too0afGW4IysoeybQ6wV3hJJg7uqkYhw88mHNdjxXS5hvDUFA1Q27LlEOLZ7\nG75YvJVPbzqB5ime4AIoI3q3D7Y92A7vmMnSzbvJyWoW3Pb6VYOCgX/BA6OZs3YnXbOaM+uXHewu\nKeeHlds5JDs9rAfJpFtODOt5FEt18/XHq1+XlszfEH3e/EgfXD+Efl1a8fDEpXRqlRZWYo30+c0n\nVqmOGNKjzQE31cGBrjY3lkh2A2s01fVWSxTt564aTUaah9QaBt1AoLoDYEz/zlx5fC7f3TG82kXC\nbe+OHcyse0aGlYoiu6Qd06012RmpnHFURy4e1JVnfn00t47qydFdW3H36b356d5R9O6QGfaZpy8e\nEPM7Z909kn9GPLm8eOkxTBg7OPg+dDBYZMNjq+YpYe//cs6RvHx51Seh20b15JhuWXjcLh44+wiu\nPqGyeuzEw8J7Sv39ov4c3jGT+87sw9Q7Kucgah7RkHfFcbkxz0tF9/j5RyXkOM2iNEAnmgZ31Whm\n33My8+8/pcZ0rZqnMP/+U7hl5GGICN3atKjxMxBouG2XEZha+YPrj2PW3SPjzpuIMHboIbRukVJl\nX5+OmVE+EdAuM41RfdrzwfXHBbeluF0c1aVyGcXXrxoUDKQZaV6e+XXlzeKmEYdy1fHdaWU9BRzV\npWWVropQ9SYV+qTxr4ibgd2D6aoTutO1TeUcRJF9yP9nQGfOHdA5riD/6hUDeeuaY2tMV502UX63\ntln3jOR3IyufDkfEuf7woNysmhMlkD0dSH01Ro+ZGoO7iLwiIttEJOqkGRLwtIisFJEFIlLz2Hd1\nUErzuqOOeo2mZTNvcKbAujimW+uErSJ0aLt0LhoYvb3BdnTImrget4SV0LNapHDNid0D+1zCmUd1\nCg6UaZHq4b6z+gQHpNlVUnYPjHMHdOadawfTLyf6mrspHhepnvDfaeQiMfZCMF1aV1ZXdchMo39O\nK564sD9Xn9C92nMDOKl3O9pbv89YXRKfqqYXSLc2zbnhpEOj7muR4qZdRhrtMirHZYQG+urcfcbh\ncaWz3XdmH5Y/fCq/P6VnrT739jXH8vGNx1eZ+K+u4v07qI94cvoacGo1+08DDrN+xgLP1z9bSh1Y\nxg4NVIO8duXAqPtFhEPbBQJ2bpsWYVVDXrcr2P/6tL4dATgkO5DWrnu1ux/agfOvv+rHmkfP4IkL\n+zMkyvQREHgi+NYa5/Deb4eQYfXDjgz2Fw/K4YtbhnLcIYHqm2O7Z/HtHcOD+1tFaTu46vjuYVU6\nUFnyjtYbatbdI+lgLUjTs3162NTWg3tk8d0dJwVvOif1yuZ/hx8SfD/rnpMBuCAvh3tOP5zlD59K\n/5xWdMhM46rjq7/xHNEpM2ZVSeiTwi+PnM7iB0dz1QndSfW4434ahEBvsOMObRv2NFYXoTeUeNp0\n6qvG4G6MmQpUN2vQGOANEzADaCUiHROVQaUOBD2y01nz6BkM79WOv/6qH/+IUg//1a1DmXvvqKjT\nMbds5mXWPSO5xyppjv9VP/51WR65EZO/eeNok7AN65lNp1aB0vig7ln8fnSga137zPB1a0WEXh0y\ngt1Ks1qkhJUcM0IGotnr1t53Vp+wKh37HADusL7n2O5ZzPnjycz548m0y0wLdq9t0yKV539zDJdb\ni89cMjjwr12VlNnMy52n9uadawfz+1N6BgcHpXhcXDu0R/DmNOPukdx3Vp+YTy3P/HoAXreLC/Jy\n+HHciCr7p981kosHdeWpC/sjImGrpYU+xQzMbc1z1Uy2Z9+0o5l8e+VSmX84tTfdrN/ZY+cdyYy7\nRnLnqYHf1VXHd+fGEYeRbuWhMRpUE3H76AyE9j/aYG2rMoZbRMYSKN3TtWvXBHy1Uo0v2qAVCATR\nrGrqle32AID0VA8nR1lWsR41UVw2pBsXDsyJ+chvDzizg3Q0kY3Dod1HXS4Jdp+9YGAOXrcrbMzB\nYe3S+cOpvTlnQGdat0jhwTF9eXBM3+D+yCqNvp1bxtVQ/saVg5i7fidel4t3Zq1j4sLN9O6QwZlH\ndQqmsW9yoVI8Lh45N/oU2AO6tubVKwYyoGsrWqR6ql1L4cYR4dVJD405gvs+WswLlxxDj+x0sjNS\nKdhTyvXDD+Gsfh1J8biC19qeKsS+rmP6d+Ita9qDhtaoXSGNMS8BLwHk5eXpbPvK0ebeOyruBS9G\n9G7PO7PWRV2LN14iUm1d7tDDsrn3zD5ckFf15vTlrUOj3lh+GDci6uC2aNUKIsL1w2NPJT36iPZc\nPCiHW0+uXX13y+ZeTuoVaGBN8biYuHBzsAQc6oK8LuzYV8bXS7dF3R/ppBiNtk9fPACfv4LsjFRy\nWjevcq6XDcnlsiG5wfef3nRCcK6bLq3Dn3bsy2/X0v1pTF+uObFHlem6G0IigvtGILS1qYu1TamD\nWnWl+EgPjTmCm0YcGnMK30RwuSRm42nP9tHn7s9IYH5SPW4eObd+XQnt7pzR2gkePz8wVmLF1j31\nGoNwdr9ONScK0T4zLdjYHClywXeXSxolsENigvvHwI0i8i5wLFBkjIlvxn6lFBBodI1WtaDCHdEp\nk4f/py9nHBm7We+wGDeqmjzz6wF0TvA1sBuf4xnfkWg1BncReQcYDrQVkQ3A/YAXwBjzAvAZcDqw\nEigGrmyozCqlDm4iEmygTbTQOvxEuWRwN7btKeW6aqqrGookcjX32sjLyzP5+fHNGqeUUipAROYY\nY2qc0ElHqCqllANpcFdKKQfS4K6UUg6kwV0ppRxIg7tSSjmQBnellHIgDe5KKeVAGtyVUsqBmmwQ\nk4gUAGvr+PG2wPYEZicZ6DkfHPScDw71OeduxpjsmhI1WXCvDxHJj2eElpPoOR8c9JwPDo1xzlot\no5RSDqTBXSmlHChZg/tLTZ2BJqDnfHDQcz44NPg5J2Wdu1JKqeola8ldKaVUNZIuuIvIqSKyXERW\nisi4ps5PoohIjohMEZElIrJYRG62tmeJyFcissL6t7W1XUTkaev3sEBEYi/ffgATEbeI/CQin1rv\nu4vITOu8JohIirU91Xq/0tqf25T5rg8RaSUi74vIMhFZKiJDnHydReRW6//0IhF5R0TSnHidReQV\nEdkmIotCttX6uorI5Vb6FSJyeV3zk1TBXUTcwLPAaUAf4GIR6dO0uUoYH3C7MaYPMBi4wTq3ccA3\nxpjDgG+s9xD4HRxm/YwFnm/8LCfEzcDSkPePAU8aYw4FdgJXW9uvBnZa25+00iWrvwOTjDG9gX4E\nzt+R11lEOgO/A/KMMX0BN3ARzrzOrwGnRmyr1XUVkSwCq90dCwwC7rdvCLVmjEmaH2AI8EXI+7uA\nu5o6Xw10rh8Bo4DlQEdrW0dgufX6ReDikPTBdMnyQ2Ax9W+AEcCngBAY2OGJvN7AF8AQ67XHSidN\nfQ51OOeWwC+ReXfqdQY6A+uBLOu6fQqMdup1BnKBRXW9rsDFwIsh28PS1eYnqUruVP5HsW2wtjmK\n9Sg6AJgJtDeVC45vAdpbr53wu3gKuBOosN63AXYZY3zW+9BzCp6vtb/ISp9sugMFwKtWddS/RKQF\nDr3OxpiNwF+BdcBmAtdtDs6/zrbaXteEXe9kC+6OJyLpwAfALcaY3aH7TOBW7ojuTSJyJrDNGDOn\nqfPSyDzA0cDzxpgBwD4qH9UBx13n1sAYAje1TkALqlZdHBQa+7omW3DfCOSEvO9ibXMEEfESCOxv\nGWM+tDZvFZGO1v6OwDZre7L/Lo4HzhaRNcC7BKpm/g60EhGPlSb0nILna+1vCRQ2ZoYTZAOwwRgz\n03r/PoFg79TrfDLwizGmwBhTDnxI4No7/TrbantdE3a9ky24zwYOs1raUwg0zHzcxHlKCBER4GVg\nqTHmiZBdHwN2i/nlBOri7e2XWa3ug4GikMe/A54x5i5jTBdjTC6B6zjZGPMbYApwvpUs8nzt38P5\nVvqkK90aY7YA60Wkl7VpJLAEh15nAtUxg0WkufV/3D5fR1/nELW9rl8Ap4hIa+up5xRrW+01dQNE\nHRosTgd+BlYB9zR1fhJ4XicQeGRbAMyzfk4nUN/4DbAC+BrIstILgZ5Dq4CFBHojNPl51PHchwOf\nWq97ALOAlcB/gFRre5r1fqW1v0dT57se59sfyLeu9X+B1k6+zsCDwDJgEfBvINWJ1xl4h0C7QjmB\nJ7Sr63Jdgaus818JXFnX/OgIVaWUcqBkq5ZRSikVBw3uSinlQBrclVLKgTS4K6WUA2lwV0opB9Lg\nrpRSDqTBXSmlHEiDu1JKOdD/A6LnZTpxoHcTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f138cf4cbe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "from random import sample\n",
    "\n",
    "s.run(tf.global_variables_initializer())\n",
    "\n",
    "batch_size = 32\n",
    "history = []\n",
    "\n",
    "for i in range(1000):\n",
    "    batch = to_matrix(sample(names, batch_size), max_len=MAX_LENGTH)\n",
    "    loss_i, _ = s.run([loss, optimize], {input_sequence: batch})\n",
    "    \n",
    "    history.append(loss_i)\n",
    "    \n",
    "    if (i + 1) % 100 == 0:\n",
    "        clear_output(True)\n",
    "        plt.plot(history, label='loss')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "\n",
    "assert np.mean(history[:10]) > np.mean(history[-10:]), \"RNN didn't converge\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN: sampling\n",
    "Once we've trained our network a bit, let's get to actually generating stuff. All we need is the `rnn_one_step` function you have written above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:55.341196Z",
     "start_time": "2018-08-13T20:26:55.323787Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_t = tf.placeholder(tf.int32, (1,))\n",
    "h_t = tf.Variable(np.zeros([1, rnn_num_units], np.float32))  # we will update hidden state in this variable\n",
    "\n",
    "# For sampling we need to define `rnn_one_step` tensors only once in our graph.\n",
    "# We reuse all parameters thanks to functional API usage.\n",
    "# Then we can feed appropriate tensor values using feed_dict in a loop.\n",
    "# Note how different it is from training stage, where we had to unroll the whole sequence for backprop.\n",
    "next_probs, next_h = rnn_one_step(x_t, h_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:55.346422Z",
     "start_time": "2018-08-13T20:26:55.342659Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_sample(seed_phrase=start_token, max_length=MAX_LENGTH):\n",
    "    '''\n",
    "    This function generates text given a `seed_phrase` as a seed.\n",
    "    Remember to include start_token in seed phrase!\n",
    "    Parameter `max_length` is used to set the number of characters in prediction.\n",
    "    '''\n",
    "    x_sequence = [token_to_id[token] for token in seed_phrase]\n",
    "    s.run(tf.assign(h_t, h_t.initial_value))\n",
    "    \n",
    "    # feed the seed phrase, if any\n",
    "    for ix in x_sequence[:-1]:\n",
    "         s.run(tf.assign(h_t, next_h), {x_t: [ix]})\n",
    "    \n",
    "    # start generating\n",
    "    for _ in range(max_length-len(seed_phrase)):\n",
    "        x_probs,_ = s.run([next_probs, tf.assign(h_t, next_h)], {x_t: [x_sequence[-1]]})\n",
    "        x_sequence.append(np.random.choice(n_tokens, p=x_probs[0]))\n",
    "        \n",
    "    return ''.join([tokens[ix] for ix in x_sequence if tokens[ix] != pad_token])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:26:58.458115Z",
     "start_time": "2018-08-13T20:26:55.347900Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Brin           \n",
      " Makna          \n",
      " Warla          \n",
      " Jabole         \n",
      " Toxiy          \n",
      " Arthorlin      \n",
      " Neteyne        \n",
      " Kerenan        \n",
      " Kamied         \n",
      " Gerneon        \n"
     ]
    }
   ],
   "source": [
    "# without prefix\n",
    "for _ in range(10):\n",
    "    print(generate_sample())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:27:01.986726Z",
     "start_time": "2018-08-13T20:26:58.459810Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Trumpa         \n",
      " Trumpaod       \n",
      " Trump          \n",
      " Trumpyi        \n",
      " Trumpenun      \n",
      " Trumpdi        \n",
      " Trumpina       \n",
      " Trumpy         \n",
      " Trump          \n",
      " Trumpiltia     \n"
     ]
    }
   ],
   "source": [
    "# with prefix conditioning\n",
    "for _ in range(10):\n",
    "    print(generate_sample(' Trump'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submit to Coursera"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:40:02.004926Z",
     "start_time": "2018-08-13T20:40:02.000821Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# token expires every 30 min\n",
    "COURSERA_TOKEN = \"ml6H0a3mBrhBUilI\" #YOUR TOKEN HERE ###\"\n",
    "COURSERA_EMAIL = \"ravirk17c@iiitt.ac.in\" ### YOUR EMAIL HERE ###\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:40:18.923357Z",
     "start_time": "2018-08-13T20:40:03.549343Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71dda7ef3689449094073f7f3440fab5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Submitted to Coursera platform. See results on assignment page!\n"
     ]
    }
   ],
   "source": [
    "from submit import submit_char_rnn\n",
    "samples = [generate_sample(' Al') for i in tqdm_utils.tqdm_notebook_failsafe(range(25))]\n",
    "submission = (history, samples)\n",
    "submit_char_rnn(submission, COURSERA_EMAIL, COURSERA_TOKEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Try it out!\n",
    "\n",
    "__Disclaimer:__ This part of assignment is entirely optional. You won't receive bonus points for it. However, it's a fun thing to do. Please share your results on course forums.\n",
    "\n",
    "You've just implemented a recurrent language model that can be tasked with generating any kind of sequence, so there's plenty of data you can try it on:\n",
    "\n",
    "* Novels/poems/songs of your favorite author\n",
    "* News titles/clickbait titles\n",
    "* Source code of Linux or Tensorflow\n",
    "* Molecules in [smiles](https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system) format\n",
    "* Melody in notes/chords format\n",
    "* IKEA catalog titles\n",
    "* Pokemon names\n",
    "* Cards from Magic, the Gathering / Hearthstone\n",
    "\n",
    "If you're willing to give it a try, here's what you wanna look at:\n",
    "* Current data format is a sequence of lines, so a novel can be formatted as a list of sentences. Alternatively, you can change data preprocessing altogether.\n",
    "* While some datasets are readily available, others can only be scraped from the web. Try `Selenium` or `Scrapy` for that.\n",
    "* Make sure MAX_LENGTH is adjusted for longer datasets. There's also a bonus section about dynamic RNNs at the bottom.\n",
    "* More complex tasks require larger RNN architecture, try more neurons or several layers. It would also require more training iterations.\n",
    "* Long-term dependencies in music, novels or molecules are better handled with LSTM or GRU\n",
    "\n",
    "__Good hunting!__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Bonus level: dynamic RNNs\n",
    "\n",
    "Apart from Keras, there's also a friendly TensorFlow API for recurrent neural nets. It's based around the symbolic loop function (aka [tf.scan](https://www.tensorflow.org/api_docs/python/tf/scan)).\n",
    "\n",
    "RNN loop that we implemented for training can be replaced with single TensorFlow instruction: [tf.nn.dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn).\n",
    "This interface allows for dynamic sequence length and comes with some pre-implemented architectures.\n",
    "\n",
    "Take a look at [tf.nn.rnn_cell.BasicRNNCell](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/BasicRNNCell)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:27:12.975354Z",
     "start_time": "2018-08-13T20:27:12.737529Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM outputs for each step [batch,time,n_tokens]:\n",
      "(10, 50, 55)\n"
     ]
    }
   ],
   "source": [
    "class CustomRNN(tf.nn.rnn_cell.BasicRNNCell):\n",
    "    def call(self, input, state):\n",
    "        # from docs:\n",
    "        # Returns:\n",
    "        # Output: A 2-D tensor with shape [batch_size, self.output_size].\n",
    "        # New state: Either a single 2-D tensor, or a tuple of tensors matching the arity and shapes of state.\n",
    "        return rnn_one_step(input[:, 0], state)\n",
    "    \n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return n_tokens\n",
    "    \n",
    "cell = CustomRNN(rnn_num_units)\n",
    "\n",
    "input_sequence = tf.placeholder(tf.int32, (None, None))\n",
    "    \n",
    "predicted_probas, last_state = tf.nn.dynamic_rnn(cell, input_sequence[:, :, None], dtype=tf.float32)\n",
    "\n",
    "print('LSTM outputs for each step [batch,time,n_tokens]:')\n",
    "print(predicted_probas.eval({input_sequence: to_matrix(names[:10], max_len=50)}).shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that we never used MAX_LENGTH in the code above: TF will iterate over however many time-steps you gave it.\n",
    "\n",
    "You can also use any pre-implemented RNN cell:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:27:12.981697Z",
     "start_time": "2018-08-13T20:27:12.977590Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BasicLSTMCell\tBasicRNNCell\tGRUCell\tLSTMCell\tMultiRNNCell\tRNNCell\tBasicLSTMCell\tBasicRNNCell\tBidirectionalGridLSTMCell\tCoupledInputForgetGateLSTMCell\tFusedRNNCell\tGLSTMCell\tGRUBlockCell\tGRUCell\tGridLSTMCell\tIntersectionRNNCell\tLSTMBlockCell\tLSTMBlockFusedCell\tLSTMCell\tLayerNormBasicLSTMCell\tMultiRNNCell\tNASCell\tPhasedLSTMCell\tRNNCell\tTimeFreqLSTMCell\tUGRNNCell\t"
     ]
    }
   ],
   "source": [
    "for obj in dir(tf.nn.rnn_cell) + dir(tf.contrib.rnn):\n",
    "    if obj.endswith('Cell'):\n",
    "        print(obj, end=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-13T20:27:13.168207Z",
     "start_time": "2018-08-13T20:27:12.986884Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM hidden state for each step [batch,time,rnn_num_units]:\n",
      "(10, 50, 64)\n"
     ]
    }
   ],
   "source": [
    "input_sequence = tf.placeholder(tf.int32, (None, None))\n",
    "\n",
    "inputs_embedded = embed_x(input_sequence)\n",
    "\n",
    "# standard cell returns hidden state as output!\n",
    "cell = tf.nn.rnn_cell.LSTMCell(rnn_num_units)\n",
    "\n",
    "state_sequence, last_state = tf.nn.dynamic_rnn(cell, inputs_embedded, dtype=tf.float32)\n",
    "\n",
    "s.run(tf.global_variables_initializer())\n",
    "\n",
    "print('LSTM hidden state for each step [batch,time,rnn_num_units]:')\n",
    "print(state_sequence.eval({input_sequence: to_matrix(names[:10], max_len=50)}).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}