training_tft.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:20.062715Z",
     "start_time": "2020-04-22T03:09:18.795699Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pyunpack\n",
    "import math\n",
    "import json\n",
    "\n",
    "from data.data_download import Config, download_electricity\n",
    "from data_formatters.electricity import ElectricityFormatter\n",
    "from data_formatters.base import DataTypes, InputTypes\n",
    "\n",
    "from data.custom_dataset import TFTDataset\n",
    "from models import GatedLinearUnit\n",
    "from models import GateAddNormNetwork\n",
    "from models import GatedResidualNetwork \n",
    "from models import ScaledDotProductAttention\n",
    "from models import InterpretableMultiHeadAttention\n",
    "from models import VariableSelectionNetwork\n",
    "\n",
    "from quantile_loss import QuantileLossCalculator\n",
    "from quantile_loss import NormalizedQuantileLossCalculator\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch import nn\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.callbacks import EarlyStopping\n",
    "from argparse import ArgumentParser\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-09T16:47:54.934322Z",
     "start_time": "2020-03-09T16:47:54.932271Z"
    }
   },
   "outputs": [],
   "source": [
    "config = Config('data','data/electricity.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-09T16:52:20.011881Z",
     "start_time": "2020-03-09T16:47:57.762444Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pulling data from https://archive.ics.uci.edu/ml/machine-learning-databases/00321/LD2011_2014.txt.zip to data/LD2011_2014.txt.zip\n",
      "done\n",
      "Unzipping file: data/LD2011_2014.txt.zip\n",
      "Done.\n",
      "Aggregating to hourly data\n",
      "Processing MT_001\n",
      "Processing MT_002\n",
      "Processing MT_003\n",
      "Processing MT_004\n",
      "Processing MT_005\n",
      "Processing MT_006\n",
      "Processing MT_007\n",
      "Processing MT_008\n",
      "Processing MT_009\n",
      "Processing MT_010\n",
      "Processing MT_011\n",
      "Processing MT_012\n",
      "Processing MT_013\n",
      "Processing MT_014\n",
      "Processing MT_015\n",
      "Processing MT_016\n",
      "Processing MT_017\n",
      "Processing MT_018\n",
      "Processing MT_019\n",
      "Processing MT_020\n",
      "Processing MT_021\n",
      "Processing MT_022\n",
      "Processing MT_023\n",
      "Processing MT_024\n",
      "Processing MT_025\n",
      "Processing MT_026\n",
      "Processing MT_027\n",
      "Processing MT_028\n",
      "Processing MT_029\n",
      "Processing MT_030\n",
      "Processing MT_031\n",
      "Processing MT_032\n",
      "Processing MT_033\n",
      "Processing MT_034\n",
      "Processing MT_035\n",
      "Processing MT_036\n",
      "Processing MT_037\n",
      "Processing MT_038\n",
      "Processing MT_039\n",
      "Processing MT_040\n",
      "Processing MT_041\n",
      "Processing MT_042\n",
      "Processing MT_043\n",
      "Processing MT_044\n",
      "Processing MT_045\n",
      "Processing MT_046\n",
      "Processing MT_047\n",
      "Processing MT_048\n",
      "Processing MT_049\n",
      "Processing MT_050\n",
      "Processing MT_051\n",
      "Processing MT_052\n",
      "Processing MT_053\n",
      "Processing MT_054\n",
      "Processing MT_055\n",
      "Processing MT_056\n",
      "Processing MT_057\n",
      "Processing MT_058\n",
      "Processing MT_059\n",
      "Processing MT_060\n",
      "Processing MT_061\n",
      "Processing MT_062\n",
      "Processing MT_063\n",
      "Processing MT_064\n",
      "Processing MT_065\n",
      "Processing MT_066\n",
      "Processing MT_067\n",
      "Processing MT_068\n",
      "Processing MT_069\n",
      "Processing MT_070\n",
      "Processing MT_071\n",
      "Processing MT_072\n",
      "Processing MT_073\n",
      "Processing MT_074\n",
      "Processing MT_075\n",
      "Processing MT_076\n",
      "Processing MT_077\n",
      "Processing MT_078\n",
      "Processing MT_079\n",
      "Processing MT_080\n",
      "Processing MT_081\n",
      "Processing MT_082\n",
      "Processing MT_083\n",
      "Processing MT_084\n",
      "Processing MT_085\n",
      "Processing MT_086\n",
      "Processing MT_087\n",
      "Processing MT_088\n",
      "Processing MT_089\n",
      "Processing MT_090\n",
      "Processing MT_091\n",
      "Processing MT_092\n",
      "Processing MT_093\n",
      "Processing MT_094\n",
      "Processing MT_095\n",
      "Processing MT_096\n",
      "Processing MT_097\n",
      "Processing MT_098\n",
      "Processing MT_099\n",
      "Processing MT_100\n",
      "Processing MT_101\n",
      "Processing MT_102\n",
      "Processing MT_103\n",
      "Processing MT_104\n",
      "Processing MT_105\n",
      "Processing MT_106\n",
      "Processing MT_107\n",
      "Processing MT_108\n",
      "Processing MT_109\n",
      "Processing MT_110\n",
      "Processing MT_111\n",
      "Processing MT_112\n",
      "Processing MT_113\n",
      "Processing MT_114\n",
      "Processing MT_115\n",
      "Processing MT_116\n",
      "Processing MT_117\n",
      "Processing MT_118\n",
      "Processing MT_119\n",
      "Processing MT_120\n",
      "Processing MT_121\n",
      "Processing MT_122\n",
      "Processing MT_123\n",
      "Processing MT_124\n",
      "Processing MT_125\n",
      "Processing MT_126\n",
      "Processing MT_127\n",
      "Processing MT_128\n",
      "Processing MT_129\n",
      "Processing MT_130\n",
      "Processing MT_131\n",
      "Processing MT_132\n",
      "Processing MT_133\n",
      "Processing MT_134\n",
      "Processing MT_135\n",
      "Processing MT_136\n",
      "Processing MT_137\n",
      "Processing MT_138\n",
      "Processing MT_139\n",
      "Processing MT_140\n",
      "Processing MT_141\n",
      "Processing MT_142\n",
      "Processing MT_143\n",
      "Processing MT_144\n",
      "Processing MT_145\n",
      "Processing MT_146\n",
      "Processing MT_147\n",
      "Processing MT_148\n",
      "Processing MT_149\n",
      "Processing MT_150\n",
      "Processing MT_151\n",
      "Processing MT_152\n",
      "Processing MT_153\n",
      "Processing MT_154\n",
      "Processing MT_155\n",
      "Processing MT_156\n",
      "Processing MT_157\n",
      "Processing MT_158\n",
      "Processing MT_159\n",
      "Processing MT_160\n",
      "Processing MT_161\n",
      "Processing MT_162\n",
      "Processing MT_163\n",
      "Processing MT_164\n",
      "Processing MT_165\n",
      "Processing MT_166\n",
      "Processing MT_167\n",
      "Processing MT_168\n",
      "Processing MT_169\n",
      "Processing MT_170\n",
      "Processing MT_171\n",
      "Processing MT_172\n",
      "Processing MT_173\n",
      "Processing MT_174\n",
      "Processing MT_175\n",
      "Processing MT_176\n",
      "Processing MT_177\n",
      "Processing MT_178\n",
      "Processing MT_179\n",
      "Processing MT_180\n",
      "Processing MT_181\n",
      "Processing MT_182\n",
      "Processing MT_183\n",
      "Processing MT_184\n",
      "Processing MT_185\n",
      "Processing MT_186\n",
      "Processing MT_187\n",
      "Processing MT_188\n",
      "Processing MT_189\n",
      "Processing MT_190\n",
      "Processing MT_191\n",
      "Processing MT_192\n",
      "Processing MT_193\n",
      "Processing MT_194\n",
      "Processing MT_195\n",
      "Processing MT_196\n",
      "Processing MT_197\n",
      "Processing MT_198\n",
      "Processing MT_199\n",
      "Processing MT_200\n",
      "Processing MT_201\n",
      "Processing MT_202\n",
      "Processing MT_203\n",
      "Processing MT_204\n",
      "Processing MT_205\n",
      "Processing MT_206\n",
      "Processing MT_207\n",
      "Processing MT_208\n",
      "Processing MT_209\n",
      "Processing MT_210\n",
      "Processing MT_211\n",
      "Processing MT_212\n",
      "Processing MT_213\n",
      "Processing MT_214\n",
      "Processing MT_215\n",
      "Processing MT_216\n",
      "Processing MT_217\n",
      "Processing MT_218\n",
      "Processing MT_219\n",
      "Processing MT_220\n",
      "Processing MT_221\n",
      "Processing MT_222\n",
      "Processing MT_223\n",
      "Processing MT_224\n",
      "Processing MT_225\n",
      "Processing MT_226\n",
      "Processing MT_227\n",
      "Processing MT_228\n",
      "Processing MT_229\n",
      "Processing MT_230\n",
      "Processing MT_231\n",
      "Processing MT_232\n",
      "Processing MT_233\n",
      "Processing MT_234\n",
      "Processing MT_235\n",
      "Processing MT_236\n",
      "Processing MT_237\n",
      "Processing MT_238\n",
      "Processing MT_239\n",
      "Processing MT_240\n",
      "Processing MT_241\n",
      "Processing MT_242\n",
      "Processing MT_243\n",
      "Processing MT_244\n",
      "Processing MT_245\n",
      "Processing MT_246\n",
      "Processing MT_247\n",
      "Processing MT_248\n",
      "Processing MT_249\n",
      "Processing MT_250\n",
      "Processing MT_251\n",
      "Processing MT_252\n",
      "Processing MT_253\n",
      "Processing MT_254\n",
      "Processing MT_255\n",
      "Processing MT_256\n",
      "Processing MT_257\n",
      "Processing MT_258\n",
      "Processing MT_259\n",
      "Processing MT_260\n",
      "Processing MT_261\n",
      "Processing MT_262\n",
      "Processing MT_263\n",
      "Processing MT_264\n",
      "Processing MT_265\n",
      "Processing MT_266\n",
      "Processing MT_267\n",
      "Processing MT_268\n",
      "Processing MT_269\n",
      "Processing MT_270\n",
      "Processing MT_271\n",
      "Processing MT_272\n",
      "Processing MT_273\n",
      "Processing MT_274\n",
      "Processing MT_275\n",
      "Processing MT_276\n",
      "Processing MT_277\n",
      "Processing MT_278\n",
      "Processing MT_279\n",
      "Processing MT_280\n",
      "Processing MT_281\n",
      "Processing MT_282\n",
      "Processing MT_283\n",
      "Processing MT_284\n",
      "Processing MT_285\n",
      "Processing MT_286\n",
      "Processing MT_287\n",
      "Processing MT_288\n",
      "Processing MT_289\n",
      "Processing MT_290\n",
      "Processing MT_291\n",
      "Processing MT_292\n",
      "Processing MT_293\n",
      "Processing MT_294\n",
      "Processing MT_295\n",
      "Processing MT_296\n",
      "Processing MT_297\n",
      "Processing MT_298\n",
      "Processing MT_299\n",
      "Processing MT_300\n",
      "Processing MT_301\n",
      "Processing MT_302\n",
      "Processing MT_303\n",
      "Processing MT_304\n",
      "Processing MT_305\n",
      "Processing MT_306\n",
      "Processing MT_307\n",
      "Processing MT_308\n",
      "Processing MT_309\n",
      "Processing MT_310\n",
      "Processing MT_311\n",
      "Processing MT_312\n",
      "Processing MT_313\n",
      "Processing MT_314\n",
      "Processing MT_315\n",
      "Processing MT_316\n",
      "Processing MT_317\n",
      "Processing MT_318\n",
      "Processing MT_319\n",
      "Processing MT_320\n",
      "Processing MT_321\n",
      "Processing MT_322\n",
      "Processing MT_323\n",
      "Processing MT_324\n",
      "Processing MT_325\n",
      "Processing MT_326\n",
      "Processing MT_327\n",
      "Processing MT_328\n",
      "Processing MT_329\n",
      "Processing MT_330\n",
      "Processing MT_331\n",
      "Processing MT_332\n",
      "Processing MT_333\n",
      "Processing MT_334\n",
      "Processing MT_335\n",
      "Processing MT_336\n",
      "Processing MT_337\n",
      "Processing MT_338\n",
      "Processing MT_339\n",
      "Processing MT_340\n",
      "Processing MT_341\n",
      "Processing MT_342\n",
      "Processing MT_343\n",
      "Processing MT_344\n",
      "Processing MT_345\n",
      "Processing MT_346\n",
      "Processing MT_347\n",
      "Processing MT_348\n",
      "Processing MT_349\n",
      "Processing MT_350\n",
      "Processing MT_351\n",
      "Processing MT_352\n",
      "Processing MT_353\n",
      "Processing MT_354\n",
      "Processing MT_355\n",
      "Processing MT_356\n",
      "Processing MT_357\n",
      "Processing MT_358\n",
      "Processing MT_359\n",
      "Processing MT_360\n",
      "Processing MT_361\n",
      "Processing MT_362\n",
      "Processing MT_363\n",
      "Processing MT_364\n",
      "Processing MT_365\n",
      "Processing MT_366\n",
      "Processing MT_367\n",
      "Processing MT_368\n",
      "Processing MT_369\n",
      "Processing MT_370\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "download_electricity(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:29.304537Z",
     "start_time": "2020-04-22T03:09:20.960449Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/numpy/lib/arraysetops.py:569: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Formatting train-valid-test splits.\n",
      "Setting scalers with training data...\n"
     ]
    }
   ],
   "source": [
    "electricity = pd.read_csv('data/electricity.csv', index_col = 0)\n",
    "data_formatter = ElectricityFormatter()\n",
    "train, valid, test = data_formatter.split_data(electricity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:56:55.662935Z",
     "start_time": "2020-04-22T02:56:55.659314Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1923536, 13), (274536, 13), (123984, 13))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, valid.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:56:55.700490Z",
     "start_time": "2020-04-22T02:56:55.664219Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>days_from_start</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>1096</td>\n",
       "      <td>8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>1097</td>\n",
       "      <td>8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>1098</td>\n",
       "      <td>8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>1099</td>\n",
       "      <td>8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>1100</td>\n",
       "      <td>8400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1310</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1311</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1312</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1313</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1314</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>219 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index  days_from_start\n",
       "218   1096             8400\n",
       "206   1097             8400\n",
       "208   1098             8400\n",
       "209   1099             8400\n",
       "210   1100             8400\n",
       "..     ...              ...\n",
       "17    1310             8856\n",
       "18    1311             8856\n",
       "20    1312             8856\n",
       "2     1313             8856\n",
       "0     1314             8856\n",
       "\n",
       "[219 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.days_from_start.value_counts().to_frame().reset_index().sort_values(by=['index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:56:55.714718Z",
     "start_time": "2020-04-22T02:56:55.702319Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>days_from_start</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1308</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1309</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1310</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1311</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1312</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1313</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1314</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1315</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1316</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1317</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1318</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1319</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1320</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1321</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1322</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1323</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1324</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1325</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1326</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1327</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1328</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1329</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1330</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1331</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1332</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1333</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1334</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1335</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1336</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1337</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1338</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  days_from_start\n",
       "30   1308             8856\n",
       "2    1309             8856\n",
       "3    1310             8856\n",
       "4    1311             8856\n",
       "5    1312             8856\n",
       "6    1313             8856\n",
       "7    1314             8856\n",
       "8    1315             8856\n",
       "9    1316             8856\n",
       "10   1317             8856\n",
       "11   1318             8856\n",
       "12   1319             8856\n",
       "13   1320             8856\n",
       "14   1321             8856\n",
       "1    1322             8856\n",
       "15   1323             8856\n",
       "17   1324             8856\n",
       "18   1325             8856\n",
       "19   1326             8856\n",
       "20   1327             8856\n",
       "21   1328             8856\n",
       "22   1329             8856\n",
       "23   1330             8856\n",
       "24   1331             8856\n",
       "25   1332             8856\n",
       "26   1333             8856\n",
       "27   1334             8856\n",
       "28   1335             8856\n",
       "29   1336             8856\n",
       "16   1337             8856\n",
       "0    1338             8856"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.days_from_start.value_counts().to_frame().reset_index().sort_values(by=['index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:56:55.725026Z",
     "start_time": "2020-04-22T02:56:55.715893Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>days_from_start</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1332</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1333</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1334</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1335</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1336</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1337</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1338</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1339</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1340</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1341</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1342</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1343</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1344</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1345</td>\n",
       "      <td>8856</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index  days_from_start\n",
       "11   1332             8856\n",
       "10   1333             8856\n",
       "9    1334             8856\n",
       "8    1335             8856\n",
       "7    1336             8856\n",
       "6    1337             8856\n",
       "5    1338             8856\n",
       "4    1339             8856\n",
       "3    1340             8856\n",
       "2    1341             8856\n",
       "1    1342             8856\n",
       "0    1343             8856\n",
       "13   1344             8856\n",
       "12   1345             8856"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.days_from_start.value_counts().to_frame().reset_index().sort_values(by=['index'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reviewing Test dataset error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:20.633823Z",
     "start_time": "2020-03-29T01:17:20.591206Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power_usage</th>\n",
       "      <th>t</th>\n",
       "      <th>days_from_start</th>\n",
       "      <th>categorical_id</th>\n",
       "      <th>date</th>\n",
       "      <th>id</th>\n",
       "      <th>hour</th>\n",
       "      <th>day</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>month</th>\n",
       "      <th>hours_from_start</th>\n",
       "      <th>categorical_day_of_week</th>\n",
       "      <th>categorical_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.313606</td>\n",
       "      <td>31968.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 00:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.661325</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.001283</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.160683</td>\n",
       "      <td>31969.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 01:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.516862</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.001942</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.160683</td>\n",
       "      <td>31970.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 02:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.372399</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.002601</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.313606</td>\n",
       "      <td>31971.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 03:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.227936</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.003260</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.237144</td>\n",
       "      <td>31972.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 04:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.083473</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.003920</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>0.929284</td>\n",
       "      <td>32299.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>368</td>\n",
       "      <td>2014-09-07 19:00:00</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>1.083473</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.219437</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>0.579135</td>\n",
       "      <td>32300.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>368</td>\n",
       "      <td>2014-09-07 20:00:00</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>1.227936</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.220096</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>0.761504</td>\n",
       "      <td>32301.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>368</td>\n",
       "      <td>2014-09-07 21:00:00</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>1.372399</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.220755</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>0.666672</td>\n",
       "      <td>32302.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>368</td>\n",
       "      <td>2014-09-07 22:00:00</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>1.516862</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.221415</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>0.743267</td>\n",
       "      <td>32303.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>368</td>\n",
       "      <td>2014-09-07 23:00:00</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>1.661325</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.222074</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        power_usage        t  days_from_start  categorical_id  \\\n",
       "0          3.313606  31968.0             1332               0   \n",
       "1          3.160683  31969.0             1332               0   \n",
       "2          3.160683  31970.0             1332               0   \n",
       "3          3.313606  31971.0             1332               0   \n",
       "4          3.237144  31972.0             1332               0   \n",
       "...             ...      ...              ...             ...   \n",
       "123979     0.929284  32299.0             1345             368   \n",
       "123980     0.579135  32300.0             1345             368   \n",
       "123981     0.761504  32301.0             1345             368   \n",
       "123982     0.666672  32302.0             1345             368   \n",
       "123983     0.743267  32303.0             1345             368   \n",
       "\n",
       "                       date      id      hour  day  day_of_week  month  \\\n",
       "0       2014-08-25 00:00:00  MT_001 -1.661325   25    -1.503741      8   \n",
       "1       2014-08-25 01:00:00  MT_001 -1.516862   25    -1.503741      8   \n",
       "2       2014-08-25 02:00:00  MT_001 -1.372399   25    -1.503741      8   \n",
       "3       2014-08-25 03:00:00  MT_001 -1.227936   25    -1.503741      8   \n",
       "4       2014-08-25 04:00:00  MT_001 -1.083473   25    -1.503741      8   \n",
       "...                     ...     ...       ...  ...          ...    ...   \n",
       "123979  2014-09-07 19:00:00  MT_370  1.083473    7     1.508326      9   \n",
       "123980  2014-09-07 20:00:00  MT_370  1.227936    7     1.508326      9   \n",
       "123981  2014-09-07 21:00:00  MT_370  1.372399    7     1.508326      9   \n",
       "123982  2014-09-07 22:00:00  MT_370  1.516862    7     1.508326      9   \n",
       "123983  2014-09-07 23:00:00  MT_370  1.661325    7     1.508326      9   \n",
       "\n",
       "        hours_from_start  categorical_day_of_week  categorical_hour  \n",
       "0               2.001283                        0                 0  \n",
       "1               2.001942                        0                 1  \n",
       "2               2.002601                        0                 2  \n",
       "3               2.003260                        0                 3  \n",
       "4               2.003920                        0                 4  \n",
       "...                  ...                      ...               ...  \n",
       "123979          2.219437                        6                19  \n",
       "123980          2.220096                        6                20  \n",
       "123981          2.220755                        6                21  \n",
       "123982          2.221415                        6                22  \n",
       "123983          2.222074                        6                23  \n",
       "\n",
       "[123984 rows x 13 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = test.reset_index(drop=True)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:21.707585Z",
     "start_time": "2020-03-29T01:17:21.674517Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power_usage</th>\n",
       "      <th>t</th>\n",
       "      <th>days_from_start</th>\n",
       "      <th>categorical_id</th>\n",
       "      <th>date</th>\n",
       "      <th>id</th>\n",
       "      <th>hour</th>\n",
       "      <th>day</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>month</th>\n",
       "      <th>hours_from_start</th>\n",
       "      <th>categorical_day_of_week</th>\n",
       "      <th>categorical_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.313606</td>\n",
       "      <td>31968.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 00:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.661325</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.001283</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.160683</td>\n",
       "      <td>31969.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 01:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.516862</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.001942</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.160683</td>\n",
       "      <td>31970.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 02:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.372399</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.002601</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.313606</td>\n",
       "      <td>31971.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 03:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.227936</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.003260</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.237144</td>\n",
       "      <td>31972.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-08-25 04:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>-1.083473</td>\n",
       "      <td>25</td>\n",
       "      <td>-1.503741</td>\n",
       "      <td>8</td>\n",
       "      <td>2.003920</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>3.237144</td>\n",
       "      <td>32299.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-09-07 19:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>1.083473</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.219437</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>0.025749</td>\n",
       "      <td>32300.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-09-07 20:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>1.227936</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.220096</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>1.325600</td>\n",
       "      <td>32301.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-09-07 21:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>1.372399</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.220755</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>3.237144</td>\n",
       "      <td>32302.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-09-07 22:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>1.516862</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.221415</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>3.237144</td>\n",
       "      <td>32303.0</td>\n",
       "      <td>1345</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-09-07 23:00:00</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>1.661325</td>\n",
       "      <td>7</td>\n",
       "      <td>1.508326</td>\n",
       "      <td>9</td>\n",
       "      <td>2.222074</td>\n",
       "      <td>6</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>336 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     power_usage        t  days_from_start  categorical_id  \\\n",
       "0       3.313606  31968.0             1332               0   \n",
       "1       3.160683  31969.0             1332               0   \n",
       "2       3.160683  31970.0             1332               0   \n",
       "3       3.313606  31971.0             1332               0   \n",
       "4       3.237144  31972.0             1332               0   \n",
       "..           ...      ...              ...             ...   \n",
       "331     3.237144  32299.0             1345               0   \n",
       "332     0.025749  32300.0             1345               0   \n",
       "333     1.325600  32301.0             1345               0   \n",
       "334     3.237144  32302.0             1345               0   \n",
       "335     3.237144  32303.0             1345               0   \n",
       "\n",
       "                    date      id      hour  day  day_of_week  month  \\\n",
       "0    2014-08-25 00:00:00  MT_001 -1.661325   25    -1.503741      8   \n",
       "1    2014-08-25 01:00:00  MT_001 -1.516862   25    -1.503741      8   \n",
       "2    2014-08-25 02:00:00  MT_001 -1.372399   25    -1.503741      8   \n",
       "3    2014-08-25 03:00:00  MT_001 -1.227936   25    -1.503741      8   \n",
       "4    2014-08-25 04:00:00  MT_001 -1.083473   25    -1.503741      8   \n",
       "..                   ...     ...       ...  ...          ...    ...   \n",
       "331  2014-09-07 19:00:00  MT_001  1.083473    7     1.508326      9   \n",
       "332  2014-09-07 20:00:00  MT_001  1.227936    7     1.508326      9   \n",
       "333  2014-09-07 21:00:00  MT_001  1.372399    7     1.508326      9   \n",
       "334  2014-09-07 22:00:00  MT_001  1.516862    7     1.508326      9   \n",
       "335  2014-09-07 23:00:00  MT_001  1.661325    7     1.508326      9   \n",
       "\n",
       "     hours_from_start  categorical_day_of_week  categorical_hour  \n",
       "0            2.001283                        0                 0  \n",
       "1            2.001942                        0                 1  \n",
       "2            2.002601                        0                 2  \n",
       "3            2.003260                        0                 3  \n",
       "4            2.003920                        0                 4  \n",
       "..                ...                      ...               ...  \n",
       "331          2.219437                        6                19  \n",
       "332          2.220096                        6                20  \n",
       "333          2.220755                        6                21  \n",
       "334          2.221415                        6                22  \n",
       "335          2.222074                        6                23  \n",
       "\n",
       "[336 rows x 13 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[test.categorical_id == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:20:33.732401Z",
     "start_time": "2020-03-29T01:20:33.704106Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "336.0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.groupby(['categorical_id']).apply(lambda x: x.shape[0]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:23.049046Z",
     "start_time": "2020-03-29T01:17:23.042974Z"
    }
   },
   "outputs": [],
   "source": [
    "g = test.groupby(['categorical_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:23.496912Z",
     "start_time": "2020-03-29T01:17:23.492365Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_formatter.get_time_steps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:24.633923Z",
     "start_time": "2020-03-29T01:17:24.102559Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>init_abs</th>\n",
       "      <th>end_abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>123979</td>\n",
       "      <td>124171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>123980</td>\n",
       "      <td>124172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>123981</td>\n",
       "      <td>124173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>123982</td>\n",
       "      <td>124174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>123983</td>\n",
       "      <td>124175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        init_abs  end_abs\n",
       "0              0      192\n",
       "1              1      193\n",
       "2              2      194\n",
       "3              3      195\n",
       "4              4      196\n",
       "...          ...      ...\n",
       "123979    123979   124171\n",
       "123980    123980   124172\n",
       "123981    123981   124173\n",
       "123982    123982   124174\n",
       "123983    123983   124175\n",
       "\n",
       "[123984 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_index_abs = g[['categorical_id']].transform(lambda x: x.index+data_formatter.get_time_steps()) \\\n",
    "                        .reset_index() \\\n",
    "                        .rename(columns={'index':'init_abs',\n",
    "                                         'categorical_id':'end_abs'})\n",
    "df_index_abs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:25.227946Z",
     "start_time": "2020-03-29T01:17:24.829277Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>init_rel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        init_rel\n",
       "0              0\n",
       "1              1\n",
       "2              2\n",
       "3              3\n",
       "4              4\n",
       "...          ...\n",
       "123979       331\n",
       "123980       332\n",
       "123981       333\n",
       "123982       334\n",
       "123983       335\n",
       "\n",
       "[123984 rows x 1 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_index_rel_init = g[['categorical_id']].transform(lambda x: x.reset_index(drop=True).index) \\\n",
    "                        .rename(columns={'categorical_id':'init_rel'})\n",
    "df_index_rel_init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:25.741907Z",
     "start_time": "2020-03-29T01:17:25.283520Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_rel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        end_rel\n",
       "0           192\n",
       "1           193\n",
       "2           194\n",
       "3           195\n",
       "4           196\n",
       "...         ...\n",
       "123979      523\n",
       "123980      524\n",
       "123981      525\n",
       "123982      526\n",
       "123983      527\n",
       "\n",
       "[123984 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_index_rel_end = g[['categorical_id']].transform(lambda x: x.reset_index(drop=True).index+data_formatter.get_time_steps()) \\\n",
    "                .rename(columns={'categorical_id':'end_rel'})\n",
    "df_index_rel_end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:26.025561Z",
     "start_time": "2020-03-29T01:17:26.019175Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "145"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "336 - 192 + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:27.362184Z",
     "start_time": "2020-03-29T01:17:27.050941Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        group_count\n",
       "0               145\n",
       "1               145\n",
       "2               145\n",
       "3               145\n",
       "4               145\n",
       "...             ...\n",
       "123979          145\n",
       "123980          145\n",
       "123981          145\n",
       "123982          145\n",
       "123983          145\n",
       "\n",
       "[123984 rows x 1 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_total_count = g[['categorical_id']].transform(lambda x: x.shape[0] - data_formatter.get_time_steps() + 1) \\\n",
    "                .rename(columns = {'categorical_id':'group_count'})\n",
    "df_total_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:17:28.518757Z",
     "start_time": "2020-03-29T01:17:28.484203Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>init_abs</th>\n",
       "      <th>end_abs</th>\n",
       "      <th>init_rel</th>\n",
       "      <th>end_rel</th>\n",
       "      <th>id</th>\n",
       "      <th>group_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>193</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>194</td>\n",
       "      <td>2</td>\n",
       "      <td>194</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>195</td>\n",
       "      <td>3</td>\n",
       "      <td>195</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>196</td>\n",
       "      <td>4</td>\n",
       "      <td>196</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123979</th>\n",
       "      <td>123979</td>\n",
       "      <td>124171</td>\n",
       "      <td>331</td>\n",
       "      <td>523</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123980</th>\n",
       "      <td>123980</td>\n",
       "      <td>124172</td>\n",
       "      <td>332</td>\n",
       "      <td>524</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123981</th>\n",
       "      <td>123981</td>\n",
       "      <td>124173</td>\n",
       "      <td>333</td>\n",
       "      <td>525</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123982</th>\n",
       "      <td>123982</td>\n",
       "      <td>124174</td>\n",
       "      <td>334</td>\n",
       "      <td>526</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123983</th>\n",
       "      <td>123983</td>\n",
       "      <td>124175</td>\n",
       "      <td>335</td>\n",
       "      <td>527</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>123984 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        init_abs  end_abs  init_rel  end_rel      id  group_count\n",
       "0              0      192         0      192  MT_001          145\n",
       "1              1      193         1      193  MT_001          145\n",
       "2              2      194         2      194  MT_001          145\n",
       "3              3      195         3      195  MT_001          145\n",
       "4              4      196         4      196  MT_001          145\n",
       "...          ...      ...       ...      ...     ...          ...\n",
       "123979    123979   124171       331      523  MT_370          145\n",
       "123980    123980   124172       332      524  MT_370          145\n",
       "123981    123981   124173       333      525  MT_370          145\n",
       "123982    123982   124174       334      526  MT_370          145\n",
       "123983    123983   124175       335      527  MT_370          145\n",
       "\n",
       "[123984 rows x 6 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_test = pd.concat([df_index_abs, \n",
    "                       df_index_rel_init,\n",
    "                       df_index_rel_end,\n",
    "                       test[['id']], \n",
    "                       df_total_count], axis = 1).reset_index(drop = True)\n",
    "new_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T01:22:25.641796Z",
     "start_time": "2020-03-29T01:22:25.602541Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>init_abs</th>\n",
       "      <th>end_abs</th>\n",
       "      <th>init_rel</th>\n",
       "      <th>end_rel</th>\n",
       "      <th>id</th>\n",
       "      <th>group_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>193</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>194</td>\n",
       "      <td>2</td>\n",
       "      <td>194</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>195</td>\n",
       "      <td>3</td>\n",
       "      <td>195</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>196</td>\n",
       "      <td>4</td>\n",
       "      <td>196</td>\n",
       "      <td>MT_001</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53131</th>\n",
       "      <td>123787</td>\n",
       "      <td>123787</td>\n",
       "      <td>123979</td>\n",
       "      <td>139</td>\n",
       "      <td>331</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53132</th>\n",
       "      <td>123788</td>\n",
       "      <td>123788</td>\n",
       "      <td>123980</td>\n",
       "      <td>140</td>\n",
       "      <td>332</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53133</th>\n",
       "      <td>123789</td>\n",
       "      <td>123789</td>\n",
       "      <td>123981</td>\n",
       "      <td>141</td>\n",
       "      <td>333</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53134</th>\n",
       "      <td>123790</td>\n",
       "      <td>123790</td>\n",
       "      <td>123982</td>\n",
       "      <td>142</td>\n",
       "      <td>334</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53135</th>\n",
       "      <td>123791</td>\n",
       "      <td>123791</td>\n",
       "      <td>123983</td>\n",
       "      <td>143</td>\n",
       "      <td>335</td>\n",
       "      <td>MT_370</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>53136 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        index  init_abs  end_abs  init_rel  end_rel      id  group_count\n",
       "0           0         0      192         0      192  MT_001          145\n",
       "1           1         1      193         1      193  MT_001          145\n",
       "2           2         2      194         2      194  MT_001          145\n",
       "3           3         3      195         3      195  MT_001          145\n",
       "4           4         4      196         4      196  MT_001          145\n",
       "...       ...       ...      ...       ...      ...     ...          ...\n",
       "53131  123787    123787   123979       139      331  MT_370          145\n",
       "53132  123788    123788   123980       140      332  MT_370          145\n",
       "53133  123789    123789   123981       141      333  MT_370          145\n",
       "53134  123790    123790   123982       142      334  MT_370          145\n",
       "53135  123791    123791   123983       143      335  MT_370          145\n",
       "\n",
       "[53136 rows x 7 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_test[new_test.end_rel < test.groupby(['categorical_id']).apply(lambda x: x.shape[0]).mean()].reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.347675Z",
     "start_time": "2020-04-22T03:09:29.305754Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = TFTDataset(train)\n",
    "valid_dataset = TFTDataset(valid)\n",
    "test_dataset = TFTDataset(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:57:13.230902Z",
     "start_time": "2020-04-22T02:57:13.227547Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1841286, 203688, 53136)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataset), len(valid_dataset), len(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:57:13.238349Z",
     "start_time": "2020-04-22T02:57:13.232842Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1841286, 203688, 53136)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_dataset), len(valid_dataset), len(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T02:57:13.252589Z",
     "start_time": "2020-04-22T02:57:13.239632Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((192, 5), (24, 1), (24, 1))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset[0][0].shape, test_dataset[0][1].shape, test_dataset[0][2].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Temporal Fusion Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.415971Z",
     "start_time": "2020-04-22T03:09:35.349298Z"
    },
    "code_folding": [
     1,
     98,
     114,
     134,
     144,
     153,
     174,
     188,
     196,
     202,
     207,
     217,
     226,
     242,
     301,
     446,
     449,
     452,
     467,
     478,
     483,
     491,
     497,
     526
    ]
   },
   "outputs": [],
   "source": [
    "class TemporalFusionTransformer(pl.LightningModule):\n",
    "    def __init__(self, hparams):\n",
    "        super(TemporalFusionTransformer, self).__init__()\n",
    "        \n",
    "        self.hparams = hparams\n",
    "        \n",
    "        self.name = self.__class__.__name__\n",
    "\n",
    "        # Data parameters\n",
    "        self.time_steps = int(hparams.total_time_steps)#int(params['total_time_steps'])\n",
    "        self.input_size = int(hparams.input_size)#int(params['input_size'])\n",
    "        self.output_size = int(hparams.output_size)#int(params['output_size'])\n",
    "        self.category_counts = json.loads(str(hparams.category_counts))#json.loads(str(params['category_counts']))\n",
    "        self.num_categorical_variables = len(self.category_counts)\n",
    "        self.num_regular_variables = self.input_size - self.num_categorical_variables\n",
    "        self.n_multiprocessing_workers = int(hparams.multiprocessing_workers) #int(params['multiprocessing_workers'])\n",
    "\n",
    "        # Relevant indices for TFT\n",
    "        self._input_obs_loc = json.loads(str(hparams.input_obs_loc))#json.loads(str(params['input_obs_loc']))\n",
    "        self._static_input_loc = json.loads(str(hparams.static_input_loc))#json.loads(str(params['static_input_loc']))\n",
    "        self._known_regular_input_idx = json.loads(str(hparams.known_regular_inputs))#json.loads(str(params['known_regular_inputs']))\n",
    "        self._known_categorical_input_idx = json.loads(str(hparams.known_categorical_inputs))#json.loads(str(params['known_categorical_inputs']))\n",
    "        \n",
    "        self.num_non_static_historical_inputs = self.get_historical_num_inputs()\n",
    "        self.num_non_static_future_inputs = self.get_future_num_inputs()\n",
    "        \n",
    "        self.column_definition = [\n",
    "                                  ('id', DataTypes.REAL_VALUED, InputTypes.ID),\n",
    "                                  ('hours_from_start', DataTypes.REAL_VALUED, InputTypes.TIME),\n",
    "                                  ('power_usage', DataTypes.REAL_VALUED, InputTypes.TARGET),\n",
    "                                  ('hour', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),\n",
    "                                  ('day_of_week', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),\n",
    "                                  ('hours_from_start', DataTypes.REAL_VALUED, InputTypes.KNOWN_INPUT),\n",
    "                                  ('categorical_id', DataTypes.CATEGORICAL, InputTypes.STATIC_INPUT),\n",
    "                                ]\n",
    "\n",
    "        # Network params\n",
    "        self.quantiles = [0.1, 0.5, 0.9]\n",
    "#         self.use_cudnn = use_cudnn  # Whether to use GPU optimised LSTM\n",
    "        self.hidden_layer_size = int(hparams.hidden_layer_size)#int(params['hidden_layer_size'])\n",
    "        self.dropout_rate = float(hparams.dropout_rate)#float(params['dropout_rate'])\n",
    "        self.max_gradient_norm = float(hparams.max_gradient_norm)#float(params['max_gradient_norm'])\n",
    "        self.learning_rate = float(hparams.learning_rate)#float(params['learning_rate'])\n",
    "        self.minibatch_size = int(hparams.minibatch_size)#int(params['minibatch_size'])\n",
    "        self.num_epochs = int(hparams.num_epochs)#int(params['num_epochs'])\n",
    "        self.early_stopping_patience = int(hparams.early_stopping_patience)#int(params['early_stopping_patience'])\n",
    "\n",
    "        self.num_encoder_steps = int(hparams.num_encoder_steps)#int(params['num_encoder_steps'])\n",
    "        self.num_stacks = int(hparams.stack_size)#int(params['stack_size'])\n",
    "        self.num_heads = int(hparams.num_heads)#int(params['num_heads'])\n",
    "\n",
    "        # Serialisation options\n",
    "#         self._temp_folder = os.path.join(params['model_folder'], 'tmp')\n",
    "#         self.reset_temp_folder()\n",
    "\n",
    "        # Extra components to store Tensorflow nodes for attention computations\n",
    "        self._input_placeholder = None\n",
    "        self._attention_components = None\n",
    "        self._prediction_parts = None\n",
    "\n",
    "        print('*** {} params ***'.format(self.name))\n",
    "        for k in vars(hparams):\n",
    "            print('# {} = {}'.format(k, vars(hparams)[k]))\n",
    "            \n",
    "        self.train_criterion = QuantileLossCalculator(self.quantiles, self.output_size)\n",
    "        self.test_criterion = NormalizedQuantileLossCalculator(self.quantiles, self.output_size)\n",
    "\n",
    "        # Build model\n",
    "        ## Build embeddings\n",
    "        self.build_embeddings()\n",
    "        \n",
    "        ## Build Static Contex Networks\n",
    "        self.build_static_context_networks()\n",
    "        \n",
    "        ## Building Variable Selection Networks\n",
    "        self.build_variable_selection_networks()\n",
    "        \n",
    "        ## Build Lstm\n",
    "        self.build_lstm()\n",
    "        \n",
    "        ## Build GLU for after lstm encoder decoder and layernorm\n",
    "        self.build_post_lstm_gate_add_norm()\n",
    "        \n",
    "        ## Build Static Enrichment Layer\n",
    "        self.build_static_enrichment()\n",
    "        \n",
    "        ## Building decoder multihead attention\n",
    "        self.build_temporal_self_attention()\n",
    "        \n",
    "        ## Building positionwise decoder\n",
    "        self.build_position_wise_feed_forward()\n",
    "        \n",
    "        ## Build output feed forward\n",
    "        self.build_output_feed_forward()\n",
    "        \n",
    "        ## Initializing remaining weights\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        for name, p in self.named_parameters():\n",
    "            if ('lstm' in name and 'ih' in name) and 'bias' not in name:\n",
    "                #print(name)\n",
    "                #print(p.shape)\n",
    "                torch.nn.init.xavier_uniform_(p)\n",
    "#                 torch.nn.init.kaiming_normal_(p, a=0, mode='fan_in', nonlinearity='sigmoid')\n",
    "            elif ('lstm' in name and 'hh' in name) and 'bias' not in name:\n",
    "        \n",
    "                 torch.nn.init.orthogonal_(p)\n",
    "            \n",
    "            elif 'lstm' in name and 'bias' in name:\n",
    "                #print(name)\n",
    "                #print(p.shape)\n",
    "                torch.nn.init.zeros_(p)\n",
    "        \n",
    "    def get_historical_num_inputs(self):\n",
    "        \n",
    "        obs_inputs = [i for i in self._input_obs_loc]\n",
    "        \n",
    "        known_regular_inputs = [i for i in self._known_regular_input_idx\n",
    "                                if i not in self._static_input_loc]\n",
    "            \n",
    "        known_categorical_inputs = [i for i in self._known_categorical_input_idx\n",
    "                                    if i + self.num_regular_variables not in self._static_input_loc]\n",
    "        \n",
    "        wired_embeddings = [i for i in range(self.num_categorical_variables)\n",
    "                            if i not in self._known_categorical_input_idx \n",
    "                            and i not in self._input_obs_loc]\n",
    "\n",
    "        unknown_inputs = [i for i in range(self.num_regular_variables)\n",
    "                          if i not in self._known_regular_input_idx\n",
    "                          and i not in self._input_obs_loc]\n",
    "\n",
    "        return len(obs_inputs+known_regular_inputs+known_categorical_inputs+wired_embeddings+unknown_inputs)\n",
    "    \n",
    "    def get_future_num_inputs(self):\n",
    "            \n",
    "        known_regular_inputs = [i for i in self._known_regular_input_idx\n",
    "                                if i not in self._static_input_loc]\n",
    "            \n",
    "        known_categorical_inputs = [i for i in self._known_categorical_input_idx\n",
    "                                    if i + self.num_regular_variables not in self._static_input_loc]\n",
    "\n",
    "        return len(known_regular_inputs + known_categorical_inputs)\n",
    "    \n",
    "    def build_embeddings(self):\n",
    "        self.categorical_var_embeddings = nn.ModuleList([nn.Embedding(self.category_counts[i], \n",
    "                                                                      self.hidden_layer_size) \n",
    "                                                     for i in range(self.num_categorical_variables)])\n",
    "\n",
    "        self.regular_var_embeddings = nn.ModuleList([nn.Linear(1, \n",
    "                                                              self.hidden_layer_size) \n",
    "                                                  for i in range(self.num_regular_variables)])\n",
    "\n",
    "    def build_variable_selection_networks(self):\n",
    "        \n",
    "        self.static_vsn = VariableSelectionNetwork(hidden_layer_size = self.hidden_layer_size,\n",
    "                                                   input_size = self.hidden_layer_size * len(self._static_input_loc),\n",
    "                                                   output_size = len(self._static_input_loc),\n",
    "                                                   dropout_rate = self.dropout_rate)\n",
    "        \n",
    "        self.temporal_historical_vsn = VariableSelectionNetwork(hidden_layer_size = self.hidden_layer_size,\n",
    "                                                                input_size = self.hidden_layer_size *\n",
    "                                                                        self.num_non_static_historical_inputs,\n",
    "                                                                output_size = self.num_non_static_historical_inputs,\n",
    "                                                                dropout_rate = self.dropout_rate,\n",
    "                                                                additional_context=self.hidden_layer_size)\n",
    "        \n",
    "        self.temporal_future_vsn = VariableSelectionNetwork(hidden_layer_size = self.hidden_layer_size,\n",
    "                                                            input_size = self.hidden_layer_size *\n",
    "                                                                        self.num_non_static_future_inputs,\n",
    "                                                            output_size = self.num_non_static_future_inputs,\n",
    "                                                            dropout_rate = self.dropout_rate,\n",
    "                                                            additional_context=self.hidden_layer_size)\n",
    "        \n",
    "    def build_static_context_networks(self):\n",
    "        \n",
    "        self.static_context_variable_selection_grn = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                                          dropout_rate=self.dropout_rate)\n",
    "        \n",
    "        self.static_context_enrichment_grn = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                              dropout_rate=self.dropout_rate)\n",
    "\n",
    "        self.static_context_state_h_grn = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                           dropout_rate=self.dropout_rate)\n",
    "        \n",
    "        self.static_context_state_c_grn = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                           dropout_rate=self.dropout_rate)\n",
    "        \n",
    "    def build_lstm(self):\n",
    "        self.historical_lstm = nn.LSTM(input_size = self.hidden_layer_size,\n",
    "                                       hidden_size = self.hidden_layer_size,\n",
    "                                       batch_first = True)\n",
    "        self.future_lstm = nn.LSTM(input_size = self.hidden_layer_size,\n",
    "                                   hidden_size = self.hidden_layer_size,\n",
    "                                   batch_first = True)\n",
    "        \n",
    "    def build_post_lstm_gate_add_norm(self):\n",
    "        self.post_seq_encoder_gate_add_norm = GateAddNormNetwork(self.hidden_layer_size,\n",
    "                                                                 self.hidden_layer_size,\n",
    "                                                                 self.dropout_rate,\n",
    "                                                                 activation = None)\n",
    "        \n",
    "    def build_static_enrichment(self):\n",
    "        self.static_enrichment = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                      dropout_rate = self.dropout_rate,\n",
    "                                                      additional_context=self.hidden_layer_size)\n",
    "        \n",
    "    def build_temporal_self_attention(self):\n",
    "        self.self_attn_layer = InterpretableMultiHeadAttention(n_head = self.num_heads, \n",
    "                                                               d_model = self.hidden_layer_size,\n",
    "                                                               dropout = self.dropout_rate)\n",
    "        \n",
    "        self.post_attn_gate_add_norm = GateAddNormNetwork(self.hidden_layer_size,\n",
    "                                                           self.hidden_layer_size,\n",
    "                                                           self.dropout_rate,\n",
    "                                                           activation = None)\n",
    "        \n",
    "    def build_position_wise_feed_forward(self):\n",
    "        self.GRN_positionwise = GatedResidualNetwork(self.hidden_layer_size,\n",
    "                                                     dropout_rate = self.dropout_rate)\n",
    "        \n",
    "        self.post_tfd_gate_add_norm = GateAddNormNetwork(self.hidden_layer_size,\n",
    "                                                         self.hidden_layer_size,\n",
    "                                                         self.dropout_rate,\n",
    "                                                         activation = None)\n",
    "        \n",
    "    def build_output_feed_forward(self):\n",
    "        self.output_feed_forward = torch.nn.Linear(self.hidden_layer_size, \n",
    "                                                   self.output_size * len(self.quantiles))\n",
    "         \n",
    "    def get_decoder_mask(self, self_attn_inputs):\n",
    "        \"\"\"Returns causal mask to apply for self-attention layer.\n",
    "        Args:\n",
    "        self_attn_inputs: Inputs to self attention layer to determine mask shape\n",
    "        \"\"\"\n",
    "        len_s = self_attn_inputs.shape[1]\n",
    "        bs = self_attn_inputs.shape[0]\n",
    "        mask = torch.cumsum(torch.eye(len_s), 0)\n",
    "        mask = mask.repeat(bs,1,1).to(torch.float32)\n",
    "\n",
    "        return mask.to(DEVICE)\n",
    "    \n",
    "    def get_tft_embeddings(self, regular_inputs, categorical_inputs):\n",
    "        # Static input\n",
    "        if self._static_input_loc:\n",
    "            static_regular_inputs = [self.regular_var_embeddings[i](regular_inputs[:, 0, i:i + 1]) \n",
    "                                    for i in range(self.num_regular_variables)\n",
    "                                    if i in self._static_input_loc]\n",
    "            #print('static_regular_inputs')\n",
    "            #print([print(emb.shape) for emb in static_regular_inputs])\n",
    "            \n",
    "            static_categorical_inputs = [self.categorical_var_embeddings[i](categorical_inputs[Ellipsis, i])[:,0,:] \n",
    "                                         for i in range(self.num_categorical_variables)\n",
    "                                         if i + self.num_regular_variables in self._static_input_loc]\n",
    "            #print('static_categorical_inputs')\n",
    "            #print([print(emb.shape) for emb in static_categorical_inputs])\n",
    "            static_inputs = torch.stack(static_regular_inputs + static_categorical_inputs, axis = 1)\n",
    "        else:\n",
    "            static_inputs = None\n",
    "            \n",
    "        # Target input\n",
    "        obs_inputs = torch.stack([self.regular_var_embeddings[i](regular_inputs[Ellipsis, i:i + 1])\n",
    "                                     for i in self._input_obs_loc], axis=-1)\n",
    "        \n",
    "        # Observed (a prioir unknown) inputs\n",
    "        wired_embeddings = []\n",
    "        for i in range(self.num_categorical_variables):\n",
    "            if i not in self._known_categorical_input_idx \\\n",
    "            and i not in self._input_obs_loc:\n",
    "                e = self.categorical_var_embeddings[i](categorical_inputs[:, :, i])\n",
    "                wired_embeddings.append(e)\n",
    "\n",
    "        unknown_inputs = []\n",
    "        for i in range(self.num_regular_variables):\n",
    "            if i not in self._known_regular_input_idx \\\n",
    "            and i not in self._input_obs_loc:\n",
    "                e = self.regular_var_embeddings[i](regular_inputs[Ellipsis, i:i + 1])\n",
    "                unknown_inputs.append(e)\n",
    "                \n",
    "        if unknown_inputs + wired_embeddings:\n",
    "            unknown_inputs = torch.stack(unknown_inputs + wired_embeddings, axis=-1)\n",
    "        else:\n",
    "            unknown_inputs = None\n",
    "            \n",
    "        # A priori known inputs\n",
    "        known_regular_inputs = [self.regular_var_embeddings[i](regular_inputs[Ellipsis, i:i + 1])\n",
    "                                for i in self._known_regular_input_idx\n",
    "                                if i not in self._static_input_loc]\n",
    "        #print('known_regular_inputs')\n",
    "        #print([print(emb.shape) for emb in known_regular_inputs])\n",
    "        \n",
    "        known_categorical_inputs = [self.categorical_var_embeddings[i](categorical_inputs[Ellipsis, i])\n",
    "                                    for i in self._known_categorical_input_idx\n",
    "                                    if i + self.num_regular_variables not in self._static_input_loc]\n",
    "       #print('known_categorical_inputs')\n",
    "       #print([print(emb.shape) for emb in known_categorical_inputs])\n",
    "\n",
    "        known_combined_layer = torch.stack(known_regular_inputs + known_categorical_inputs, axis=-1)\n",
    "        \n",
    "        return unknown_inputs, known_combined_layer, obs_inputs, static_inputs\n",
    "        \n",
    "    def forward(self, all_inputs):\n",
    "\n",
    "        regular_inputs = all_inputs[:, :, :self.num_regular_variables].to(torch.float)\n",
    "        #print('regular_inputs')\n",
    "        #print(regular_inputs.shape)\n",
    "        categorical_inputs = all_inputs[:, :, self.num_regular_variables:].to(torch.long)\n",
    "        #print('categorical_inputs')\n",
    "        #print(categorical_inputs.shape)\n",
    "        \n",
    "        unknown_inputs, known_combined_layer, obs_inputs, static_inputs \\\n",
    "            = self.get_tft_embeddings(regular_inputs, categorical_inputs)\n",
    "        \n",
    "        # Isolate known and observed historical inputs.\n",
    "        if unknown_inputs is not None:\n",
    "              historical_inputs = torch.cat([\n",
    "                  unknown_inputs[:, :self.num_encoder_steps, :],\n",
    "                  known_combined_layer[:, :self.num_encoder_steps, :],\n",
    "                  obs_inputs[:, :self.num_encoder_steps, :]\n",
    "              ], axis=-1)\n",
    "        else:\n",
    "              historical_inputs = torch.cat([\n",
    "                  known_combined_layer[:, :self.num_encoder_steps, :],\n",
    "                  obs_inputs[:, :self.num_encoder_steps, :]\n",
    "              ], axis=-1)\n",
    "                \n",
    "        #print('historical_inputs')\n",
    "        #print(historical_inputs.shape)\n",
    "        \n",
    "        # Isolate only known future inputs.\n",
    "        future_inputs = known_combined_layer[:, self.num_encoder_steps:, :]\n",
    "        #print('future_inputs')\n",
    "        #print(future_inputs.shape)\n",
    "              \n",
    "        #print('static_inputs')\n",
    "        #print(static_inputs.shape)\n",
    "        \n",
    "        static_encoder, sparse_weights = self.static_vsn(static_inputs)\n",
    "        \n",
    "        #print('static_encoder')\n",
    "        #print(static_encoder.shape)\n",
    "        \n",
    "        #print('sparse_weights')\n",
    "        #print(sparse_weights.shape)\n",
    "        \n",
    "        static_context_variable_selection = self.static_context_variable_selection_grn(static_encoder)\n",
    "        #print('static_context_variable_selection')\n",
    "        #print(static_context_variable_selection.shape)\n",
    "        static_context_enrichment = self.static_context_enrichment_grn(static_encoder)\n",
    "        #print('static_context_enrichment')\n",
    "        #print(static_context_enrichment.shape)\n",
    "        static_context_state_h = self.static_context_state_h_grn(static_encoder)\n",
    "        #print('static_context_state_h')\n",
    "        #print(static_context_state_h.shape)\n",
    "        static_context_state_c = self.static_context_state_c_grn(static_encoder)\n",
    "        #print('static_context_state_c')\n",
    "        #print(static_context_state_c.shape)\n",
    "        \n",
    "        historical_features, historical_flags \\\n",
    "        = self.temporal_historical_vsn((historical_inputs,\n",
    "                                        static_context_variable_selection))\n",
    "        #print('historical_features')\n",
    "        #print(historical_features.shape)\n",
    "        #print('historical_flags')\n",
    "        #print(historical_flags.shape)\n",
    "        \n",
    "        future_features, future_flags \\\n",
    "        = self.temporal_future_vsn((future_inputs,\n",
    "                                    static_context_variable_selection))\n",
    "        #print('future_features')\n",
    "        #print(future_features.shape)\n",
    "        #print('future_flags')\n",
    "        #print(future_flags.shape)\n",
    "        \n",
    "        history_lstm, (state_h, state_c) \\\n",
    "        = self.historical_lstm(historical_features,\n",
    "                               (static_context_state_h.unsqueeze(0),\n",
    "                                static_context_state_c.unsqueeze(0)))\n",
    "        #print('history_lstm')\n",
    "        #print(history_lstm.shape)\n",
    "        #print('state_h')\n",
    "        #print(state_h.shape)\n",
    "        #print('state_c')\n",
    "        #print(state_c.shape)\n",
    "        \n",
    "        future_lstm, _ = self.future_lstm(future_features,\n",
    "                                          (state_h,\n",
    "                                           state_c))\n",
    "        #print('future_lstm')\n",
    "        #print(future_lstm.shape)\n",
    "        \n",
    "        # Apply gated skip connection\n",
    "        input_embeddings = torch.cat((historical_features, future_features), axis=1)\n",
    "        #print('input_embeddings')\n",
    "        #print(input_embeddings.shape) \n",
    "        \n",
    "        lstm_layer = torch.cat((history_lstm, future_lstm), axis=1)\n",
    "        #print('lstm_layer')\n",
    "        #print(lstm_layer.shape) \n",
    "        \n",
    "        temporal_feature_layer = self.post_seq_encoder_gate_add_norm(lstm_layer, input_embeddings)\n",
    "        #print('temporal_feature_layer')\n",
    "        #print(temporal_feature_layer.shape)  \n",
    "        \n",
    "        # Static enrichment layers\n",
    "        expanded_static_context = static_context_enrichment.unsqueeze(1)\n",
    "        \n",
    "        enriched = self.static_enrichment((temporal_feature_layer, expanded_static_context))\n",
    "        #print('enriched')\n",
    "        #print(enriched.shape)    \n",
    "        \n",
    "        # Decoder self attention\n",
    "        #self.mask = self.get_decoder_mask(enriched)\n",
    "        #print('enriched')\n",
    "        #print(enriched.shape)\n",
    "        x, self_att = self.self_attn_layer(enriched, \n",
    "                                           enriched, \n",
    "                                           enriched,\n",
    "                                           mask = self.get_decoder_mask(enriched))\n",
    "        #print('x')\n",
    "        #print(x.shape)\n",
    "        #print('self_att')\n",
    "        #print(self_att.shape)\n",
    "        \n",
    "        x = self.post_attn_gate_add_norm(x, enriched)\n",
    "        #print('x')\n",
    "        #print(x.shape)\n",
    "        \n",
    "        # Nonlinear processing on outputs\n",
    "        decoder = self.GRN_positionwise(x)\n",
    "        #print('decoder')\n",
    "        #print(decoder.shape)\n",
    "        \n",
    "        # Final skip connection\n",
    "        transformer_layer = self.post_tfd_gate_add_norm(decoder, temporal_feature_layer)\n",
    "        #print('transformer_layer')\n",
    "        #print(transformer_layer.shape)\n",
    "        \n",
    "        outputs = self.output_feed_forward(transformer_layer[Ellipsis, self.num_encoder_steps:, :])\n",
    "        #print('outputs')\n",
    "        #print(outputs.shape)\n",
    "        \n",
    "        #ipdb.set_trace()\n",
    "        \n",
    "        return outputs\n",
    "    \n",
    "    def loss(self, y_hat, y):\n",
    "        return self.train_criterion.apply(y_hat, y)\n",
    "    \n",
    "    def test_loss(self, y_hat, y):\n",
    "        return self.test_criterion.apply(y_hat, y, self.quantiles[1])\n",
    "    \n",
    "    def training_step(self, batch, batch_nb):\n",
    "        x, y, _ = batch\n",
    "        \n",
    "        x = x.to(torch.float)\n",
    "        y = y.to(torch.float)\n",
    "#         print('y')\n",
    "#         print(y.shape)\n",
    "        y_hat = self.forward(x)\n",
    "#         print('y_hat')\n",
    "#         print(y_hat.shape)\n",
    "        loss = self.loss(y_hat, torch.cat([y, y, y], dim = -1))\n",
    "        #print(loss.shape)\n",
    "        tensorboard_logs = {'train_loss': loss}\n",
    "        return {'loss': loss, 'log': tensorboard_logs}\n",
    "    \n",
    "    def validation_step(self, batch, batch_nb):\n",
    "        x, y, _ = batch\n",
    "        x = x.to(torch.float)\n",
    "        y = y.to(torch.float)\n",
    "        y_hat = self.forward(x)\n",
    "        #print(y_hat.shape)\n",
    "        #print(torch.cat([y, y, y], dim = -1).shape)\n",
    "        loss = self.loss(y_hat, torch.cat([y, y, y], dim = -1))\n",
    "        #print(loss)\n",
    "        return {'val_loss': loss}\n",
    "    \n",
    "    def validation_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n",
    "        tensorboard_logs = {'val_loss': avg_loss}\n",
    "        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n",
    "    \n",
    "    def test_step(self, batch, batch_idx):\n",
    "        # OPTIONAL\n",
    "        x, y, _ = batch\n",
    "        x = x.to(torch.float)\n",
    "        y = y.to(torch.float)\n",
    "        y_hat = self.forward(x)\n",
    "        return {'test_loss': self.test_loss(y_hat[Ellipsis, 1], y[Ellipsis, 0])}\n",
    "\n",
    "    def test_end(self, outputs):\n",
    "        # OPTIONAL\n",
    "        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()\n",
    "        tensorboard_logs = {'test_loss': avg_loss}\n",
    "        return {'avg_test_loss': avg_loss, 'log': tensorboard_logs}\n",
    "    \n",
    "    def configure_optimizers(self):\n",
    "        # REQUIRED\n",
    "        # can return multiple optimizers and learning_rate schedulers\n",
    "        # (LBFGS it is automatically supported, no need for closure function)\n",
    "        return [torch.optim.Adam(self.parameters(), lr=self.learning_rate)]\n",
    "    \n",
    "    def plot_grad_flow(self, named_parameters):\n",
    "        ave_grads = []\n",
    "        layers = []\n",
    "        for name, p in named_parameters:\n",
    "            if p.grad is not None:\n",
    "                if (p.requires_grad) and (\"bias\" not in name):\n",
    "                    layers.append(name)\n",
    "                    ave_grads.append(p.grad.abs().mean())\n",
    "                    self.logger.experiment.add_histogram(tag=name, values=p.grad,\n",
    "                                                         global_step=self.trainer.global_step)\n",
    "            else:\n",
    "                 print('{} - {}'.format(name, p.requires_grad))\n",
    "            \n",
    "        plt.plot(ave_grads, alpha=0.3, color=\"b\")\n",
    "        plt.hlines(0, 0, len(ave_grads), linewidth=1, color=\"k\" )\n",
    "        plt.xticks(list(range(0,len(ave_grads), 1)), layers, rotation='vertical')\n",
    "        plt.xlim(left=0, right=len(ave_grads))\n",
    "        plt.xlabel(\"Layers\")\n",
    "        plt.ylabel(\"average gradient\")\n",
    "        plt.title(\"Gradient flow\")\n",
    "        plt.grid(True)\n",
    "        plt.rcParams[\"figure.figsize\"] = (20, 5)\n",
    "    \n",
    "    def on_after_backward(self):\n",
    "        # example to inspect gradient information in tensorboard\n",
    "        if self.trainer.global_step % 25 == 0:  \n",
    "            self.plot_grad_flow(self.named_parameters())\n",
    "    \n",
    "    def train_dataloader(self):\n",
    "        # REQUIRED\n",
    "        return DataLoader(train_dataset, batch_size = self.minibatch_size, shuffle=True, drop_last=True, num_workers=1)\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        # OPTIONAL\n",
    "        return DataLoader(valid_dataset, batch_size = self.minibatch_size, shuffle=True, drop_last=True, num_workers=1)\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        # OPTIONAL\n",
    "        return DataLoader(test_dataset, batch_size = self.minibatch_size, shuffle=True, drop_last=True, num_workers=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setting Device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.443503Z",
     "start_time": "2020-04-22T03:09:35.417309Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DEVICE = torch.device(\"cuda: 0\" if torch.cuda.is_available() else \"cpu\")\n",
    "DEVICE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.450301Z",
     "start_time": "2020-04-22T03:09:35.445130Z"
    }
   },
   "outputs": [],
   "source": [
    "params = data_formatter.get_experiment_params()\n",
    "params.update(data_formatter.get_default_model_params())\n",
    "\n",
    "parser = ArgumentParser(add_help=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.455442Z",
     "start_time": "2020-04-22T03:09:35.451554Z"
    }
   },
   "outputs": [],
   "source": [
    "for k in params:\n",
    "    if type(params[k]) in [int, float]:\n",
    "        #if k == 'minibatch_size':\n",
    "        #    parser.add_argument('--{}'.format(k), type=type(params[k]), default = 256)\n",
    "        #else:\n",
    "        parser.add_argument('--{}'.format(k), type=type(params[k]), default = params[k])\n",
    "    else:\n",
    "        parser.add_argument('--{}'.format(k), type=str, default = str(params[k]))\n",
    "hparams = parser.parse_known_args()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:35.947089Z",
     "start_time": "2020-04-22T03:09:35.903770Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** TemporalFusionTransformer params ***\n",
      "# total_time_steps = 192\n",
      "# num_encoder_steps = 168\n",
      "# num_epochs = 100\n",
      "# early_stopping_patience = 5\n",
      "# multiprocessing_workers = 5\n",
      "# column_definition = [('id', <DataTypes.REAL_VALUED: 0>, <InputTypes.ID: 4>), ('hours_from_start', <DataTypes.REAL_VALUED: 0>, <InputTypes.TIME: 5>), ('power_usage', <DataTypes.REAL_VALUED: 0>, <InputTypes.TARGET: 0>), ('hour', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('day_of_week', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('hours_from_start', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('categorical_id', <DataTypes.CATEGORICAL: 1>, <InputTypes.STATIC_INPUT: 3>)]\n",
      "# input_size = 5\n",
      "# output_size = 1\n",
      "# category_counts = [369]\n",
      "# input_obs_loc = [0]\n",
      "# static_input_loc = [4]\n",
      "# known_regular_inputs = [1, 2, 3]\n",
      "# known_categorical_inputs = [0]\n",
      "# dropout_rate = 0.1\n",
      "# hidden_layer_size = 160\n",
      "# learning_rate = 0.001\n",
      "# minibatch_size = 64\n",
      "# max_gradient_norm = 0.01\n",
      "# num_heads = 4\n",
      "# stack_size = 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TemporalFusionTransformer(\n",
       "  (categorical_var_embeddings): ModuleList(\n",
       "    (0): Embedding(369, 160)\n",
       "  )\n",
       "  (regular_var_embeddings): ModuleList(\n",
       "    (0): Linear(in_features=1, out_features=160, bias=True)\n",
       "    (1): Linear(in_features=1, out_features=160, bias=True)\n",
       "    (2): Linear(in_features=1, out_features=160, bias=True)\n",
       "    (3): Linear(in_features=1, out_features=160, bias=True)\n",
       "  )\n",
       "  (static_context_variable_selection_grn): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (static_context_enrichment_grn): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (static_context_state_h_grn): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (static_context_state_c_grn): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (static_vsn): VariableSelectionNetwork(\n",
       "    (flattened_grn): GatedResidualNetwork(\n",
       "      (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (skip_linear): Linear(in_features=160, out_features=1, bias=True)\n",
       "      (glu_add_norm): GateAddNormNetwork(\n",
       "        (GLU): GatedLinearUnit(\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "          (W4): Linear(in_features=160, out_features=1, bias=True)\n",
       "          (W5): Linear(in_features=160, out_features=1, bias=True)\n",
       "          (sigmoid): Sigmoid()\n",
       "        )\n",
       "        (LayerNorm): LayerNorm((1,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (per_feature_grn): ModuleList(\n",
       "      (0): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (temporal_historical_vsn): VariableSelectionNetwork(\n",
       "    (flattened_grn): GatedResidualNetwork(\n",
       "      (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W2): Linear(in_features=640, out_features=160, bias=True)\n",
       "      (W3): Linear(in_features=160, out_features=160, bias=False)\n",
       "      (skip_linear): Linear(in_features=640, out_features=4, bias=True)\n",
       "      (glu_add_norm): GateAddNormNetwork(\n",
       "        (GLU): GatedLinearUnit(\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "          (W4): Linear(in_features=160, out_features=4, bias=True)\n",
       "          (W5): Linear(in_features=160, out_features=4, bias=True)\n",
       "          (sigmoid): Sigmoid()\n",
       "        )\n",
       "        (LayerNorm): LayerNorm((4,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (per_feature_grn): ModuleList(\n",
       "      (0): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (1): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (2): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (3): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (temporal_future_vsn): VariableSelectionNetwork(\n",
       "    (flattened_grn): GatedResidualNetwork(\n",
       "      (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W2): Linear(in_features=480, out_features=160, bias=True)\n",
       "      (W3): Linear(in_features=160, out_features=160, bias=False)\n",
       "      (skip_linear): Linear(in_features=480, out_features=3, bias=True)\n",
       "      (glu_add_norm): GateAddNormNetwork(\n",
       "        (GLU): GatedLinearUnit(\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "          (W4): Linear(in_features=160, out_features=3, bias=True)\n",
       "          (W5): Linear(in_features=160, out_features=3, bias=True)\n",
       "          (sigmoid): Sigmoid()\n",
       "        )\n",
       "        (LayerNorm): LayerNorm((3,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (per_feature_grn): ModuleList(\n",
       "      (0): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (1): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "      (2): GatedResidualNetwork(\n",
       "        (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (glu_add_norm): GateAddNormNetwork(\n",
       "          (GLU): GatedLinearUnit(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "            (sigmoid): Sigmoid()\n",
       "          )\n",
       "          (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (historical_lstm): LSTM(160, 160, batch_first=True)\n",
       "  (future_lstm): LSTM(160, 160, batch_first=True)\n",
       "  (post_seq_encoder_gate_add_norm): GateAddNormNetwork(\n",
       "    (GLU): GatedLinearUnit(\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "      (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (sigmoid): Sigmoid()\n",
       "    )\n",
       "    (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (static_enrichment): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W3): Linear(in_features=160, out_features=160, bias=False)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (self_attn_layer): InterpretableMultiHeadAttention(\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (v_layer): Linear(in_features=160, out_features=40, bias=False)\n",
       "    (q_layers): ModuleList(\n",
       "      (0): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (1): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (2): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (3): Linear(in_features=160, out_features=40, bias=False)\n",
       "    )\n",
       "    (k_layers): ModuleList(\n",
       "      (0): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (1): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (2): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (3): Linear(in_features=160, out_features=40, bias=False)\n",
       "    )\n",
       "    (v_layers): ModuleList(\n",
       "      (0): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (1): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (2): Linear(in_features=160, out_features=40, bias=False)\n",
       "      (3): Linear(in_features=160, out_features=40, bias=False)\n",
       "    )\n",
       "    (attention): ScaledDotProductAttention(\n",
       "      (dropout): Dropout(p=0, inplace=False)\n",
       "      (softmax): Softmax(dim=2)\n",
       "    )\n",
       "    (w_h): Linear(in_features=40, out_features=160, bias=False)\n",
       "  )\n",
       "  (post_attn_gate_add_norm): GateAddNormNetwork(\n",
       "    (GLU): GatedLinearUnit(\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "      (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (sigmoid): Sigmoid()\n",
       "    )\n",
       "    (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (GRN_positionwise): GatedResidualNetwork(\n",
       "    (W1): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (W2): Linear(in_features=160, out_features=160, bias=True)\n",
       "    (glu_add_norm): GateAddNormNetwork(\n",
       "      (GLU): GatedLinearUnit(\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "        (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "        (sigmoid): Sigmoid()\n",
       "      )\n",
       "      (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (post_tfd_gate_add_norm): GateAddNormNetwork(\n",
       "    (GLU): GatedLinearUnit(\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "      (W4): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (W5): Linear(in_features=160, out_features=160, bias=True)\n",
       "      (sigmoid): Sigmoid()\n",
       "    )\n",
       "    (LayerNorm): LayerNorm((160,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (output_feed_forward): Linear(in_features=160, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tft = TemporalFusionTransformer(hparams)#.to(DEVICE)\n",
    "tft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-22T03:09:37.696931Z",
     "start_time": "2020-04-22T03:09:37.694439Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "early_stop_callback = EarlyStopping(monitor = 'val_loss',\n",
    "                                    min_delta = 1e-4,\n",
    "                                    patience = tft.early_stopping_patience,\n",
    "                                    verbose=False,\n",
    "                                    mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-22T03:09:48.709Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:lightning:GPU available: True, used: True\n",
      "INFO:lightning:VISIBLE GPUS: 0\n",
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: RuntimeWarning: You have defined a `val_dataloader()` and have defined a `validation_step()`, you may also want to define `validation_epoch_end()` for accumulating stats.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: RuntimeWarning: You have defined a `test_dataloader()` and have defined a `test_step()`, you may also want to define `test_epoch_end()` for accumulating stats.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "INFO:lightning:\n",
      "    | Name                                                               | Type                            | Params\n",
      "-------------------------------------------------------------------------------------------------------------------\n",
      "0   | categorical_var_embeddings                                         | ModuleList                      | 59 K  \n",
      "1   | categorical_var_embeddings.0                                       | Embedding                       | 59 K  \n",
      "2   | regular_var_embeddings                                             | ModuleList                      | 1 K   \n",
      "3   | regular_var_embeddings.0                                           | Linear                          | 320   \n",
      "4   | regular_var_embeddings.1                                           | Linear                          | 320   \n",
      "5   | regular_var_embeddings.2                                           | Linear                          | 320   \n",
      "6   | regular_var_embeddings.3                                           | Linear                          | 320   \n",
      "7   | static_context_variable_selection_grn                              | GatedResidualNetwork            | 103 K \n",
      "8   | static_context_variable_selection_grn.W1                           | Linear                          | 25 K  \n",
      "9   | static_context_variable_selection_grn.W2                           | Linear                          | 25 K  \n",
      "10  | static_context_variable_selection_grn.glu_add_norm                 | GateAddNormNetwork              | 51 K  \n",
      "11  | static_context_variable_selection_grn.glu_add_norm.GLU             | GatedLinearUnit                 | 51 K  \n",
      "12  | static_context_variable_selection_grn.glu_add_norm.GLU.dropout     | Dropout                         | 0     \n",
      "13  | static_context_variable_selection_grn.glu_add_norm.GLU.W4          | Linear                          | 25 K  \n",
      "14  | static_context_variable_selection_grn.glu_add_norm.GLU.W5          | Linear                          | 25 K  \n",
      "15  | static_context_variable_selection_grn.glu_add_norm.GLU.sigmoid     | Sigmoid                         | 0     \n",
      "16  | static_context_variable_selection_grn.glu_add_norm.LayerNorm       | LayerNorm                       | 320   \n",
      "17  | static_context_enrichment_grn                                      | GatedResidualNetwork            | 103 K \n",
      "18  | static_context_enrichment_grn.W1                                   | Linear                          | 25 K  \n",
      "19  | static_context_enrichment_grn.W2                                   | Linear                          | 25 K  \n",
      "20  | static_context_enrichment_grn.glu_add_norm                         | GateAddNormNetwork              | 51 K  \n",
      "21  | static_context_enrichment_grn.glu_add_norm.GLU                     | GatedLinearUnit                 | 51 K  \n",
      "22  | static_context_enrichment_grn.glu_add_norm.GLU.dropout             | Dropout                         | 0     \n",
      "23  | static_context_enrichment_grn.glu_add_norm.GLU.W4                  | Linear                          | 25 K  \n",
      "24  | static_context_enrichment_grn.glu_add_norm.GLU.W5                  | Linear                          | 25 K  \n",
      "25  | static_context_enrichment_grn.glu_add_norm.GLU.sigmoid             | Sigmoid                         | 0     \n",
      "26  | static_context_enrichment_grn.glu_add_norm.LayerNorm               | LayerNorm                       | 320   \n",
      "27  | static_context_state_h_grn                                         | GatedResidualNetwork            | 103 K \n",
      "28  | static_context_state_h_grn.W1                                      | Linear                          | 25 K  \n",
      "29  | static_context_state_h_grn.W2                                      | Linear                          | 25 K  \n",
      "30  | static_context_state_h_grn.glu_add_norm                            | GateAddNormNetwork              | 51 K  \n",
      "31  | static_context_state_h_grn.glu_add_norm.GLU                        | GatedLinearUnit                 | 51 K  \n",
      "32  | static_context_state_h_grn.glu_add_norm.GLU.dropout                | Dropout                         | 0     \n",
      "33  | static_context_state_h_grn.glu_add_norm.GLU.W4                     | Linear                          | 25 K  \n",
      "34  | static_context_state_h_grn.glu_add_norm.GLU.W5                     | Linear                          | 25 K  \n",
      "35  | static_context_state_h_grn.glu_add_norm.GLU.sigmoid                | Sigmoid                         | 0     \n",
      "36  | static_context_state_h_grn.glu_add_norm.LayerNorm                  | LayerNorm                       | 320   \n",
      "37  | static_context_state_c_grn                                         | GatedResidualNetwork            | 103 K \n",
      "38  | static_context_state_c_grn.W1                                      | Linear                          | 25 K  \n",
      "39  | static_context_state_c_grn.W2                                      | Linear                          | 25 K  \n",
      "40  | static_context_state_c_grn.glu_add_norm                            | GateAddNormNetwork              | 51 K  \n",
      "41  | static_context_state_c_grn.glu_add_norm.GLU                        | GatedLinearUnit                 | 51 K  \n",
      "42  | static_context_state_c_grn.glu_add_norm.GLU.dropout                | Dropout                         | 0     \n",
      "43  | static_context_state_c_grn.glu_add_norm.GLU.W4                     | Linear                          | 25 K  \n",
      "44  | static_context_state_c_grn.glu_add_norm.GLU.W5                     | Linear                          | 25 K  \n",
      "45  | static_context_state_c_grn.glu_add_norm.GLU.sigmoid                | Sigmoid                         | 0     \n",
      "46  | static_context_state_c_grn.glu_add_norm.LayerNorm                  | LayerNorm                       | 320   \n",
      "47  | static_vsn                                                         | VariableSelectionNetwork        | 155 K \n",
      "48  | static_vsn.flattened_grn                                           | GatedResidualNetwork            | 52 K  \n",
      "49  | static_vsn.flattened_grn.W1                                        | Linear                          | 25 K  \n",
      "50  | static_vsn.flattened_grn.W2                                        | Linear                          | 25 K  \n",
      "51  | static_vsn.flattened_grn.skip_linear                               | Linear                          | 161   \n",
      "52  | static_vsn.flattened_grn.glu_add_norm                              | GateAddNormNetwork              | 324   \n",
      "53  | static_vsn.flattened_grn.glu_add_norm.GLU                          | GatedLinearUnit                 | 322   \n",
      "54  | static_vsn.flattened_grn.glu_add_norm.GLU.dropout                  | Dropout                         | 0     \n",
      "55  | static_vsn.flattened_grn.glu_add_norm.GLU.W4                       | Linear                          | 161   \n",
      "56  | static_vsn.flattened_grn.glu_add_norm.GLU.W5                       | Linear                          | 161   \n",
      "57  | static_vsn.flattened_grn.glu_add_norm.GLU.sigmoid                  | Sigmoid                         | 0     \n",
      "58  | static_vsn.flattened_grn.glu_add_norm.LayerNorm                    | LayerNorm                       | 2     \n",
      "59  | static_vsn.per_feature_grn                                         | ModuleList                      | 103 K \n",
      "60  | static_vsn.per_feature_grn.0                                       | GatedResidualNetwork            | 103 K \n",
      "61  | static_vsn.per_feature_grn.0.W1                                    | Linear                          | 25 K  \n",
      "62  | static_vsn.per_feature_grn.0.W2                                    | Linear                          | 25 K  \n",
      "63  | static_vsn.per_feature_grn.0.glu_add_norm                          | GateAddNormNetwork              | 51 K  \n",
      "64  | static_vsn.per_feature_grn.0.glu_add_norm.GLU                      | GatedLinearUnit                 | 51 K  \n",
      "65  | static_vsn.per_feature_grn.0.glu_add_norm.GLU.dropout              | Dropout                         | 0     \n",
      "66  | static_vsn.per_feature_grn.0.glu_add_norm.GLU.W4                   | Linear                          | 25 K  \n",
      "67  | static_vsn.per_feature_grn.0.glu_add_norm.GLU.W5                   | Linear                          | 25 K  \n",
      "68  | static_vsn.per_feature_grn.0.glu_add_norm.GLU.sigmoid              | Sigmoid                         | 0     \n",
      "69  | static_vsn.per_feature_grn.0.glu_add_norm.LayerNorm                | LayerNorm                       | 320   \n",
      "70  | temporal_historical_vsn                                            | VariableSelectionNetwork        | 571 K \n",
      "71  | temporal_historical_vsn.flattened_grn                              | GatedResidualNetwork            | 157 K \n",
      "72  | temporal_historical_vsn.flattened_grn.W1                           | Linear                          | 25 K  \n",
      "73  | temporal_historical_vsn.flattened_grn.W2                           | Linear                          | 102 K \n",
      "74  | temporal_historical_vsn.flattened_grn.W3                           | Linear                          | 25 K  \n",
      "75  | temporal_historical_vsn.flattened_grn.skip_linear                  | Linear                          | 2 K   \n",
      "76  | temporal_historical_vsn.flattened_grn.glu_add_norm                 | GateAddNormNetwork              | 1 K   \n",
      "77  | temporal_historical_vsn.flattened_grn.glu_add_norm.GLU             | GatedLinearUnit                 | 1 K   \n",
      "78  | temporal_historical_vsn.flattened_grn.glu_add_norm.GLU.dropout     | Dropout                         | 0     \n",
      "79  | temporal_historical_vsn.flattened_grn.glu_add_norm.GLU.W4          | Linear                          | 644   \n",
      "80  | temporal_historical_vsn.flattened_grn.glu_add_norm.GLU.W5          | Linear                          | 644   \n",
      "81  | temporal_historical_vsn.flattened_grn.glu_add_norm.GLU.sigmoid     | Sigmoid                         | 0     \n",
      "82  | temporal_historical_vsn.flattened_grn.glu_add_norm.LayerNorm       | LayerNorm                       | 8     \n",
      "83  | temporal_historical_vsn.per_feature_grn                            | ModuleList                      | 413 K \n",
      "84  | temporal_historical_vsn.per_feature_grn.0                          | GatedResidualNetwork            | 103 K \n",
      "85  | temporal_historical_vsn.per_feature_grn.0.W1                       | Linear                          | 25 K  \n",
      "86  | temporal_historical_vsn.per_feature_grn.0.W2                       | Linear                          | 25 K  \n",
      "87  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm             | GateAddNormNetwork              | 51 K  \n",
      "88  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.GLU         | GatedLinearUnit                 | 51 K  \n",
      "89  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.GLU.dropout | Dropout                         | 0     \n",
      "90  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.GLU.W4      | Linear                          | 25 K  \n",
      "91  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.GLU.W5      | Linear                          | 25 K  \n",
      "92  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.GLU.sigmoid | Sigmoid                         | 0     \n",
      "93  | temporal_historical_vsn.per_feature_grn.0.glu_add_norm.LayerNorm   | LayerNorm                       | 320   \n",
      "94  | temporal_historical_vsn.per_feature_grn.1                          | GatedResidualNetwork            | 103 K \n",
      "95  | temporal_historical_vsn.per_feature_grn.1.W1                       | Linear                          | 25 K  \n",
      "96  | temporal_historical_vsn.per_feature_grn.1.W2                       | Linear                          | 25 K  \n",
      "97  | temporal_historical_vsn.per_feature_grn.1.glu_add_norm             | GateAddNormNetwork              | 51 K  \n",
      "98  | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.GLU         | GatedLinearUnit                 | 51 K  \n",
      "99  | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.GLU.dropout | Dropout                         | 0     \n",
      "100 | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.GLU.W4      | Linear                          | 25 K  \n",
      "101 | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.GLU.W5      | Linear                          | 25 K  \n",
      "102 | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.GLU.sigmoid | Sigmoid                         | 0     \n",
      "103 | temporal_historical_vsn.per_feature_grn.1.glu_add_norm.LayerNorm   | LayerNorm                       | 320   \n",
      "104 | temporal_historical_vsn.per_feature_grn.2                          | GatedResidualNetwork            | 103 K \n",
      "105 | temporal_historical_vsn.per_feature_grn.2.W1                       | Linear                          | 25 K  \n",
      "106 | temporal_historical_vsn.per_feature_grn.2.W2                       | Linear                          | 25 K  \n",
      "107 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm             | GateAddNormNetwork              | 51 K  \n",
      "108 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.GLU         | GatedLinearUnit                 | 51 K  \n",
      "109 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.GLU.dropout | Dropout                         | 0     \n",
      "110 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.GLU.W4      | Linear                          | 25 K  \n",
      "111 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.GLU.W5      | Linear                          | 25 K  \n",
      "112 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.GLU.sigmoid | Sigmoid                         | 0     \n",
      "113 | temporal_historical_vsn.per_feature_grn.2.glu_add_norm.LayerNorm   | LayerNorm                       | 320   \n",
      "114 | temporal_historical_vsn.per_feature_grn.3                          | GatedResidualNetwork            | 103 K \n",
      "115 | temporal_historical_vsn.per_feature_grn.3.W1                       | Linear                          | 25 K  \n",
      "116 | temporal_historical_vsn.per_feature_grn.3.W2                       | Linear                          | 25 K  \n",
      "117 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm             | GateAddNormNetwork              | 51 K  \n",
      "118 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.GLU         | GatedLinearUnit                 | 51 K  \n",
      "119 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.GLU.dropout | Dropout                         | 0     \n",
      "120 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.GLU.W4      | Linear                          | 25 K  \n",
      "121 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.GLU.W5      | Linear                          | 25 K  \n",
      "122 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.GLU.sigmoid | Sigmoid                         | 0     \n",
      "123 | temporal_historical_vsn.per_feature_grn.3.glu_add_norm.LayerNorm   | LayerNorm                       | 320   \n",
      "124 | temporal_future_vsn                                                | VariableSelectionNetwork        | 440 K \n",
      "125 | temporal_future_vsn.flattened_grn                                  | GatedResidualNetwork            | 130 K \n",
      "126 | temporal_future_vsn.flattened_grn.W1                               | Linear                          | 25 K  \n",
      "127 | temporal_future_vsn.flattened_grn.W2                               | Linear                          | 76 K  \n",
      "128 | temporal_future_vsn.flattened_grn.W3                               | Linear                          | 25 K  \n",
      "129 | temporal_future_vsn.flattened_grn.skip_linear                      | Linear                          | 1 K   \n",
      "130 | temporal_future_vsn.flattened_grn.glu_add_norm                     | GateAddNormNetwork              | 972   \n",
      "131 | temporal_future_vsn.flattened_grn.glu_add_norm.GLU                 | GatedLinearUnit                 | 966   \n",
      "132 | temporal_future_vsn.flattened_grn.glu_add_norm.GLU.dropout         | Dropout                         | 0     \n",
      "133 | temporal_future_vsn.flattened_grn.glu_add_norm.GLU.W4              | Linear                          | 483   \n",
      "134 | temporal_future_vsn.flattened_grn.glu_add_norm.GLU.W5              | Linear                          | 483   \n",
      "135 | temporal_future_vsn.flattened_grn.glu_add_norm.GLU.sigmoid         | Sigmoid                         | 0     \n",
      "136 | temporal_future_vsn.flattened_grn.glu_add_norm.LayerNorm           | LayerNorm                       | 6     \n",
      "137 | temporal_future_vsn.per_feature_grn                                | ModuleList                      | 310 K \n",
      "138 | temporal_future_vsn.per_feature_grn.0                              | GatedResidualNetwork            | 103 K \n",
      "139 | temporal_future_vsn.per_feature_grn.0.W1                           | Linear                          | 25 K  \n",
      "140 | temporal_future_vsn.per_feature_grn.0.W2                           | Linear                          | 25 K  \n",
      "141 | temporal_future_vsn.per_feature_grn.0.glu_add_norm                 | GateAddNormNetwork              | 51 K  \n",
      "142 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.GLU             | GatedLinearUnit                 | 51 K  \n",
      "143 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.GLU.dropout     | Dropout                         | 0     \n",
      "144 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.GLU.W4          | Linear                          | 25 K  \n",
      "145 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.GLU.W5          | Linear                          | 25 K  \n",
      "146 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.GLU.sigmoid     | Sigmoid                         | 0     \n",
      "147 | temporal_future_vsn.per_feature_grn.0.glu_add_norm.LayerNorm       | LayerNorm                       | 320   \n",
      "148 | temporal_future_vsn.per_feature_grn.1                              | GatedResidualNetwork            | 103 K \n",
      "149 | temporal_future_vsn.per_feature_grn.1.W1                           | Linear                          | 25 K  \n",
      "150 | temporal_future_vsn.per_feature_grn.1.W2                           | Linear                          | 25 K  \n",
      "151 | temporal_future_vsn.per_feature_grn.1.glu_add_norm                 | GateAddNormNetwork              | 51 K  \n",
      "152 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.GLU             | GatedLinearUnit                 | 51 K  \n",
      "153 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.GLU.dropout     | Dropout                         | 0     \n",
      "154 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.GLU.W4          | Linear                          | 25 K  \n",
      "155 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.GLU.W5          | Linear                          | 25 K  \n",
      "156 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.GLU.sigmoid     | Sigmoid                         | 0     \n",
      "157 | temporal_future_vsn.per_feature_grn.1.glu_add_norm.LayerNorm       | LayerNorm                       | 320   \n",
      "158 | temporal_future_vsn.per_feature_grn.2                              | GatedResidualNetwork            | 103 K \n",
      "159 | temporal_future_vsn.per_feature_grn.2.W1                           | Linear                          | 25 K  \n",
      "160 | temporal_future_vsn.per_feature_grn.2.W2                           | Linear                          | 25 K  \n",
      "161 | temporal_future_vsn.per_feature_grn.2.glu_add_norm                 | GateAddNormNetwork              | 51 K  \n",
      "162 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.GLU             | GatedLinearUnit                 | 51 K  \n",
      "163 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.GLU.dropout     | Dropout                         | 0     \n",
      "164 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.GLU.W4          | Linear                          | 25 K  \n",
      "165 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.GLU.W5          | Linear                          | 25 K  \n",
      "166 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.GLU.sigmoid     | Sigmoid                         | 0     \n",
      "167 | temporal_future_vsn.per_feature_grn.2.glu_add_norm.LayerNorm       | LayerNorm                       | 320   \n",
      "168 | historical_lstm                                                    | LSTM                            | 206 K \n",
      "169 | future_lstm                                                        | LSTM                            | 206 K \n",
      "170 | post_seq_encoder_gate_add_norm                                     | GateAddNormNetwork              | 51 K  \n",
      "171 | post_seq_encoder_gate_add_norm.GLU                                 | GatedLinearUnit                 | 51 K  \n",
      "172 | post_seq_encoder_gate_add_norm.GLU.dropout                         | Dropout                         | 0     \n",
      "173 | post_seq_encoder_gate_add_norm.GLU.W4                              | Linear                          | 25 K  \n",
      "174 | post_seq_encoder_gate_add_norm.GLU.W5                              | Linear                          | 25 K  \n",
      "175 | post_seq_encoder_gate_add_norm.GLU.sigmoid                         | Sigmoid                         | 0     \n",
      "176 | post_seq_encoder_gate_add_norm.LayerNorm                           | LayerNorm                       | 320   \n",
      "177 | static_enrichment                                                  | GatedResidualNetwork            | 128 K \n",
      "178 | static_enrichment.W1                                               | Linear                          | 25 K  \n",
      "179 | static_enrichment.W2                                               | Linear                          | 25 K  \n",
      "180 | static_enrichment.W3                                               | Linear                          | 25 K  \n",
      "181 | static_enrichment.glu_add_norm                                     | GateAddNormNetwork              | 51 K  \n",
      "182 | static_enrichment.glu_add_norm.GLU                                 | GatedLinearUnit                 | 51 K  \n",
      "183 | static_enrichment.glu_add_norm.GLU.dropout                         | Dropout                         | 0     \n",
      "184 | static_enrichment.glu_add_norm.GLU.W4                              | Linear                          | 25 K  \n",
      "185 | static_enrichment.glu_add_norm.GLU.W5                              | Linear                          | 25 K  \n",
      "186 | static_enrichment.glu_add_norm.GLU.sigmoid                         | Sigmoid                         | 0     \n",
      "187 | static_enrichment.glu_add_norm.LayerNorm                           | LayerNorm                       | 320   \n",
      "188 | self_attn_layer                                                    | InterpretableMultiHeadAttention | 64 K  \n",
      "189 | self_attn_layer.dropout                                            | Dropout                         | 0     \n",
      "190 | self_attn_layer.v_layer                                            | Linear                          | 6 K   \n",
      "191 | self_attn_layer.q_layers                                           | ModuleList                      | 25 K  \n",
      "192 | self_attn_layer.q_layers.0                                         | Linear                          | 6 K   \n",
      "193 | self_attn_layer.q_layers.1                                         | Linear                          | 6 K   \n",
      "194 | self_attn_layer.q_layers.2                                         | Linear                          | 6 K   \n",
      "195 | self_attn_layer.q_layers.3                                         | Linear                          | 6 K   \n",
      "196 | self_attn_layer.k_layers                                           | ModuleList                      | 25 K  \n",
      "197 | self_attn_layer.k_layers.0                                         | Linear                          | 6 K   \n",
      "198 | self_attn_layer.k_layers.1                                         | Linear                          | 6 K   \n",
      "199 | self_attn_layer.k_layers.2                                         | Linear                          | 6 K   \n",
      "200 | self_attn_layer.k_layers.3                                         | Linear                          | 6 K   \n",
      "201 | self_attn_layer.v_layers                                           | ModuleList                      | 6 K   \n",
      "202 | self_attn_layer.attention                                          | ScaledDotProductAttention       | 0     \n",
      "203 | self_attn_layer.attention.dropout                                  | Dropout                         | 0     \n",
      "204 | self_attn_layer.attention.softmax                                  | Softmax                         | 0     \n",
      "205 | self_attn_layer.w_h                                                | Linear                          | 6 K   \n",
      "206 | post_attn_gate_add_norm                                            | GateAddNormNetwork              | 51 K  \n",
      "207 | post_attn_gate_add_norm.GLU                                        | GatedLinearUnit                 | 51 K  \n",
      "208 | post_attn_gate_add_norm.GLU.dropout                                | Dropout                         | 0     \n",
      "209 | post_attn_gate_add_norm.GLU.W4                                     | Linear                          | 25 K  \n",
      "210 | post_attn_gate_add_norm.GLU.W5                                     | Linear                          | 25 K  \n",
      "211 | post_attn_gate_add_norm.GLU.sigmoid                                | Sigmoid                         | 0     \n",
      "212 | post_attn_gate_add_norm.LayerNorm                                  | LayerNorm                       | 320   \n",
      "213 | GRN_positionwise                                                   | GatedResidualNetwork            | 103 K \n",
      "214 | GRN_positionwise.W1                                                | Linear                          | 25 K  \n",
      "215 | GRN_positionwise.W2                                                | Linear                          | 25 K  \n",
      "216 | GRN_positionwise.glu_add_norm                                      | GateAddNormNetwork              | 51 K  \n",
      "217 | GRN_positionwise.glu_add_norm.GLU                                  | GatedLinearUnit                 | 51 K  \n",
      "218 | GRN_positionwise.glu_add_norm.GLU.dropout                          | Dropout                         | 0     \n",
      "219 | GRN_positionwise.glu_add_norm.GLU.W4                               | Linear                          | 25 K  \n",
      "220 | GRN_positionwise.glu_add_norm.GLU.W5                               | Linear                          | 25 K  \n",
      "221 | GRN_positionwise.glu_add_norm.GLU.sigmoid                          | Sigmoid                         | 0     \n",
      "222 | GRN_positionwise.glu_add_norm.LayerNorm                            | LayerNorm                       | 320   \n",
      "223 | post_tfd_gate_add_norm                                             | GateAddNormNetwork              | 51 K  \n",
      "224 | post_tfd_gate_add_norm.GLU                                         | GatedLinearUnit                 | 51 K  \n",
      "225 | post_tfd_gate_add_norm.GLU.dropout                                 | Dropout                         | 0     \n",
      "226 | post_tfd_gate_add_norm.GLU.W4                                      | Linear                          | 25 K  \n",
      "227 | post_tfd_gate_add_norm.GLU.W5                                      | Linear                          | 25 K  \n",
      "228 | post_tfd_gate_add_norm.GLU.sigmoid                                 | Sigmoid                         | 0     \n",
      "229 | post_tfd_gate_add_norm.LayerNorm                                   | LayerNorm                       | 320   \n",
      "230 | output_feed_forward                                                | Linear                          | 483   \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validation sanity check', layout=Layout(flex='2'), max=5.…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b998265860634849b491266cde10c2ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max=1.0), HTML(value='')), …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: RuntimeWarning: Displayed epoch numbers in the progress bar start from \"1\" until v0.6.x, but will start from \"0\" in v0.8.0.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/pytorch_lightning/utilities/warnings.py:18: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ae70cc229fe4ee0a2aed87773dfca26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Validating', layout=Layout(flex='2'), max=31.0, style=Pro…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer = pl.Trainer(max_nb_epochs = tft.num_epochs,\n",
    "                     gpus = 1, \n",
    "                     track_grad_norm = 2, \n",
    "                     gradient_clip_val = tft.max_gradient_norm,\n",
    "                     early_stop_callback = early_stop_callback,\n",
    "                     #train_percent_check = 0.01,\n",
    "                     #val_percent_check = 0.01,\n",
    "                     #test_percent_check = 0.01,\n",
    "                     overfit_pct=0.01,\n",
    "                     #fast_dev_run=True,\n",
    "                     profiler=True,\n",
    "                     #print_nan_grads = True,\n",
    "                     #distributed_backend='dp'\n",
    "                    )    \n",
    "trainer.fit(tft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-04-22T03:10:15.247Z"
    }
   },
   "outputs": [],
   "source": [
    "trainer.test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T22:13:11.878369Z",
     "start_time": "2020-03-29T22:13:11.376438Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/daniel/github/temporal_fusion_transformer_pytorch\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T23:01:53.459938Z",
     "start_time": "2020-03-29T23:01:53.362912Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** TemporalFusionTransformer params ***\n",
      "# total_time_steps = 192\n",
      "# num_encoder_steps = 168\n",
      "# num_epochs = 100\n",
      "# early_stopping_patience = 5\n",
      "# multiprocessing_workers = 5\n",
      "# column_definition = [('id', <DataTypes.REAL_VALUED: 0>, <InputTypes.ID: 4>), ('hours_from_start', <DataTypes.REAL_VALUED: 0>, <InputTypes.TIME: 5>), ('power_usage', <DataTypes.REAL_VALUED: 0>, <InputTypes.TARGET: 0>), ('hour', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('day_of_week', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('hours_from_start', <DataTypes.REAL_VALUED: 0>, <InputTypes.KNOWN_INPUT: 2>), ('categorical_id', <DataTypes.CATEGORICAL: 1>, <InputTypes.STATIC_INPUT: 3>)]\n",
      "# input_size = 5\n",
      "# output_size = 1\n",
      "# category_counts = [369]\n",
      "# input_obs_loc = [0]\n",
      "# static_input_loc = [4]\n",
      "# known_regular_inputs = [1, 2, 3]\n",
      "# known_categorical_inputs = [0]\n",
      "# dropout_rate = 0.1\n",
      "# hidden_layer_size = 160\n",
      "# learning_rate = 0.001\n",
      "# minibatch_size = 64\n",
      "# max_gradient_norm = 0.01\n",
      "# num_heads = 4\n",
      "# stack_size = 1\n",
      "# on_gpu = False\n"
     ]
    }
   ],
   "source": [
    "model = tft.load_from_metrics(\n",
    "                             weights_path='lightning_logs/version_18/checkpoints/epoch=6.ckpt',\n",
    "                             tags_csv='lightning_logs/version_18/meta_tags.csv',\n",
    "                             #on_gpu=True,\n",
    "                             map_location=None\n",
    "                            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T23:02:02.935152Z",
     "start_time": "2020-03-29T23:02:02.932563Z"
    }
   },
   "outputs": [],
   "source": [
    "q_risk = NormalizedQuantileLossCalculator([0.1, 0.5, 0.9], 1)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size = 64, shuffle=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T23:02:10.945537Z",
     "start_time": "2020-03-29T23:02:04.985297Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2., dtype=torch.float64, grad_fn=<DivBackward0>)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = []\n",
    "batches = 0\n",
    "for i, (batch, target, _ )in enumerate(test_dataloader):\n",
    "    if i < 5:\n",
    "        t = target\n",
    "        batches += 1\n",
    "        output = tft(batch)\n",
    "        loss.append(q_risk.apply(output[Ellipsis, 1], target[Ellipsis, 0], 0.5))\n",
    "    else:\n",
    "        break\n",
    "mean_loss = sum(loss) / batches\n",
    "mean_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T22:57:40.648319Z",
     "start_time": "2020-03-29T22:57:40.642766Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor(2., dtype=torch.float64, grad_fn=<DivBackward0>)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T22:57:17.348076Z",
     "start_time": "2020-03-29T22:57:17.344899Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([64, 24, 1])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[Ellipsis, 1:2].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-29T22:55:27.672365Z",
     "start_time": "2020-03-29T22:55:27.669081Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 24, 1]), torch.Size([64, 24, 3]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape, output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:34.676782Z",
     "start_time": "2020-03-26T16:18:34.667335Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_decoder_mask(self_attn_inputs):\n",
    "    \"\"\"Returns causal mask to apply for self-attention layer.\n",
    "    Args:\n",
    "    self_attn_inputs: Inputs to self attention layer to determine mask shape\n",
    "    \"\"\"\n",
    "    len_s = self_attn_inputs.shape[1]\n",
    "    bs = self_attn_inputs.shape[0]\n",
    "    mask = torch.cumsum(torch.eye(len_s), 0)\n",
    "    mask = mask.repeat(bs,1,1).to(torch.float32)\n",
    "\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:20:27.615226Z",
     "start_time": "2020-03-26T16:20:27.609822Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.4701, -0.5985, -0.9775,  0.3386],\n",
       "         [-0.6160, -0.3554,  0.8719,  2.0412],\n",
       "         [ 1.3902,  2.1303, -1.2918,  0.5983],\n",
       "         [ 1.0158,  0.5428,  0.8793,  0.9450],\n",
       "         [-0.0350,  0.8870, -1.7651, -0.4571],\n",
       "         [-0.8908,  0.1620, -1.3613, -0.4487]],\n",
       "\n",
       "        [[-0.7297, -0.7173,  0.5279, -0.0354],\n",
       "         [ 0.9267, -1.7141,  2.7410, -0.5471],\n",
       "         [ 0.1380, -0.8149,  1.9545, -0.3068],\n",
       "         [-0.6078,  1.2512,  0.1963, -0.5023],\n",
       "         [ 0.0187, -0.2740,  0.8720,  1.1292],\n",
       "         [ 1.2788,  0.0790, -0.9275, -0.1276]]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn((2,6,4))\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:20:27.956883Z",
     "start_time": "2020-03-26T16:20:27.942930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1., 0., 0., 0., 0., 0.],\n",
       "         [1., 1., 0., 0., 0., 0.],\n",
       "         [1., 1., 1., 0., 0., 0.],\n",
       "         [1., 1., 1., 1., 0., 0.],\n",
       "         [1., 1., 1., 1., 1., 0.],\n",
       "         [1., 1., 1., 1., 1., 1.]],\n",
       "\n",
       "        [[1., 0., 0., 0., 0., 0.],\n",
       "         [1., 1., 0., 0., 0., 0.],\n",
       "         [1., 1., 1., 0., 0., 0.],\n",
       "         [1., 1., 1., 1., 0., 0.],\n",
       "         [1., 1., 1., 1., 1., 0.],\n",
       "         [1., 1., 1., 1., 1., 1.]]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = get_decoder_mask(a)\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:37.428095Z",
     "start_time": "2020-03-26T16:18:37.420599Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 6, 6])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:37.783682Z",
     "start_time": "2020-03-26T16:18:37.778133Z"
    }
   },
   "outputs": [],
   "source": [
    "linear = nn.Linear(4, 2, bias = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:38.026534Z",
     "start_time": "2020-03-26T16:18:38.018281Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 6, 2])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a_lin = linear(a)\n",
    "a_lin.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:43:34.121081Z",
     "start_time": "2020-03-26T16:43:34.116672Z"
    }
   },
   "outputs": [],
   "source": [
    "to_attn = torch.bmm(a_lin, a_lin.permute(0,2,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:44:27.955457Z",
     "start_time": "2020-03-26T16:44:27.938023Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 4.9358e-01, -1.0000e+09, -1.0000e+09, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [ 6.6717e-01,  2.3156e+00, -1.0000e+09, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [ 6.1026e-02, -2.8282e-01,  1.0194e-01, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [ 4.5336e-01,  9.2999e-01, -2.5902e-02,  4.8758e-01, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [-1.9531e-01, -7.7465e-01,  1.0780e-01, -2.9396e-01,  2.6173e-01,\n",
       "          -1.0000e+09],\n",
       "         [ 9.6977e-03,  7.2699e-01, -1.8326e-01,  1.6906e-01, -2.6169e-01,\n",
       "           3.6066e-01]],\n",
       "\n",
       "        [[ 1.0421e+00, -1.0000e+09, -1.0000e+09, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [-1.4668e-02,  1.8592e+00, -1.0000e+09, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [-4.1870e-01, -1.8913e-01,  1.8869e-01, -1.0000e+09, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [-1.6112e+00,  2.2147e+00,  4.1740e-01,  5.0758e+00, -1.0000e+09,\n",
       "          -1.0000e+09],\n",
       "         [ 9.0954e-01, -1.5431e+00, -2.0489e-01, -3.2107e+00,  2.0536e+00,\n",
       "          -1.0000e+09],\n",
       "         [-3.4914e-01, -3.6158e-01,  1.7872e-01,  1.0765e-01, -3.0122e-03,\n",
       "           1.8922e-01]]], grad_fn=<MaskedFillBackward0>)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "masked_attn = to_attn.masked_fill(mask == 0, -1e9)\n",
    "masked_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:53:56.737081Z",
     "start_time": "2020-03-26T16:53:56.734449Z"
    }
   },
   "outputs": [],
   "source": [
    "softmax = nn.Softmax(dim = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:57:44.273261Z",
     "start_time": "2020-03-26T16:57:44.266922Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.1613, 0.8387, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.3635, 0.2578, 0.3787, 0.0000, 0.0000, 0.0000],\n",
       "         [0.2345, 0.3777, 0.1452, 0.2426, 0.0000, 0.0000],\n",
       "         [0.1852, 0.1038, 0.2508, 0.1678, 0.2925, 0.0000],\n",
       "         [0.1383, 0.2834, 0.1141, 0.1622, 0.1055, 0.1965]],\n",
       "\n",
       "        [[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.1331, 0.8669, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.2443, 0.3073, 0.4484, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0012, 0.0536, 0.0089, 0.9364, 0.0000, 0.0000],\n",
       "         [0.2188, 0.0188, 0.0718, 0.0036, 0.6870, 0.0000],\n",
       "         [0.1192, 0.1177, 0.2021, 0.1882, 0.1685, 0.2042]]],\n",
       "       grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sft_attn = softmax(masked_attn)\n",
    "sft_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:58:14.286401Z",
     "start_time": "2020-03-26T16:58:14.275870Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 6, 2])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.bmm(sft_attn, a_lin).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:41.373265Z",
     "start_time": "2020-03-26T16:18:41.366287Z"
    }
   },
   "outputs": [],
   "source": [
    "scaled_att = ScaledDotProductAttention()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-26T16:18:41.691534Z",
     "start_time": "2020-03-26T16:18:41.679174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first bmm\n",
      "torch.Size([2, 6, 6])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[[ 0.0166, -0.7024],\n",
       "          [ 0.9273, -0.8692],\n",
       "          [ 0.2318, -0.5408],\n",
       "          [ 0.4313, -0.6560],\n",
       "          [ 0.0198, -0.3175],\n",
       "          [ 0.3680, -0.4154]],\n",
       " \n",
       "         [[ 0.9537,  0.3642],\n",
       "          [-0.1946,  1.0787],\n",
       "          [-0.0389,  0.3842],\n",
       "          [-1.7977,  0.9332],\n",
       "          [ 0.8576, -0.2872],\n",
       "          [-0.2122,  0.1365]]], grad_fn=<BmmBackward>),\n",
       " tensor([[[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.2376, 0.7624, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.3554, 0.2787, 0.3659, 0.0000, 0.0000, 0.0000],\n",
       "          [0.2417, 0.3385, 0.1722, 0.2476, 0.0000, 0.0000],\n",
       "          [0.1917, 0.1273, 0.2375, 0.1788, 0.2648, 0.0000],\n",
       "          [0.1479, 0.2457, 0.1291, 0.1656, 0.1221, 0.1896]],\n",
       " \n",
       "         [[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.2100, 0.7900, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.2693, 0.3168, 0.4138, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0075, 0.1122, 0.0315, 0.8488, 0.0000, 0.0000],\n",
       "          [0.2544, 0.0449, 0.1157, 0.0138, 0.5712, 0.0000],\n",
       "          [0.1322, 0.1310, 0.1920, 0.1826, 0.1688, 0.1934]]],\n",
       "        grad_fn=<SoftmaxBackward>))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_att(a_lin, a_lin, a_lin, mask = get_decoder_mask(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nlp] *",
   "language": "python",
   "name": "conda-env-nlp-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "309px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}