Using DL-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.dataset import *\n",
    "from fastai.learner import *\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "import feather\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data/elo/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep = 'target'\n",
    "df_raw = feather.read_dataframe('train_df_3691')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = feather.read_dataframe('test_df_3691')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_raw.drop('outliers', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_raw, df_test]:\n",
    "    for f in ['purchase_date_max','purchase_date_min','purchase_date_max_old',\\\n",
    "                     'purchase_date_min_old']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_excluded = ['purchase_date_max', 'purchase_date_max_old', 'card_id', 'first_active_month',\n",
    "                 'target','outliers','card_id_size', 'card_id_size_old', \n",
    "                 'purchase_date_min', 'purchase_date_min_old','first_active_monthYear',\n",
    "                 'first_active_monthMonth',\n",
    "                 'first_active_monthWeek',\n",
    "                 'first_active_monthDay',\n",
    "                 'first_active_monthDayofweek',\n",
    "                 'first_active_monthDayofyear',\n",
    "                 'first_active_monthIs_month_end',\n",
    "                 'first_active_monthIs_month_start',\n",
    "                 'first_active_monthIs_quarter_end',\n",
    "                 'first_active_monthIs_quarter_start',\n",
    "                 'first_active_monthIs_year_end',\n",
    "                 'first_active_monthIs_year_start']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_indep = df_raw.drop(dep,axis=1)\n",
    "n_valid = 40000\n",
    "n_trn = len(df_raw)-n_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'feature_1 feature_2 feature_3 subsector_id_nunique merchant_category_id_nunique purchase_Month_min purchase_Month_max purchase_Week_nunique purchase_Dayofweek_min purchase_Dayofweek_max purchase_Day_nunique purchase_Day_min purchase_Day_max purchase_Hour_nunique purchase_Hour_min purchase_Hour_max installments_max month_lag_max month_lag_min subsector_id_nunique_old purchase_Month_nunique purchase_Month_min_old purchase_Month_max_old purchase_Week_min_old purchase_Week_max_old purchase_Day_nunique_old purchase_Day_min_old purchase_Hour_nunique_old purchase_Hour_min_old purchase_Hour_max_old installments_max_old month_lag_max_old month_lag_min_old month_diff_max month_diff_min quarter feature_sum feature_mean feature_max feature_min feature_var'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_flds = [n for n in df_indep.columns if df_raw[n].nunique()<50 and n not in cols_excluded]\n",
    "' '.join(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['first_active_month', 'card_id']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n for n in df_indep.drop(cat_flds,axis=1).columns if not is_numeric_dtype(df_raw[n])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for df in [df_raw, df_indep, df_test]:\n",
    "#     for f in ['purchase_date_max','purchase_date_min','purchase_date_max_old',\\\n",
    "#                      'purchase_date_min_old', 'first_active_month']:\n",
    "#         df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['first_active_month', 'card_id']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n for n in df_indep.drop(cat_flds,axis=1).columns if not is_numeric_dtype(df_raw[n])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_drop = ['first_active_monthDayofweek',\n",
    " 'first_active_monthIs_month_end',\n",
    " 'first_active_monthIs_quarter_end',\n",
    " 'first_active_monthIs_year_end']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_indep.drop(cols_to_drop, axis=1, inplace=True)\n",
    "df_test.drop(cols_to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'first_active_monthElapsed transactions_count merchant_id_nunique purchase_Month_mean purchase_Week_mean purchase_Week_min purchase_Week_max purchase_Dayofweek_mean purchase_Day_mean purchase_Hour_mean purchase_amount_sum purchase_amount_max purchase_amount_min purchase_amount_mean purchase_amount_var purchase_amount_skew installments_sum installments_mean installments_var installments_skew month_lag_mean month_lag_var month_lag_skew month_diff_mean month_diff_var month_diff_skew purchased_on_weekend_mean category_1_mean category_2_mean category_3_mean card_id_count price_mean price_max price_min price_var Christmas_Day_2017_mean Children_day_2017_mean Black_Friday_2017_mean Mothers_Day_2018_mean duration_mean duration_min duration_max duration_var duration_skew amount_month_ratio_mean amount_month_ratio_min amount_month_ratio_max amount_month_ratio_var amount_month_ratio_skew category_2_mean_mean category_3_mean_mean purchase_date_diff purchase_date_average purchase_date_uptonow purchase_date_uptomin transactions_count_old merchant_id_nunique_old merchant_category_id_nunique_old purchase_Month_mean_old purchase_Week_nunique_old purchase_Week_mean_old purchase_Dayofweek_mean_old purchase_Day_mean_old purchase_Hour_mean_old purchase_amount_sum_old purchase_amount_max_old purchase_amount_min_old purchase_amount_mean_old purchase_amount_var_old purchase_amount_skew_old installments_sum_old installments_mean_old installments_var_old installments_skew_old month_lag_mean_old month_lag_var_old month_lag_skew_old month_diff_mean_old month_diff_var_old month_diff_skew_old authorized_flag_mean purchased_on_weekend_mean_old category_1_mean_old category_2_mean_old category_3_mean_old card_id_count_old price_sum price_mean_old price_max_old price_min_old price_var_old Christmas_Day_2017_mean_old Mothers_Day_2017_mean fathers_day_2017_mean Children_day_2017_mean_old Valentine_Day_2017_mean Black_Friday_2017_mean_old Mothers_Day_2018_mean_old duration_mean_old duration_min_old duration_max_old duration_var_old duration_skew_old amount_month_ratio_mean_old amount_month_ratio_min_old amount_month_ratio_max_old amount_month_ratio_var_old amount_month_ratio_skew_old category_2_mean_mean_old category_3_mean_mean_old purchase_date_diff_old purchase_date_average_old purchase_date_uptonow_old purchase_date_uptomin_old card_id_total card_id_count_total card_id_count_ratio purchase_amount_total purchase_amount_total_mean purchase_amount_total_max purchase_amount_total_min purchase_amount_sum_ratio hist_first_buy new_first_buy hist_last_buy new_last_buy month_diff_ratio installments_total installments_ratio price_total CLV CLV_old CLV_ratio elapsed_time days_feature1 days_feature2 days_feature3 days_feature1_ratio days_feature2_ratio days_feature3_ratio'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for n in cat_flds: df_raw[n] = df_raw[n].astype('category').cat.as_ordered()\n",
    "\n",
    "cont_flds = [n for n in df_indep.columns if n not in cat_flds and n not in cols_excluded]\n",
    "' '.join(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.replace(np.inf, 0, inplace=True)\n",
    "df_raw.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = df_raw[cat_flds+cont_flds+[dep]]\n",
    "df, y, nas, mapper = proc_df(df_raw, 'target', do_scale=True)\n",
    "\n",
    "val_idx = list(range(n_trn, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.replace(np.inf, 0, inplace=True)\n",
    "df_test.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_cols = [c for c in df_raw.columns if not c in ['target']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(181, (201917, 182), (123623, 197))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_cols), df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t, _, nas, mapper = proc_df(df_test[test_cols], do_scale=True, mapper=mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 270), (123623, 287))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_cats(df_t, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_t = df_t[df.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((123623, 270), (201917, 270))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_t.shape, df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ColumnarModelData.from_data_frame(PATH, val_idx, df, y, cat_flds=cat_flds, bs=64, test_df=df_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(x,y): return math.sqrt(((x-y)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in cat_flds: df[n] = df[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'feature_1': 6,\n",
       " 'feature_2': 4,\n",
       " 'feature_3': 3,\n",
       " 'subsector_id_nunique': 25,\n",
       " 'merchant_category_id_nunique': 40,\n",
       " 'purchase_Month_min': 13,\n",
       " 'purchase_Month_max': 13,\n",
       " 'purchase_Week_nunique': 11,\n",
       " 'purchase_Dayofweek_min': 9,\n",
       " 'purchase_Dayofweek_max': 9,\n",
       " 'purchase_Day_nunique': 31,\n",
       " 'purchase_Day_min': 32,\n",
       " 'purchase_Day_max': 32,\n",
       " 'purchase_Hour_nunique': 24,\n",
       " 'purchase_Hour_min': 26,\n",
       " 'purchase_Hour_max': 26,\n",
       " 'installments_max': 27,\n",
       " 'month_lag_max': 14,\n",
       " 'month_lag_min': 16,\n",
       " 'subsector_id_nunique_old': 35,\n",
       " 'purchase_Month_nunique': 12,\n",
       " 'purchase_Month_min_old': 12,\n",
       " 'purchase_Month_max_old': 12,\n",
       " 'purchase_Week_min_old': 49,\n",
       " 'purchase_Week_max_old': 49,\n",
       " 'purchase_Day_nunique_old': 32,\n",
       " 'purchase_Day_min_old': 31,\n",
       " 'purchase_Hour_nunique_old': 25,\n",
       " 'purchase_Hour_min_old': 24,\n",
       " 'purchase_Hour_max_old': 23,\n",
       " 'installments_max_old': 14,\n",
       " 'month_lag_max_old': 13,\n",
       " 'month_lag_min_old': 14,\n",
       " 'month_diff_max': 15,\n",
       " 'month_diff_min': 15,\n",
       " 'quarter': 5,\n",
       " 'feature_sum': 8,\n",
       " 'feature_mean': 8,\n",
       " 'feature_max': 6,\n",
       " 'feature_min': 3,\n",
       " 'feature_var': 10}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_c = {n: len(c.cat.categories)+1 for n,c in df[cat_flds].items()}\n",
    "emb_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = [(c, min(50, (c+1)//2)) for _,c in emb_c.items()]\n",
    "metrics=[rmse]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_range=(np.min(y)*1,np.max(y)*1.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(6, 3),\n",
       " (4, 2),\n",
       " (3, 2),\n",
       " (25, 13),\n",
       " (40, 20),\n",
       " (13, 7),\n",
       " (13, 7),\n",
       " (11, 6),\n",
       " (9, 5),\n",
       " (9, 5),\n",
       " (31, 16),\n",
       " (32, 16),\n",
       " (32, 16),\n",
       " (24, 12),\n",
       " (26, 13),\n",
       " (26, 13),\n",
       " (27, 14),\n",
       " (14, 7),\n",
       " (16, 8),\n",
       " (35, 18),\n",
       " (12, 6),\n",
       " (12, 6),\n",
       " (12, 6),\n",
       " (49, 25),\n",
       " (49, 25),\n",
       " (32, 16),\n",
       " (31, 16),\n",
       " (25, 13),\n",
       " (24, 12),\n",
       " (23, 12),\n",
       " (14, 7),\n",
       " (13, 7),\n",
       " (14, 7),\n",
       " (15, 8),\n",
       " (15, 8),\n",
       " (5, 3),\n",
       " (8, 4),\n",
       " (8, 4),\n",
       " (6, 3),\n",
       " (3, 2),\n",
       " (10, 5)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_szs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Embedding-1',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 18)])),\n",
       "             ('Embedding-2',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 8)])),\n",
       "             ('Embedding-3',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-4',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 13]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 325)])),\n",
       "             ('Embedding-5',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 20]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 800)])),\n",
       "             ('Embedding-6',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 91)])),\n",
       "             ('Embedding-7',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 91)])),\n",
       "             ('Embedding-8',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 6]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 66)])),\n",
       "             ('Embedding-9',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 5]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 45)])),\n",
       "             ('Embedding-10',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 5]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 45)])),\n",
       "             ('Embedding-11',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 496)])),\n",
       "             ('Embedding-12',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 512)])),\n",
       "             ('Embedding-13',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 512)])),\n",
       "             ('Embedding-14',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 12]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 288)])),\n",
       "             ('Embedding-15',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 13]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 338)])),\n",
       "             ('Embedding-16',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 13]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 338)])),\n",
       "             ('Embedding-17',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 14]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 378)])),\n",
       "             ('Embedding-18',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 98)])),\n",
       "             ('Embedding-19',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 128)])),\n",
       "             ('Embedding-20',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 18]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 630)])),\n",
       "             ('Embedding-21',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 6]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 72)])),\n",
       "             ('Embedding-22',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 6]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 72)])),\n",
       "             ('Embedding-23',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 6]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 72)])),\n",
       "             ('Embedding-24',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 25]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1225)])),\n",
       "             ('Embedding-25',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 25]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 1225)])),\n",
       "             ('Embedding-26',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 512)])),\n",
       "             ('Embedding-27',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 16]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 496)])),\n",
       "             ('Embedding-28',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 13]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 325)])),\n",
       "             ('Embedding-29',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 12]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 288)])),\n",
       "             ('Embedding-30',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 12]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 276)])),\n",
       "             ('Embedding-31',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 98)])),\n",
       "             ('Embedding-32',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 91)])),\n",
       "             ('Embedding-33',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 7]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 98)])),\n",
       "             ('Embedding-34',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 120)])),\n",
       "             ('Embedding-35',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 8]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 120)])),\n",
       "             ('Embedding-36',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 15)])),\n",
       "             ('Embedding-37',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 32)])),\n",
       "             ('Embedding-38',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 4]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 32)])),\n",
       "             ('Embedding-39',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 18)])),\n",
       "             ('Embedding-40',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-41',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 5]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 50)])),\n",
       "             ('Dropout-42',\n",
       "              OrderedDict([('input_shape', [-1, 398]),\n",
       "                           ('output_shape', [-1, 398]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('BatchNorm1d-43',\n",
       "              OrderedDict([('input_shape', [-1, 229]),\n",
       "                           ('output_shape', [-1, 229]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 458)])),\n",
       "             ('Linear-44',\n",
       "              OrderedDict([('input_shape', [-1, 627]),\n",
       "                           ('output_shape', [-1, 1000]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 628000)])),\n",
       "             ('Dropout-45',\n",
       "              OrderedDict([('input_shape', [-1, 1000]),\n",
       "                           ('output_shape', [-1, 1000]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-46',\n",
       "              OrderedDict([('input_shape', [-1, 1000]),\n",
       "                           ('output_shape', [-1, 500]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 500500)])),\n",
       "             ('Dropout-47',\n",
       "              OrderedDict([('input_shape', [-1, 500]),\n",
       "                           ('output_shape', [-1, 500]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-48',\n",
       "              OrderedDict([('input_shape', [-1, 500]),\n",
       "                           ('output_shape', [-1, 1]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 501)]))])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# m = md.get_learner(emb_szs, len(cont_flds), 0.05, 1, [500,250], [0.5,0.05],\n",
    "#                    y_range=y_range, use_bn=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "140"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a154ac1fa0ce422289e8ea4613c8bf51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2530 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCTensorCopy.cu:204",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-0510517b8367>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mlr_find\u001b[0;34m(self, start_lr, end_lr, wds, linear, **kwargs)\u001b[0m\n\u001b[1;32m    343\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLR_Finder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrn_dl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlinear\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'tmp'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    247\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, visualize, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/column_data.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x_cat, x_cont)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_cat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_cont\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_emb\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memb_drop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/column_data.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_cat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_cont\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_emb\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memb_drop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/sparse.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    101\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m             \u001b[0mpadding_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_norm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm_type\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscale_grad_by_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m         )\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/_functions/thnn/sparse.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(cls, ctx, indices, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_contiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m             \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m             \u001b[0mindices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_indices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCTensorCopy.cu:204"
     ]
    }
   ],
   "source": [
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEOCAYAAACetPCkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xd4VGX6//H3nQpJICAJSJUiqIBgQSyggqJib7jquq5tdfXrz7pFXF1dcRX96upa1nVd6xZd61ddQVBRiqggKFJFOgIBEkoS0svz++OchCSkzCSZmSTzeV3XXDPnmXNm7jkMueepx5xziIiIBCom0gGIiEjrosQhIiJBUeIQEZGgKHGIiEhQlDhERCQoShwiIhIUJQ4REQmKEoeIiARFiUNERIKixCEiIkGJi3QATZGWlub69u0b6TBERFqVhQsXZjnn0ht7fKtOHH379mXBggWRDkNEpFUxsw1NOV5NVSIiEhQlDhERCYoSh4iIBEWJQ0REgqLEISIiQVHiEBGRoERl4liflce0pVspK9dlc0VEghWViWP6sq1c/6+FFJaURToUEZFWJyoTR0Kc97GLS8sjHImISOsT3YmjTIlDRCRY0Zk4Yr2PXVSixCEiEqyoTByJ8bEAFJepj0NEJFhRmTgqaxzq4xARCVpUJo5EdY6LiDRaVCYOjaoSEWm8qEwcFTUONVWJiAQvKhOHahwiIo0X3YlD8zhERIIWnYkjVjUOEZHGClniMLMXzWy7mS2tUrafmX1sZqv8+85+uZnZk2a22swWm9kRoYoL9s7jKCrVPA4RkWCFssbxMjC+RtlEYIZzbiAww98GOB0Y6N+uA/4awrhU4xARaYKQJQ7n3GxgZ43ic4FX/MevAOdVKf+H83wFdDKz7qGKLUGjqkREGi3cfRzdnHMZAP59V7+8J/Bjlf02+WUhUVHjKCnT9ThERILVUjrHrZayWv+qm9l1ZrbAzBZkZmY26s00HFdEpPHCnTi2VTRB+ffb/fJNQO8q+/UCttT2As6555xzI5xzI9LT0xsVRGyMEWNQouG4IiJBC3fieB+4wn98BfBelfKf+6OrjgGyK5q0QiUhLkbzOEREGiEuVC9sZq8BY4A0M9sE3As8BLxhZtcAG4GL/N2nAmcAq4F84KpQxVUhITZGTVUiIo0QssThnLu0jqdOrmVfB9wYqlhqoxqHiEjjtJTO8bBTjUNEpHGiNnHEx8Woc1xEpBGiNnGoxiEi0jjRmzhU4xARaZSoTRzxsTFackREpBGiNnEkxKmpSkSkMaI3ccSqqUpEpDGiN3FoHoeISKNEbeKIjzVKSrU6rohIsKI2cSTExarGISLSCNGbODSPQ0SkUaI3ccSZahwiIo0QvYlDNQ4RkUaJ2sQRr+G4IiKNErWJQxMARUQaJ6oTR2m5o7xcQ3JFRIIRtYkjPtb76OogFxEJTtQmjsQ4JQ4RkcaI2sRRUeMoUT+HiEhQojZxJKjGISLSKNGbOCprHOocFxEJRtQmjvjKGkdZhCMREWldojZxVHSOF5aoqUpEJBhRmziSE+IAyC9WjUNEJBhRmzjaJ8QCkF9cGuFIRERal6hNHEl+4ihQjUNEJChRnzjylDhERIIStYmjfWWNQ01VIiLBiNrEoc5xEZHGidrE0T6+onNciUNEJBhRmzhiYox28TEUlChxiIgEI2oTB0BSQhx5RerjEBEJRlQnjvbxsRqOKyISpIgkDjO7zcyWmdlSM3vNzNqZWT8zm2dmq8zsdTNLCHUcSQmx6uMQEQlS2BOHmfUEbgZGOOeGArHAJcDDwOPOuYHALuCaUMeSlBhHvvo4RESCEqmmqjigvZnFAUlABnAS8Jb//CvAeaEOIik+VvM4RESCFPbE4ZzbDDwKbMRLGNnAQmC3c67ir/gmoGeoY0lKiCWvSDUOEZFgRKKpqjNwLtAP6AEkA6fXsmutV1gys+vMbIGZLcjMzGxSLO0TYjUcV0QkSJFoqhoHrHPOZTrnSoB3gOOATn7TFUAvYEttBzvnnnPOjXDOjUhPT29SIF7nuJqqRESCEYnEsRE4xsySzMyAk4HlwGfABH+fK4D3Qh1IUkKcRlWJiAQpEn0c8/A6wb8BlvgxPAfcAdxuZquBLsALoY4lKcGbx+GcrjsuIhKouIZ3aX7OuXuBe2sUrwVGhjOOpIRYSssdxWXlJMbFhvOtRURareieOe6vkKvZ4yIigWswcZhZspnF+I8Hmdk5ZhYf+tBCLylBK+SKiAQrkBrHbKCdP+N7BnAV8HIogwoXJQ4RkeAFkjjMOZcPXAA85Zw7Hxgc2rDCI0lNVSIiQQsocZjZscBlwBS/LCKd6s1t73XHNZdDRCRQgSSOW4E7gf9zzi0zs/54cy5avb3XHVeNQ0QkUA3WHJxzs4BZAH4neZZz7uZQBxYOKYnex1eNQ0QkcIGMqnrVzDqaWTLeDO+VZvab0IcWehWJY0+hEoeISKACaaoa7JzLwVvmfCrQB7g8pFGFSUo7L3HkKnGIiAQskMQR78/bOA94z1+YsE2s0ZHij6rK1XXHRUQCFkji+BuwHm/589lmdgCQE8qgwiUmxkhJjFNTlYhIEALpHH8SeLJK0QYzGxu6kMIrJTGOPUUlkQ5DRKTVCKRzPNXMHqu4eJKZ/Qmv9tEmpLSLUx+HiEgQAmmqehHIBX7i33KAl0IZVDh165hIRnZhpMMQEWk1ApkBPsA5d2GV7fvMbFGoAgq37qntmbs6K9JhiIi0GoHUOArMbHTFhpmNAgpCF1J4pSTGkadRVSIiAQukxnED8IqZpQIG7ASuDGVQ4ZSUEEtBiZYcEREJVCCjqhYBw82so7/dJobiVkhKiKWkzFFcWk5CXFRf10pEJCB1Jg4zu72OcgCcc4+FKKawqlhaPb+4lIS4hAhHIyLS8tVX4+gQtigiKDmxYmn1MjolRTgYEZFWoM7E4Zy7L5yBRMreizmpg1xEJBBR36hfeTGnInWQi4gEQokjQdfkEBEJRtQnjoo+jnzVOEREAtLgcFwzSwQuBPpW3d85Nyl0YYVPRVPVK1+uZ9zgbpENRkSkFQhkAuB7QDawECgKbTjhV9FUNWdVFqVl5cTFRn0lTESkXoEkjl7OufEhjyRCkhP3noI9RaV0StJcDhGR+gTy8/oLMzs05JFESGr7eK48ri8A2QW6LoeISEMCSRyjgYVmttLMFpvZEjNbHOrAwum4AV0AyCnQyCoRkYYE0lR1esijiLCO7eMByClUjUNEpCEN1jiccxuATsDZ/q2TX9ZmpPqJQ01VIiINC+TSsbcA/wa6+rd/mdlNoQ4snCprHEocIiINCqSp6hrgaOdcHoCZPQx8CTwVysDCKVVNVSIiAQukc9yAqtOqy/yyRjOzTmb2lpl9b2YrzOxYM9vPzD42s1X+feemvEcwkhNiiTE1VYmIBCKQxPESMM/M/mBmfwC+Al5o4vs+AUxzzh0MDAdWABOBGc65gcAMfzsszIyO7eM1qkpEJACBXAHwMTObiTcs14CrnHPfNvYN/SsJnoB/+VnnXDFQbGbnAmP83V4BZgJ3NPZ9gpXaPl5NVSIiAajvCoAdnXM5ZrYfsN6/VTy3n3NuZyPfsz+QCbxkZsPxljK5BejmnMsAcM5lmFnXRr5+o3RsF6+mKhGRANRX43gVOAvvD7urUm7+dv8mvOcRwE3OuXlm9gRBNEuZ2XXAdQB9+vRpZAj76tg+TqOqREQCUGcfh3PuLP++n3Ouf5VbP+dcY5MGwCZgk3Nunr/9Fl4i2WZm3QH8++11xPWcc26Ec25Eenp6E8KozmuqUh+HiEhDApnHMSOQskA557YCP5rZQX7RycBy4H3gCr/sCrxVecNGTVUiIoGpr4+jHZAEpPlDYyuG4HYEejTxfW8C/m1mCcBa4Cq8JPaGmV0DbAQuauJ7BGW/5AR25RVTUlZOvJZWFxGpU319HL8EbsVLEgvZmzhygL805U2dc4uAEbU8dXJTXrcp+qUlU1ru2LyrgL5pyZEKQ0SkxaszcTjnngCeMLObnHNtZpZ4XdJSEgHYlV9MX5Q4RETqEsg8jqfMbCgwGGhXpfwfoQws3FKTvGVHduern0NEpD6BXHP8XryJeYOBqXjLrH8OtKnE0clfr2p3QXGEIxERadkC6QWegNf3sNU5dxXeEiGJIY0qAiouGasah4hI/QJJHAXOuXKg1F8uZDuNn/zXYlWskKvEISJSv0CWVV9gZp2Av+ONrtoDzA9pVBEQG2N0bBenuRwiIg0IpHP8f/yHz5rZNKCjc65NXXO8QqekBHblq49DRKQ+9U0APKK+55xz34QmpMjpnBSvpioRkQbUV+P4k3/fDm+y3nd4kwCHAfPwlllvU1KTEtitpioRkXrVt8jhWOfcWGADcIS/sOCRwOHA6nAFGE6d2sezW01VIiL1CmRU1cHOuSUVG865pcBhoQspcvqnJ7NxZ746yEVE6hHIqKoVZvY88C+863D8DO9Sr21O//QUnIPM3MLK4bkiIlJdIDWOq4BleFfpuxVvCfSrQhlUpKQle5MA12bmRTgSEZGWK5DhuIXA4/6tTRveuxMASzdnc+qQ/SMcjYhIy1TfcNw3nHM/MbMlVL90LADOuWEhjSwCkhPjSEtJJHNPUaRDERFpseqrcdzi358VjkBaivQOiWzPUeIQEalLfdfjyPDvN4QvnMjLKShhRUYOW3YX0KNT+0iHIyLS4tTZOW5muWaWU8st18xywhlkOG3eXQDAlMUZEY5ERKRlqq/G0SGcgbQ0xWXlkQ5BRKRFCmQ4LgBm1tXM+lTcQhlUJL3xy2MBKFHiEBGpVYOJw8zOMbNVwDpgFrAe+DDEcUXMyH770Tkpnu256iAXEalNIDWO+4FjgB+cc/3wrgY4N6RRRVjXDu3IVOIQEalVIImjxDm3A4gxsxjn3Ge00bWqKnTtmKjEISJSh0DWqtptZinAbODfZrYdKA1tWJGVnpKoZUdEROoQSI3jXCAfuA2YBqwBzg5lUJHWvVM7MrIL2JWnJdZFRGoKJHFcB/RwzpU6515xzj3pN121WaMGpFHuYHlGm52uIiLSaIEkjo7AdDObY2Y3mlm3UAcVaQO6pgCwNkvNVSIiNTWYOJxz9znnhgA3Aj2AWWb2Scgji6CuHRJJSohlnfo5RET2EfAEQGA7sBXYAXQNTTgtg5nRLy2ZtVl7Ih2KiEiLE8gEwBvMbCYwA0gDrm2LS6rX1C8tmXVqqhIR2Ucgw3EPAG51zi0KdTAtSf+0ZKYuyaC4tJyEuGAqZiIibVsgfRwToy1pAPRLT6bcwcad+ZEORUSkRdFP6Tr0S/NGVqm5SkSkuoglDjOLNbNvzewDf7ufmc0zs1Vm9rqZJUQqNoB+XZIBWKcOchGRaiJZ47gFWFFl+2HgcefcQGAXcE1EovKlJsUTH2s8OPV73lu0OZKhiIi0KBFJHGbWCzgTeN7fNuAk4C1/l1eA8yIRW1UxZgA8P2ddhCMREWk5IlXj+DPwW6DiakldgN3OuYrFEzcBPSMRWFVPXno4AGkpEW01ExFpUcKeOMzsLGC7c25h1eJadnV1HH+dmS0wswWZmZkhibHCaUP2Z9wh3ViRkYtztYYjIhJ1IlHjGAWcY2brgf/gNVH9GehkZhXzSnoBW2o72Dn3nHNuhHNuRHp6esiDPengrmzNKWTJ5uyQv5eISGsQ9sThnLvTOdfLOdcXuAT41Dl3GfAZMMHf7QrgvXDHVptxh3irq8xd3aYXBBYRCVhLmsdxB3C7ma3G6/N4IcLxANC1Yzu6p7Zj1bbcSIciItIiBLLkSMg452YCM/3Ha4GRkYynLgd2TWHVds3nEBGBllXjaLEGpKewNnOPOshFRFDiCEj/9GTyisvYnlsU6VBERCJOiSMA/dK85Ud+/+7SCEciIhJ5ShwBOKJPZwA+Wr6NBet3RjgaEZHIUuIIQHJiHOkdEgGYvmxrhKMREYksJY4AfX3XOIb3StVEQBGJekocQRjSM5Wv1u4ku6Ak0qGIiESMEkcQDuvVCYBnZ62JcCQiIpGjxBGECUf2AiBTw3JFJIopcQQhJsY4fmAa32/NiXQoIiIRo8QRpKE9U1mRkcv2nMJIhyIiEhFKHEE6Y2h3ysodX67VarkiEp2UOII0uEdHUhLj+Hj5tkiHIiISEUocQYqNMcYd0pUPFmewbIvmdIhI9FHiaIS7zhwMwG/eXBzhSEREwk+JoxEqlh9ZnpHDGU/MobCkLMIRiYiEjxJHI9188kDASx4H/34aN7/2Ldn5Jbpmh4i0eUocjXT7KYO4YcyAyu33v9vC8Ekf8fIX6yMXlIhIGChxNMFt4wbx+MXDefSi4ZVl9/13uZquRKRNi+g1x1u7hLgYzj+8F2Xlju8zcli8OZv563YybelWzju8Z6TDExEJCdU4mkFsjHH3WYN57dpjALj19UV8sTorwlGJiISGEkczio0xLh3ZG4CfPj+P/OLSCEckItL8lDia2eQLhlU+/miZZpeLSNujxBECax48A/CarHILS/hidZZqHyLSZihxhEBsjHHZ0X0AeGbmGn76/Dz+OGVFhKMSEWkeShwhMuncoQD8daZ3tcCpSzIiGY6ISLNR4giR2Bjj0J6pldu780vIzte1ykWk9VPiCKH/XHcMSQmxldvDJ31E34lTWL4lh/F/ns3X63dGMDoRkcax1ry20ogRI9yCBQsiHUaDnHP0u3Nqrc+tf+jMMEcjItHOzBY650Y09njVOMLAzConB9b02ffbwxyNiEjTKHGEybEDurD+oTN54pLDqpVf9fLXlJSVRygqEZHgKXGE2bhDunH8wDQeOH8oYw9KB2DgXR9SWkfy2LQrn74Tp3DSn2ayaltuOEMVEamVEkeYJSfG8c9rjuayow/gxSuPqix/ce66Wvef/OH3AKzNzOOUx2eHJcZIeP3rjZz11Bz2FLW+iZLZ+SXMW7sj0mGIhE3YE4eZ9Tazz8xshZktM7Nb/PL9zOxjM1vl33cOd2zhZmasm3wGCXExTFmyFWCfJdkLi6tvP/3pKhb9uJuy8tY7qKGm0rJy7nh7CUs35zD03unsyiuOdEiVCkvK2J1fTHFpea3n/IdtuQyf9BEXP/cVM1eqv0qiQ9hHVZlZd6C7c+4bM+sALATOA64EdjrnHjKziUBn59wd9b1WaxlV1ZBnZ63hIb9mAWAGn9x+In+duYa3Fm5iSI+OvPM/x3HQ3dMq9/np0X148PxDIxFu0DbsyGPzrgIGdE2hW8d2+zz/zcZdXPDMF9XK+uyXxIe3HM/iTdlc+vevAFj74BnExFhYYgZYuGEnF/71y8rtMw/tzl8uOwKApZuz2ZpdyFsLNzFt2dbKff77/0ZzaK/UfV4rGHuKSskuKKFnp/ZNeh2RujR1VFXEh+Oa2XvA0/5tjHMuw08uM51zB9V3bFtJHJt25TP64c/qfP7pnx7OWcN68PHybVz7j72fd9whXUlJjOPhCcNIjIut8/hI2p5TyMgHZ1Ru333mIfzi+P7V9jn6wU/YllPEJ7efyLjHZtX5Wlce15djB3ThtCH7hyxe8GoZM1ZsZ9YP23ljwaZqz63843gSYmMY+eAMMnOLiIsxSqvUREYfmMZvxx/EsF6dgnpP5xwlZY6EuBhOenQma7Py+O6eU0lNim+WzyRSVatOHGbWF5gNDAU2Ouc6VXlul3Ou3uaqtpI4AD5flcUfpyynuKyctZl5leWPTBjGRSN6V9t37uosLnt+XrWyAenJzPjVmHCECsCSTdk8OHUFj/5kOM453lywietO6E9GdgFdkhPpnJwAwOUvzGPOqurXJmkfH8vFR/Xm+605XHb0Adz02reAN6clp7CE0Q99Sk5h3X0dC+8eR5eUxJB9tn9+tYHfv7u0cvvSkb0Zd0g3rnml9u+aGSy651R+/uJ8vvtxNwDjh+zPXy47gtg6akhrM/fwweIMFm/K5t6zB/PNxl3c8p9FHNAliQ078gG49+zBXDWqX63H/7gznydnrOIP5wwhOVHXY5PgtNrEYWYpwCzgAefcO2a2O5DEYWbXAdcB9OnT58gNGzaELeZw2bgjnxMe8Wogs34zhgO6JO+zz2vzN3LnO0uqlX326zH0S9t33+ZWUlbOwLs+DOqY+XedzJdrdnDLfxbV+vzVo/pxz9mDASgq9fp1jn/4M7bnFvH8z0eQGB/D5S/M98oHpvHUpYfTKclLTgs37GLGim385rSDMGt6U9Yhv59Ggd/X9OhFw5lwZC/yi0sZfM/0ffZN75DIS1cexdCeqTw3ew0PTt3b5HjzSQdy+6m1V5r7TpxS+Xj8kP1ZnbmH1dv3VNvnuAFdeLXG/B/nHA9PW8lLc9dRVOqNxFs3+Yxm+dxSv/Jyx9w1WfRPT2n1zYitcgKgmcUDbwP/ds694xdv85uoKvpBau1pdM4955wb4ZwbkZ6eHp6Aw6xPlyT+dvmR3DBmAH32S6p1n0uO6s2dpx/MIxOG0T/dSxZjH51J34lTyC3cuyZWebkL+BroX6zJIiO7oMH9Ply6tcF9qrrl5IF07dCOc4b3YEiPjrXu8+vTBlU+ToyLJTEulnm/O5l1k89g3OBuHD8wnQ9uGk23jonMWZXFYZM+ZtMu75f53e8u5ZmZa5jSDAtJzl2dRUFJGT1S23HH+IM5e3h3AJIS4hjYNaXavsvuO42v7xrHUH9NsutOGMD0W0/g2P5dAHjy09Vs2V1AaVk5z81eQ2ZuEc45/vLZ6mqvM23ZVlZv38PFfs3SDC4d2Ycv1uzguMkzWLo5u3LfjOxCnp21pjJpALxZozlNmkdxaTmrtuVS8eP6nveXcvkL8xn10Kcc9cAnFBQH9v+qLYpE57gBr+B1hN9apfwRYEeVzvH9nHO/re+12lJTVVMdNukjdldZRPHUwd24ZGRvnpu9lq/W7mTGr05kQHpKrcdu3l3A+Mdnk+sPhV39wOnExdb9m6Li1/Kb1x/LRc96nce/HX8Q//5qI2cN684vju/Pba8vIiEuhrgY4+mfHkFCnPd65eWOPcWlLNmUzTcbdrErv4SUxNg6f5nXtHFHPhf8dS5Ze4q5alRfbjhxQLU+lJV/HF+tvye3sITFm7IpLCkjJsaYt3Ynd4yvvWZy82vf8v53WwB478ZRDO+9bz9Feblj1fY9rMncwxmHdq8zztXb9zDusVnce/Zg8ovLeGT6yn32eemqo4iPieFnL3jNjusmn8GOvGI6tIvjx50F1fp7vr9/PO3iY/n5i/OZ/UMmAB/fdkLlEO0Vk8bTPqFl9HOVl7uwDmIIlYc+/J5nZ3mrW//t8iP55T8XVnv+6lH96J+ezIwV23jxyqNY9ONuSssdR/XdLxLhBqXVNVWZ2WhgDrAEqPjZ9DtgHvAG0AfYCFzknKt3FUAljr1WZORw/wfL+WJN3fMJThyUzrqsPF7/5TEkxsWyeVcBh/ZKZfyfZ/P91r2TC1+5eiQnDtpbm9uVV8z0ZVvZsDOfY/t34ecvzqdnp/bMnXgSX6zO4oGpK3j9l8eSEsa29mte/poZ329n7EHpfLYys7L8mcuOqPyDXtcaYTefPJDOSfFceVzfygRSsymqOdYQO/XxWWTmFpFXVEZxjQmeyQmxzL9rHMmJcdz472/olBTPAzVGyW3NLuSYyV5S/MXofuyf2q7yui4VPwQmT13B32av5ZrR/fj9WYNrjSOnsISE2BjaxYc+scxZlVnZpDj5gkO5dGSfoF8jO7+E4ZM+4qxh3XlkwnDaJ8TyxCerOLBrCmcOqztZN7eqzYkVrj2+H/slJ/LwtO9rOcIz9qB0nv7pEazLyuOQ7h3r7OcKxM68YlIS4yp/eDWXVpc4mpMSx77eW7SZ/522ks27vSanzknx7ApwOfdrj+/H3+d4ExFHHdiFGDNOOrgr9/13+T779k9P5tMwdsbXVHOo7NL7TuPkP83k0J6p/P3nI3j8k1UkJcRWG+bcPj62su8C4NVfHM1xB6YBsGxLNmc++TkAf//5CE4Z3K3JMT728Q88OWMVAG/fcGxlvPefN5TLjzkgoNeorT+pa4dE5k48iXi/VjjknmnkFZdV1kqqWro5m7Oe+pzjB6YxrFcqpwzen8NqqUk1VU5hCec+PZd1WXnVyudOPKnB/oD/+3YT//ftFp64+DA6Jyfwr682cHeVwQlL7zuNofd6SX3d5DNYuS2X1Pbx7N+xHXnFZSH5wZJXVMqQe/ft05p/18l07dCO/8zfyMQafYy1Gdy9I1eP7keX5ATGHtw1qBiqfsen33oCB+3fIajj69Mq+zgkdM49rCdzJ57EnN+O5Z/XjOTbe05lxaTxdPFHOdXlhStGcNeZg5lwZC8A5q7ewZxVWbUmDfD6WCLpyAP24ycjvFjHHpROSmIc5x3ek89WZjJ7VRZPzlhVmTRuP2UQH95yPCvuH8/Qnnv7WH76/DyWb8lhfVYeN/77GwAW3D2uWZIGwDX+iKi0lASG9+rEb047iPvOGRJw0gCIj42pVvvrkdqO+XeNq0waAPf5Fw2b5TdhOef4cWc+z89Zy/nPzAVgzqos/vLZGs77y1w27crnh225FJc23xppn32/vVrSGO0n5He/3dzgsbe9/h2zf8jk8Ps/JiO7gGVbcgAv0QP8j/9vA9DvzqmM//Mcjp38Kb9+czFD753Ohh15tb5uY7254MfKpHHv2YN59mdHAnDn6QfTtYM3D+nio3qTEBdD56R4htczb2d5Rg6/fvM7rnr5a1Zty+W52WvoO3EKT3+6qsG+x6ojJx+ZvrJa32WkqcYRRbZmF/LjrvzKfokrj+vLy1+s5/iBafzj6pGYGWXljlMfn8WeolK25RRVHjvqwC5cf+IAbnv9Ox6ZMCzoX0+hUF7uWLktlwO7phAfG8P6rDzGPDpzn/3WPHhGZXNBUWkZ+UVlPDFjFS9/sb7afpcc1ZuHLhzWrDFW/P9qyqin7PwSjntoBtefOIArRvWlY7vqcztKyso5YtLH5BaVMnfiSXzw3ZbKpWoa8u3vT6kcOt1Y5eWO/r/zmgTf/3+jKuewnP3U5yzZnM1FR/bikYuGVzvGOceSzdn06pzEuMdmsbPGagE9O7Xn8zvGMubRmZXDk+ty4qB0Xrl6ZJM+Q4UNO/KNZgd/AAAOeklEQVQ48ZGZldvLJ51G+/hYZq7M5LgDu1TrP6voyykrdzjnKvsFpy7JqJbs6vPujaM4rHcnikvLKSoto4P/b5tTWMKwP3xUbd8TBqXzj6tHsiIjh9OfmMMXE0+iRyNHdzW1xqEB4FFk/9R27J/arlr7/R/OGVJtn9gYq5wPMmVxBje++g1vXn9sZYffgrvHhS3ehsTEGId031uD6JuWzMCuKazyh7W+e+Mo8otKq7UxV4zY+v1Zg5n1Q2a1X8kVl/ttTs0xTDY1KZ5lk8bX+Xx8bAxnDe/Ba/M3MuqhT+naofocl6pNkAmxMdX6W3715ne8cMWIfeLcU1TaYBPQjj1FjH9iDpm5e39gVJ34OOHIXizZnM2bCzfx5sJNld+78nLHoLs/rDZx8trj+/H1+l0s8ufBjOjbGTPjnOE9eOrT1Yw4oDM9OrXn/e+28LNj+hAfG8NLc9cDXk1r4458+nSpfQRiTTmFJSxcv2ufHz+lZeWVtR2ASecOISnBOwe1/VCqGADgfb/2nr+xB3XlqL6due2UQRw3wKt5Hf+/n/Ljzn1HLJ73l7msm3wGt72xiCmLM1h232kkJ8YxzV+CaNwh3bjl5IGc/fTnzP4hk9zCEp7xL0c9belWrh5d+zyfUFONQ+q1PbewsnreGjjn2Lgzn7VZeYw9KLBa0WMfraRvWjIXHNErxNGFTm3zTH596iAuPLIX3Tq0o7isnI078xnUrQMrt+YSH2v866uNvDh3HV2SE5jxqxNJiIshKSGOWT9kcsWL87lhzADuGH9w5es55yguKycxLpYF63cy4dkvq71fbSPR7v9gOS987iWtig79ihFnVU2/9QQGdUvh3UWb6ZHanuG9O9EuPpb1WXlc/NyXTDp3KKMPTGNrTmHl6MCv1u4gv7iUq19ewK9OGcRNJw+s9xwt25LN8i05/OatxUD1CZZfrMnip3+fxxXHHsArX25g9m/GBpyIAnXff5fx0tz13HXGIVx7Qn9ufPUbpizOYPIFh1abk3VU3858n5FLblEpM389hr5pySzcsIsL/1p9WZ6mDOBQ57gShwhQfeLoSQd3rbb6cm1qSzbTbj2ec5+eWzlP5KUrj6JLSgI784p5YsYqvt24mwfOH8qC9bv4P7//ovd+7fnlCQP4WR19N9tzCxn5gDc67IIjevLON95xU24ezTOfrWFE3851zpAPxMV/+5LMPUXMuP1E5qzKYsvuAmJijAuP6FWttlnbKKmKAQVnPTWHpZu92oYZLL731Mpmo+binGN3fkll02B2QQnD7/uo3mN++OPpJMTF4JzXHFj1z7USRyMpcYhU9/X6nby9cBOTzh0a0BDOguIyDrlnWoP71SXQ+SM1m2riY42V95/eLPM93vj6R3779uJ9ys89rAdPXHI4AHe+s4TX5m8EvJGG5x/eixfnruOJSw7jhc/XsXjT3kmWNWtaoVR17tBPRvSqtjZabeu6DbrrQ4rLyokxWDtZiaNRlDhEms45x/bcIi569ks27sxnSI+OvHvjKIbcO32fkVcXHN6Td/yaxuDuHZl6y/EBvUdpWTkHVhlWPP93J9O1lpWSG6OuobPgNddde0L/ypWln/3ZkYwfuj/l5Y5D7plWbQb+vWcP5sCuKYw+MC2sS7h8vX4nJWXlHNOvC7mFpazbkce0pVu57ZSB+yxeWl7u+PbH3XRKiq9zQm8glDiUOESazTcbdzGoW4fKjvEPl2SQ1iGRXp3b89/vtvCL0f1xeDPjg51X4Jzjn19tYOxBXeldx1I6jfXHD5bz/OfrKvsPvt24i/P9pforBgQ8+7MjGD907wTCP3/yA3/+xJtn8+zPjuS0Id2iZs0vJQ4lDpGoV17uKCotr9Zs9vmqrMrlXAAW3XNK5cKYFaYv20rPTu0r1xuLFpoAKCJRLybG9ulrGXVgF847rAfgTQKtmTQAThuyf9QljeagGoeISJRRjUNERMJKiUNERIKixCEiIkFR4hARkaAocYiISFCUOEREJChKHCIiEhQlDhERCUqrngBoZpnAbiC7xlOpNcqqbqcBWSEMq+Z7N/dxDe1X1/OBlje0rfMXXFk0nb9gnwv2/LXlc1ff86H4v3uAcy6dxnLOteob8FxDZVW3gQXhjqc5j2tov7qeD7Q8gG2dvyDKoun8Bfuc/u82/rvX0LmqY7vZzl9baKr6bwBlte0TKo19r0CPa2i/up4PtDyS564p7xeJ89fSvntNeb/mOH/BPtfSzl9r+u7VVh62c9eqm6oaw8wWuCas0RLtdP6aRuev8XTumqY5z19bqHEE67lIB9DK6fw1jc5f4+ncNU2znb+oq3GIiEjTRGONQ0REmkCJQ0REgqLEISIiQVHiqMLMxpjZHDN71szGRDqe1sjMks1soZmdFelYWhMzO8T/3r1lZjdEOp7WxszOM7O/m9l7ZnZqpONpbcysv5m9YGZvBbJ/m0kcZvaimW03s6U1yseb2UozW21mExt4GQfsAdoBm0IVa0vUTOcP4A7gjdBE2TI1x7lzzq1wzl0P/ASIqiGnzXT+3nXOXQtcCVwcwnBbnGY6f2udc9cE/J5tZVSVmZ2A90f/H865oX5ZLPADcApeIvgauBSIBSbXeImrgSznXLmZdQMec85dFq74I62Zzt8wvGUN2uGdyw/CE31kNce5c85tN7NzgInA0865V8MVf6Q11/nzj/sT8G/n3DdhCj/imvn8veWcm9DQe8Y1X/iR5ZybbWZ9axSPBFY759YCmNl/gHOdc5OB+ppSdgGJoYizpWqO82dmY4FkYDBQYGZTnXPlIQ28BWiu755z7n3gfTObAkRN4mim754BDwEfRlPSgGb/2xeQNpM46tAT+LHK9ibg6Lp2NrMLgNOATsDToQ2tVQjq/Dnn7gIwsyvxa28hja5lC/a7Nwa4AO8Hy9SQRtY6BHX+gJuAcUCqmR3onHs2lMG1AsF+/7oADwCHm9mdfoKpU1tPHFZLWZ1tc865d4B3QhdOqxPU+avcwbmXmz+UVifY795MYGaogmmFgj1/TwJPhi6cVifY87cDuD7QF28zneN12AT0rrLdC9gSoVhaI52/xtO5axqdv6YJ6flr64nja2CgmfUzswTgEuD9CMfUmuj8NZ7OXdPo/DVNSM9fm0kcZvYa8CVwkJltMrNrnHOlwP8DpgMrgDecc8siGWdLpfPXeDp3TaPz1zSROH9tZjiuiIiER5upcYiISHgocYiISFCUOEREJChKHCIiEhQlDhERCYoSh4iIBEWJQ8LKzPaE4T3OCXAJ+OZ8zzFmdlwjjjvczJ73H19pZi1ijTQz61tzme5a9kk3s2nhiklaDiUOaZX8ZaNr5Zx73zn3UAjes7613cYAQScO4HfAU40KKMKcc5lAhpmNinQsEl5KHBIxZvYbM/vazBab2X1Vyt817yqCy8zsuirle8xskpnNA441s/Vmdp+ZfWNmS8zsYH+/yl/uZvaymT1pZl+Y2Vozm+CXx5jZM/57fGBmUyueqxHjTDN70MxmAbeY2dlmNs/MvjWzT8ysm7+k9fXAbWa2yMyO93+Nv+1/vq9r++NqZh2AYc6572p57gAzm+Gfmxlm1scvH2BmX/mvOam2Gpx5V2GcYmbfmdlSM7vYLz/KPw/fmdl8M+vg1yzm+Ofwm9pqTWYWa2aPVPm3+mWVp98Foua6NeJzzummW9huwB7//lTgObxVPGOAD4AT/Of28+/bA0uBLv62A35S5bXWAzf5j/8HeN5/fCXexZAAXgbe9N9jMN41CgAm4C1fHgPsj3cNlgm1xDsTeKbKdmf2rrjwC+BP/uM/AL+ust+rwGj/cR9gRS2vPRZ4u8p21bj/C1zhP74aeNd//AFwqf/4+orzWeN1LwT+XmU7FUgA1gJH+WUd8VbHTgLa+WUDgQX+477AUv/xdcDd/uNEYAHQz9/uCSyJ9PdKt/De2vqy6tJynerfvvW3U/D+cM0Gbjaz8/3y3n75DqAMeLvG61Qsg78Q73oWtXnXedcGWW7e1R0BRgNv+uVbzeyzemJ9vcrjXsDrZtYd74/xujqOGQcMNqtc3bqjmXVwzuVW2ac7kFnH8cdW+Tz/BP63Svl5/uNXgUdrOXYJ8KiZPQx84JybY2aHAhnOua8BnHM54NVOgKfN7DC88zuoltc7FRhWpUaWivdvsg7YDvSo4zNIG6XEIZFiwGTn3N+qFXoXNBoHHOucyzezmXiXogUodM6V1XidIv++jLq/z0VVHluN+0DkVXn8FN5lhd/3Y/1DHcfE4H2Ggnpet4C9n60hAS8q55z7wcyOBM4AJpvZR3hNSrW9xm3ANmC4H3NhLfsYXs1uei3PtcP7HBJF1MchkTIduNrMUgDMrKeZdcX7NbvLTxoHA8eE6P0/By70+zq64XVuByIV2Ow/vqJKeS7Qocr2R3irkwLg/6KvaQVwYB3v8wXeUtjg9SF87j/+Cq8piirPV2NmPYB859y/8GokRwDfAz3M7Ch/nw5+Z38qXk2kHLgc75rUNU0HbjCzeP/YQX5NBbwaSr2jr6TtUeKQiHDOfYTX1PKlmS0B3sL7wzsNiDOzxcD9eH8oQ+FtvIvdLAX+BswDsgM47g/Am2Y2B8iqUv5f4PyKznHgZmCE35m8nFquruac+x7vUqcdaj7nH3+Vfx4uB27xy28Fbjez+XhNXbXFfCgw38wWAXcBf3TOFQMXA0+Z2XfAx3i1hWeAK8zsK7wkkFfL6z0PLAe+8Yfo/o29tbuxwJRajpE2TMuqS9QysxTn3B7zrrc8HxjlnNsa5hhuA3Kdc88HuH8SUOCcc2Z2CV5H+bkhDbL+eGYD5zrndkUqBgk/9XFINPvAzDrhdXLfH+6k4fsrcFEQ+x+J15ltwG68EVcRYWbpeP09ShpRRjUOEREJivo4REQkKEocIiISFCUOEREJihKHiIgERYlDRESCosQhIiJB+f8Y0VpBN3PTXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=5e-5; wd=1e-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a0eefd7dc4140cda7c62c2086d84fff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2530 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCTensorCopy.c:20",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<timed eval>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    300\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    247\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, visualize, **kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mall_val\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mval_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mIterBatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_dl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 138\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    139\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    977\u001b[0m \"\"\", fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m    978\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 979\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    980\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    981\u001b[0m                 \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mchunk_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_sampler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_workers\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m                     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                         \u001b[0;32myield\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36mget_tensor\u001b[0;34m(batch, pin, half)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"batch must contain numbers, dicts or lists; found {type(batch)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"batch must contain numbers, dicts or lists; found {type(batch)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36mget_tensor\u001b[0;34m(batch, pin, half)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/core.py\u001b[0m in \u001b[0;36mto_gpu\u001b[0;34m(x, *args, **kwargs)\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0;34m'''puts pytorch variable to gpu, if cuda is available and USE_GPU is set to true. '''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mUSE_GPU\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mnoop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/_utils.py\u001b[0m in \u001b[0;36m_cuda\u001b[0;34m(self, device, async)\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0mnew_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnew_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCTensorCopy.c:20"
     ]
    }
   ],
   "source": [
    "%time m.fit(lr, 2, wd, cycle_len=1, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4a5f38282454f6d9511c14f83cd2b7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      13.666312  14.463123  3.627514  \n",
      "    1      13.294603  14.287758  3.604412                      \n",
      "    2      12.96801   14.76587   3.668159                      \n",
      "    3      11.516536  14.419428  3.617812                      \n",
      "    4      11.561937  14.160295  3.584021                      \n",
      "    5      13.417909  14.108828  3.581043                      \n",
      "    6      12.919792  14.414855  3.62646                       \n",
      "    7      13.810862  14.42516   3.635437                      \n",
      "    8      13.725518  14.387083  3.627153                      \n",
      "    9      14.166614  14.239976  3.606354                      \n",
      "    10     13.626104  14.262966  3.597994                      \n",
      "    11     10.214616  14.128918  3.583085                      \n",
      "    12     10.178611  14.150596  3.584987                      \n",
      "    13     11.047064  14.140439  3.587595                      \n",
      "\n",
      "CPU times: user 3min 15s, sys: 3.4 s, total: 3min 18s\n",
      "Wall time: 3min 15s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([14.14044]), 3.587595124999163]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 3, wd, cycle_len=2, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCTensorCopy.cu:204",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-894287fc7838>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, is_test, use_swa)\u001b[0m\n\u001b[1;32m    370\u001b[0m         \u001b[0mdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_dl\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_test\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_dl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 372\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_with_targs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_swa\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m     \u001b[0mpreda\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict_with_targs_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    250\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict_with_targs_\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    259\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mget_prediction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mVV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/column_data.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x_cat, x_cont)\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_emb\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrops\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_bn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m    833\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    834\u001b[0m         \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 835\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    837\u001b[0m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCTensorCopy.cu:204"
     ]
    }
   ],
   "source": [
    "preds = m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4590)\n",
    "oof = np.zeros(len(df_raw))\n",
    "predictions = np.zeros(len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliers = df_indep['outliers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = df_raw['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[189917, 189918, 189919, 189920, 189921]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_idx[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Get the val_idx\n",
    "2. Get the model data set \n",
    "`ColumnarModelData.from_data_frame(PATH, val_idx, df, y, cat_flds=cat_flds, bs=64)`\n",
    "3. Get the learner from model `md.get_learner(`\n",
    "4. Fit the model `m.fit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21ba1f084944444b8ba0192207466453",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      15.90532   14.590436  3.730362  \n",
      "    1      16.636636  14.521373  3.723079                      \n",
      "    2      14.094885  14.287253  3.691638                      \n",
      "    3      15.200652  14.473827  3.726571                      \n",
      "    4      13.269479  13.989985  3.654138                      \n",
      "    5      14.168423  14.049192  3.6646                        \n",
      "    6      14.935375  13.990876  3.657652                      \n",
      "    7      15.293805  14.116217  3.679225                      \n",
      "    8      14.136285  14.037943  3.664146                      \n",
      "    9      15.499863  13.766563  3.631436                      \n",
      "    10     12.939522  13.711981  3.618812                      \n",
      "    11     12.937443  13.63897   3.608274                      \n",
      "    12     12.376907  13.645307  3.611287                      \n",
      "    13     12.040647  13.648799  3.611737                      \n",
      "\n",
      "fold 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83ffce96d9434c5d8f0dd056b017fb95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      19.169962  14.847157  3.772192  \n",
      "    1      15.924142  14.515021  3.725014                      \n",
      "    2      14.702569  14.379131  3.706228                      \n",
      "    3      12.261706  14.182392  3.681849                      \n",
      "    4      13.598058  14.034147  3.665217                      \n",
      "    5      13.969396  13.991572  3.657983                      \n",
      "    6      14.463711  13.808887  3.630376                      \n",
      "    7      12.045211  13.826998  3.633127                      \n",
      "    8      14.481098  13.875959  3.645563                      \n",
      "    9      13.372849  13.666987  3.614234                      \n",
      "    10     14.334264  13.624824  3.606543                      \n",
      "    11     13.514668  13.661471  3.614602                      \n",
      "    12     12.351634  13.65184   3.613192                      \n",
      "    13     12.021324  13.665567  3.615174                      \n",
      "\n",
      "fold 2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "270a8ac3e1b740d1b3eacc7b2f3da881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      15.553468  14.57795   3.726842  \n",
      "    1      13.320078  14.381167  3.698273                      \n",
      "    2      13.914944  14.187339  3.671701                      \n",
      "    3      15.009974  14.003612  3.650309                      \n",
      "    4      13.781466  13.74567   3.605476                      \n",
      "    5      13.03731   13.771313  3.61168                       \n",
      "    6      12.017088  13.770312  3.606614                      \n",
      "    7      12.378191  13.730836  3.599832                      \n",
      "    8      12.774747  13.62501   3.585738                      \n",
      "    9      12.31523   13.593068  3.581531                      \n",
      "    10     13.785336  13.598255  3.587389                      \n",
      "    11     13.400499  13.647203  3.594277                      \n",
      "    12     12.431539  13.623571  3.591423                      \n",
      "    13     13.510818  13.625425  3.591425                      \n",
      "\n",
      "fold 3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd6d5fc09d9541c4817e849c610d7c3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      17.959838  14.66542   3.740677  \n",
      "    1      17.467989  14.459046  3.710467                      \n",
      "    2      14.977177  14.816611  3.767583                      \n",
      "    3      15.048977  14.100075  3.666597                      \n",
      "    4      15.086512  14.104984  3.667835                      \n",
      "    5      14.13241   14.085708  3.665205                      \n",
      "    6      14.249057  13.813915  3.61566                       \n",
      "    7      13.745444  14.011514  3.653542                      \n",
      "    8      13.475943  13.76562   3.617357                      \n",
      "    9      12.987171  13.717956  3.610853                      \n",
      "    10     12.266481  13.760248  3.617033                      \n",
      "    11     13.265814  13.816967  3.6283                        \n",
      "    12     12.593482  13.709094  3.611805                      \n",
      "    13     14.119061  13.738945  3.616306                      \n",
      "\n",
      "fold 4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eeba002d1ef146a0bc2768123de096b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=14, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      17.889693  14.860595  3.767871  \n",
      "    1      15.449988  14.36485   3.693957                      \n",
      "    2      15.120486  14.269434  3.683128                      \n",
      "    3      14.170923  14.042828  3.653109                      \n",
      "    4      15.182405  13.962082  3.642103                      \n",
      "    5      15.33342   13.877547  3.629072                      \n",
      "    6      14.934545  14.093177  3.665605                      \n",
      "    7      13.151111  13.674127  3.598654                      \n",
      "    8      13.633256  13.663314  3.597758                      \n",
      "    9      13.987799  13.635043  3.599197                      \n",
      "    10     13.093681  13.591654  3.590982                      \n",
      "    11     12.651642  13.655931  3.602316                      \n",
      "    12     13.899274  13.70562   3.61044                       \n",
      "    13     12.193685  13.665955  3.604333                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3.6971527526968155"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(df, outliers.values)):\n",
    "    print(\"fold {}\".format(fold_))\n",
    "    md = ColumnarModelData.from_data_frame(PATH, val_idx, df, target.astype(np.float32), \n",
    "                                           cat_flds=cat_flds, bs=128, test_df=df_t)\n",
    "    m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.1, 1, [1000,500], [0.1,0.2], y_range=y_range)\n",
    "    m.fit(lr, 3, wd, cycle_len=2, cycle_mult=2, metrics=[rmse])\n",
    "    oof[val_idx] = m.predict().reshape(1, -1)[0]\n",
    "#     predictions += m.predict(is_test=True) / folds.n_splits\n",
    "\n",
    "rmse(oof, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.715945356390822"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1.9276787482334348**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.715945356390822"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse(oof, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "q =p.reshape(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCTensorCopy.c:20",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-126-5a63bb45c9fe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, is_test, use_swa)\u001b[0m\n\u001b[1;32m    370\u001b[0m         \u001b[0mdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_dl\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_test\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_dl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 372\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_with_targs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_swa\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m     \u001b[0mpreda\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict_with_targs_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    250\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict_with_targs_\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    259\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mget_prediction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mVV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mchunk_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_sampler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_workers\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m                     \u001b[0;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                         \u001b[0;32myield\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36mget_tensor\u001b[0;34m(batch, pin, half)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"batch must contain numbers, dicts or lists; found {type(batch)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSequence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mget_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"batch must contain numbers, dicts or lists; found {type(batch)}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/dataloader.py\u001b[0m in \u001b[0;36mget_tensor\u001b[0;34m(batch, pin, half)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhalf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpin\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstring_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/core.py\u001b[0m in \u001b[0;36mto_gpu\u001b[0;34m(x, *args, **kwargs)\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0;34m'''puts pytorch variable to gpu, if cuda is available and USE_GPU is set to true. '''\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mUSE_GPU\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mnoop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/_utils.py\u001b[0m in \u001b[0;36m_cuda\u001b[0;34m(self, device, async)\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0mnew_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnew_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCTensorCopy.c:20"
     ]
    }
   ],
   "source": [
    "p =m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "915"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((40384, 1), (201917,), (40384,))"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p.shape, oof.shape, q[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.77326,  0.07696,  0.78907, -0.14393, -2.32876, -0.19552], dtype=float32),\n",
       " array([0., 0., 0., 0., 0., 0.]))"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q[0][:6], oof[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40384"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(oof[val_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161533"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trn_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature_1</th>\n",
       "      <th>feature_2</th>\n",
       "      <th>feature_3</th>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <th>...</th>\n",
       "      <th>days_since_last_transaction_na</th>\n",
       "      <th>repurchase_merchant_rate_na</th>\n",
       "      <th>merchant_category_repurchase_na</th>\n",
       "      <th>avg_spend_per_merchant_na</th>\n",
       "      <th>avg_trans_per_merchant_na</th>\n",
       "      <th>avg_spend_per_transaction_na</th>\n",
       "      <th>card_id_total_na</th>\n",
       "      <th>purchase_amount_total_na</th>\n",
       "      <th>installments_total_na</th>\n",
       "      <th>new_first_buy_na</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201704</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201710</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201725</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201737</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201743</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201750</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201753</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201756</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201766</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201769</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201777</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201778</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201787</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201821</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201838</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201843</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201860</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201861</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201862</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201863</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201871</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "      <td>2.864773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201875</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201876</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201877</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201881</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201888</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201897</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201898</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201910</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201913</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "      <td>-0.349068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40384 rows × 160 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        feature_1  feature_2  feature_3  first_active_monthYear  \\\n",
       "2               3          1          1                       6   \n",
       "4               1          3          1                       7   \n",
       "12              5          1          2                       7   \n",
       "18              3          2          1                       7   \n",
       "19              4          2          1                       7   \n",
       "23              3          1          1                       7   \n",
       "31              5          1          2                       7   \n",
       "32              5          2          2                       7   \n",
       "39              2          3          2                       6   \n",
       "45              4          1          1                       6   \n",
       "53              3          1          1                       6   \n",
       "54              3          2          1                       7   \n",
       "56              2          2          2                       7   \n",
       "66              3          2          1                       7   \n",
       "71              3          1          1                       6   \n",
       "94              3          1          1                       7   \n",
       "98              2          3          2                       4   \n",
       "104             3          1          1                       6   \n",
       "108             5          2          2                       7   \n",
       "117             2          1          2                       7   \n",
       "124             5          1          2                       6   \n",
       "125             2          3          2                       5   \n",
       "127             4          3          1                       7   \n",
       "128             3          2          1                       7   \n",
       "131             3          2          1                       7   \n",
       "132             5          2          2                       7   \n",
       "135             3          1          1                       5   \n",
       "140             3          3          1                       7   \n",
       "143             2          3          2                       7   \n",
       "152             5          2          2                       6   \n",
       "...           ...        ...        ...                     ...   \n",
       "201704          3          1          1                       6   \n",
       "201710          5          2          2                       7   \n",
       "201725          2          1          2                       7   \n",
       "201737          5          2          2                       7   \n",
       "201743          3          2          1                       7   \n",
       "201750          3          3          1                       7   \n",
       "201753          4          1          1                       6   \n",
       "201756          2          1          2                       7   \n",
       "201766          3          3          1                       7   \n",
       "201769          5          2          2                       5   \n",
       "201777          2          1          2                       7   \n",
       "201778          2          3          2                       5   \n",
       "201787          5          1          2                       6   \n",
       "201821          2          1          2                       6   \n",
       "201838          5          2          2                       7   \n",
       "201843          3          2          1                       7   \n",
       "201860          2          1          2                       7   \n",
       "201861          2          1          2                       7   \n",
       "201862          3          2          1                       7   \n",
       "201863          2          1          2                       7   \n",
       "201871          4          1          1                       7   \n",
       "201875          3          1          1                       6   \n",
       "201876          3          1          1                       6   \n",
       "201877          1          2          1                       7   \n",
       "201881          4          1          1                       7   \n",
       "201888          4          1          1                       6   \n",
       "201897          5          1          2                       6   \n",
       "201898          1          2          1                       7   \n",
       "201910          3          3          1                       7   \n",
       "201913          2          2          2                       5   \n",
       "\n",
       "        first_active_monthMonth  first_active_monthWeek  \\\n",
       "2                             8                      12   \n",
       "4                            11                      17   \n",
       "12                            9                      13   \n",
       "18                           11                      17   \n",
       "19                            1                      20   \n",
       "23                           10                      15   \n",
       "31                            7                      10   \n",
       "32                            1                      20   \n",
       "39                            4                       4   \n",
       "45                           12                      18   \n",
       "53                           12                      18   \n",
       "54                            6                       8   \n",
       "56                           10                      15   \n",
       "66                           10                      15   \n",
       "71                           10                      15   \n",
       "94                            6                       8   \n",
       "98                            8                      12   \n",
       "104                          11                      17   \n",
       "108                           8                      12   \n",
       "117                           5                       7   \n",
       "124                           7                      10   \n",
       "125                          10                      16   \n",
       "127                           3                       3   \n",
       "128                          11                      17   \n",
       "131                           4                       4   \n",
       "132                           9                      13   \n",
       "135                          11                      17   \n",
       "140                          11                      17   \n",
       "143                          11                      17   \n",
       "152                           1                      21   \n",
       "...                         ...                     ...   \n",
       "201704                        6                       8   \n",
       "201710                       12                      18   \n",
       "201725                        8                      12   \n",
       "201737                        8                      12   \n",
       "201743                        4                       4   \n",
       "201750                        8                      12   \n",
       "201753                        7                      10   \n",
       "201756                        4                       4   \n",
       "201766                       10                      15   \n",
       "201769                       10                      16   \n",
       "201777                       10                      15   \n",
       "201778                        1                       1   \n",
       "201787                       12                      18   \n",
       "201821                        4                       4   \n",
       "201838                        7                      10   \n",
       "201843                        7                      10   \n",
       "201860                        9                      13   \n",
       "201861                        5                       7   \n",
       "201862                        1                      20   \n",
       "201863                        6                       8   \n",
       "201871                       11                      17   \n",
       "201875                        7                      10   \n",
       "201876                       10                      15   \n",
       "201877                        6                       8   \n",
       "201881                       12                      18   \n",
       "201888                        8                      12   \n",
       "201897                       11                      17   \n",
       "201898                        6                       8   \n",
       "201910                       10                      15   \n",
       "201913                       10                      16   \n",
       "\n",
       "        first_active_monthDay  first_active_monthDayofweek  \\\n",
       "2                           1                            1   \n",
       "4                           1                            3   \n",
       "12                          1                            5   \n",
       "18                          1                            3   \n",
       "19                          1                            7   \n",
       "23                          1                            7   \n",
       "31                          1                            6   \n",
       "32                          1                            7   \n",
       "39                          1                            5   \n",
       "45                          1                            4   \n",
       "53                          1                            4   \n",
       "54                          1                            4   \n",
       "56                          1                            7   \n",
       "66                          1                            7   \n",
       "71                          1                            6   \n",
       "94                          1                            4   \n",
       "98                          1                            5   \n",
       "104                         1                            2   \n",
       "108                         1                            2   \n",
       "117                         1                            1   \n",
       "124                         1                            5   \n",
       "125                         1                            4   \n",
       "127                         1                            3   \n",
       "128                         1                            3   \n",
       "131                         1                            6   \n",
       "132                         1                            5   \n",
       "135                         1                            7   \n",
       "140                         1                            3   \n",
       "143                         1                            3   \n",
       "152                         1                            5   \n",
       "...                       ...                          ...   \n",
       "201704                      1                            3   \n",
       "201710                      1                            5   \n",
       "201725                      1                            2   \n",
       "201737                      1                            2   \n",
       "201743                      1                            6   \n",
       "201750                      1                            2   \n",
       "201753                      1                            5   \n",
       "201756                      1                            6   \n",
       "201766                      1                            7   \n",
       "201769                      1                            4   \n",
       "201777                      1                            7   \n",
       "201778                      1                            4   \n",
       "201787                      1                            4   \n",
       "201821                      1                            5   \n",
       "201838                      1                            6   \n",
       "201843                      1                            6   \n",
       "201860                      1                            5   \n",
       "201861                      1                            1   \n",
       "201862                      1                            7   \n",
       "201863                      1                            4   \n",
       "201871                      1                            3   \n",
       "201875                      1                            5   \n",
       "201876                      1                            6   \n",
       "201877                      1                            4   \n",
       "201881                      1                            5   \n",
       "201888                      1                            1   \n",
       "201897                      1                            2   \n",
       "201898                      1                            4   \n",
       "201910                      1                            7   \n",
       "201913                      1                            4   \n",
       "\n",
       "        first_active_monthDayofyear  first_active_monthIs_month_end  \\\n",
       "2                                14                               1   \n",
       "4                                19                               1   \n",
       "12                               15                               1   \n",
       "18                               19                               1   \n",
       "19                                1                               1   \n",
       "23                               17                               1   \n",
       "31                               11                               1   \n",
       "32                                1                               1   \n",
       "39                                6                               1   \n",
       "45                               22                               1   \n",
       "53                               22                               1   \n",
       "54                                9                               1   \n",
       "56                               17                               1   \n",
       "66                               17                               1   \n",
       "71                               18                               1   \n",
       "94                                9                               1   \n",
       "98                               13                               1   \n",
       "104                              20                               1   \n",
       "108                              13                               1   \n",
       "117                               7                               1   \n",
       "124                              12                               1   \n",
       "125                              17                               1   \n",
       "127                               3                               1   \n",
       "128                              19                               1   \n",
       "131                               5                               1   \n",
       "132                              15                               1   \n",
       "135                              19                               1   \n",
       "140                              19                               1   \n",
       "143                              19                               1   \n",
       "152                               1                               1   \n",
       "...                             ...                             ...   \n",
       "201704                           10                               1   \n",
       "201710                           21                               1   \n",
       "201725                           13                               1   \n",
       "201737                           13                               1   \n",
       "201743                            5                               1   \n",
       "201750                           13                               1   \n",
       "201753                           12                               1   \n",
       "201756                            5                               1   \n",
       "201766                           17                               1   \n",
       "201769                           17                               1   \n",
       "201777                           17                               1   \n",
       "201778                            1                               1   \n",
       "201787                           22                               1   \n",
       "201821                            6                               1   \n",
       "201838                           11                               1   \n",
       "201843                           11                               1   \n",
       "201860                           15                               1   \n",
       "201861                            7                               1   \n",
       "201862                            1                               1   \n",
       "201863                            9                               1   \n",
       "201871                           19                               1   \n",
       "201875                           12                               1   \n",
       "201876                           18                               1   \n",
       "201877                            9                               1   \n",
       "201881                           21                               1   \n",
       "201888                           14                               1   \n",
       "201897                           20                               1   \n",
       "201898                            9                               1   \n",
       "201910                           17                               1   \n",
       "201913                           17                               1   \n",
       "\n",
       "              ...         days_since_last_transaction_na  \\\n",
       "2             ...                              -0.349068   \n",
       "4             ...                              -0.349068   \n",
       "12            ...                              -0.349068   \n",
       "18            ...                              -0.349068   \n",
       "19            ...                              -0.349068   \n",
       "23            ...                              -0.349068   \n",
       "31            ...                              -0.349068   \n",
       "32            ...                              -0.349068   \n",
       "39            ...                              -0.349068   \n",
       "45            ...                              -0.349068   \n",
       "53            ...                              -0.349068   \n",
       "54            ...                              -0.349068   \n",
       "56            ...                              -0.349068   \n",
       "66            ...                              -0.349068   \n",
       "71            ...                              -0.349068   \n",
       "94            ...                              -0.349068   \n",
       "98            ...                              -0.349068   \n",
       "104           ...                              -0.349068   \n",
       "108           ...                              -0.349068   \n",
       "117           ...                              -0.349068   \n",
       "124           ...                              -0.349068   \n",
       "125           ...                               2.864773   \n",
       "127           ...                              -0.349068   \n",
       "128           ...                              -0.349068   \n",
       "131           ...                               2.864773   \n",
       "132           ...                              -0.349068   \n",
       "135           ...                              -0.349068   \n",
       "140           ...                              -0.349068   \n",
       "143           ...                              -0.349068   \n",
       "152           ...                              -0.349068   \n",
       "...           ...                                    ...   \n",
       "201704        ...                              -0.349068   \n",
       "201710        ...                              -0.349068   \n",
       "201725        ...                              -0.349068   \n",
       "201737        ...                              -0.349068   \n",
       "201743        ...                              -0.349068   \n",
       "201750        ...                              -0.349068   \n",
       "201753        ...                              -0.349068   \n",
       "201756        ...                              -0.349068   \n",
       "201766        ...                              -0.349068   \n",
       "201769        ...                              -0.349068   \n",
       "201777        ...                              -0.349068   \n",
       "201778        ...                              -0.349068   \n",
       "201787        ...                               2.864773   \n",
       "201821        ...                              -0.349068   \n",
       "201838        ...                              -0.349068   \n",
       "201843        ...                              -0.349068   \n",
       "201860        ...                              -0.349068   \n",
       "201861        ...                              -0.349068   \n",
       "201862        ...                              -0.349068   \n",
       "201863        ...                              -0.349068   \n",
       "201871        ...                               2.864773   \n",
       "201875        ...                              -0.349068   \n",
       "201876        ...                              -0.349068   \n",
       "201877        ...                              -0.349068   \n",
       "201881        ...                              -0.349068   \n",
       "201888        ...                              -0.349068   \n",
       "201897        ...                              -0.349068   \n",
       "201898        ...                              -0.349068   \n",
       "201910        ...                              -0.349068   \n",
       "201913        ...                              -0.349068   \n",
       "\n",
       "        repurchase_merchant_rate_na  merchant_category_repurchase_na  \\\n",
       "2                         -0.349068                        -0.349068   \n",
       "4                         -0.349068                        -0.349068   \n",
       "12                        -0.349068                        -0.349068   \n",
       "18                        -0.349068                        -0.349068   \n",
       "19                        -0.349068                        -0.349068   \n",
       "23                        -0.349068                        -0.349068   \n",
       "31                        -0.349068                        -0.349068   \n",
       "32                        -0.349068                        -0.349068   \n",
       "39                        -0.349068                        -0.349068   \n",
       "45                        -0.349068                        -0.349068   \n",
       "53                        -0.349068                        -0.349068   \n",
       "54                        -0.349068                        -0.349068   \n",
       "56                        -0.349068                        -0.349068   \n",
       "66                        -0.349068                        -0.349068   \n",
       "71                        -0.349068                        -0.349068   \n",
       "94                        -0.349068                        -0.349068   \n",
       "98                        -0.349068                        -0.349068   \n",
       "104                       -0.349068                        -0.349068   \n",
       "108                       -0.349068                        -0.349068   \n",
       "117                       -0.349068                        -0.349068   \n",
       "124                       -0.349068                        -0.349068   \n",
       "125                        2.864773                         2.864773   \n",
       "127                       -0.349068                        -0.349068   \n",
       "128                       -0.349068                        -0.349068   \n",
       "131                        2.864773                         2.864773   \n",
       "132                       -0.349068                        -0.349068   \n",
       "135                       -0.349068                        -0.349068   \n",
       "140                       -0.349068                        -0.349068   \n",
       "143                       -0.349068                        -0.349068   \n",
       "152                       -0.349068                        -0.349068   \n",
       "...                             ...                              ...   \n",
       "201704                    -0.349068                        -0.349068   \n",
       "201710                    -0.349068                        -0.349068   \n",
       "201725                    -0.349068                        -0.349068   \n",
       "201737                    -0.349068                        -0.349068   \n",
       "201743                    -0.349068                        -0.349068   \n",
       "201750                    -0.349068                        -0.349068   \n",
       "201753                    -0.349068                        -0.349068   \n",
       "201756                    -0.349068                        -0.349068   \n",
       "201766                    -0.349068                        -0.349068   \n",
       "201769                    -0.349068                        -0.349068   \n",
       "201777                    -0.349068                        -0.349068   \n",
       "201778                    -0.349068                        -0.349068   \n",
       "201787                     2.864773                         2.864773   \n",
       "201821                    -0.349068                        -0.349068   \n",
       "201838                    -0.349068                        -0.349068   \n",
       "201843                    -0.349068                        -0.349068   \n",
       "201860                    -0.349068                        -0.349068   \n",
       "201861                    -0.349068                        -0.349068   \n",
       "201862                    -0.349068                        -0.349068   \n",
       "201863                    -0.349068                        -0.349068   \n",
       "201871                     2.864773                         2.864773   \n",
       "201875                    -0.349068                        -0.349068   \n",
       "201876                    -0.349068                        -0.349068   \n",
       "201877                    -0.349068                        -0.349068   \n",
       "201881                    -0.349068                        -0.349068   \n",
       "201888                    -0.349068                        -0.349068   \n",
       "201897                    -0.349068                        -0.349068   \n",
       "201898                    -0.349068                        -0.349068   \n",
       "201910                    -0.349068                        -0.349068   \n",
       "201913                    -0.349068                        -0.349068   \n",
       "\n",
       "        avg_spend_per_merchant_na  avg_trans_per_merchant_na  \\\n",
       "2                       -0.349068                  -0.349068   \n",
       "4                       -0.349068                  -0.349068   \n",
       "12                      -0.349068                  -0.349068   \n",
       "18                      -0.349068                  -0.349068   \n",
       "19                      -0.349068                  -0.349068   \n",
       "23                      -0.349068                  -0.349068   \n",
       "31                      -0.349068                  -0.349068   \n",
       "32                      -0.349068                  -0.349068   \n",
       "39                      -0.349068                  -0.349068   \n",
       "45                      -0.349068                  -0.349068   \n",
       "53                      -0.349068                  -0.349068   \n",
       "54                      -0.349068                  -0.349068   \n",
       "56                      -0.349068                  -0.349068   \n",
       "66                      -0.349068                  -0.349068   \n",
       "71                      -0.349068                  -0.349068   \n",
       "94                      -0.349068                  -0.349068   \n",
       "98                      -0.349068                  -0.349068   \n",
       "104                     -0.349068                  -0.349068   \n",
       "108                     -0.349068                  -0.349068   \n",
       "117                     -0.349068                  -0.349068   \n",
       "124                     -0.349068                  -0.349068   \n",
       "125                      2.864773                   2.864773   \n",
       "127                     -0.349068                  -0.349068   \n",
       "128                     -0.349068                  -0.349068   \n",
       "131                      2.864773                   2.864773   \n",
       "132                     -0.349068                  -0.349068   \n",
       "135                     -0.349068                  -0.349068   \n",
       "140                     -0.349068                  -0.349068   \n",
       "143                     -0.349068                  -0.349068   \n",
       "152                     -0.349068                  -0.349068   \n",
       "...                           ...                        ...   \n",
       "201704                  -0.349068                  -0.349068   \n",
       "201710                  -0.349068                  -0.349068   \n",
       "201725                  -0.349068                  -0.349068   \n",
       "201737                  -0.349068                  -0.349068   \n",
       "201743                  -0.349068                  -0.349068   \n",
       "201750                  -0.349068                  -0.349068   \n",
       "201753                  -0.349068                  -0.349068   \n",
       "201756                  -0.349068                  -0.349068   \n",
       "201766                  -0.349068                  -0.349068   \n",
       "201769                  -0.349068                  -0.349068   \n",
       "201777                  -0.349068                  -0.349068   \n",
       "201778                  -0.349068                  -0.349068   \n",
       "201787                   2.864773                   2.864773   \n",
       "201821                  -0.349068                  -0.349068   \n",
       "201838                  -0.349068                  -0.349068   \n",
       "201843                  -0.349068                  -0.349068   \n",
       "201860                  -0.349068                  -0.349068   \n",
       "201861                  -0.349068                  -0.349068   \n",
       "201862                  -0.349068                  -0.349068   \n",
       "201863                  -0.349068                  -0.349068   \n",
       "201871                   2.864773                   2.864773   \n",
       "201875                  -0.349068                  -0.349068   \n",
       "201876                  -0.349068                  -0.349068   \n",
       "201877                  -0.349068                  -0.349068   \n",
       "201881                  -0.349068                  -0.349068   \n",
       "201888                  -0.349068                  -0.349068   \n",
       "201897                  -0.349068                  -0.349068   \n",
       "201898                  -0.349068                  -0.349068   \n",
       "201910                  -0.349068                  -0.349068   \n",
       "201913                  -0.349068                  -0.349068   \n",
       "\n",
       "        avg_spend_per_transaction_na  card_id_total_na  \\\n",
       "2                          -0.349068         -0.349068   \n",
       "4                          -0.349068         -0.349068   \n",
       "12                         -0.349068         -0.349068   \n",
       "18                         -0.349068         -0.349068   \n",
       "19                         -0.349068         -0.349068   \n",
       "23                         -0.349068         -0.349068   \n",
       "31                         -0.349068         -0.349068   \n",
       "32                         -0.349068         -0.349068   \n",
       "39                         -0.349068         -0.349068   \n",
       "45                         -0.349068         -0.349068   \n",
       "53                         -0.349068         -0.349068   \n",
       "54                         -0.349068         -0.349068   \n",
       "56                         -0.349068         -0.349068   \n",
       "66                         -0.349068         -0.349068   \n",
       "71                         -0.349068         -0.349068   \n",
       "94                         -0.349068         -0.349068   \n",
       "98                         -0.349068         -0.349068   \n",
       "104                        -0.349068         -0.349068   \n",
       "108                        -0.349068         -0.349068   \n",
       "117                        -0.349068         -0.349068   \n",
       "124                        -0.349068         -0.349068   \n",
       "125                         2.864773          2.864773   \n",
       "127                        -0.349068         -0.349068   \n",
       "128                        -0.349068         -0.349068   \n",
       "131                         2.864773          2.864773   \n",
       "132                        -0.349068         -0.349068   \n",
       "135                        -0.349068         -0.349068   \n",
       "140                        -0.349068         -0.349068   \n",
       "143                        -0.349068         -0.349068   \n",
       "152                        -0.349068         -0.349068   \n",
       "...                              ...               ...   \n",
       "201704                     -0.349068         -0.349068   \n",
       "201710                     -0.349068         -0.349068   \n",
       "201725                     -0.349068         -0.349068   \n",
       "201737                     -0.349068         -0.349068   \n",
       "201743                     -0.349068         -0.349068   \n",
       "201750                     -0.349068         -0.349068   \n",
       "201753                     -0.349068         -0.349068   \n",
       "201756                     -0.349068         -0.349068   \n",
       "201766                     -0.349068         -0.349068   \n",
       "201769                     -0.349068         -0.349068   \n",
       "201777                     -0.349068         -0.349068   \n",
       "201778                     -0.349068         -0.349068   \n",
       "201787                      2.864773          2.864773   \n",
       "201821                     -0.349068         -0.349068   \n",
       "201838                     -0.349068         -0.349068   \n",
       "201843                     -0.349068         -0.349068   \n",
       "201860                     -0.349068         -0.349068   \n",
       "201861                     -0.349068         -0.349068   \n",
       "201862                     -0.349068         -0.349068   \n",
       "201863                     -0.349068         -0.349068   \n",
       "201871                      2.864773          2.864773   \n",
       "201875                     -0.349068         -0.349068   \n",
       "201876                     -0.349068         -0.349068   \n",
       "201877                     -0.349068         -0.349068   \n",
       "201881                     -0.349068         -0.349068   \n",
       "201888                     -0.349068         -0.349068   \n",
       "201897                     -0.349068         -0.349068   \n",
       "201898                     -0.349068         -0.349068   \n",
       "201910                     -0.349068         -0.349068   \n",
       "201913                     -0.349068         -0.349068   \n",
       "\n",
       "        purchase_amount_total_na  installments_total_na  new_first_buy_na  \n",
       "2                      -0.349068              -0.349068         -0.349068  \n",
       "4                      -0.349068              -0.349068         -0.349068  \n",
       "12                     -0.349068              -0.349068         -0.349068  \n",
       "18                     -0.349068              -0.349068         -0.349068  \n",
       "19                     -0.349068              -0.349068         -0.349068  \n",
       "23                     -0.349068              -0.349068         -0.349068  \n",
       "31                     -0.349068              -0.349068         -0.349068  \n",
       "32                     -0.349068              -0.349068         -0.349068  \n",
       "39                     -0.349068              -0.349068         -0.349068  \n",
       "45                     -0.349068              -0.349068         -0.349068  \n",
       "53                     -0.349068              -0.349068         -0.349068  \n",
       "54                     -0.349068              -0.349068         -0.349068  \n",
       "56                     -0.349068              -0.349068         -0.349068  \n",
       "66                     -0.349068              -0.349068         -0.349068  \n",
       "71                     -0.349068              -0.349068         -0.349068  \n",
       "94                     -0.349068              -0.349068         -0.349068  \n",
       "98                     -0.349068              -0.349068         -0.349068  \n",
       "104                    -0.349068              -0.349068         -0.349068  \n",
       "108                    -0.349068              -0.349068         -0.349068  \n",
       "117                    -0.349068              -0.349068         -0.349068  \n",
       "124                    -0.349068              -0.349068         -0.349068  \n",
       "125                     2.864773               2.864773          2.864773  \n",
       "127                    -0.349068              -0.349068         -0.349068  \n",
       "128                    -0.349068              -0.349068         -0.349068  \n",
       "131                     2.864773               2.864773          2.864773  \n",
       "132                    -0.349068              -0.349068         -0.349068  \n",
       "135                    -0.349068              -0.349068         -0.349068  \n",
       "140                    -0.349068              -0.349068         -0.349068  \n",
       "143                    -0.349068              -0.349068         -0.349068  \n",
       "152                    -0.349068              -0.349068         -0.349068  \n",
       "...                          ...                    ...               ...  \n",
       "201704                 -0.349068              -0.349068         -0.349068  \n",
       "201710                 -0.349068              -0.349068         -0.349068  \n",
       "201725                 -0.349068              -0.349068         -0.349068  \n",
       "201737                 -0.349068              -0.349068         -0.349068  \n",
       "201743                 -0.349068              -0.349068         -0.349068  \n",
       "201750                 -0.349068              -0.349068         -0.349068  \n",
       "201753                 -0.349068              -0.349068         -0.349068  \n",
       "201756                 -0.349068              -0.349068         -0.349068  \n",
       "201766                 -0.349068              -0.349068         -0.349068  \n",
       "201769                 -0.349068              -0.349068         -0.349068  \n",
       "201777                 -0.349068              -0.349068         -0.349068  \n",
       "201778                 -0.349068              -0.349068         -0.349068  \n",
       "201787                  2.864773               2.864773          2.864773  \n",
       "201821                 -0.349068              -0.349068         -0.349068  \n",
       "201838                 -0.349068              -0.349068         -0.349068  \n",
       "201843                 -0.349068              -0.349068         -0.349068  \n",
       "201860                 -0.349068              -0.349068         -0.349068  \n",
       "201861                 -0.349068              -0.349068         -0.349068  \n",
       "201862                 -0.349068              -0.349068         -0.349068  \n",
       "201863                 -0.349068              -0.349068         -0.349068  \n",
       "201871                  2.864773               2.864773          2.864773  \n",
       "201875                 -0.349068              -0.349068         -0.349068  \n",
       "201876                 -0.349068              -0.349068         -0.349068  \n",
       "201877                 -0.349068              -0.349068         -0.349068  \n",
       "201881                 -0.349068              -0.349068         -0.349068  \n",
       "201888                 -0.349068              -0.349068         -0.349068  \n",
       "201897                 -0.349068              -0.349068         -0.349068  \n",
       "201898                 -0.349068              -0.349068         -0.349068  \n",
       "201910                 -0.349068              -0.349068         -0.349068  \n",
       "201913                 -0.349068              -0.349068         -0.349068  \n",
       "\n",
       "[40384 rows x 160 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[val_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.60878],\n",
       "       [ 0.67478],\n",
       "       [-0.46273],\n",
       "       ...,\n",
       "       [-0.98494],\n",
       "       [-3.06388],\n",
       "       [ 0.05033]], dtype=float32)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-85-2ed5a4731b02>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mval_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, is_test, use_swa)\u001b[0m\n\u001b[1;32m    368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    369\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_test\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_swa\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 370\u001b[0;31m         \u001b[0mdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_dl\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_test\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mval_dl\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    371\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    372\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__nonzero__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1574\u001b[0m         raise ValueError(\"The truth value of a {0} is ambiguous. \"\n\u001b[1;32m   1575\u001b[0m                          \u001b[0;34m\"Use a.empty, a.bool(), a.item(), a.any() or a.all().\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1576\u001b[0;31m                          .format(self.__class__.__name__))\n\u001b[0m\u001b[1;32m   1577\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1578\u001b[0m     \u001b[0m__bool__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m__nonzero__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
     ]
    }
   ],
   "source": [
    "m.predict(df.iloc[val_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(val_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40384,)"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 161534 is out of bounds for axis 1 with size 161533",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-217-ee0f192463ca>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mval_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msplit_by_idx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrn_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/dataset.py\u001b[0m in \u001b[0;36msplit_by_idx\u001b[0;34m(idxs, *a)\u001b[0m\n\u001b[1;32m    604\u001b[0m     \"\"\"\n\u001b[1;32m    605\u001b[0m     \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 606\u001b[0;31m     \u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    607\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 161534 is out of bounds for axis 1 with size 161533"
     ]
    }
   ],
   "source": [
    "((val_df, trn_df), (val_y, trn_y)) = split_by_idx(val_idx, df.iloc[trn_idx], y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "first_active_month                          0\n",
       "feature_1                                   0\n",
       "feature_2                                   0\n",
       "feature_3                                   0\n",
       "first_active_monthYear                      0\n",
       "first_active_monthMonth                     0\n",
       "first_active_monthWeek                      0\n",
       "first_active_monthDay                       0\n",
       "first_active_monthDayofweek                 0\n",
       "first_active_monthDayofyear                 0\n",
       "first_active_monthIs_month_end              0\n",
       "first_active_monthIs_month_start            0\n",
       "first_active_monthIs_quarter_end            0\n",
       "first_active_monthIs_quarter_start          0\n",
       "first_active_monthIs_year_end               0\n",
       "first_active_monthIs_year_start             0\n",
       "first_active_monthElapsed                   0\n",
       "transactions_count                      21931\n",
       "city_id_nunique                         21931\n",
       "merchant_category_id_nunique            21931\n",
       "merchant_id_nunique                     21931\n",
       "state_id_nunique                        21931\n",
       "subsector_id_nunique                    21931\n",
       "purchase_Year_nunique                   21931\n",
       "purchase_Month_nunique                  21931\n",
       "purchase_Week_nunique                   21931\n",
       "purchase_Day_nunique                    21931\n",
       "purchase_amount_sum                     21931\n",
       "purchase_amount_max                     21931\n",
       "purchase_amount_min                     21931\n",
       "                                        ...  \n",
       "month_diff_mean_old                         0\n",
       "month_diff_std_old                          0\n",
       "month_diff_var_old                          0\n",
       "authorized_flag_sum_old                     0\n",
       "authorized_flag_mean_old                    0\n",
       "purchased_on_weekend_sum_old                0\n",
       "purchased_on_weekend_mean_old               0\n",
       "category_1_sum_old                          0\n",
       "category_1_mean_old                         0\n",
       "card_id_size_old                            0\n",
       "category_2_mean_mean_old                    0\n",
       "category_2_sum_sum_old                      0\n",
       "category_3_mean_mean_old                    0\n",
       "category_3_sum_sum_old                      0\n",
       "purchase_date_diff_old                      0\n",
       "purchase_date_average_old                   0\n",
       "purchase_date_uptonow_old                   0\n",
       "inverse_avg_transactions_per_day_old        0\n",
       "days_since_last_transaction_old             0\n",
       "repurchase_merchant_rate_old                0\n",
       "merchant_category_repurchase_old            0\n",
       "avg_spend_per_merchant_old                  0\n",
       "avg_trans_per_merchant_old                  0\n",
       "avg_spend_per_transaction_old               0\n",
       "elapsed_time                                0\n",
       "card_id_total                           21931\n",
       "purchase_amount_total                   21931\n",
       "installments_total                      21931\n",
       "hist_first_buy                              0\n",
       "new_first_buy                           21931\n",
       "Length: 123, dtype: int64"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}