Using DL last try.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from fastai.imports import *\n",
    "from fastai.torch_imports import *\n",
    "from fastai.dataset import *\n",
    "from fastai.learner import *\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data/elo/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/pyarrow/pandas_compat.py:752: FutureWarning: .labels was deprecated in version 0.24.0. Use .codes instead.\n",
      "  labels, = index.labels\n"
     ]
    }
   ],
   "source": [
    "dep = 'target'\n",
    "df_raw = feather.read_dataframe('train_df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df2= feather.read_dataframe('train_df_alpha')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = feather.read_dataframe('test_df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_raw.drop('outliers', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_raw, df_test]:\n",
    "    for f in ['purchase_date_max','purchase_date_min','purchase_date_max_old',\\\n",
    "                     'purchase_date_min_old']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_excluded = ['purchase_date_max', 'purchase_date_max_old', 'card_id', 'first_active_month',\n",
    "                 'target','outliers','card_id_size', 'card_id_size_old', \n",
    "                 'purchase_date_min', 'purchase_date_min_old','first_active_monthYear',\n",
    "                 'first_active_monthMonth',\n",
    "                 'first_active_monthWeek',\n",
    "                 'first_active_monthDay',\n",
    "                 'first_active_monthDayofweek',\n",
    "                 'first_active_monthDayofyear',\n",
    "                 'first_active_monthIs_month_end',\n",
    "                 'first_active_monthIs_month_start',\n",
    "                 'first_active_monthIs_quarter_end',\n",
    "                 'first_active_monthIs_quarter_start',\n",
    "                 'first_active_monthIs_year_end',\n",
    "                 'first_active_monthIs_year_start']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_included = ['feature_1','feature_2','feature_3','transactions_count','subsector_id_nunique','merchant_id_nunique','merchant_category_id_nunique','purchase_Month_mean','purchase_Month_min','purchase_Month_max','purchase_Week_nunique','purchase_Week_mean','purchase_Week_min','purchase_Week_max','purchase_Dayofweek_mean','purchase_Dayofweek_min','purchase_Dayofweek_max','purchase_Day_nunique','purchase_Day_mean','purchase_Day_min','purchase_Day_max','purchase_Hour_nunique','purchase_Hour_mean','purchase_Hour_min','purchase_Hour_max','purchase_amount_sum','purchase_amount_max','purchase_amount_min','purchase_amount_mean','purchase_amount_var','purchase_amount_skew','installments_sum','installments_max','installments_mean','installments_var','installments_skew','month_lag_max','month_lag_min','month_lag_mean','month_lag_var','month_lag_skew','month_diff_mean','month_diff_var','month_diff_skew','purchased_on_weekend_mean','category_1_mean','category_2_mean','category_3_mean','card_id_count','price_mean','price_max','price_min','price_var','Christmas_Day_2017_mean','Children_day_2017_mean','Black_Friday_2017_mean','Mothers_Day_2018_mean','duration_mean','duration_min','duration_max','duration_var','duration_skew','amount_month_ratio_mean','amount_month_ratio_min','amount_month_ratio_max','amount_month_ratio_var','amount_month_ratio_skew','category_2_mean_mean','category_3_mean_mean','purchase_date_diff','purchase_date_average','purchase_date_uptonow','purchase_date_uptomin','transactions_count_old','subsector_id_nunique_old','merchant_id_nunique_old','merchant_category_id_nunique_old','purchase_Month_nunique','purchase_Month_mean_old','purchase_Month_min_old','purchase_Month_max_old','purchase_Week_nunique_old','purchase_Week_mean_old','purchase_Week_min_old','purchase_Week_max_old','purchase_Dayofweek_mean_old','purchase_Day_nunique_old','purchase_Day_mean_old','purchase_Day_min_old','purchase_Hour_nunique_old','purchase_Hour_mean_old','purchase_Hour_min_old','purchase_Hour_max_old','purchase_amount_sum_old','purchase_amount_max_old','purchase_amount_min_old','purchase_amount_mean_old','purchase_amount_var_old','purchase_amount_skew_old','installments_sum_old','installments_max_old','installments_mean_old','installments_var_old','installments_skew_old','month_lag_max_old','month_lag_min_old','month_lag_mean_old','month_lag_var_old','month_lag_skew_old','month_diff_max','month_diff_min','month_diff_mean_old','month_diff_var_old','month_diff_skew_old','authorized_flag_mean','purchased_on_weekend_mean_old','category_1_mean_old','category_2_mean_old','category_3_mean_old','card_id_count_old','price_sum','price_mean_old','price_max_old','price_min_old','price_var_old','Christmas_Day_2017_mean_old','Mothers_Day_2017_mean','fathers_day_2017_mean','Children_day_2017_mean_old','Valentine_Day_2017_mean','Black_Friday_2017_mean_old','Mothers_Day_2018_mean_old','duration_mean_old','duration_min_old','duration_max_old','duration_var_old','duration_skew_old','amount_month_ratio_mean_old','amount_month_ratio_min_old','amount_month_ratio_max_old','amount_month_ratio_var_old','amount_month_ratio_skew_old','category_2_mean_mean_old','category_3_mean_mean_old','purchase_date_diff_old','purchase_date_average_old','purchase_date_uptonow_old','purchase_date_uptomin_old','quarter','observed_elapsed_time','days_feature1','days_feature2','days_feature3','days_feature1_ratio','days_feature2_ratio','days_feature3_ratio','feature_sum','feature_mean','feature_max','feature_min','feature_var','card_id_total','card_id_count_total','card_id_count_ratio','purchase_amount_total','purchase_amount_total_mean','purchase_amount_total_max','purchase_amount_total_min','purchase_amount_sum_ratio','hist_first_buy','new_first_buy','hist_last_buy','new_last_buy','month_diff_ratio','installments_total','installments_ratio','price_total','CLV','CLV_old','CLV_ratio']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_columns = [c for c in df_raw.columns.values if c not in cols_excluded]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_indep = df_raw[df_train_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "n_valid = 12000\n",
    "n_trn = len(df_raw)-n_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'feature_1 feature_2 feature_3 purchase_Year_nunique purchase_Month_nunique month_lag_max month_lag_min authorized_flag_mean purchase_Year_nunique_old'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_flds = [n for n in df_indep.columns if df_raw[n].nunique()<6 and n != 'outliers']\n",
    "' '.join(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[n for n in df_indep.drop(cat_flds,axis=1).columns if not is_numeric_dtype(df_raw[n])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'first_active_monthElapsed transactions_count city_id_nunique merchant_category_id_nunique merchant_id_nunique state_id_nunique subsector_id_nunique purchase_Week_nunique purchase_Day_nunique purchase_amount_sum purchase_amount_max purchase_amount_min purchase_amount_mean purchase_amount_var installments_sum installments_max installments_min installments_mean installments_var month_lag_mean month_lag_var month_diff_mean month_diff_std month_diff_var authorized_flag_sum purchased_on_weekend_sum purchased_on_weekend_mean category_1_sum category_1_mean card_id_count category_2_mean_mean category_3_mean_mean purchase_date_diff purchase_date_average purchase_date_uptonow purchase_date_uptomin inverse_avg_transactions_per_day days_since_last_transaction repurchase_merchant_rate merchant_category_repurchase avg_spend_per_merchant avg_trans_per_merchant avg_spend_per_transaction transactions_count_old city_id_nunique_old merchant_category_id_nunique_old merchant_id_nunique_old state_id_nunique_old subsector_id_nunique_old purchase_Month_nunique_old purchase_Week_nunique_old purchase_Day_nunique_old purchase_amount_sum_old purchase_amount_max_old purchase_amount_min_old purchase_amount_mean_old purchase_amount_var_old installments_sum_old installments_max_old installments_min_old installments_mean_old installments_var_old month_lag_max_old month_lag_min_old month_lag_mean_old month_lag_var_old month_diff_mean_old month_diff_std_old month_diff_var_old authorized_flag_sum_old authorized_flag_mean_old purchased_on_weekend_sum_old purchased_on_weekend_mean_old category_1_sum_old category_1_mean_old card_id_count_old category_2_mean_mean_old category_3_mean_mean_old purchase_date_diff_old purchase_date_average_old purchase_date_uptonow_old purchase_date_uptomin_old inverse_avg_transactions_per_day_old days_since_last_transaction_old repurchase_merchant_rate_old merchant_category_repurchase_old avg_spend_per_merchant_old avg_trans_per_merchant_old avg_spend_per_transaction_old elapsed_time card_id_total card_id_count_total purchase_amount_total purchase_amount_total_mean installments_total hist_first_buy new_first_buy hist_last_buy new_last_buy purchase_amount_max_total days days_feature1 days_feature2 days_feature3'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for n in cat_flds: df_raw[n] = df_raw[n].astype('category').cat.as_ordered()\n",
    "\n",
    "cont_flds = [n for n in df_indep.columns if n not in cat_flds and n!= 'outliers']\n",
    "' '.join(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw.replace(np.inf, 0, inplace=True)\n",
    "df_raw.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.replace(np.inf, 0, inplace=True)\n",
    "df_test.replace(-np.inf, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 135), (123623, 133))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['target'] = 0\n",
    "df_test['outliers'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_raw.columns.values if c not in df_test.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# df_raw.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_test.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 135), (123623, 135))"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in cat_flds: df_raw[n] = df_raw[n].astype('category').cat.as_ordered()\n",
    "for n in cat_flds: df_test[n] = df_test[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in cont_flds: df_raw[n] = df_raw[n].fillna(0).astype('float32')\n",
    "for n in cont_flds: df_test[n] = df_test[n].fillna(0).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['feature_1',\n",
       " 'feature_2',\n",
       " 'feature_3',\n",
       " 'purchase_Year_nunique',\n",
       " 'purchase_Month_nunique',\n",
       " 'month_lag_max',\n",
       " 'month_lag_min',\n",
       " 'authorized_flag_mean',\n",
       " 'purchase_Year_nunique_old']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_flds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in cont_flds: df_raw[n] = df_raw[n].fillna(0).astype('float32')\n",
    "for n in cont_flds: df_test[n] = df_test[n].fillna(0).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_flds = ['target',\n",
    "            'first_active_monthWeek',\n",
    "            'first_active_monthDay',\n",
    "            'first_active_monthDayofweek',\n",
    "            'first_active_monthDayofyear',\n",
    "            'first_active_monthMonth',\n",
    "            'first_active_monthYear']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "for n in conv_flds: df_raw[n] = df_raw[n].astype('float32')\n",
    "for n in conv_flds: df_test[n] = df_test[n].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag_mean             category\n",
       "authorized_flag_mean_old          float32\n",
       "authorized_flag_sum               float32\n",
       "authorized_flag_sum_old           float32\n",
       "avg_spend_per_merchant            float32\n",
       "avg_spend_per_merchant_old        float32\n",
       "avg_spend_per_transaction         float32\n",
       "avg_spend_per_transaction_old     float32\n",
       "avg_trans_per_merchant            float32\n",
       "avg_trans_per_merchant_old        float32\n",
       "card_id                            object\n",
       "card_id_count                     float32\n",
       "card_id_count_old                 float32\n",
       "card_id_count_total               float32\n",
       "card_id_size                      float64\n",
       "card_id_size_old                    int64\n",
       "card_id_total                     float32\n",
       "category_1_mean                   float32\n",
       "category_1_mean_old               float32\n",
       "category_1_sum                    float32\n",
       "category_1_sum_old                float32\n",
       "category_2_mean_mean              float32\n",
       "category_2_mean_mean_old          float32\n",
       "category_3_mean_mean              float32\n",
       "category_3_mean_mean_old          float32\n",
       "city_id_nunique                   float32\n",
       "city_id_nunique_old               float32\n",
       "days                              float32\n",
       "days_feature1                     float32\n",
       "days_feature2                     float32\n",
       "                                   ...   \n",
       "purchase_amount_sum_old           float32\n",
       "purchase_amount_total             float32\n",
       "purchase_amount_total_mean        float32\n",
       "purchase_amount_var               float32\n",
       "purchase_amount_var_old           float32\n",
       "purchase_date_average             float32\n",
       "purchase_date_average_old         float32\n",
       "purchase_date_diff                float32\n",
       "purchase_date_diff_old            float32\n",
       "purchase_date_max                 float64\n",
       "purchase_date_max_old             float64\n",
       "purchase_date_min                 float64\n",
       "purchase_date_min_old             float64\n",
       "purchase_date_uptomin             float32\n",
       "purchase_date_uptomin_old         float32\n",
       "purchase_date_uptonow             float32\n",
       "purchase_date_uptonow_old         float32\n",
       "purchased_on_weekend_mean         float32\n",
       "purchased_on_weekend_mean_old     float32\n",
       "purchased_on_weekend_sum          float32\n",
       "purchased_on_weekend_sum_old      float32\n",
       "repurchase_merchant_rate          float32\n",
       "repurchase_merchant_rate_old      float32\n",
       "state_id_nunique                  float32\n",
       "state_id_nunique_old              float32\n",
       "subsector_id_nunique              float32\n",
       "subsector_id_nunique_old          float32\n",
       "target                            float32\n",
       "transactions_count                float32\n",
       "transactions_count_old            float32\n",
       "Length: 135, dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw.dtypes.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag_mean             category\n",
       "authorized_flag_mean_old          float32\n",
       "authorized_flag_sum               float32\n",
       "authorized_flag_sum_old           float32\n",
       "avg_spend_per_merchant            float32\n",
       "avg_spend_per_merchant_old        float32\n",
       "avg_spend_per_transaction         float32\n",
       "avg_spend_per_transaction_old     float32\n",
       "avg_trans_per_merchant            float32\n",
       "avg_trans_per_merchant_old        float32\n",
       "card_id                            object\n",
       "card_id_count                     float32\n",
       "card_id_count_old                 float32\n",
       "card_id_count_total               float32\n",
       "card_id_size                      float64\n",
       "card_id_size_old                    int64\n",
       "card_id_total                     float32\n",
       "category_1_mean                   float32\n",
       "category_1_mean_old               float32\n",
       "category_1_sum                    float32\n",
       "category_1_sum_old                float32\n",
       "category_2_mean_mean              float32\n",
       "category_2_mean_mean_old          float32\n",
       "category_3_mean_mean              float32\n",
       "category_3_mean_mean_old          float32\n",
       "city_id_nunique                   float32\n",
       "city_id_nunique_old               float32\n",
       "days                              float32\n",
       "days_feature1                     float32\n",
       "days_feature2                     float32\n",
       "                                   ...   \n",
       "purchase_amount_sum_old           float32\n",
       "purchase_amount_total             float32\n",
       "purchase_amount_total_mean        float32\n",
       "purchase_amount_var               float32\n",
       "purchase_amount_var_old           float32\n",
       "purchase_date_average             float32\n",
       "purchase_date_average_old         float32\n",
       "purchase_date_diff                float32\n",
       "purchase_date_diff_old            float32\n",
       "purchase_date_max                 float64\n",
       "purchase_date_max_old             float64\n",
       "purchase_date_min                 float64\n",
       "purchase_date_min_old             float64\n",
       "purchase_date_uptomin             float32\n",
       "purchase_date_uptomin_old         float32\n",
       "purchase_date_uptonow             float32\n",
       "purchase_date_uptonow_old         float32\n",
       "purchased_on_weekend_mean         float32\n",
       "purchased_on_weekend_mean_old     float32\n",
       "purchased_on_weekend_sum          float32\n",
       "purchased_on_weekend_sum_old      float32\n",
       "repurchase_merchant_rate          float32\n",
       "repurchase_merchant_rate_old      float32\n",
       "state_id_nunique                  float32\n",
       "state_id_nunique_old              float32\n",
       "subsector_id_nunique              float32\n",
       "subsector_id_nunique_old          float32\n",
       "target                            float32\n",
       "transactions_count                float32\n",
       "transactions_count_old            float32\n",
       "Length: 135, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.dtypes.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_raw.dtypes) == type(df_test.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_test.columns.values if c not in df_raw.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df_raw.dtypes.sort_index() == df_test.dtypes.sort_index()).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 104)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw_copy = df_raw[cat_flds+cont_flds+[dep]].copy()\n",
    "df, y, nas, mapper_t = proc_df(df_raw_copy, 'target', do_scale=True)\n",
    "\n",
    "val_idx = list(range(n_trn, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_copy = df_test[df_train_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 113), (201917, 135))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_test_copy.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for n in cat_flds: df_test_copy[n] = df_test_copy[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test_copy = df_test[cat_flds+cont_flds+[dep]].copy()\n",
    "df_t, _, nas, mapper = proc_df(df_test_copy,'target', do_scale=True, mapper=mapper_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 114)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_copy.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 113), (123623, 113))"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[c for c in df_t.columns.values if c not in df.columns.values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = df[df_train_columns]\n",
    "# df_t = df_test[df_train_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "feature_1                                  int8\n",
       "feature_2                                  int8\n",
       "feature_3                                  int8\n",
       "purchase_Year_nunique                      int8\n",
       "purchase_Month_nunique                     int8\n",
       "month_lag_max                              int8\n",
       "month_lag_min                              int8\n",
       "authorized_flag_mean                       int8\n",
       "purchase_Year_nunique_old                  int8\n",
       "first_active_monthElapsed               float64\n",
       "transactions_count                      float64\n",
       "city_id_nunique                         float64\n",
       "merchant_category_id_nunique            float64\n",
       "merchant_id_nunique                     float64\n",
       "state_id_nunique                        float64\n",
       "subsector_id_nunique                    float64\n",
       "purchase_Week_nunique                   float64\n",
       "purchase_Day_nunique                    float64\n",
       "purchase_amount_sum                     float64\n",
       "purchase_amount_max                     float64\n",
       "purchase_amount_min                     float64\n",
       "purchase_amount_mean                    float64\n",
       "purchase_amount_var                     float64\n",
       "installments_sum                        float64\n",
       "installments_max                        float64\n",
       "installments_min                        float64\n",
       "installments_mean                       float64\n",
       "installments_var                        float64\n",
       "month_lag_mean                          float64\n",
       "month_lag_var                           float64\n",
       "                                         ...   \n",
       "category_1_mean_old                     float64\n",
       "card_id_count_old                       float64\n",
       "category_2_mean_mean_old                float64\n",
       "category_3_mean_mean_old                float64\n",
       "purchase_date_diff_old                  float64\n",
       "purchase_date_average_old               float64\n",
       "purchase_date_uptonow_old               float64\n",
       "purchase_date_uptomin_old               float64\n",
       "inverse_avg_transactions_per_day_old    float64\n",
       "days_since_last_transaction_old         float64\n",
       "repurchase_merchant_rate_old            float64\n",
       "merchant_category_repurchase_old        float64\n",
       "avg_spend_per_merchant_old              float64\n",
       "avg_trans_per_merchant_old              float64\n",
       "avg_spend_per_transaction_old           float64\n",
       "elapsed_time                            float64\n",
       "card_id_total                           float64\n",
       "card_id_count_total                     float64\n",
       "purchase_amount_total                   float64\n",
       "purchase_amount_total_mean              float64\n",
       "installments_total                      float64\n",
       "hist_first_buy                          float64\n",
       "new_first_buy                           float64\n",
       "hist_last_buy                           float64\n",
       "new_last_buy                            float64\n",
       "purchase_amount_max_total               float64\n",
       "days                                    float64\n",
       "days_feature1                           float64\n",
       "days_feature2                           float64\n",
       "days_feature3                           float64\n",
       "Length: 113, dtype: object"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.dtypes == df_t.dtypes).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_cats(df_t, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((123623, 113), (201917, 113))"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_t.shape, df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for n in cat_flds: df[n] = df[n].astype('category').cat.as_ordered()\n",
    "# for n in cat_flds: df_t[n] = df_t[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = df.copy()\n",
    "df_t_copy = df_t.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = df_copy\n",
    "# df_t = df_t_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idx = list(range(n_trn, len(df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 113), (123623, 113), (201917,))"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for n in cat_flds: df[n] = df[n].astype('category').cat.as_ordered()\n",
    "# for n in cat_flds: df_t[n] = df_t[n].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ColumnarModelData.from_data_frame(PATH, val_idx, df, y.astype(np.float32), cat_flds=cat_flds, bs=64, test_df=df_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(x,y): return math.sqrt(((x-y)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_trn_flds = ['feature_1',\n",
    " 'feature_2',\n",
    " 'feature_3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'feature_1': 6,\n",
       " 'feature_2': 4,\n",
       " 'feature_3': 3,\n",
       " 'purchase_Year_nunique': 3,\n",
       " 'purchase_Month_nunique': 3,\n",
       " 'month_lag_max': 3,\n",
       " 'month_lag_min': 3,\n",
       " 'authorized_flag_mean': 2,\n",
       " 'purchase_Year_nunique_old': 3}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_c = {n: len(c.cat.categories)+1 for n,c in df_raw[cat_flds].items()}\n",
    "emb_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = [(c, min(10, (c+1)//2)) for _,c in emb_c.items()]\n",
    "metrics=[rmse]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_range=(np.min(y)*1,np.max(y)*1.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(6, 3), (4, 2), (3, 2), (3, 2), (3, 2), (3, 2), (3, 2), (2, 1), (3, 2)]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_szs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(189917, 1)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md.trn_ds.y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "185232"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(md.trn_ds.y, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11760"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(md.val_ds.y, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 104, 113)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds), len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.14 s, sys: 264 ms, total: 1.41 s\n",
      "Wall time: 1.47 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.04, 1, [500], [0.01], y_range=y_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('Embedding-1',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 3]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 18)])),\n",
       "             ('Embedding-2',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 8)])),\n",
       "             ('Embedding-3',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-4',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-5',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-6',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-7',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Embedding-8',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 1]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 2)])),\n",
       "             ('Embedding-9',\n",
       "              OrderedDict([('input_shape', [-1]),\n",
       "                           ('output_shape', [-1, 2]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 6)])),\n",
       "             ('Dropout-10',\n",
       "              OrderedDict([('input_shape', [-1, 18]),\n",
       "                           ('output_shape', [-1, 18]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('BatchNorm1d-11',\n",
       "              OrderedDict([('input_shape', [-1, 104]),\n",
       "                           ('output_shape', [-1, 104]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 208)])),\n",
       "             ('Linear-12',\n",
       "              OrderedDict([('input_shape', [-1, 122]),\n",
       "                           ('output_shape', [-1, 500]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 61500)])),\n",
       "             ('Dropout-13',\n",
       "              OrderedDict([('input_shape', [-1, 500]),\n",
       "                           ('output_shape', [-1, 500]),\n",
       "                           ('nb_params', 0)])),\n",
       "             ('Linear-14',\n",
       "              OrderedDict([('input_shape', [-1, 500]),\n",
       "                           ('output_shape', [-1, 1]),\n",
       "                           ('trainable', True),\n",
       "                           ('nb_params', 501)]))])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# m = md.get_learner(emb_szs, len(cont_flds), 0.05, 1, [500,250], [0.5,0.05],\n",
    "#                    y_range=y_range, use_bn=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 3, 2, 3, 3, 3, 3, 2, 2])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[cat_flds].nunique().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 3, 2, 3, 3, 3, 3, 2, 2])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_t[cat_flds].nunique().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 104)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cat_flds), len(cont_flds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 113), (123623, 113))"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca39b99c901743c7a2f49a8f122096ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 2571/2968 [00:06<00:01, 393.25it/s, loss=18.9]\n",
      "                                                               \r"
     ]
    }
   ],
   "source": [
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEOCAYAAACTqoDjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8XNWZ//HPo26r2bIkW65yww2wAWEghiw9QKi7SZY0CGFDQtgNye6yQPJLSCNLdpdll7AplCyEwCYESCDUEEIvBtm494rlXmVZlq32/P6YKyPbI2ksdHVnNN/36zWvuXNumeeozDP3nnPPMXdHRETkUBlRByAiIslJCUJEROJSghARkbiUIEREJC4lCBERiUsJQkRE4lKCEBGRuJQgREQkLiUIERGJSwlCRETiyoo6gA+jtLTUKysrow5DRCSlzJo1a5u7l3W1XUoniMrKSqqrq6MOQ0QkpZjZ2kS20yUmERGJSwlCRETiUoIQEZG4lCBERCQuJQgREYkrtARhZnlm9o6ZzTWzhWb2vaD8fjNbbWZzgse0oNzM7E4zW2Fm88zs+LBiExGRroXZzXU/cKa77zGzbOB1M3s2WHeDuz96yPbnA+ODx0nAz4LnUKzeVk95YS75uSnd01dEJDShnUF4zJ7gZXbw6GwC7EuAXwX7vQ0MMLOKMGJrbXXO+I+X+eL974ZxeBGRPiHUNggzyzSzOcAW4AV3nxmsujW4jHSHmeUGZcOAde12rwnKetzufU0AzFy9I4zDi4j0CaEmCHdvcfdpwHBgupkdDdwMTAROBEqAG4PNLd4hDi0ws2vMrNrMqrdu3dqtuLbXN3ZrPxGRdNIrvZjcfRfwMnCeu28MLiPtB/4XmB5sVgOMaLfbcGBDnGPd7e5V7l5VVtblUCJxbd/zQYJ4dVn3koyISF8XZi+mMjMbECz3A84GlrS1K5iZAZcCC4JdngSuCHoznQzUuvvGMGLbUb//wPIVv3yHnTqjEBE5TJhnEBXAS2Y2D3iXWBvEU8BDZjYfmA+UAj8Mtn8GWAWsAO4BvhpWYOMHF3LNR8cceH3T4/PCeisRkZQVWh9Pd58HHBen/MwOtnfgurDiaW9sWQHfvGASz8zfSM3OBjbV7uuNtxURSSlpfSf1KzecAUBpQW4XW4qIpJ+0ThCZGcbZkwazfldD1KGIiCSdtE4QEGuwXrKpju179ne9sYhIGkn7BPHxY4cC8OyCTRFHIiKSXNI+QXxxRiWFeVks3rg76lBERJJK2icIM6NuXzMPzXyfumAIDhERUYIAYMa4QQBUr9kZcSQiIslDCQL40WXHAHCVRncVETlACQIYWdIfgH7ZmRFHIiKSPJQgiLVDfPKE4TQ0tbBB90SIiABKEAdkZ8V+FA+8uSbaQEREkoQSROA7F04GIDdLPxIREVCCOCAvO5MRJf1Ys31v1KGIiCQFJYh2Kgfls2Z7fdRhiIgkBSWIdioH5bN6az2trYfNdCoiknaUINqZNmIAdfubWbxJw26IiChBtDOxohCAuetqI45ERCR6ShDtjAhumLvlyQVdbCki0vcpQbRTlJfN8IH9yM8NbSZWEZGUoQRxiC98pJJde5tYt0PdXUUkvYWWIMwsz8zeMbO5ZrbQzL4XlI82s5lmttzMfmtmOUF5bvB6RbC+MqzYOnPymNjIrnNrdkXx9iIiSSPMM4j9wJnuPhWYBpxnZicDPwbucPfxwE7g6mD7q4Gd7j4OuCPYrteNKy8gM8M0gZCIpL3QEoTH7AleZgcPB84EHg3KHwAuDZYvCV4TrD/LzCys+DqSl53J2LJ8Fm+s6+23FhFJKqG2QZhZppnNAbYALwArgV3u3hxsUgMMC5aHAesAgvW1wKAw4+vIpIoinUGISNoLNUG4e4u7TwOGA9OBSfE2C57jnS0cdkuzmV1jZtVmVr1169aeC7adSRVFbKzdx669jaEcX0QkFfRKLyZ33wW8DJwMDDCztn6kw4ENwXINMAIgWF8M7IhzrLvdvcrdq8rKykKJd3JFEQCLdBYhImkszF5MZWY2IFjuB5wNLAZeAj4RbHYl8ESw/GTwmmD9X9w9kkGRJgUJQu0QIpLOwrwjrAJ4wMwyiSWiR9z9KTNbBPzGzH4IvAfcF2x/H/Cgma0gduZweYixdaqsMJfSgly1Q4hIWgstQbj7POC4OOWriLVHHFq+D/hkWPEcqUkVhUoQIpLWdCd1ByZXFLF88x6aWlqjDkVEJBJKEB2YVFFEY0srK7fu6XpjEZE+SAmiAx80VOsyk4ikJyWIDowpyyc3K4O3Vx7W01ZEJC0oQXQgOzODE0YN1OxyIpK2lCA6Mb68gJVb9hDR7RgiIpFSgujE6NJ86htbWLm1PupQRER6nRJEJyoG9APgjj8vizgSEZHepwTRiXMnDwbghYWbI45ERKT3KUF0wsw4sXIgjS2tNDbrhjkRSS9KEF343MmjAFi1TTfMiUh6UYLogm6YE5F0pQTRhTGl+eRkZrBEQ3+LSJpRguhCVmYG4wcXaPIgEUk7ShAJmFRRxJJNOoMQkfSiBJGAiUMK2Vq3n2179kcdiohIr1GCSEDbHNXzanZFHImISO9RgkjA8aMGYgbza9QOISLpQwkiAXnZmQwf2E+TB4lIWlGCSNCY0gIlCBFJK6ElCDMbYWYvmdliM1toZtcH5d81s/VmNid4XNBun5vNbIWZLTWzj4UVW3eMLStg1dZ6Wls19LeIpIesEI/dDPyTu882s0Jglpm9EKy7w93/o/3GZjYZuByYAgwF/mxmR7l7S4gxJmxseT4NTS1s3L2PYcEoryIifVloZxDuvtHdZwfLdcBiYFgnu1wC/Mbd97v7amAFMD2s+I7U2LICAFZu0WUmEUkPvdIGYWaVwHHAzKDo781snpn90swGBmXDgHXtdquh84TSqw4kCLVDiEiaCD1BmFkB8BjwdXffDfwMGAtMAzYCt7dtGmf3wy74m9k1ZlZtZtVbt24NKerDlRbkUJSXpQQhImkj1ARhZtnEksND7v44gLtvdvcWd28F7uGDy0g1wIh2uw8HNhx6THe/292r3L2qrKwszPAPYmaMLS9ghS4xiUiaCLMXkwH3AYvd/T/blVe02+wyYEGw/CRwuZnlmtloYDzwTljxdce4MiUIEUkfYfZimgF8HphvZnOCsm8CnzazacQuH60Bvgzg7gvN7BFgEbEeUNclSw+mNhOGFPK7WTVs27Of0oLcqMMREQlVlwnCzPKBBndvNbOjgInAs+7e1Nl+7v468dsVnulkn1uBW7uKKSoTh8TGZFq6qY7ScUoQItK3JXKJ6VUgz8yGAS8CVwH3hxlUspowpBBAQ3+LSFpIJEGYu+8F/hr4ibtfBkwON6zkVFaYy6D8HJZu0qB9ItL3JZQgzOwU4LPA00FZmG0XSW3CkEKW6gxCRNJAIgni68DNwO+DhuQxwEvhhpW8JgwpZNnmPRqTSUT6vC7PBNz9FeAVADPLALa5+9fCDixZTRxSSENTC+t27mXUoPyowxERCU2XZxBm9rCZFQW9mRYBS83shvBDS07jymMN1bofQkT6ukQuMU0Ohsi4lFgX1ZHE7m9IS6MG9Qfg/R17I45ERCRciSSI7GDIjEuBJ4L7H9L2Avyg/BxKC3JZsF49mUSkb0skQfyC2B3P+cCrZjYKSNtPRzNjbFk+q7fpEpOI9G1dJgh3v9Pdh7n7BR6zFjijF2JLWlOGFjN/fS2Nza1RhyIiEppEGqmLzew/24bYNrPbiZ1NpK1jhxfT1OKs0lmEiPRhiVxi+iVQB3wqeOwG/jfMoJLdMcOLAZi7blfEkYiIhCeRBDHW3W9x91XB43vAmLADS2ZjSvMZ0D+b2WuVIESk70okQTSY2altL8xsBtAQXkjJz8w4bsQAZr2/M+pQRERCk8iYStcCD5hZMbHhu3cAXwgzqFRw3MiBvLR0K3v2N1OQm7ZDU4lIH5bIUBtzgKlmVhS8Ttsuru1NDIb+Xr65juNGDow4GhGRntdhgjCzf+ygHID204imo6MGtyWIPUoQItIndXYGUdhrUaSgESX9yc3KYNlmDf0tIn1Thwki6K0kHcjMMMaWFbBiq+6FEJG+KZFeTNKB8YMLWL5ZCUJE+qbQEoSZjTCzl8xssZktNLPrg/ISM3vBzJYHzwODcjOzO81shZnNM7Pjw4qtp4wvL2D9rgbq9zdHHYqISI8L8wyiGfgnd58EnAxcZ2aTgZuAF919PPBi8BrgfGB88LgG+FmIsfWI8UFD9bya2ogjERHpeV12czWzXOBvgMr227v79zvbz903AhuD5TozWwwMAy4BTg82ewB4GbgxKP+VuzvwtpkNMLOK4DhJqWpUrPfSwg21nDJ2UMTRiIj0rETu8HoCqAVmAfu78yZmVgkcB8wEBrd96Lv7RjMrDzYbBqxrt1tNUJa0CWJQQS4l+TmsVEO1iPRBiSSI4e5+XnffwMwKgMeAr7v77rb7KOJtGqfssImJzOwaYpegGDlyZHfD6jFjy/JZuaU+6jBERHpcIm0Qb5rZMd05eDAT3WPAQ+7+eFC82cwqgvUVwJagvAYY0W734cCGQ4/p7ne7e5W7V5WVlXUnrB41tqxAZxAi0iclkiBOBWaZ2dKgd9F8M5vX1U4WO1W4D1h8yF3XTwJXBstXEruE1VZ+RdCb6WSgNpnbH9qMLStge30jO+sbow5FRKRHJXKJ6fxuHnsG8HlgvpnNCcq+CdwGPGJmVwPvA58M1j0DXACsAPYCV3XzfXvV2PLY3Emrtu3hhPySiKMREek5iQzWt9bMpgKnBUWvufvcBPZ7nfjtCgBnxdnegeu6Om6yGVtWAMDKLfWcMEoJQkT6jkSmHL0eeAgoDx6/NrN/CDuwVDF8YH9yMjPUDiEifU4il5iuBk5y93oAM/sx8BbwkzADSxWZGcaYsnyWatA+EeljEmmkNqCl3esWOr50lJYmVRSxdJMShIj0LYmcQfwvMNPMfh+8vpRY7yQJTBhSyO/fW8+uvY0M6J8TdTgiIj2iyzOIoIvqVcSmGt0JXOXu/xV2YKmkbXY5nUWISF/S2YxyRcGdzyXAmuDRtq7E3XeEH15qmDikCIAlm+o4aYzGZBKRvqGzS0wPAxcSG4Op/ZAXFrweE2JcKWVwUS7F/bJZojMIEelDOptR7sLgeXTvhZOazIyJQwpZsml31KGIiPSYRO6DeDGRsnQ3cUghyzbV0dp62PiCIiIpqbM2iDygP1AazPrW1rW1CBjaC7GllIkVRdQ3trB+VwMjSvpHHY6IyIfWWRvEl4GvE0sGs/ggQewG/ifkuFLOhKAn0+KNu5UgRKRP6PASk7v/d9D+8M/uPsbdRwePqe5+Vy/GmBKOCqYfXaY7qkWkj0hksL6fmNnRwGQgr135r8IMLNUU5GYxtDiPFVs0JpOI9A2JzEl9C7E5pCcTG5L7fOB1QAniEGPLC1ihQftEpI9IZCymTxAbnnuTu18FTAVyQ40qRY0rL2Dllnr1ZBKRPiGRBNHg7q1As5kVEZsiVDfJxTGuvICGplhPJhGRVJdIgqg2swHAPcR6M80G3gk1qhQ1Lpg8SJeZRKQvSKSR+qvB4s/N7DmgyN27nJM6HY0rb5tdbg9nTCiPOBoRkQ+nsxvlju9snbvPDiek1DWoIJeS/Bx1dRWRPqGzM4jbg+c8oAqYS+xmuWOBmcCp4YaWmiZXFLFwg8ZkEpHU19mNcme4+xnAWuB4d69y9xOA44AVvRVgqpkyrIhlm+tobG6NOhQRkQ8lkUbqie4+v+2Fuy8ApnW1k5n90sy2mNmCdmXfNbP1ZjYneFzQbt3NZrbCzJaa2ceOtCLJ4uihxTS1uC4ziUjKSyRBLDaze83sdDP7KzO7B1icwH73A+fFKb/D3acFj2cAzGwycDkwJdjnp2aWmVgVksvkoR9MHiQiksoSSRBXAQuB64kN3rcoKOuUu79KbJrSRFwC/Mbd97v7amKXsKYnuG9SGVXSn+xM05AbIpLyEunmug+4I3j0hL83syuAauCf3H0nMAx4u902NUHZYczsGuAagJEjR/ZQSD0nKzOD0aX5rNiiMwgRSW0dnkGY2SPB83wzm3foo5vv9zNgLLE2jI180FPK4mwbd7wKd787aDCvKisr62YY4RpfXsiiDbtx15AbIpK6OjuDuD54vrCn3szdN7ctB20ZTwUva4AR7TYdDmzoqfftbSeNKeHp+Rt5f8deRg3KjzocEZFu6ayb68bgeW28R3fezMwq2r28DGjr4fQkcLmZ5ZrZaGA8KTycR9WoEgDmrNsVcSQiIt3X2Z3UdcS/zGOAu3tRZwc2s/8jNkx4qZnVALcAp5vZtOC4a4jNWoe7LwwuaS0CmoHr3L3liGuTJI4aXED/nEzee38Xl0yL25QiIpL0OkwQ7l74YQ7s7p+OU3xfJ9vfCtz6Yd4zWWRlZnD00GLmr6+NOhQRkW5LpJsrAGZWbmYj2x5hBtUXVJb2Z+32vVGHISLSbV0mCDO72MyWA6uBV4hdGno25LhS3qhB+Wzbs5/6/c1RhyIi0i2JnEH8ADgZWObuo4nNLvdGqFH1AW1Dfy/VkBsikqISSRBN7r4dyDCzDHd/iQTGYkp3U4IhNzSyq4ikqi7vpAZ2mVkB8CrwkJltIdbTSDoxbEA/ivtls2iDGqpFJDUlcgZxCbAX+AbwHLASuCjMoPoCM2PKUM0NISKpK5EEcQ0w1N2b3f0Bd78zuOQkXZgytIglm+poatHcECKSehJJEEXA82b2mpldZ2aDww6qr5gytJjG5lZWbtXIriKSerpMEO7+PXefAlwHDAVeMbM/hx5ZH3CgoXq9LjOJSOpJ+EY5YAuwCdgOlIcTTt8ypqyAvOwMtUOISEpK5Ea5a83sZeBFoBT4krsfG3ZgfUFmhjFxSBEL1JNJRFJQIt1cRwFfd/c5YQfTF00bMYBHqtfR0upkZsSb9kJEJDkl0gZxk5JD9x09rJi9jS2s3lYfdSgiIkfkSNogpBvaGqrnr9fcECKSWpQgQja+PNZQPXed2iFEJLUoQYSsbW6IeTU6gxCR1KIE0QumjhjAwg27dUe1iKQUJYhecOzwYvY3t7JMQ3+LSApRgugFU4cPAGBejdohRCR1KEH0glGD+lPcL1vtECKSUkJLEGb2SzPbYmYL2pWVmNkLZrY8eB4YlJuZ3WlmK8xsnpkdH1ZcUTAzjh1erJ5MIpJSwjyDuB8475Cym4AX3X08saE7bgrKzwfGB49rgJ+FGFckjh1ezNLNdTQ0tkQdiohIQkJLEO7+KrDjkOJLgAeC5QeAS9uV/8pj3gYGmFlFWLFFYerwAbS0Oos26ixCRFJDb7dBDHb3jQDBc9uosMOAde22qwnK+oypI2IN1brMJCKpIlkaqeONYudxNzS7xsyqzax669atIYfVcwYX5TG4KFcN1SKSMno7QWxuu3QUPG8JymuAEe22Gw5siHcAd7/b3avcvaqsrCzUYHvascMHqKuriKSM3k4QTwJXBstXAk+0K78i6M10MlDbdimqLzlpdAmrttWzRiO7ikgKCLOb6/8BbwETzKzGzK4GbgPOMbPlwDnBa4BngFXACuAe4KthxRWl0yfEmlzeXrU94khERLqWyIRB3eLun+5g1VlxtnVic173aWPL8hnYP5tZa3dy+fSRUYcjItKpZGmkTgtmxgmjSnh3zaG9f0VEko8SRC87eUwJa7bvZVPtvqhDERHplBJELztp9CAA3tFZhIgkOSWIXjapopD8nEzeXa0EISLJTQmil2VlZnD8qIFqhxCRpKcEEYETK0tYurmO2r1NUYciItIhJYgInFhZgjtUr9VZhIgkLyWICEwbMYDsTOPdNTujDkVEpENKEBHol5PJ0cOK1Q4hIklNCSIi0ytLmFezi31NmkBIRJKTEkRETqwsoanFmbNOw3+LSHJSgohIVeVAAN0PISJJSwkiIgP653DU4ALu/MtyYmMViogkFyWICJ1/dAVNLc47OosQkSSkBBGhL//VGApzs3j4nfejDkVE5DBKEBHqn5PFhVOH8sKizexvVm8mEUkuShARO2dyOXsbW3hzpWaZE5HkogQRsRnjSinKy+Le11ZFHYqIyEGUICKWm5XJqeNLeWPFdr7zxIKowxEROUAJIgl8/5KjAfjVW2tZt2NvxNGIiMREkiDMbI2ZzTezOWZWHZSVmNkLZrY8eB4YRWxRKC3I5Y2bzsQMHp1VE3U4IiJAtGcQZ7j7NHevCl7fBLzo7uOBF4PXaWPYgH6cMHIg//3ictZur486HBGRpLrEdAnwQLD8AHBphLFE4rozxwHw4FtrI45ERCS6BOHAn8xslpldE5QNdveNAMFzeUSxReaMCeV8/NgKHp1dQ2Nza9ThiEiaiypBzHD344HzgevM7KOJ7mhm15hZtZlVb926NbwII/I3xw9j194mnpizPupQRCTNRZIg3H1D8LwF+D0wHdhsZhUAwfOWDva9292r3L2qrKyst0LuNaeNL6MkP4cbHp3Hvz23JOpwRCSN9XqCMLN8MytsWwbOBRYATwJXBptdCTzR27Elg+zMDB6/9iMA/PTllWzevS/iiEQkXUVxBjEYeN3M5gLvAE+7+3PAbcA5ZrYcOCd4nZYqS/N54RsfJTPD+MFTi2huUXuEiPS+rN5+Q3dfBUyNU74dOKu340lW4wcX8pnpI3nw7bU8NW8j91xRxTmTB0cdloikkWTq5iqH+O7FU/j62eMB+NKvqplfUxtxRCIStTXb6vnbX7zF4o27Q38vJYgklplhfP3so3j2+tMAuOiu17nt2SWagU4kjf1lyRZmrt7B9/+4KPT3UoJIAZMqinjw6ukA/PyVlXz23pm0tipJiKSj0sJcAL4wozL091KCSBGnjS9j6Q/PY3BRLm+u3M4v31gddUgiEoG2KwjjywtCfy8liBSSm5XJ2zefxfEjB/DDpxdrDgmRNNQSXD3IMAv9vZQgUoyZ8bPPnUBBbhY/fHoxlTc9zSPV66IOS0R6SdvV5cwMJQiJY3BRHrO/fQ7TK0sA+JdH51F509NM+vZz7N7XdNj2T87dwM9eXslDM9dy/A9e4Lfvvg+gxm6RFNTW/tgLJxC9fx+E9IycrAwe+copLN1Ux8f+61UAGppaOPlHL1LcL5tvnHMUNz42j3g54MbH5nPjY/MB+LtTR3PT+RPJyox9V9i9r4n+2ZkHXkvX1u3Yy++q1/G5U0ZRXph30Lrte/ZT29DEmLIPrhe7O9Yb/93SJ7UG/9S9cQZhqfwtsqqqyqurq6MOI3INjS0s2ljLgvW7ueXJhXG3ufb0sTw7fyOfnj6S382qYcWWPYdtU5Kfw476RvrnZPLGjWcyMD8HgGWb61i3Yy8zxpWSl50Zal2SnbvT0ups2LWPigF5ZJhR9cMX2Ln3gzO32z85lbHlBTS3tPKJn78FwAmjBnL9WeOZ/f5O/uvPywG4/MQR3HLRFPrlZLJ2ez3DB/bvlX96SW0PzVzLt36/gJnfPIvBRXld7xCHmc1qNxdPx9spQfQtO+sb+W31Om57dgnnTRnChVMrOHvS4MM+2JtbWlm/q4Fz73iVIcV5rN1+8FSnBblZHDu8mKrKEu58cfmB8qe/diqTK4rS7hvwa8u38tyCTTw08/2460eW9Of9TqaLzc40mlo6/1/Lycrg4qlD+c5FkynKy/5Q8Urf9eDba/n2Hxbw7rfOpizo8nqklCDkiMxau4NZa3fypdPG8PzCTVz70Oy4l6faK8zL4v6rTqS4Xw7F/bJxdwrysnh+4SZOrCxh+MD+uDvukPEhvxk3t7TS3Oq9egazYH0tq7fV88aKbfzm3Q86AuTnZPLVM8bx788vPVC25AfnkZuVwdLNdbyxYju1DU28umwr3/r4JE6sLKG2oYlbnljAwg27ueIjlZw5sZx7Xl3F/W+uifveD//dSXxkXGmHsdXta2JvYwv9cjKVTI5AT/09dseKLXtYuKGW4n7ZnDqutNuXcR94cw23PLmQWf/vbAYVKEF0SAkiPKu31bNxVwNzanYxqaKIMyaU88KizdzyxAI21H64EWZvOn8iZ00sZ1x5wUFnIo3NrVSv3UHVqBJysjJYvrmOytJ8xn/r2QPbnDBqICu27KEgN4v1uxqYXlnCF2ZUcuvTi9nf3Mq2Pfvpl53J508ZxT+fO4GcrMT+CVdsqWNrXSO52Rls2b2PW59ZzLodDQdtc/KYEu678kTyc2NNd/uaWsjNyvhQZ1OLN+5mdGk+edmZ1O9v5qcvr+B/Xlp50Db3XFHFoIIcXlu2jc+ePJK7/rLioMRy8/kT+fJfjT3w+v3te/m355dw1YxKjh8Zm9p9/a4Grv31bCpL87ntr485UIeetK+phRcXb+HY4cUMLspL+GffW56cu4Gv/d97QGyK38e/+hEWrK/l6XkbufWyY+iXE86Xj4Ubanl+wSbu/MuKA2Ul+Tk8cd0MRpT0Z+66Xdz+wjLq9jWRn5PFhCGFHD2siJPHDKKiuN9hx7vv9dX84KlFzPnOOQzon9OtmJQgJDQtrU5TSyt/eG89Nz0ea+wuL8xlS91+IHY5JS87k7p9zV0eq7hfNteePpbbnu35uS/yczL5xjlH8ZmTRpKblUltQxMFuVkHfXBt2NXA8ws38b0Ohi34VNVwHqmu4ZmvncbkoUU9HmM882tqueiu149onxEl/di+p5G9jS0Jbf/g1dM5bXwZj1SvY3JFEfe8toon5mzg1HGlfOmjYzhu5ICEzkxmrtrOW6u2M2VoMT9+bsmBtq1x5QU887XTDksSD898n6WbdnPLRVO69S1+z/5mfv32Wi48toLBRXlkH/ItvKMOAK2tTtWtf2ZHfWPc404bMYBHv3JKj3bOeGPFNj5778yDysoLczljQjmPv1fD0cOKmTp8QIdnkW3652Ryw8cmkJ+TxaKNuw9sP++753b77FEJQnqFu1O3vznuH6q7s2jjbnbUN3La+DL2NjazeONuvvrQbDbv3h/3eIOLcg+sG5SfQ2VpPqUFOfz35ceRl51Ja6uzflcDudkZtLQ6c9fV8uisGnKyjNs/OY2NtQ1UDsrnx88v4RevHNmNhKUFOeza28T5x1QwKD+Hb14wKbJvwe6b2mtjAAANK0lEQVROU4vT0NjCR257kfrGFr570WTeWbODuetqeeCL0xlXXkDdviYuvusNVm+rP2j/i6YO5Y9zNxxU9uWPjmFQQQ4/eubIknHloP789sunkJ2ZwefuncnEIYXccvEULvzJa4edZV1xyih+++469gdT5v7LeRO4dNowdu9r4rL/eZOGpg8SWEFuFnv2N5Ofk8lrN57Jzr2NjC2Lf3ewu/Obd9dxc/CFpL1vXjCROet28cz8TQAMLc7j99fNONCA29TSyo2PzuPx99Zz62VH85npI/n2Ewt4et5Gdu5tomrUQKrX7gRgxrhBzBhXysQhhZw5Mf7oyTvqG7nhd3OZOmIAa7bV8/h768nLzmBfU6zOmRnG9MoS3lq1/cA+l0wbyvcvPpri/rH/kwffWsO3n/igQ8m/nDeB997fxfY9+zEzZgXxdGbNbR/vcpuOKEFI0tvX1MLqbfX86q01XHFKJeWFuZQEPafm1dQyZWjRh/pG9/zCTfz42SXsamjq8JsjxD5Q7v/idI4aXNjt9wpTV91iG5tbyTDYXt/IvqYWhg7od+Cb9b6mlsPabRZuqOVz98480PMqJzODC4+t4KbzJ3LHn5dT29B44MM2EdNHl5CVYXz19HGcOj7WbnLbs0v4+Ssr425fmJtF3f74Z5fnTh5Mq0NuVgYzV+9g2579nDJmEO+t23ngAxhg+MB+1OxsiHuMNoPyc/jORZO5/jdzgNhlpZdvOD3uWcc//W4uj88+fJrf718yheNHDmT5ljoWrt/Nva8f2RA3915Rxdlxhul3d65+oJo3VmzjlRvOYEjx4b2Rtu3Zz8ote5g6YgD3vraKu15awdWnjmZIUR6fPWnUh2pHUYIQ6cDexmb652RRt6+Jllbv9nXcVNbQ2MKyzXUcNbiw02vvra3Oo7Nr+JdH5wHw0aPKOGl0yYEG+lU/uqDDD6qGxhYuv+dt5q7bBcAXZ4zmOxdNBuDVZVtpbG5lxrhSfv7KSv67XU+5ztz56eO44OghB7oDr95Wz4+eWcKQ4lw+VTWCiUOK+PFzS7jvkA/y848ewu2fmkr/nI7bXppaWvn97PX8adFmFqyvZVMnszkOG9CP8qJcPjN9JCePGcSIkv4H1r33/k7+/uH3ePDq6Qfd/5JMlCBEpMf0Vu+fX7+9ltnv72R6ZQnjygsYX17Iy8u2sKl2H186bUzC71+/v5mFG3bzzPyNfLJqOFOGFh9xLHsbm7n217N5ZdlWhhTlUVqYwydPGMHZkwczbMDhjcepRAlCRETiSjRBJFc/NBERSRpKECIiElfSJQgzO8/MlprZCjO7Kep4RETSVVIlCDPLBP4HOB+YDHzazCZHG5WISHpKqgQBTAdWuPsqd28EfgNcEnFMIiJpKdkSxDCg/fRoNUGZiIj0smRLEPE6OR/UD9fMrjGzajOr3rp1ay+FJSKSfpItQdQAI9q9Hg4cNKCMu9/t7lXuXlVWVtarwYmIpJOkulHOzLKAZcBZwHrgXeAz7h53mjQz2wrsAmrbFRd38rptuRTY1gMhH/pe3d0u3vpEyuLV7dDlZKprR+uSqa6J1rOrbROt65G81t9v96XL329ncbY3yt27/oYdu4U+eR7ABcSSxErgWwlsf3eir9uWgeoeivXuntgu3vpEyuLVLc5y0tS1o3XJVNdE69lTddXfr/5+o6prIo+enzXkQ3L3Z4BnjmCXPx7B60PXfViJHq+r7eKtT6Sso7r1dD2P5JidbdfRumSq65Ecryfqqr/fzpd7Srr8/fboMZPqElNvMbNqT2Ackr5Ade170qWeoLpGLdkaqXvL3VEH0ItU174nXeoJqmuk0vIMQkREupauZxAiItIFJQgREYlLCUJEROJSgjiEmZ1uZq+Z2c/N7PSo4wmbmeWb2SwzuzDqWMJiZpOC3+ejZnZt1PGEycwuNbN7zOwJMzs36njCZGZjzOw+M3s06lh6WvB/+UDwu/xsVHH0qQRhZr80sy1mtuCQ8iOZY8KBPUAesaE/klIP1RXgRuCRcKL88Hqinu6+2N2/AnwKSKpuhO31UF3/4O5fAr4A/G2I4X4oPVTXVe5+dbiR9pwjrPNfA48Gv8uLez3YNj11x10yPICPAscDC9qVZRK7K3sMkAPMJTbXxDHAU4c8yoGMYL/BwENR1ynkup4NXE7sw+TCqOsUVj2DfS4G3iQ2dEvk9QqzrsF+twPHR12nXqrro1HXJ4Q63wxMC7Z5OKqYk+5O6g/D3V81s8pDig/MMQFgZr8BLnH3fwU6u6yyE8gNI86e0BN1NbMzgHxif5ANZvaMu7eGGvgR6qnfqbs/CTxpZk8DD4cXcff10O/UgNuAZ919drgRd18P/6+mhCOpM7GrF8OBOUR4padPJYgOxJtj4qSONjazvwY+BgwA7go3tB53RHV1928BmNkXgG3Jlhw6caS/09OJnbLncmTDuCSDI6or8A/EzgyLzWycu/88zOB62JH+XgcBtwLHmdnNQSJJNR3V+U7gLjP7OOEMx5GQdEgQXc4xcdAK98eBx8MLJ1RHVNcDG7jf3/OhhOpIf6cvAy+HFUzIjrSudxL7cElFR1rX7cBXwgunV8Sts7vXA1f1djCH6lON1B3oco6JPiRd6pou9QTVta/WtU1S1zkdEsS7wHgzG21mOcQaZZ+MOKawpEtd06WeoLr21bq2Seo696kEYWb/B7wFTDCzGjO72t2bgb8HngcWA494BxMQpZJ0qWu61BNU175a1zapWGcN1iciInH1qTMIERHpOUoQIiISlxKEiIjEpQQhIiJxKUGIiEhcShAiIhKXEoT0KjPb0wvvcXGCQ5335HuebmYf6cZ+x5nZvcHyF8wsKcb/MrPKQ4eljrNNmZk911sxSe9TgpCUZGaZHa1z9yfd/bYQ3rOzsctOB444QQDfBH7SrYAi5u5bgY1mNiPqWCQcShASGTO7wczeNbN5Zva9duV/sNgsdwvN7Jp25XvM7PtmNhM4xczWmNn3zGy2mc03s4nBdge+iZvZ/WZ2p5m9aWarzOwTQXmGmf00eI+nzOyZtnWHxPiymf3IzF4Brjezi8xsppm9Z2Z/NrPBwRDOXwG+YWZzzOy04Nv1Y0H93o33IWpmhcCx7j43zrpRZvZi8LN50cxGBuVjzezt4Jjfj3dGZrHZyJ42s7lmtsDM/jYoPzH4Ocw1s3fMrDA4U3gt+BnOjncWZGaZZvbv7X5XX263+g9AZDOeSciinkRDj/R6AHuC53OBu4mNZplBbBKYjwbrSoLnfsACYFDw2oFPtTvWGuAfguWvAvcGy18A7gqW7wd+F7zHZGJj7wN8gtjQ3xnAEGLzf3wiTrwvAz9t93ogH4xA8HfA7cHyd4F/brfdw8CpwfJIYHGcY58BPNbudfu4/whcGSx/EfhDsPwU8Olg+SttP89Djvs3wD3tXhcTm4xmFXBiUFZEbDTn/kBeUDYeqA6WKwkmtgGuAf5fsJwLVAOjg9fDgPlR/13pEc4jHYb7luR0bvB4L3hdQOwD6lXga2Z2WVA+IijfDrQAjx1ynLah2WcRm/Mhnj94bK6LRWY2OCg7FfhdUL7JzF7qJNbftlseDvzWzCqIfeiu7mCfs4HJZgdGcy4ys0J3r2u3TQWwtYP9T2lXnweBf2tXfmmw/DDwH3H2nQ/8h5n9GHjK3V8zs2OAje7+LoC774bY2QaxeQemEfv5HhXneOcCx7Y7wyom9jtZDWwBhnZQB0lxShASFQP+1d1/cVBhbHKfs4FT3H2vmb1MbH5wgH3u3nLIcfYHzy10/Pe8v92yHfKciPp2yz8B/tPdnwxi/W4H+2QQq0NDJ8dt4IO6dSXhQdPcfZmZnQBcAPyrmf2J2KWgeMf4BrAZmBrEvC/ONkbsTO35OOvyiNVD+iC1QUhUnge+aGYFAGY2zMzKiX073Rkkh4nAySG9/+vA3wRtEYOJNTInohhYHyxf2a68Dihs9/pPxEbpBCD4hn6oxcC4Dt7nTWJDP0PsGv/rwfLbxC4h0W79QcxsKLDX3X9N7AzjeGAJMNTMTgy2KQwa3YuJnVm0Ap8nNkfyoZ4HrjWz7GDfo4IzD4idcXTa20lSlxKERMLd/0TsEslbZjYfeJTYB+xzQJaZzQN+QOwDMQyPEZusZQHwC2AmUJvAft8FfmdmrwHb2pX/EbisrZEa+BpQFTTqLiLOzGfuvoTY1KCFh64L9r8q+Dl8Hrg+KP868I9m9g6xS1TxYj4GeMfM5gDfAn7o7o3A3wI/MbO5wAvEvv3/FLjSzN4m9mFfH+d49wKLgNlB19df8MHZ2hnA03H2kT5Aw31L2jKzAnffY7G5jd8BZrj7pl6O4RtAnbvfm+D2/YEGd3czu5xYg/UloQbZeTyvApe4+86oYpDwqA1C0tlTZjaAWGPzD3o7OQR+BnzyCLY/gVijsgG7iPVwioSZlRFrj1Fy6KN0BiEiInGpDUJEROJSghARkbiUIEREJC4lCBERiUsJQkRE4lKCEBGRuP4/izyI2/0r7vkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=5e-4; wd=1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8329f6296544634acb5505bdddc67a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      15.227886  15.488525  3.771931  \n",
      "    1      12.535917  15.55474   3.755456                      \n",
      "    2      13.935416  14.529819  3.625766                      \n",
      "\n",
      "CPU times: user 26.1 s, sys: 801 ms, total: 26.9 s\n",
      "Wall time: 25.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([14.52982]), 3.625765886647851]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 2, wd, cycle_len=1, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "# m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 1)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((201917, 113), (123623, 113))"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, df_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623, 135)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.3.1.post2'"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCGeneral.c:844",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-5d114aaa87de>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mcds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mColumnarDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_data_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_t\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcat_flds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcat_flds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_dl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/learner.py\u001b[0m in \u001b[0;36mpredict_dl\u001b[0;34m(self, dl)\u001b[0m\n\u001b[1;32m    377\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mpredict_with_targs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    378\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 379\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_dl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mpredict_with_targs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict_with_targs\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpredict_with_targs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 265\u001b[0;31m     \u001b[0mpreda\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtarga\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict_with_targs_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    266\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreda\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarga\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/model.py\u001b[0m in \u001b[0;36mpredict_with_targs_\u001b[0;34m(m, dl)\u001b[0m\n\u001b[1;32m    259\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reset'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mget_prediction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mVV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/column_data.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x_cat, x_cont)\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_emb\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0me\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memb_drop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_cont\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (59) : device-side assert triggered at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/THCGeneral.c:844"
     ]
    }
   ],
   "source": [
    "# cds = ColumnarDataset.from_data_frame(df_t,cat_flds=cat_flds)\n",
    "# dl = DataLoader(cds)\n",
    "# predictions = m.predict_dl(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NotImplementedError",
     "evalue": "object",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNotImplementedError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-bf6b5e43d109>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_t\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcat_flds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf_t\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcont_flds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/kaggle/fastai/column_data.py\u001b[0m in \u001b[0;36mpredict_array\u001b[0;34m(self, x_cat, x_cont)\u001b[0m\n\u001b[1;32m    167\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx_cont\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 169\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mto_np\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_cont\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/private/kaggle/fastai/core.py\u001b[0m in \u001b[0;36mT\u001b[0;34m(a, half, cuda)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m             \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_half\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhalf\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFloatTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNotImplementedError\u001b[0m: object"
     ]
    }
   ],
   "source": [
    "# m.predict_array(df_t[cat_flds].values, df_t[cont_flds].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l = m.predict(is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4341532700a64091bc2acb0125c4364c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=6, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      13.294061  14.742921  3.654648  \n",
      "    1      13.896752  14.331877  3.594383                      \n",
      "    2      11.770199  15.011213  3.677315                      \n",
      "    3      17.282892  14.602047  3.665574                      \n",
      "    4      14.269538  14.164847  3.57803                       \n",
      "    5      11.953786  14.124732  3.566191                      \n",
      "\n",
      "CPU times: user 54.1 s, sys: 1.62 s, total: 55.8 s\n",
      "Wall time: 52.5 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([14.12473]), 3.5661906499500207]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time m.fit(lr, 2, wd, cycle_len=2, cycle_mult=2, metrics=[rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4590)\n",
    "oof = np.zeros(len(df_raw))\n",
    "predictions = np.zeros(len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliers = df_raw['outliers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = df_raw['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[189917, 189918, 189919, 189920, 189921]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_idx[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Get the val_idx\n",
    "2. Get the model data set \n",
    "`ColumnarModelData.from_data_frame(PATH, val_idx, df, y, cat_flds=cat_flds, bs=64)`\n",
    "3. Get the learner from model `md.get_learner(`\n",
    "4. Fit the model `m.fit`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = m.predict(is_test=True).reshape(1, -1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(123623,)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37c55da28ad14d08936d8b32686f45d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      13.252438  14.272474  3.691864  \n",
      "    1      13.7039    13.951992  3.646141                      \n",
      "    2      13.002302  13.77174   3.624562                      \n",
      "    3      12.838314  13.655751  3.607016                      \n",
      "\n",
      "fold 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce5260be962b41df8be76817a6d54d8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.071089  14.71018   3.753652  \n",
      "    1      13.673916  13.851845  3.63836                       \n",
      "    2      14.276303  13.883272  3.646653                      \n",
      "    3      11.945822  13.58156   3.599526                      \n",
      "\n",
      "fold 2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bd81c4c2b64452596f436b7b12f4245",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      13.237116  14.003774  3.633716  \n",
      "    1      13.646907  13.654926  3.591715                      \n",
      "    2      14.129115  13.696076  3.595298                      \n",
      "    3      13.826535  13.522576  3.574844                      \n",
      "\n",
      "fold 3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "997e1c535a4a46a5b38b6dbf9af5ef5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.58264   14.360641  3.688091  \n",
      "    1      13.27546   13.862714  3.627557                      \n",
      "    2      12.960227  13.753141  3.609715                      \n",
      "    3      12.819354  13.662845  3.598971                      \n",
      "\n",
      "fold 4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17a24fe718e04db7b91fcc6ac774a81e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse                          \n",
      "    0      14.371841  14.305679  3.678577  \n",
      "    1      13.163964  13.824649  3.619093                      \n",
      "    2      13.281218  13.670398  3.595098                      \n",
      "    3      13.150278  13.593109  3.587094                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.9204810121779068"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(df, outliers.values)):\n",
    "    print(\"fold {}\".format(fold_))\n",
    "    md = ColumnarModelData.from_data_frame(PATH, val_idx, df, target.astype(np.float32), \n",
    "                                           cat_flds=cat_flds, bs=128, test_df=df_t)\n",
    "    m = md.get_learner(emb_szs, len(df.columns)-len(cat_flds),\n",
    "                   0.04, 1, [1000,500], [0.001,0.01], y_range=y_range)\n",
    "    m.fit(lr, 2, wd, cycle_len=2, cycle_mult=1, metrics=[rmse])\n",
    "    oof[val_idx] = m.predict().reshape(1, -1)[0]\n",
    "    predictions += m.predict(is_test=True).reshape(1, -1)[0] / folds.n_splits\n",
    "\n",
    "rmse(oof, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.6882473181358773"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1.9204810121779068**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((123623,), (201917,), (201917,))"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape, oof.shape, target.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}