mercari-tabular-text-version-2-complete.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train tabular model/ text model/ new tabular+text model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From https://nbviewer.jupyter.org/gist/joshfp/b62b76eae95e6863cb511997b5a63118/5.full-deep-learning.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.callbacks.tracker import *\n",
    "from fastai.text import *\n",
    "from fastai.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_seed(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "#     tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "reset_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mercari_path = Path('data/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "```text\n",
      "=== Software === \n",
      "python        : 3.7.1\n",
      "fastai        : 1.0.54\n",
      "fastprogress  : 0.1.21\n",
      "torch         : 1.1.0.dev20190418\n",
      "nvidia driver : 410.104\n",
      "torch cuda    : 9.0.176 / is available\n",
      "torch cudnn   : 7402 / is enabled\n",
      "\n",
      "=== Hardware === \n",
      "nvidia gpus   : 1\n",
      "torch devices : 1\n",
      "  - gpu0      : 8116MB | GeForce GTX 1080\n",
      "\n",
      "=== Environment === \n",
      "platform      : Linux-4.15.0-52-generic-x86_64-with-debian-stretch-sid\n",
      "distro        : #56~16.04.1-Ubuntu SMP Thu Jun 6 12:03:31 UTC 2019\n",
      "conda env     : python37\n",
      "python        : /home/quantran/anaconda3/envs/python37/bin/python\n",
      "sys.path      : \n",
      "/home/quantran/kwon/kaggle/mercari\n",
      "/home/quantran/anaconda3/envs/python37/lib/python37.zip\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/lib-dynload\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/site-packages\n",
      "/home/quantran/anaconda3/envs/python37/lib/python3.7/site-packages/IPython/extensions\n",
      "/home/quantran/.ipython\n",
      "```\n",
      "\n",
      "Please make sure to include opening/closing ``` when you paste into forums/github to make the reports appear formatted as code sections.\n",
      "\n",
      "Optional package(s) to enhance the diagnostics can be installed with:\n",
      "pip install distro\n",
      "Once installed, re-run this utility to get the additional information\n"
     ]
    }
   ],
   "source": [
    "from fastai.utils.collect_env import show_install\n",
    "show_install()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text_cols(df: pd.DataFrame) -> pd.DataFrame:\n",
    "  \n",
    "    df['category_name'] = df['category_name'].fillna('//')\n",
    "    df['category1'] = df['category_name'].apply(lambda x : x.split('/')[0].strip())\n",
    "    df.loc[df.category1=='','category1']= np.NaN\n",
    "    df['category2'] = df['category_name'].apply(lambda x : x.split('/')[1].strip())\n",
    "    df.loc[df.category2=='','category2']= np.NaN\n",
    "    df['category3'] = df['category_name'].apply(lambda x : x.split('/')[2].strip())\n",
    "    df.loc[df.category3=='','category3']= np.NaN\n",
    "    df['category_name'] = df['category_name'].apply( lambda x : ' '.join( x.split('/') ).strip() )\n",
    "    df.loc[df.category_name=='','category_name']= 'No category' # let this info in when concatenating text for RNN\n",
    "    \n",
    "    df_bn_fillna = df['brand_name'].fillna('No brand')\n",
    "    df['text'] = (df['name'].fillna('No name') + '. ' + df_bn_fillna + '. ' + \n",
    "                  df['category_name'] + '. ' + df['item_description'].fillna('No description'))\n",
    "    return df[['category1','category2','category3','brand_name', 'text', 'shipping', 'item_condition_id','price']]\n",
    "\n",
    "def preprocess_all(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "    \n",
    "    test = pd.read_table(mercari_path/'test_stg2.tsv').drop('test_id',axis=1)\n",
    "    test['price'] = np.NAN\n",
    "    train = train[train['price'] > 0].reset_index(drop=True)\n",
    "    all_df = pd.concat([train,test],axis=0).reset_index(drop=True)\n",
    "    del train\n",
    "    del test\n",
    "    gc.collect()\n",
    "\n",
    "    all_df = preprocess_text_cols(all_df)\n",
    "    train_df = all_df[~all_df.price.isnull()]\n",
    "    test_df = all_df[all_df.price.isnull()]\n",
    "    del all_df\n",
    "    gc.collect()\n",
    "    \n",
    "    if sample:\n",
    "        np.random.seed(42)\n",
    "        sample = np.random.permutation(sample)\n",
    "        train_df = train_df.loc[sample].reset_index(drop=True)\n",
    "        \n",
    "    test_df= test_df.drop('price',axis=1)    \n",
    "    return train_df,test_df\n",
    "\n",
    "def preprocess_train(sample=None):\n",
    "    train = pd.read_table(mercari_path/'train.tsv').drop('train_id',axis=1)\n",
    "    price = train.price.values\n",
    "    train=train.drop('price',axis=1)\n",
    "    train['price']=price\n",
    "\n",
    "    if sample:\n",
    "        np.random.seed(42)\n",
    "        sample = np.random.permutation(sample)\n",
    "        train = train.loc[sample].reset_index(drop=True)\n",
    "\n",
    "    train = preprocess_text_cols(train)\n",
    "\n",
    "    return train\n",
    "def get_val_idxs(train,n_splits=20):\n",
    "    np.random.seed(42)\n",
    "    cv = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    train_idxs, valid_idxs = next(cv.split(train))\n",
    "    return train_idxs,valid_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=1482535 # train shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((29650, 8), (3460725, 7))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df,test_df = preprocess_all(int(0.02*n)) # only use 2% for testing\n",
    "\n",
    "train_df.shape,test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_df = preprocess_train()\n",
    "# train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['category1', 'category2', 'category3', 'brand_name', 'text', 'shipping',\n",
       "       'item_condition_id', 'price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.price = np.log1p(train_df['price']) # Metric for this is RMSLE, we log them so we can use MSE in NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category1</th>\n",
       "      <th>category2</th>\n",
       "      <th>category3</th>\n",
       "      <th>brand_name</th>\n",
       "      <th>text</th>\n",
       "      <th>shipping</th>\n",
       "      <th>item_condition_id</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Beauty</td>\n",
       "      <td>Makeup</td>\n",
       "      <td>Face</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Primer &amp; Brush Duo. No brand. Beauty Makeup Fa...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.833213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Women</td>\n",
       "      <td>Dresses</td>\n",
       "      <td>Above Knee, Mini</td>\n",
       "      <td>Bebe</td>\n",
       "      <td>Bebe black feather dress. Bebe. Women Dresses ...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3.555348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Women</td>\n",
       "      <td>Athletic Apparel</td>\n",
       "      <td>Jackets</td>\n",
       "      <td>NaN</td>\n",
       "      <td>For Kayla-do not buy. No brand. Women Athletic...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.258097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vintage &amp; Collectibles</td>\n",
       "      <td>Accessories</td>\n",
       "      <td>Keychain</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Rhinestone choker &amp; pom pom keychain. No brand...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.397895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Beauty</td>\n",
       "      <td>Makeup</td>\n",
       "      <td>Eyes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FREE SHP GREEN CONTAC LENSES. No brand. Beauty...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.302585</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                category1         category2         category3 brand_name  \\\n",
       "0                  Beauty            Makeup              Face        NaN   \n",
       "1                   Women           Dresses  Above Knee, Mini       Bebe   \n",
       "2                   Women  Athletic Apparel           Jackets        NaN   \n",
       "3  Vintage & Collectibles       Accessories          Keychain        NaN   \n",
       "4                  Beauty            Makeup              Eyes        NaN   \n",
       "\n",
       "                                                text  shipping  \\\n",
       "0  Primer & Brush Duo. No brand. Beauty Makeup Fa...         0   \n",
       "1  Bebe black feather dress. Bebe. Women Dresses ...         0   \n",
       "2  For Kayla-do not buy. No brand. Women Athletic...         1   \n",
       "3  Rhinestone choker & pom pom keychain. No brand...         1   \n",
       "4  FREE SHP GREEN CONTAC LENSES. No brand. Beauty...         1   \n",
       "\n",
       "   item_condition_id     price  \n",
       "0                  2  2.833213  \n",
       "1                  2  3.555348  \n",
       "2                  1  3.258097  \n",
       "3                  1  2.397895  \n",
       "4                  1  2.302585  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Primer & Brush Duo. No brand. Beauty Makeup Face. Younique face and eye primer : used once Brush: never used\n",
      "--------------------\n",
      "Bebe black feather dress. Bebe. Women Dresses Above Knee, Mini. Bebe black feather dress strapless around the neck decorative front bejeweled bow tie in the back satin beautiful dress like new no flaws size extra small women's\n",
      "--------------------\n",
      "For Kayla-do not buy. No brand. Women Athletic Apparel Jackets. New with tags and long sleeve. No free shipping and firm on price. Also have available in a size medium. No free shipping and firm on price.\n",
      "--------------------\n",
      "Rhinestone choker & pom pom keychain. No brand. Vintage & Collectibles Accessories Keychain. 1x rhinestone choker ( handmade) 1x pink pom pom keychain Brand new I ship fast\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "    print(train_df.text[i])\n",
    "    print('-'*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of continuous feas: 1\n",
      "# of categorical feas: 5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_names=['category1','category2','category3','brand_name','shipping']\n",
    "cont_names= list(set(train_df.columns) - set(cat_names) - {'price','text'})\n",
    "print(f'# of continuous feas: {len(cont_names)}')\n",
    "print(f'# of categorical feas: {len(cat_names)}')\n",
    "dep_var = 'price'\n",
    "procs = [FillMissing,Categorify, Normalize]\n",
    "\n",
    "txt_cols=['text']\n",
    "\n",
    "len(cat_names) + len(cont_names) + 2 == train_df.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare validation indices using kfold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([    0,     1,     2,     3, ..., 29646, 29647, 29648, 29649]),\n",
       " array([    6,    34,    92,    96, ..., 29586, 29597, 29634, 29642]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "((28167,), (1483,))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_idxs,val_idxs = get_val_idxs(train_df,n_splits=20)\n",
    "# train_idxs,val_idxs = get_val_idxs(train_df,n_splits=5)\n",
    "train_idxs,val_idxs\n",
    "train_idxs.shape,val_idxs.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get databunches and learners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tabular_databunch(train_df,bs=100,val_idxs=val_idxs,path = mercari_path):\n",
    "    return (TabularList.from_df(train_df, cat_names, cont_names, procs=procs, path=path)\n",
    "                            .split_by_idx(val_idxs)\n",
    "                            .label_from_df(cols=dep_var,label_cls=FloatList)\n",
    "#                             .add_test(TabularTextList.from_df(test_df, cat_names, cont_names, txt_cols,path=path))\n",
    "                            .databunch(bs=bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text_databunch(train_df,bs=100,val_idxs=val_idxs,path = mercari_path):\n",
    "    data_lm = load_data(path, 'data_lm.pkl', bs=bs)\n",
    "    return (TextList.from_df(train_df, cols = txt_cols[0], vocab=data_lm.vocab, path=path)\n",
    "                            .split_by_idx(val_idxs)\n",
    "                            .label_from_df(cols=dep_var,label_cls=FloatList)\n",
    "                            .databunch(bs=bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tabular_learner(data,params,seed=42):\n",
    "    return tabular_learner(data,metrics=[root_mean_squared_error],**params)\n",
    "\n",
    "def get_text_learner(data,params,seed=42):\n",
    "\n",
    "    learn = text_classifier_learner(data,AWD_LSTM,metrics=root_mean_squared_error,**params)\n",
    "    \n",
    "    # language model trained in 'mercari language model' notebook\n",
    "    \n",
    "    #TODO: new AWD_LSTM structure causing mismatch. Retrain language model!\n",
    "#     encoder_name = 'bs60-awdlstm-enc-stage2'\n",
    "#     learn.load_encoder(encoder_name) \n",
    "    return learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai_tabtext2 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_databunches(bs=64,path= mercari_path):\n",
    "    # get tabtext databunch, tabular databunch (for tabular model) and text databunch (for text model)\n",
    "    tab_db = get_tabular_databunch(train_df[cat_names + cont_names+ [dep_var]])\n",
    "    text_db = get_text_databunch(train_df[txt_cols +[dep_var]])\n",
    "    \n",
    "    train_ds = ConcatDataset(tab_db.train_ds.x, text_db.train_ds.x, tab_db.train_ds.y)\n",
    "    valid_ds = ConcatDataset(tab_db.valid_ds.x, text_db.valid_ds.x, tab_db.valid_ds.y)\n",
    "    \n",
    "    train_sampler = SortishSampler(text_db.train_ds.x, key=lambda t: len(text_db.train_ds[t][0].data), bs=bs//2)\n",
    "    valid_sampler = SortSampler(text_db.valid_ds.x, key=lambda t: len(text_db.valid_ds[t][0].data))\n",
    "\n",
    "#     train_dl = DataLoader(train_ds, bs//2, sampler=train_sampler)\n",
    "    train_dl = DataLoader(train_ds, bs//2, sampler=train_sampler,shuffle=True)\n",
    "    valid_dl = DataLoader(valid_ds, bs, sampler=valid_sampler)\n",
    "    data = DataBunch(train_dl, valid_dl, device=defaults.device, collate_fn=tabtext_collate, path=path)\n",
    "    return data,tab_db,text_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data,tab_db,text_db = get_databunches(bs=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab_params={\n",
    "    'layers':[500],\n",
    "    'emb_drop': 0.,\n",
    "    'y_range': [0,6],\n",
    "    'use_bn': True,    \n",
    "    }\n",
    "\n",
    "text_params={\n",
    "    #     'lin_ftrs':[1000],\n",
    "    #     'ps': [0.001,0,0],\n",
    "        'bptt':70,\n",
    "        'max_len':20*70,\n",
    "        'drop_mult': 1., \n",
    "    #     'use_bn': True,    \n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab_learner = get_tabular_learner(tab_db,tab_params)\n",
    "text_learner = get_text_learner(text_db,text_params)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(33781, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(33781, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.4)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=1, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_learner.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get tabular_text learner and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "lin_layers=[500]\n",
    "ps=[0.]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 50 is the default lin_ftrs in AWD_LSTM\n",
    "lin_layers[-1]+= 50 if 'lin_ftrs' not in text_params else text_params['lin_ftrs']\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# be careful here. If no lin_ftrs is specified, the default lin_ftrs is 50\n",
    "\n",
    "learner = get_tabtext_learner(data,tab_learner,text_learner,lin_layers ,ps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ConcatModel(\n",
       "  (mod_tab): TabularModel(\n",
       "    (embeds): ModuleList(\n",
       "      (0): Embedding(11, 6)\n",
       "      (1): Embedding(110, 22)\n",
       "      (2): Embedding(590, 57)\n",
       "      (3): Embedding(1201, 85)\n",
       "      (4): Embedding(3, 3)\n",
       "    )\n",
       "    (emb_drop): Dropout(p=0.0)\n",
       "    (bn_cont): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (layers): Sequential(\n",
       "      (0): Linear(in_features=174, out_features=500, bias=True)\n",
       "      (1): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (mod_nlp): SequentialRNN(\n",
       "    (0): MultiBatchEncoder(\n",
       "      (module): AWD_LSTM(\n",
       "        (encoder): Embedding(33781, 400, padding_idx=1)\n",
       "        (encoder_dp): EmbeddingDropout(\n",
       "          (emb): Embedding(33781, 400, padding_idx=1)\n",
       "        )\n",
       "        (rnns): ModuleList(\n",
       "          (0): WeightDropout(\n",
       "            (module): LSTM(400, 1152, batch_first=True)\n",
       "          )\n",
       "          (1): WeightDropout(\n",
       "            (module): LSTM(1152, 1152, batch_first=True)\n",
       "          )\n",
       "          (2): WeightDropout(\n",
       "            (module): LSTM(1152, 400, batch_first=True)\n",
       "          )\n",
       "        )\n",
       "        (input_dp): RNNDropout()\n",
       "        (hidden_dps): ModuleList(\n",
       "          (0): RNNDropout()\n",
       "          (1): RNNDropout()\n",
       "          (2): RNNDropout()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (1): PoolingLinearClassifier(\n",
       "      (layers): Sequential(\n",
       "        (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.4)\n",
       "        (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "        (3): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (layers): Sequential(\n",
       "    (0): BatchNorm1d(550, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (1): Linear(in_features=550, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[Sequential(\n",
       "   (0): Embedding(33781, 400, padding_idx=1)\n",
       "   (1): Embedding(33781, 400, padding_idx=1)\n",
       " ), Sequential(\n",
       "   (0): LSTM(400, 1152, batch_first=True)\n",
       "   (1): ParameterModule()\n",
       "   (2): RNNDropout()\n",
       " ), Sequential(\n",
       "   (0): LSTM(1152, 1152, batch_first=True)\n",
       "   (1): ParameterModule()\n",
       "   (2): RNNDropout()\n",
       " ), Sequential(\n",
       "   (0): LSTM(1152, 400, batch_first=True)\n",
       "   (1): ParameterModule()\n",
       "   (2): RNNDropout()\n",
       " ), Sequential(\n",
       "   (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "   (1): Dropout(p=0.4)\n",
       "   (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "   (3): ReLU(inplace)\n",
       "   (4): Embedding(11, 6)\n",
       "   (5): Embedding(110, 22)\n",
       "   (6): Embedding(590, 57)\n",
       "   (7): Embedding(1201, 85)\n",
       "   (8): Embedding(3, 3)\n",
       "   (9): Dropout(p=0.0)\n",
       "   (10): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "   (11): Linear(in_features=174, out_features=500, bias=True)\n",
       "   (12): ReLU(inplace)\n",
       "   (13): BatchNorm1d(550, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "   (14): Linear(in_features=550, out_features=1, bias=True)\n",
       " )]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(learner.layer_groups)\n",
    "learner.layer_groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learner.freeze()\n",
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt81NWd//HXZzK53wMJhAQMoCKigBAEtbq2Xmpbq1K11dX9abXrttuLqz/rb3fdbXe321bbbe3abtfaLmq9tdVqq662UluloqLhpoig3C8CCRCSAMlkLuf3x0xCxCQEmJnvTL7v5+MxD2a+8505n8N3Mp8553y/55hzDhER8a+A1wGIiIi3lAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeCXgcwFCNHjnQNDQ1ehyEiklUWL1680zlXfaj9UpYIzGwecCHQ7Jw7qc/2LwNfAiLA/zrnbj3UezU0NNDU1JSqUEVEhiUz2ziU/VLZNXQfcEHfDWb2YeBiYKpzbgrwHyksX0REhiBlicA5twDYfdDmLwC3O+dCiX2aU1W+iIgMTboHi48HzjSzRWb2opnNGmhHM7vBzJrMrKmlpSWNIYqI+Eu6E0EQqATmAF8FfmVm1t+Ozrl7nHONzrnG6upDjnWIiMgRSnci2AI87uJeA2LAyDTHICIifaQ7EfwG+AiAmR0P5AE70xyDiIj0kcrTRx8BzgZGmtkW4OvAPGCema0AuoFrnJZIExHxVMoSgXPuygGeujpVZYqIDBfb2jp5eNEm5p5Sx4TqkpSWpSkmREQy0NbWTn74xzVsae1MeVlKBCIiGSgUiQGQH0z917QSgYhIBuruSQS5OSkvS4lARCQDhSJRQC0CERHfUteQiIjPhcLqGhIR8TV1DYmI+FxP11CeEoGIiD9pjEBExOdC4XjXUF6OEoGIiC+FIjHygwEGmKk/qZQIREQyUE8iSAclAhGRDBSKxNJy6igoEYiIZKRQJKoWgYiIn6lrSETE50LhGPlBdQ2JiPhWKBIlP1ctAhER3wpFYmm5hgCUCEREMpLOGhIR8blQWGcNiYj4WrfOGhIR8bf46aPqGhIR8a34GIFaBCIivqUri0VEfE5dQyIiPuac02CxiIif9a5OpjECERF/6l2vWFcWi4j4UygSX6ZSVxaLiPhUKJy+hetBiUBEJOP0jhEoEYiI+FNv15BOHxUR8adunTUkIuJv6hoSEfG5A4kgy7uGzGyemTWb2Yp+nrvFzJyZjUxV+SIi2SoU7hkjyP4WwX3ABQdvNLOxwHnAphSWLSKStXpaBAXZPkbgnFsA7O7nqTuBWwGXqrJFRLLZgSuLs7xrqD9mdhGw1Tm3PJ3liohkkwNXFqfnKzqYllIAMysCbgPOH+L+NwA3AIwbNy6FkYmIZJbhfGXxRGA8sNzMNgD1wBIzG93fzs65e5xzjc65xurq6jSGKSLirXSfNZS2FoFz7k2gpudxIhk0Oud2pisGEZFs0HNBWV62twjM7BHgFWCSmW0xs+tTVZaIyHASikTJzTFyApaW8lLWInDOXXmI5xtSVbaISDZL5zKVoCuLRUQyTjoXrgclAhGRjBMKp2+9YlAiEBHJOKFILG0DxaBEICKSceJdQxojEBHxrVAklrarikGJQEQk42iMQETE59Q1JCLic91RtQhERHwtFNYYgYiIr+nKYhERn9OVxSIiPhdvESgRiIj4ViisK4tFRHzLOafTR0VE/CwSc8Rc+papBCUCEZGM0rtMpU4fFRHxp1A4CqRvvWJQIhARySjd0Z6F69UiEBHxpVBYXUMiIr7WO0agriEREX8KRXrGCNQiEBHxJbUIRER8rmeMQFcWi4j4lLqGRER8TheUiYj43IEWgcYIRER8qfc6AnUNiYj4k64sFhHxuQNXFqtrSETEl3TWkIiIz4UiMQIGwYClrUwlAhGRDBKKxJepNFMiEBHxpVA4vctUghKBiEhGCUViaR0fACUCEZGMEorE0npVMaQwEZjZPDNrNrMVfbZ918xWmdkbZvaEmVWkqnwRkWwUigyvrqH7gAsO2jYfOMk5NxV4B/iHFJYvIpJ1QuFh1DXknFsA7D5o23POuUji4atAfarKFxHJRt3RYZQIhuA64FkPyxcRyTjxFsHw6RoakJndBkSAhwbZ5wYzazKzppaWlvQFJyLioVAkOnwGiwdiZtcAFwJXOefcQPs55+5xzjU65xqrq6vTF6CIiIe8OH00mM7CzOwC4P8Bf+Gc25/OskVEskH8yuJh0jVkZo8ArwCTzGyLmV0P/AgoBeab2TIzuztV5YuIZKP4lcXDpEXgnLuyn83/k6ryRESGA11ZLCLic/FEMEy6hkRE5PBl7FlDZjbRzPIT9882s69oeggRkeSKxRzhqMvYrqFfA1EzO5Z4P/944OGURSUi4kMH1ivOzK6hWGJqiLnAD5xzNwG1qQtLRMR/etcrztAWQdjMrgSuAZ5ObMtNTUgiIv7Uu15xJo4RAJ8FTgO+6Zxbb2bjgQdTF5aIiP+EIt50DQ3pOgLn3ErgKwBmVgmUOuduT2VgIiJ+09MiyMvEriEze8HMysysClgO3Gtm309taCIi/tKV4WME5c65duBTwL3OuZnAuakLS0TEfw50DWVmIgiaWS3waQ4MFouISBL1DhZn6Omj/wb8HljrnHvdzCYA76YuLBER/+ltEaT5rKGhDhY/Cjza5/E64NJUBSUi4kcZfR2BmdWb2RNm1mxmO8zs12am9YZFRJIo068svhd4EhgD1AFPJbaJiEiShMI9YwQZ2CIAqp1z9zrnIonbfYDWjxQRSSKvxgiGWtpOM7vazHISt6uBXakMTETEb3oTQU5mdg1dR/zU0e3ANuAy4tNOiIhIkmT0XEPOuU3OuYucc9XOuRrn3CXELy4TEZEk6TlrKC8nAxPBAG5OWhQiIkIoEiMvJ0AgYGkt92gSQXojFREZ5kKRaNrPGIKjSwQuaVGIiEh84fo0jw/AIa4sNrMO+v/CN6AwJRGJiPhUdySW9ovJ4BCJwDlXmq5ARET8LhSJZV3XkIiIJFEoHE37ojSgRCAikjHiYwTp7xpSIhARyRChSJT8NF9DAEoEIiIZw6uzhpQIREQyRCiswWIREV+LX1CmMQIREd/S6aMiIj6nMQIREZ/z6spiJQIRkQyRjZPOiYhIkjjnht8YgZnNM7NmM1vRZ1uVmc03s3cT/1amqnwRkWwSjjqcY9hdWXwfcMFB2/4eeN45dxzwfOKxiIjv9SxTme7VySCFicA5twDYfdDmi4H7E/fvBy5JVfkiItmkd+F6H5w1NMo5tw0g8W9NmssXEclIvYlgOI0RHC0zu8HMmsysqaWlxetwRERSKhSOdw354fTRHWZWC5D4t3mgHZ1z9zjnGp1zjdXV1WkLUETEC52JRFDgg66hJ4FrEvevAX6b5vJFRDJSe2cEgLKC3LSXncrTRx8BXgEmmdkWM7seuB04z8zeBc5LPBYR8b32rjAAZYXpTwSDrll8NJxzVw7w1DmpKlNEJFu1dcYTQbkHiSBjB4tFRPykPZEIhlXXkIiIDF17ZxgzKC1IWUfNgJQIREQyQHtXhJL8IIGApb1sJQIRkQzQ1hn2ZHwAlAhERDJCe2fYk/EBUCIQEckIahGIiPhce1eYssL0DxSDEoGISEZQiyBFojGHc87rMEREDqm9M+LZGIE37ZA0mffSev64qpnbPjGZk+rKD/v1LR0htrd1YQZmEDBjXFURxfnD+r/tsDjnWLhmFz/847us2t5BSX6Q0oIgZQW5jCzNY1xVMQ0jihg3ooiC3Bxa93XTuj/Mnv3dNIwo5uxJ1QQ9WIhDJJN0R2J0hqOetQiG9TdaeVEuq3d08MkfvcSlM+q55fxJjC4vGHB/5xxrmvfy3ModzF+5g2Wb93xgn9L8IFecOpZrzxhPXUXhYcXTFY4vTG02+HnC7+7o4JHXNvPsim3sDUVwLh5bIGDMHl/FeSeO4pzJoxhZkn9Y5SeTc44X3mnhruffZemmPYwuK+CT02rZ3x2loytCR1eYVds6mL9yB+HowK2y2vICrpg1jitOHcuosoGPTX/lH+r/MVlCkSirtnVQnJ9DRVEeFYW5vcnLOUck5ojG3JCOrUh/vJxnCMCyoeuksbHRNTU1HdFr2zrD/PhPa7h34QZyAsYnp9UyoiSf0oIgpflBOsNR1rXsi9927mXn3m4AptaXc97kUZxQW4ZzjpiDSCzG79/awTNvbgPgEyfXcuHUWibXllFXUfiBC0H2hSIs2dTKy2t38fLaXby5ZQ+VRXnMnlDFnAkjmNVQRW5OgLbOblr3hdnW1slvlr3H4o2t5OYYHzmhhjEVhRhGwGBfd5QF77SwdU8nZjB9bAUn1pYxfmQxE6tLGFsVT0zdEUc4GiMSe/+xLS/M5diakiP6f+xr+eY9fOPplTRtbKWuopAvnD2Ryxvr+51HPRpzvLenk02799MdiVFRlEtVcR7lhbm8um43Dy3ayJ/f3UlOwJgwspiSgiAl+fFbXjBATsDIMSMnYOzZH2Z7exc72rto6QhRWhBkbFURYyuLqK8spLo0nxEleYwojv87pryQiqLcI/pyjsYcr6zdxZPLt/K7Fdtp74q87/nC3ByiMUd3NPa+7XnBAPnBAOWFuZw0ppypY8uZVl/B1PpySj1q9kvmW9uyl3O+9yI/+Mx0LjmlLmnva2aLnXONh9xvuCeCHpt37+eO361i4ZqddHRF3vclOaI4jwnVxUwYWcLJ9eWcO3nUoC2HLa37uf/lDTzy2mb2huJfEMV5ORw/upQcM1r2htjZEWJfd3x+8WDAmD62glnjq9jR1sWr63bxXltXv+89sbqYK2aNY+6Mun5/8TvneHtbB8+t3M6Cd1pY07z3A19Sg7lo2hj+6cLJ1JQO/dd3jx3tXdzxu1U8vmQrI0vyuOm847l85ljyjnJFpQ079/HLps2sb9nHvu4IHV0R9oYihKMxoolf29GYo7wwl1FlBYwqK6C6NJ/2rjBbWjvZsns/W1o7P/ClDPHjUl9ZRG1FATlmdEdjRKKOmHOMqypi0uhSJo0uZVxVEeta9vHm1jbe3NrGko2t7NrXTUl+kPOnjOKcE0YRicVo6wzTui9MR1eYYE6AvBwjNydAIGB0R2KEIjFCkSgtHSHe3NrGxl37gXiC+MTJtVw1exwzj6lUy0HeZ+mmVub++GXmXdvIR04YlbT3VSIYhHOOUCRGe1eYvJwAFUV5R/Q+nd1R3t7ezurtHaza1s7qHR0YxsjSfEaW5DGyJJ8pY8qY1VD1vnEF5xxbWjtp2ribgFlvd0NVcR71lYWH9SXhnKN1f5j1O/eypbWTgMW/mHJz4r+i+77X4o2t3P3CWvJzA9z60Un85exjyBnkcvZde0Os3NbO29vaWfleO8+t3EEk6vjshxr40oePzahfuM452jsj7NoXYte+bnbtDbF1TxdbWuNJYltbJwDBQIC8nAAOx/qd+3pbgD3MYPzIYqbWlfPRKaP58Ak1FOQe+YpRe/Z388aWNuav3METS7eyNxRh0qhSLpxay5iKQmrK8qkpLaC+slBjTz72wupmrr33dX79hdOYeUxV0t5XiUD6ta5lL//82xUsXLOL40eV0NhQxaRRpRw3qoSyglyWbmqlaWMrize2sqW1s/d1teUFzB5fxU3nHc8xI4o9rEFy7dwb4p0dHWzevZ+GEcWcOKYsZQluXyjCU8vf4+HXNvHGlrb3PWcGx9WUMK2+gunjKhhbWUQkFuvt5qstL+CUcZWDJm7JXk8uf4+vPLKUP9x8FsfWlCbtfZUIZEDOOZ5c/h4PvrqR1ds7PtC1VFOaT2NDJaeMrWTKmDIm15ZRWXxkrSbp375QhOaOEM3tXTR3hFjXso9lm1tZtnkPrfvD/b6mujSfC6aM5mMnj+bUhiqdbTWMPPDqRv75Nyt47R/PoeYwTpo4lKEmArVFfcjMuHh6HRdPr8M5R3NH/FdxW2eYafUVh909JYevOD/I+Pwg40e+v3XlnGPT7v00d4R6u/hycwKs3t7Bsyu28ejizTzw6kYqi3I5e1IN50yu4azjqz07/1ySo3ctAp0+Kl4ws94BWPGemXHMiOIPdL8dP6qUT04bw/7uCC+ubmH+yh38aXUzTyzdSjBgTK0vZ/rYSqaNLeeUsZWMrVIyzybtnWHygoGjGo86GkoEIlmkKC/Ix06u5WMn1xKNOZZuauUPbzfTtCF+Ku68hfEzp04YXcrfnXsc55842pP57eXwtHd5N70EKBGIZK2cgNHYUEVjQ/wsk3A0xurtHSze2Mp9L2/g8w8uYXJtGTeecxznnzhKCSGDtXWGKfNgZbIeGm0SGSZycwKcVFfONac3MP+ms7jzM9PoCkf5/IOLuezul1m1vd3rEGUA7Z0RT1sESgQiw1AwJ8DcU+qZf9NZfOeyqWzYtZ8L73qJ259dRWfiQkfJHG2dYc8GikGJQGRYC+YE+HTjWJ6/+S/41Iw67n5xLefd+SLPv71DM/NmEK/HCJQIRHygsjiP71w2jV/cMIf8YIDr72/is/e9ztqWvV6HJvSMESgRiEgazJkwgmdvPIt/+sRkFm9o5aN3LuCb/7uyd84sSb/49ChqEYhIGuUFA3zuzAn88ZazuXRGPT97aT2f+vFCNu/e73VovrQ3FCHm8GyZSlAiEPGt6tJ87rhsKg9eP5vtbV1c8l8LWbxxt9dh+U7PFC9qEYiIZ844diRPfPEMSguCXPnTRfx22VavQ/KVtsTcUhojEBFPTawu4Ym/PYNTxlZw4y+W8a1n3qY78sH1HST5elYnU4tARDxXWZzHA9fP5uo547hnwTouv/tlNu3SuEGqtXk84RwoEYhIH3nBAP9+ycn891UzWL9zHx+/68/qKkqxnplH1SIQkYzysZNreebGM5k0upQbf7GMu55/1+uQhq3eFoHGCEQk09RXFvHLG+bwqVPq+P78d3h40SavQxqW2rsimEGph5POafZRERlQMCfAHZdNpXV/N//0mzcZUZLHR6eM9jqsYaW9M0xJftDT2WHVIhCRQeXmBPivq2Ywtb6CLz+ylEXrdnkd0rDi9VXF4FEiMLObzOwtM1thZo+YmZbHEslgRXlB7r12FmMrC/ncz5tY+Z6mtE4Wr+cZAg8SgZnVAV8BGp1zJwE5wBXpjkNEDk9lcR4/v342xXlB/s+8RZqwLkm8nnkUvOsaCgKFZhYEioD3PIpDRA5DXUUhD/31bACu+ukizU+UBPG1CLwdrk17InDObQX+A9gEbAPanHPPpTsOETkyE6tLeOD62XSGo1z1s0Vsb+vyOqSs5vXqZOBN11AlcDEwHhgDFJvZ1f3sd4OZNZlZU0tLS7rDFJFBTK4t4/7rTmXX3hBX/exVJYOj4MsxAuBcYL1zrsU5FwYeB04/eCfn3D3OuUbnXGN1dXXagxSRwU0fW8G8a2exra2Lj9/1Z15Y3ex1SFmnOxKjMxz1X4uAeJfQHDMrMjMDzgHe9iAOETlKsyeM4MkvfYia0nyuvfd1bn92FeGoJqsbqp4J57ycZwg8uKDMObfIzB4DlgARYClwT7rjEJHkOLamhN988Qz+9am3uPvFtSxcs5Pa8gJ2dIRobu9i975uivODlBUEKS/Mpba8kFsvmMSE6hKvQ/dcJswzBB5dWeyc+zrwdS/KFpHkK8jN4dufmsqcCSP4/vx36I7EqCnL57iakVQV57EvFKGtM0xbZ5hX1u3ioh8t5I5Lp/KJqbVeh+6pAzOPenvWkKaYEJGkuXh6HRdPrxt0n/f2dPLFh5fwxYeX8PqGBv7x45PJC/pzkoNMWJ0MNMWEiKTZmIpCfnnDaVx3xnjue3kDn/7JK7R0hLwOyxOZMPMoKBGIiAfyggG+9skT+fFVM1i9vYPLfLoITqaMESgRiIhnPn5yLQ/99WzaOsNcevfLvL3NX3MYZcLqZKBEICIemzGukkf/5jSCAePTP3mF19bv9jqktGnvCpMXDFCQm+NpHEoEIuK540aV8tgXTqe6NJ+rf7aI7z23ms7uqNdhpVx7BlxVDEoEIpIh6ioK+fXnT+fCqbX88I9rOPf7L/LcW9txznkdWsrE5xny/uRNJQIRyRiVxXl8/zPT+eUNcyjJD3LDA4v5q/95jceXbKFtf9jr8JIuPvOo9y0C71ORiMhBZk8YwdNf+RA/f2UjP12wjpt/tZxgwDht4gjOO3EUp46v4viaUk+Xd0yG9q4wVcV5XoehRCAimSk3J8D1HxrPZ09vYPmWPfzure38fsV2vvbbtwAoKwjS2FDFOZNruHLWuKxMCm2dYRpGFHsdhhKBiGS2QMA4ZVwlp4yr5O8vOIFNu/fTtKGV1zfs5rX1u7ntiRX8aVUz3/v0dM/Pxz9cmbBeMSgRiEgWMTOOGVHMMSOKuXRmPc45fv7KRr7x9Eou+tFL3H31TCbXlnkd5pA452jving+zxBosFhEspiZcc3pDfzyb+bQFY4y98cL+VXT5qw402hfd5RozGVEi0CJQESy3sxjqnj6y2cyfWwFtz72Bn/500Wsae7wOqxBvbMjHl9FofeDxUoEIjIsVJfm89Dn5vDvl5zEym3tXPCDP/PtZ99mR3sXG3buY9nmPbywupmNu/Z5HSpd4Si3PvYGNaX5nHfiKK/D0RiBiAwfOQHj6jnH8LGTRnP7s6v4yYvr+MmL6963T14wwLfnnsylM+s9ihK+/czbrGneywPXn0qlTh8VEUm+ESX5fPfyafzl7HEs27yH8sJcKopyKc4L8oM/vMv/fXQ5b25t47ZPTCY3J70dI39a3cz9r2zkujPGc+ZxmbEeuxKBiAxbPaed9vXA9ZV865lVzFu4nre3tfPNuSdRVpBLfm4Ohbk5KV0kZ9feEF999A1OGF3KrRdMSlk5h0uJQER8JZgTXwvhpLoy/uHxNzn3+wve9/yE6mKumDWWS2fUM6IkH4BINMbija28tGYnJ9eVc/6U0YdVZtv+MG9ta+O/X1hLe2eYBz93quczjvZl2XCaVWNjo2tqavI6DBEZZta27GXppj10haN0haN0dkd58Z0Wmja2kptjfHTKaHICxgurW3rXDgD41Iw6/vWiKZT2mTk0Eo2xcO0uNuzcx869IXbu7aalo4vVOzrYvLuzd79vXDyFvzqtIS31M7PFzrnGQ+6nRCAi8n7v7Ojgkdc28fiSrQQDxodPqOGcE2o4feJI5i1czw//+C5jKgq58zPTqSzK49HFm3l8ydbeJTcDBlXF+YwsyWNiTQlTxpQxZUw5U8aUMTLRykgHJQIRkaMUi8W/Hw+ex2jxxlZu+uUyNrfux7n42UofOaGGy2fWM+OYSiqL8sjJgLmPhpoINEYgIjKAgSaym3lMJc/ceCb3LFhHcV4Oc2fUUVNakObokkeJQETkCJTkB7n5vOO9DiMpdGWxiIjPKRGIiPicEoGIiM8pEYiI+JwSgYiIzykRiIj4nBKBiIjPKRGIiPhcVkwxYWYtwMaDNpcDbYfY1vfxoe6PBHYeYYj9xTLUfQ5Vj6HUKdPrcPBjL4/FYM8PFvNgj7Pp83Tw48E+T+DNscimz9Ng+2TCsTjGOXfoRQ+cc1l5A+451La+jw91H2hKZixD3edQ9RhKnTK9Dpl0LAZ7frCYB3ucTZ+nIf7/992W9mORTZ+nbDkWh7plc9fQU0PY9tRh3k9mLEPd51D1GEqdMr0OBz/28lgM9vxgMQ/2OJs+Twc/TtXnaSjvMxw+T4Ptk0nHYlBZ0TWUDmbW5IYwS18mGw51gOFRj+FQBxge9RgOdYDU1iObWwTJdo/XASTBcKgDDI96DIc6wPCox3CoA6SwHmoRiIj4nFoEIiI+NywTgZnNM7NmM1txBK+daWZvmtkaM7vLzKzPc182s9Vm9paZfSe5UX8gjqTXwcz+xcy2mtmyxO3jyY/8A7Gk5Fgknr/FzJyZjUxexP3GkYpj8Q0zeyNxHJ4zszHJj/x9caSiDt81s1WJejxhZhXJj/wDsaSiHpcn/qZjZpaysYSjiX2A97vGzN5N3K7ps33Qv5t+pep0JC9vwFnADGDFEbz2NeA0wIBngY8ltn8Y+AOQn3hck4V1+Bfglmw/FonnxgK/J359ychsqwNQ1mefrwB3Z2EdzgeCift3AHdk4+cJmAxMAl4AGjMt9kRcDQdtqwLWJf6tTNyvHKyeg92GZYvAObcA2N13m5lNNLPfmdliM/uzmZ1w8OvMrJb4H+grLv4/+nPgksTTXwBud86FEmU0Z2Ed0i6F9bgTuBVI+SBXKurgnGvvs2sxKa5HiurwnHMuktj1VaA+lXVIlJmKerztnFudqbEP4KPAfOfcbudcKzAfuOBI//6HZSIYwD3Al51zM4FbgB/3s08dsKXP4y2JbQDHA2ea2SIze9HMZqU02v4dbR0AvpRoys8zs8rUhTqoo6qHmV0EbHXOLU91oIM46mNhZt80s83AVcDXUhjrQJLxeepxHfFfn15IZj3SbSix96cO2NzncU99jqievliz2MxKgNOBR/t0l+X3t2s/23p+qQWJN8HmALOAX5nZhETWTbkk1eG/gW8kHn8D+B7xP+C0Odp6mFkRcBvxbglPJOlY4Jy7DbjNzP4B+BLw9SSHOqBk1SHxXrcBEeChZMY4FMmsR7oNFruZfRa4MbHtWOAZM+sG1jvn5jJwfY6onr5IBMRbPnucc9P7bjSzHGBx4uGTxL8o+zZv64H3Eve3AI8nvvhfM7MY8bk/WlIZeB9HXQfn3I4+r/sp8HQqAx7A0dZjIjAeWJ7446kHlpjZqc657SmOvUcyPk99PQz8L2lMBCSpDolByguBc9L1o+ggyT4W6dRv7ADOuXuBewHM7AXgWufchj67bAHO7vO4nvhYwhaOpJ6pGhjx+gY00GdQBngZuDxx34BpA7zudeK/+nsGWj6e2P554N8S948n3iyzLKtDbZ99bgJ+kY3H4qB9NpDiweIUHYvj+uzzZeCxLKzDBcBKoDodn6NUf55I8WDxkcbOwIPF64n3UlQm7lcNpZ79xpXOA5jGD8ojwDYgTDxDXk/8V+TvgOWJD+/XBnhtI7ACWAv8iAMX3eUrMCrzAAADi0lEQVQBDyaeWwJ8JAvr8ADwJvAG8V9JtamsQ6rqcdA+G0j9WUOpOBa/Tmx/g/h8MnVZWIc1xH8QLUvcUnrmUwrrMTfxXiFgB/D7TIqdfhJBYvt1iWOwBvjs4fzdHHzTlcUiIj7np7OGRESkH0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBJKVzGxvmsv7mZmdmKT3ilp81tEVZvbUoWbtNLMKM/vbZJQt0h+dPipZycz2OudKkvh+QXdgArWU6hu7md0PvOOc++Yg+zcATzvnTkpHfOI/ahHIsGFm1Wb2azN7PXE7I7H9VDN72cyWJv6dlNh+rZk9amZPAc+Z2dlm9oKZPWbxefYf6pnLPbG9MXF/b2LCuOVm9qqZjUpsn5h4/LqZ/dsQWy2vcGAyvRIze97Mllh8PvmLE/vcDkxMtCK+m9j3q4ly3jCzf03if6P4kBKBDCf/CdzpnJsFXAr8LLF9FXCWc+4U4rN8fqvPa04DrnHOfSTx+BTg74ATgQnAGf2UUwy86pybBiwA/rpP+f+ZKP+Q87sk5sM5h/hV3gBdwFzn3Azi6198L5GI/h5Y65yb7pz7qpmdDxwHnApMB2aa2VmHKk9kIH6ZdE784VzgxD4zOZaZWSlQDtxvZscRn4kxt89r5jvn+s4R/5pzbguAmS0jPjfMSweV082BCfsWA+cl7p/GgbnfHwb+Y4A4C/u892Lic8lDfG6YbyW+1GPEWwqj+nn9+Ynb0sTjEuKJYcEA5YkMSolAhpMAcJpzrrPvRjP7IfAn59zcRH/7C32e3nfQe4T63I/S/99I2B0YXBton8F0Ouemm1k58YTyReAu4usSVAMznXNhM9sAFPTzegO+7Zz7yWGWK9IvdQ3JcPIc8Xn9ATCznul9y4GtifvXprD8V4l3SQFccaidnXNtxJepvMXMconH2ZxIAh8Gjkns2gGU9nnp74HrEvPZY2Z1ZlaTpDqIDykRSLYqMrMtfW43E/9SbUwMoK4kPnU4wHeAb5vZQiAnhTH9HXCzmb0G1AJth3qBc24p8ZknryC+sEujmTURbx2sSuyzC1iYON30u86554h3Pb1iZm8Cj/H+RCFyWHT6qEiSJFZP63TOOTO7ArjSOXfxoV4n4jWNEYgkz0zgR4kzffaQ5mVARY6UWgQiIj6nMQIREZ9TIhAR8TklAhERn1MiEBHxOSUCERGfUyIQEfG5/w+eHec0Ih9gFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot(skip_end=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>root_mean_squared_error</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.466214</td>\n",
       "      <td>0.417982</td>\n",
       "      <td>0.642091</td>\n",
       "      <td>00:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.358698</td>\n",
       "      <td>0.516845</td>\n",
       "      <td>0.710198</td>\n",
       "      <td>00:34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.379973</td>\n",
       "      <td>0.360047</td>\n",
       "      <td>0.593039</td>\n",
       "      <td>00:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.307417</td>\n",
       "      <td>0.324461</td>\n",
       "      <td>0.564441</td>\n",
       "      <td>00:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.283508</td>\n",
       "      <td>0.319185</td>\n",
       "      <td>0.560184</td>\n",
       "      <td>00:33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(5, 1e-2, moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('tmp1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>root_mean_squared_error</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.286906</td>\n",
       "      <td>0.322076</td>\n",
       "      <td>0.562487</td>\n",
       "      <td>00:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.267880</td>\n",
       "      <td>0.325651</td>\n",
       "      <td>0.566053</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_=learner.load('tmp1')\n",
    "learner.freeze_to(-2)\n",
    "\n",
    "learner.fit_one_cycle(2, slice(1e-03/(2.6**4),1e-03), moms=(0.8, 0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.save('tmp2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting one single item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learner.load('tmp2');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = train_df.loc[val_idxs].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category1                                                        Women\n",
       "category2                                             Women's Handbags\n",
       "category3                                                      Satchel\n",
       "brand_name                                                Michael Kors\n",
       "text                 Michael KORS purse. Michael Kors. Women Women'...\n",
       "shipping                                                             0\n",
       "item_condition_id                                                    2\n",
       "price                                                          4.99043\n",
       "Name: 9043, dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_item = val_df.loc[9043]\n",
    "one_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(one_item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred,raw_pred = predict_one_item(learner,one_item,tab_db,text_db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([4.9338]), tensor([4.9338]))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred,raw_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: for some reason, this result is slightly different from the prediction using learner.get_preds(). See more below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "End"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extra: testing predict_one_item function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load('tmp2');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_pred = learner.get_preds()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([[2.3479],\n",
       "         [2.4195],\n",
       "         [3.0596],\n",
       "         ...,\n",
       "         [3.6885],\n",
       "         [2.5004],\n",
       "         [3.5081]]),\n",
       " tensor([2.7726, 3.1355, 3.4012,  ..., 4.1897, 2.3979, 3.2189])]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4.9904)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor(198)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# val_pred[1].sort(descending=True)[0][17]\n",
    "# val_pred[1].sort(descending=True)[1][17]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.9219])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "tensor(4.9904)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_pred[0][198] #predict\n",
    "val_pred[1][198] #actual"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find that record with price (label) 4.9904"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((TabularLine category1 Women; category2 Women's Handbags; category3 Satchel; brand_name Michael Kors; shipping 0; item_condition_id 0.1024; ,\n",
       "  Text xxbos xxmaj michael xxup kors purse . xxmaj michael xxmaj kors . xxmaj women xxmaj women 's xxmaj handbags xxmaj satchel . a gorgeous gray metallic colored bag . xxmaj can be used as a cross body , it does have the strap for it . xxmaj used a couple of times . a little scuff on the left bottom corner which is shown above . xxmaj and it looks like a scratch located on the back ( last photo ) but you can barely tell it 's there . xxmaj other than that in perfect condition ! xxmaj with wallet .),\n",
       " FloatItem 4.9904327)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for i,(f,l) in enumerate(learner.data.valid_ds):\n",
    "#     if abs(l.obj-4.9904) <=0.001:\n",
    "#         print(f,l)\n",
    "#         print(i)\n",
    "#         print('-'*10)\n",
    "\n",
    "learner.data.valid_ds[436]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category1                                                        Women\n",
       "category2                                             Women's Handbags\n",
       "category3                                                      Satchel\n",
       "brand_name                                                Michael Kors\n",
       "text                 Michael KORS purse. Michael Kors. Women Women'...\n",
       "shipping                                                             0\n",
       "item_condition_id                                                    2\n",
       "price                                                          4.99043\n",
       "Name: 9043, dtype: object"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# val_df.loc[np.abs(val_df.price-4.9904)<=0.01]\n",
    "val_df.loc[9043]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Michael KORS purse. Michael Kors. Women Women's Handbags Satchel. A gorgeous gray metallic colored bag. Can be used as a cross body, it does have the strap for it. Used a couple of times. A little scuff on the left bottom corner which is shown above. And it looks like a scratch located on the back (last photo) but you can barely tell it's there. Other than that in perfect condition! With wallet.\""
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.loc[9043,txt_cols].text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Found it!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing transformation of that record between learn.data.valid_ds and databunch 's one_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([ 10, 108, 436, 711,   1]), tensor([0.1024])]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[tensor([[ 10, 108, 436, 711,   1]], device='cuda:0'),\n",
       " tensor([[0.1024]], device='cuda:0')]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.data.valid_ds[436][0][0].data\n",
    "tab_db.one_item(val_df.loc[9043])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   2,    5,  268,    6,  275,  344,    9,    5,  268,    5,  275,    9,\n",
       "            5,   14,    5,   14,   26,    5,  212,    5, 1476,    9,   20,  567,\n",
       "          313,  965,  830,  128,    9,    5,  104,   92,   58,   95,   20,  772,\n",
       "          146,   10,   34,  318,   77,   17,  489,   19,   34,    9,    5,   58,\n",
       "           20,  574,   28,  193,    9,   20,  221, 2622,   33,   17,  427,  332,\n",
       "         1945,  450,   27,  423,  346,    9,    5,   13,   34,  592,   91,   20,\n",
       "         1646, 6013,   33,   17,  168,   39,  395,  511,   36,   72,   37,  104,\n",
       "          740, 1491,   34,   26,  295,    9,    5,   51,  335,  113,   22,  141,\n",
       "           40,   15,    5,   25,  422,    9]], device='cuda:0')"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 102])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text xxbos xxmaj michael xxup kors purse . xxmaj michael xxmaj kors . xxmaj women xxmaj women 's xxmaj handbags xxmaj satchel . a gorgeous gray metallic colored bag . xxmaj can be used as a cross body , it does have the strap for it . xxmaj used a couple of times . a little scuff on the left bottom corner which is shown above . xxmaj and it looks like a scratch located on the back ( last photo ) but you can barely tell it 's there . xxmaj other than that in perfect condition ! xxmaj with wallet ."
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([   2,    5,  268,    6,  275,  344,    9,    5,  268,    5,  275,    9,    5,   14,    5,   14,   26,    5,\n",
       "        212,    5, 1476,    9,   20,  567,  313,  965,  830,  128,    9,    5])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "102"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_db.one_item(val_df.loc[9043,txt_cols])[0]\n",
    "text_db.one_item(val_df.loc[9043,txt_cols])[0].shape\n",
    "\n",
    "print('---')\n",
    "\n",
    "learner.data.valid_ds[436][0][1]\n",
    "learner.data.valid_ds[436][0][1].data[:30]\n",
    "len(learner.data.valid_ds[436][0][1].data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrap it up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = train_df.loc[val_idxs].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab_oneitem=tab_db.one_item(val_df.loc[9043],detach=True,cpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_oneitem=text_db.one_item(val_df.loc[9043,txt_cols],detach=True,cpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "_batch = [( ([tab_oneitem[0][0][0],tab_oneitem[0][1][0]],text_oneitem[0][0]), tab_oneitem[1][0] )]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabtext_onebatch = tabtext_collate( _batch )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabtext_onebatch = to_device(tabtext_onebatch,defaults.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = learner.pred_batch(batch = to_device(tabtext_onebatch,defaults.device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.9338]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_pred,x = grab_idx(result,0,batch_first=None),tabtext_onebatch[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([4.9338]),\n",
       " [[tensor([[ 10, 108, 436, 711,   1]], device='cuda:0'),\n",
       "   tensor([[0.1024]], device='cuda:0')],\n",
       "  tensor([[   2,    5,  268,    6,  275,  344,    9,    5,  268,    5,  275,    9,\n",
       "              5,   14,    5,   14,   26,    5,  212,    5, 1476,    9,   20,  567,\n",
       "            313,  965,  830,  128,    9,    5,  104,   92,   58,   95,   20,  772,\n",
       "            146,   10,   34,  318,   77,   17,  489,   19,   34,    9,    5,   58,\n",
       "             20,  574,   28,  193,    9,   20,  221, 2622,   33,   17,  427,  332,\n",
       "           1945,  450,   27,  423,  346,    9,    5,   13,   34,  592,   91,   20,\n",
       "           1646, 6013,   33,   17,  168,   39,  395,  511,   36,   72,   37,  104,\n",
       "            740, 1491,   34,   26,  295,    9,    5,   51,  335,  113,   22,  141,\n",
       "             40,   15,    5,   25,  422,    9]], device='cuda:0')])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_pred,x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.9338])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.data.single_ds.y.analyze_pred(raw_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0567])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_pred - val_pred[1][198] #actual"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Slight different here. Could be from the collate function or the sampler (SortSampler in validation set?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tabtext_onebatch[0][0][1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4.9338]], device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# raw prediction straight from the model\n",
    "learner.model(tabtext_onebatch[0][0],tabtext_onebatch[0][1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}