blog_notebooks/regression/regression_2_blog.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cuml\n",
    "import cudf\n",
    "import nvcategory\n",
    "\n",
    "import xgboost as xgb\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_squared_error, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import urllib.request\n",
    "\n",
    "data_dir = '../../data/blackfriday/'\n",
    "if not os.path.exists(data_dir):\n",
    "    print('creating black friday data directory')\n",
    "    os.system('mkdir ../../data/blackfriday')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://datahack-prod.s3.amazonaws.com/train_zip/train_oSwQCTC.zip to ../../data/blackfriday/train.zip\n"
     ]
    }
   ],
   "source": [
    "base_url = 'https://datahack-prod.s3.amazonaws.com/train_zip/'\n",
    "ofn = 'train_oSwQCTC.zip'\n",
    "fn = 'train.zip'\n",
    "if not os.path.isfile(data_dir+ofn):\n",
    "        print(f'Downloading {base_url+ofn} to {data_dir+fn}')\n",
    "        urllib.request.urlretrieve(base_url+ofn, data_dir+fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read in the data. Notice how it decompresses as it reads the data into memory. \n",
    "gdf = cudf.read_csv(data_dir+fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "537577"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>M</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>C</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7969</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID Gender   Age  Occupation City_Category  \\\n",
       "0  1000001  P00069042      F  0-17          10             A   \n",
       "1  1000001  P00248942      F  0-17          10             A   \n",
       "2  1000001  P00087842      F  0-17          10             A   \n",
       "3  1000001  P00085442      F  0-17          10             A   \n",
       "4  1000002  P00285442      M   55+          16             C   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "   Product_Category_2  Product_Category_3  Purchase  \n",
       "0                 NaN                 NaN      8370  \n",
       "1                 6.0                14.0     15200  \n",
       "2                 NaN                 NaN      1422  \n",
       "3                14.0                 NaN      1057  \n",
       "4                 NaN                 NaN      7969  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Taking a look at the data. We use \"to_pandas()\" to get the pretty printing. \n",
    "gdf.head().to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grabbing the first character of the years in city string to get rid of plus sign, and converting to int\n",
    "gdf['city_years'] = gdf.Stay_In_Current_City_Years.str.get(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we can see how we can control the value of our category variables with the replace method and turn strings to ints\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('A', '1')\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('B', '2')\n",
    "gdf['City_Category'] = gdf.City_Category.str.replace('C', '3')\n",
    "gdf['City_Category'] = gdf['City_Category'].str.stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf['Gender'] = gdf.Gender.str.replace('F', '1')\n",
    "gdf['Gender'] = gdf.Gender.str.replace('M', '0')\n",
    "gdf['Gender'] = gdf.Gender.str.stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Products: 3623\n"
     ]
    }
   ],
   "source": [
    "#Let's take a look at how many products we have\n",
    "prod_count = cudf.Series(nvcategory.from_strings(gdf.Product_ID.data).values()).unique().count() #hideous one-liner\n",
    "print(\"Unique Products: {}\".format(prod_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Product Categories: 18\n"
     ]
    }
   ],
   "source": [
    "#Let's take a look at how many primary product categories we have\n",
    "#We do it differently here because the variable is a number, not a string\n",
    "prod1_count = gdf.Product_Category_1.unique().count()\n",
    "print(\"Unique Product Categories: {}\".format(prod1_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dummy for multi-category products\n",
    "gdf['Product_Category_2'] = gdf['Product_Category_2'].fillna(0)\n",
    "gdf['Product_Category_3'] = gdf['Product_Category_3'].fillna(0)\n",
    "gdf['multi'] = ((gdf['Product_Category_2'] + gdf['Product_Category_3'])>0).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Gender/Marital Status interaction variable\n",
    "gdf['gen_mar_interaction'] = gdf['Gender']*gdf['Marital_Status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Because Occupation is a code, it should converted into indicator variables\n",
    "gdf = gdf.one_hot_encoding('Occupation', 'occ_dummy', gdf.Occupation.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dummy variable from Int\n",
    "gdf = gdf.one_hot_encoding('City_Category', 'city_cat', gdf.City_Category.unique())\n",
    "\n",
    "#Dummy from string\n",
    "cat = nvcategory.from_strings(gdf.Age.data)\n",
    "gdf['Age'] = cudf.Series(cat.values())\n",
    "gdf = gdf.one_hot_encoding('Age', 'age', gdf.Age.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Solution:\n",
    "gdf = gdf.one_hot_encoding('Product_Category_1', 'product', gdf.Product_Category_1.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We're going to drop the variables we've transformed\n",
    "drop_list = ['User_ID', 'Age', 'Stay_In_Current_City_Years', 'City_Category','Product_ID', 'Product_Category_1', 'Product_Category_2', 'Product_Category_3']\n",
    "gdf = gdf.drop(drop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We're going to make a list of all the first indicator variables in a series now so it will be\n",
    "#easier to exclude them when we're doing regressions later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummy_list = ['occ_dummy_0', 'city_cat_1', 'age_0', 'product_1', 'Purchase']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#All variables currently have to have the same type for some methods in cuML\n",
    "for col in gdf.columns.tolist():\n",
    "    gdf[col] = gdf[col].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cuml.preprocessing.model_selection.train_test_split\n",
    "test_size = round(len(gdf)*0.2)\n",
    "train_size = round(len(gdf)-test_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = gdf.iloc[0:test_size]\n",
    "gdf_train = gdf.iloc[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Deleting the main gdf because we're going to be making other subsets and other stuff, so it will be nice to have the memory. \n",
    "del(gdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = gdf_train['Purchase'].log()\n",
    "X_reg = gdf_train.drop(dummy_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_RIDGE_0.19\n"
     ]
    }
   ],
   "source": [
    "# # I'm going to perform a hyperparameter search for alpha in a ridge regression\n",
    "output_ridge = {}\n",
    "for alpha in np.around(np.arange(0.01, 1, 0.01), decimals=2):\n",
    "    \n",
    "    Ridge = cuml.Ridge(alpha=alpha, fit_intercept=False, normalize=True)\n",
    "    _fit = Ridge.fit(X_reg, y_train)\n",
    "    _y_hat = _fit.predict(X_reg)\n",
    "    _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "    output_ridge['RMSE_RIDGE_{}'.format(alpha)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_ridge, key=output_ridge.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "314.5797419772926\n"
     ]
    }
   ],
   "source": [
    "Ridge = cuml.Ridge(alpha=.1, fit_intercept=False, normalize=True)\n",
    "_fit = Ridge.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_Lasso_0.1\n"
     ]
    }
   ],
   "source": [
    "##Lasso\n",
    "output_lasso = {}\n",
    "for alpha in np.around(np.arange(0.1, 10, 0.1), decimals=2):\n",
    "    \n",
    "    Lasso = cuml.Lasso(alpha=alpha, fit_intercept=False, normalize=True)\n",
    "    _fit = Lasso.fit(X_reg, y_train)\n",
    "    _y_hat = _fit.predict(X_reg)\n",
    "    _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "    output_lasso['RMSE_Lasso_{}'.format(alpha)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_lasso, key=output_lasso.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2,949.3877330727473\n"
     ]
    }
   ],
   "source": [
    "Lasso = cuml.Lasso(alpha=.1, fit_intercept=False, normalize=True)\n",
    "_fit = Lasso.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAX RMSE: RMSE_ElasticNet_0.1_0.1\n"
     ]
    }
   ],
   "source": [
    "##Elastic Net\n",
    "output_en = {}\n",
    "for alpha in np.around(np.arange(0.1, 10, 0.1), decimals=2):\n",
    "    for ratio in np.around(np.arange(0.1, 1, 0.1), decimals=2):\n",
    "    \n",
    "        ElasticNet = cuml.ElasticNet(alpha=alpha, l1_ratio=ratio, fit_intercept=False, normalize=True)\n",
    "        _fit = ElasticNet.fit(X_reg, y_train)\n",
    "        _y_hat = _fit.predict(X_reg)\n",
    "        _mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "        output_en['RMSE_ElasticNet_{}_{}'.format(alpha, ratio)] = _mse\n",
    "\n",
    "print('MAX RMSE: {}'.format(min(output_en, key=output_en.get)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2,949.3877330727473\n"
     ]
    }
   ],
   "source": [
    "ElasticNet = cuml.ElasticNet(alpha=.1, l1_ratio=.1, fit_intercept=False, normalize=True)\n",
    "_fit = ElasticNet.fit(X_reg, y_train)\n",
    "_y_hat = _fit.predict(X_reg)\n",
    "_mse = np.sqrt((y_train.reset_index(drop=True).sub(_y_hat)**2).sum())\n",
    "print('{:,}'.format(_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_xgb = gdf_train[['Purchase']].log()\n",
    "X_xgb = gdf_train.drop('Purchase')\n",
    "xgb_train_set = xgb.DMatrix(data=X_xgb, label=y_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'nround':100,\n",
    "    'max_depth':4,\n",
    "    'max_leaves':2**4,\n",
    "    'tree_method':'gpu_hist',\n",
    "    'n_gpus':1,\n",
    "    'loss':'ls',\n",
    "    'objective':'reg:squarederror',\n",
    "    'max_features':'auto',\n",
    "    'criterion':'friedman_mse',\n",
    "    'grow_policy':'lossguide',\n",
    "    'verbose':True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_model = xgb.train(xgb_params, dtrain=xgb_train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat_xgb = xgb_model.predict(xgb_train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "RMSE = np.sqrt(mean_squared_error(y_xgb['Purchase'].to_pandas(), y_hat_xgb)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4661744\n"
     ]
    }
   ],
   "source": [
    "print(RMSE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}