A. AutoGluon - Tabular Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "169824ce",
   "metadata": {},
   "source": [
    "[![brightonr-banner-logo](images/notebanner.png)](https://www.meetup.com/en-AU/Silicon-Brighton-Brighton-R/)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a422833",
   "metadata": {},
   "source": [
    "<div style = \"text-align: right\"><font size = 5 color = \"#0077be\" face = \"verdana\"><b>AutoGluon - Get on the AutoML-wagon</b></font></div>\n",
    "<div style = \"text-align: right\"><font><i>By 'Dayo Oguntoyinbo</i></font></div>\n",
    "<div style = \"text-align: right\"><font>28th April 2022</font></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaaa8d4c",
   "metadata": {},
   "source": [
    "We will be using the Quick Start from the official documentation - https://auto.gluon.ai/stable/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "33a5935c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, roc_auc_score\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=RuntimeWarning) \n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "warnings.simplefilter(action='ignore', category=UserWarning) \n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.options.display.max_colwidth = None\n",
    "pd.set_option(\"display.float_format\", lambda x: '%.2f' % x)\n",
    "from pandas.core.common import SettingWithCopyWarning\n",
    "warnings.simplefilter(action=\"ignore\", category=SettingWithCopyWarning)\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88aea00a",
   "metadata": {},
   "source": [
    "# AutoGluon Tabular Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7306c2d",
   "metadata": {},
   "source": [
    "https://auto.gluon.ai/stable/tutorials/tabular_prediction/tabular-quickstart.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b7c3e2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -Uq pip\n",
    "# !pip install -Uq setuptools wheel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d56bd3cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # CPU version of pytorch has smaller footprint - see installation instructions in\n",
    "# # pytorch documentation - https://pytorch.org/get-started/locally/\n",
    "# !pip install -Uq torch==1.10.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html\n",
    "# !pip install -Uq autogluon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fb80cfc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from autogluon.tabular import TabularDataset, TabularPredictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "67140014",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TabularDataset(data='inputs/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "138f7c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/train.csv')\n",
    "test_data = TabularDataset('https://autogluon.s3.amazonaws.com/datasets/Inc/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "54f299ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Train data is: (39073, 15)\n",
      "Size of Test data is: (9769, 15)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Size of Train data is: {train_data.shape}\")\n",
    "print(f\"Size of Test data is: {test_data.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "862e8760",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # subsample subset of data for faster demo, try setting this to much larger values\n",
    "# subsample_size = 500\n",
    "\n",
    "# train_data = train_data.sample(n=subsample_size, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "03bf6369",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of Train data is: (39073, 15)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Size of Train data is: {train_data.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "052658b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>178478</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>13</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>61743</td>\n",
       "      <td>5th-6th</td>\n",
       "      <td>3</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Transport-moving</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46</td>\n",
       "      <td>Private</td>\n",
       "      <td>376789</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55</td>\n",
       "      <td>?</td>\n",
       "      <td>200235</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>?</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>Private</td>\n",
       "      <td>224541</td>\n",
       "      <td>7th-8th</td>\n",
       "      <td>4</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Handlers-cleaners</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>El-Salvador</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age   workclass  fnlwgt   education  education-num       marital-status  \\\n",
       "0   25     Private  178478   Bachelors             13        Never-married   \n",
       "1   23   State-gov   61743     5th-6th              3        Never-married   \n",
       "2   46     Private  376789     HS-grad              9        Never-married   \n",
       "3   55           ?  200235     HS-grad              9   Married-civ-spouse   \n",
       "4   36     Private  224541     7th-8th              4   Married-civ-spouse   \n",
       "\n",
       "           occupation    relationship    race      sex  capital-gain  \\\n",
       "0        Tech-support       Own-child   White   Female             0   \n",
       "1    Transport-moving   Not-in-family   White     Male             0   \n",
       "2       Other-service   Not-in-family   White     Male             0   \n",
       "3                   ?         Husband   White     Male             0   \n",
       "4   Handlers-cleaners         Husband   White     Male             0   \n",
       "\n",
       "   capital-loss  hours-per-week  native-country   class  \n",
       "0             0              40   United-States   <=50K  \n",
       "1             0              35   United-States   <=50K  \n",
       "2             0              15   United-States   <=50K  \n",
       "3             0              50   United-States    >50K  \n",
       "4             0              40     El-Salvador   <=50K  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e65f6a5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary of class variable: \n",
      " count      39073\n",
      "unique         2\n",
      "top        <=50K\n",
      "freq       29704\n",
      "Name: class, dtype: object\n"
     ]
    }
   ],
   "source": [
    "label = 'class'\n",
    "print(\"Summary of class variable: \\n\", train_data[label].describe())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c730cbc",
   "metadata": {},
   "source": [
    "Then, we use AutoGluon to train multiple models as follows (you can constraint your file path also):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f6ae772a",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = 'agModels-predictClass'  # specifies folder to store trained models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ada39edb",
   "metadata": {},
   "source": [
    "## Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2bf47053",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\"agModels-predictClass\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"agModels-predictClass/\"\n",
      "AutoGluon Version:  0.3.1\n",
      "Train Data Rows:    39073\n",
      "Train Data Columns: 14\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [' <=50K', ' >50K']\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type argument in fit() (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 =  >50K, class 0 =  <=50K\n",
      "\tNote: For your binary classification, AutoGluon arbitrarily selected which label-value represents positive ( >50K) vs negative ( <=50K) class.\n",
      "\tTo explicitly set the positive_class, either rename classes to 1 and 0, or specify positive_class in Predictor init.\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    64870.44 MB\n",
      "\tTrain Data (Original)  Memory Usage: 22.92 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('int', [])    : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "\t\t('object', []) : 8 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 7 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "\t\t('int', [])       : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "\t\t('int', ['bool']) : 1 | ['sex']\n",
      "\t0.2s = Fit runtime\n",
      "\t14 features in original data used to generate 14 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 2.19 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.29s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric argument of fit()\n",
      "Automatically generating train/validation split with holdout_frac=0.0639828014229775, Train Rows: 36573, Val Rows: 2500\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.7752\t = Validation score   (accuracy)\n",
      "\t0.2s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.766\t = Validation score   (accuracy)\n",
      "\t0.2s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\t0.8792\t = Validation score   (accuracy)\n",
      "\t1.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: LightGBM ...\n",
      "\t0.88\t = Validation score   (accuracy)\n",
      "\t0.38s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.864\t = Validation score   (accuracy)\n",
      "\t1.44s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8608\t = Validation score   (accuracy)\n",
      "\t1.73s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\t0.8864\t = Validation score   (accuracy)\n",
      "\t4.95s\t = Training   runtime\n",
      "\t0.01s\t = Validation runtime\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8496\t = Validation score   (accuracy)\n",
      "\t1.23s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8496\t = Validation score   (accuracy)\n",
      "\t1.33s\t = Training   runtime\n",
      "\t0.11s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training... Skipping this model.\n",
      "\t\tfuture feature annotations is not defined (dispatch.py, line 4)\n",
      "Detailed Traceback:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/tabular/trainer/abstract_trainer.py\", line 962, in _train_and_save\n",
      "    model = self._train_single(X, y, model, X_val, y_val, **model_fit_kwargs)\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/tabular/trainer/abstract_trainer.py\", line 934, in _train_single\n",
      "    model = model.fit(X=X, y=y, X_val=X_val, y_val=y_val, **model_fit_kwargs)\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/core/models/abstract/abstract_model.py\", line 522, in fit\n",
      "    out = self._fit(**kwargs)\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/tabular/models/fastainn/tabular_nn_fastai.py\", line 163, in _fit\n",
      "    try_import_fastai()\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/core/utils/try_import.py\", line 107, in try_import_fastai\n",
      "    import autogluon.tabular.models.fastainn.imports_helper\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/autogluon/tabular/models/fastainn/imports_helper.py\", line 1, in <module>\n",
      "    from fastai.tabular.all import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastai/tabular/all.py\", line 1, in <module>\n",
      "    from ..basics import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastai/basics.py\", line 1, in <module>\n",
      "    from .data.all import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastai/data/all.py\", line 1, in <module>\n",
      "    from ..torch_basics import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastai/torch_basics.py\", line 9, in <module>\n",
      "    from .imports import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastai/imports.py\", line 30, in <module>\n",
      "    from fastcore.all import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastcore/all.py\", line 3, in <module>\n",
      "    from .dispatch import *\n",
      "  File \"/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/fastcore/dispatch.py\", line 4\n",
      "    from __future__ import annotations\n",
      "                                     ^\n",
      "SyntaxError: future feature annotations is not defined\n",
      "Fitting model: XGBoost ...\n",
      "\t0.884\t = Validation score   (accuracy)\n",
      "\t0.66s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetMXNet ...\n",
      "\tWarning: Exception caused NeuralNetMXNet to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency mxnet. A quick tip is to install via `pip install mxnet --upgrade`, or `pip install mxnet_cu101 --upgrade`\n",
      "Fitting model: LightGBMLarge ...\n",
      "\t0.8784\t = Validation score   (accuracy)\n",
      "\t0.65s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8864\t = Validation score   (accuracy)\n",
      "\t1.06s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 25.76s ...\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"agModels-predictClass/\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 37s, sys: 9.29 s, total: 2min 46s\n",
      "Wall time: 25.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "predictor = (\n",
    "    TabularPredictor(label=label, path=save_path)\n",
    "    # eval_metric=eval_metric\n",
    ").fit(train_data) # , time_limit=2*60)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1cc1bb06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CatBoost</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.95</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.95</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.02</td>\n",
       "      <td>6.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.06</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.66</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LightGBM</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.38</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LightGBMXT</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>1.29</td>\n",
       "      <td>0.02</td>\n",
       "      <td>1.29</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LightGBMLarge</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.65</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.44</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.44</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.73</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.73</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.33</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.33</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  model  score_val  pred_time_val  fit_time  \\\n",
       "0              CatBoost       0.89           0.01      4.95   \n",
       "1   WeightedEnsemble_L2       0.89           0.02      6.01   \n",
       "2               XGBoost       0.88           0.02      0.66   \n",
       "3              LightGBM       0.88           0.02      0.38   \n",
       "4            LightGBMXT       0.88           0.02      1.29   \n",
       "5         LightGBMLarge       0.88           0.02      0.65   \n",
       "6      RandomForestGini       0.86           0.11      1.44   \n",
       "7      RandomForestEntr       0.86           0.11      1.73   \n",
       "8        ExtraTreesGini       0.85           0.11      1.23   \n",
       "9        ExtraTreesEntr       0.85           0.11      1.33   \n",
       "10       KNeighborsUnif       0.78           0.11      0.20   \n",
       "11       KNeighborsDist       0.77           0.11      0.20   \n",
       "\n",
       "    pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                     0.01               4.95            1       True   \n",
       "1                     0.00               1.06            2       True   \n",
       "2                     0.02               0.66            1       True   \n",
       "3                     0.02               0.38            1       True   \n",
       "4                     0.02               1.29            1       True   \n",
       "5                     0.02               0.65            1       True   \n",
       "6                     0.11               1.44            1       True   \n",
       "7                     0.11               1.73            1       True   \n",
       "8                     0.11               1.23            1       True   \n",
       "9                     0.11               1.33            1       True   \n",
       "10                    0.11               0.20            1       True   \n",
       "11                    0.11               0.20            1       True   \n",
       "\n",
       "    fit_order  \n",
       "0           7  \n",
       "1          12  \n",
       "2          10  \n",
       "3           4  \n",
       "4           3  \n",
       "5          11  \n",
       "6           5  \n",
       "7           6  \n",
       "8           8  \n",
       "9           9  \n",
       "10          1  \n",
       "11          2  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.leaderboard(silent=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a3b388c",
   "metadata": {},
   "source": [
    "## Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0633bc71",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = test_data[label]  # values to predict\n",
    "test_data_nolab = test_data.drop(columns=[label])  # delete label column to prove we're not cheating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b23d09d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31</td>\n",
       "      <td>Private</td>\n",
       "      <td>169085</td>\n",
       "      <td>11th</td>\n",
       "      <td>7</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>226203</td>\n",
       "      <td>12th</td>\n",
       "      <td>8</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Sales</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>47</td>\n",
       "      <td>Private</td>\n",
       "      <td>54260</td>\n",
       "      <td>Assoc-voc</td>\n",
       "      <td>11</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>1887</td>\n",
       "      <td>60</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>Private</td>\n",
       "      <td>176262</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>Private</td>\n",
       "      <td>241185</td>\n",
       "      <td>12th</td>\n",
       "      <td>8</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>United-States</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt      education  education-num  \\\n",
       "0   31            Private  169085           11th              7   \n",
       "1   17   Self-emp-not-inc  226203           12th              8   \n",
       "2   47            Private   54260      Assoc-voc             11   \n",
       "3   21            Private  176262   Some-college             10   \n",
       "4   17            Private  241185           12th              8   \n",
       "\n",
       "        marital-status        occupation relationship    race      sex  \\\n",
       "0   Married-civ-spouse             Sales         Wife   White   Female   \n",
       "1        Never-married             Sales    Own-child   White     Male   \n",
       "2   Married-civ-spouse   Exec-managerial      Husband   White     Male   \n",
       "3        Never-married   Exec-managerial    Own-child   White   Female   \n",
       "4        Never-married    Prof-specialty    Own-child   White     Male   \n",
       "\n",
       "   capital-gain  capital-loss  hours-per-week  native-country  \n",
       "0             0             0              20   United-States  \n",
       "1             0             0              45   United-States  \n",
       "2             0          1887              60   United-States  \n",
       "3             0             0              30   United-States  \n",
       "4             0             0              20   United-States  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_nolab.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e691e6d3",
   "metadata": {},
   "source": [
    "We can use our saved model or use it directly to make predictions on the new data and then evaluate performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7872c08e",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_direct = predictor.predict(test_data_nolab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d8e0123d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8753198894462074\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8753198894462074,\n",
      "    \"balanced_accuracy\": 0.7922548108093962,\n",
      "    \"mcc\": 0.635971335494168,\n",
      "    \"f1\": 0.7070707070707072,\n",
      "    \"precision\": 0.7989130434782609,\n",
      "    \"recall\": 0.634167385677308\n",
      "}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions:  \n",
      " 0     <=50K\n",
      "1     <=50K\n",
      "2      >50K\n",
      "3     <=50K\n",
      "4     <=50K\n",
      "Name: class, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(\"Predictions:  \\n\", y_pred_direct[0:5])\n",
    "\n",
    "perf = predictor.evaluate_predictions(\n",
    "    y_true=y_test,\n",
    "    y_pred=y_pred_direct,\n",
    "    auxiliary_metrics=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fcbd5ab3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions:  \n",
      " 0     <=50K\n",
      "1     <=50K\n",
      "2      >50K\n",
      "3     <=50K\n",
      "4     <=50K\n",
      "Name: class, dtype: object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8753198894462074\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8753198894462074,\n",
      "    \"balanced_accuracy\": 0.7922548108093962,\n",
      "    \"mcc\": 0.635971335494168,\n",
      "    \"f1\": 0.7070707070707072,\n",
      "    \"precision\": 0.7989130434782609,\n",
      "    \"recall\": 0.634167385677308\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "loaded_predictor = TabularPredictor.load(save_path)  # unnecessary, just demonstrates how to load previously-trained predictor from file\n",
    "\n",
    "y_pred = loaded_predictor.predict(test_data_nolab)\n",
    "\n",
    "print(\"Predictions:  \\n\", y_pred[0:5])\n",
    "perf = predictor.evaluate_predictions(\n",
    "    y_true=y_test,\n",
    "    y_pred=y_pred,\n",
    "    auxiliary_metrics=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "053dd8b5",
   "metadata": {},
   "source": [
    "We can evaluate the performance of individual trained model on our (labeled) test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "15145f1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.66</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CatBoost</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.95</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.01</td>\n",
       "      <td>4.95</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>6.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.06</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LightGBM</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.38</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LightGBMLarge</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.65</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LightGBMXT</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.02</td>\n",
       "      <td>1.29</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.02</td>\n",
       "      <td>1.29</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.44</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.44</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.73</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.73</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.23</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.23</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.33</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.11</td>\n",
       "      <td>1.33</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.78</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.20</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0               XGBoost        0.88       0.88            0.06           0.02   \n",
       "1              CatBoost        0.88       0.89            0.02           0.01   \n",
       "2   WeightedEnsemble_L2        0.88       0.89            0.02           0.02   \n",
       "3              LightGBM        0.87       0.88            0.02           0.02   \n",
       "4         LightGBMLarge        0.87       0.88            0.02           0.02   \n",
       "5            LightGBMXT        0.87       0.88            0.04           0.02   \n",
       "6      RandomForestGini        0.86       0.86            0.36           0.11   \n",
       "7      RandomForestEntr        0.86       0.86            0.35           0.11   \n",
       "8        ExtraTreesGini        0.85       0.85            0.56           0.11   \n",
       "9        ExtraTreesEntr        0.85       0.85            0.49           0.11   \n",
       "10       KNeighborsUnif        0.77       0.78            0.11           0.11   \n",
       "11       KNeighborsDist        0.76       0.77            0.11           0.11   \n",
       "\n",
       "    fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0       0.66                     0.06                    0.02   \n",
       "1       4.95                     0.02                    0.01   \n",
       "2       6.01                     0.00                    0.00   \n",
       "3       0.38                     0.02                    0.02   \n",
       "4       0.65                     0.02                    0.02   \n",
       "5       1.29                     0.04                    0.02   \n",
       "6       1.44                     0.36                    0.11   \n",
       "7       1.73                     0.35                    0.11   \n",
       "8       1.23                     0.56                    0.11   \n",
       "9       1.33                     0.49                    0.11   \n",
       "10      0.20                     0.11                    0.11   \n",
       "11      0.20                     0.11                    0.11   \n",
       "\n",
       "    fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0                0.66            1       True         10  \n",
       "1                4.95            1       True          7  \n",
       "2                1.06            2       True         12  \n",
       "3                0.38            1       True          4  \n",
       "4                0.65            1       True         11  \n",
       "5                1.29            1       True          3  \n",
       "6                1.44            1       True          5  \n",
       "7                1.73            1       True          6  \n",
       "8                1.23            1       True          8  \n",
       "9                1.33            1       True          9  \n",
       "10               0.20            1       True          1  \n",
       "11               0.20            1       True          2  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor.leaderboard(test_data, silent=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4496f07",
   "metadata": {},
   "source": [
    "**AutoGluon** automatically and iteratively tests values for hyperparameters to produce the best performance on the validation data. But you can tune hyperparameters also (please see the examples under the github examples).\n",
    "\n",
    "This involves repeatedly training models under different hyperparameter settings and evaluating their performance. This process can be computationally-intensive, so `fit()` can parallelize this process across multiple threads (and machines if distributed resources are available). To control runtimes, you can specify various arguments in `fit()` as demonstrated in the subsequent In-Depth tutorial.\n",
    "\n",
    "For tabular problems, `fit()` returns a Predictor object. For classification, you can easily output predicted class probabilities instead of predicted classes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e8532903",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_probs = predictor.predict_proba(test_data_nolab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bca92a90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>&lt;=50K</th>\n",
       "      <th>&gt;50K</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.91</td>\n",
       "      <td>0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.99</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    <=50K   >50K\n",
       "0    0.91   0.09\n",
       "1    0.99   0.01\n",
       "2    0.02   0.98\n",
       "3    1.00   0.00\n",
       "4    1.00   0.00"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_probs.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3df91ac1",
   "metadata": {},
   "source": [
    "Besides inference, this object can also summarize what happened during fit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "21611ed9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Summary of fit() ***\n",
      "Estimated performance of each model:\n",
      "                  model  score_val  pred_time_val  fit_time  pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  fit_order\n",
      "0              CatBoost       0.89           0.01      4.95                    0.01               4.95            1       True          7\n",
      "1   WeightedEnsemble_L2       0.89           0.02      6.01                    0.00               1.06            2       True         12\n",
      "2               XGBoost       0.88           0.02      0.66                    0.02               0.66            1       True         10\n",
      "3              LightGBM       0.88           0.02      0.38                    0.02               0.38            1       True          4\n",
      "4            LightGBMXT       0.88           0.02      1.29                    0.02               1.29            1       True          3\n",
      "5         LightGBMLarge       0.88           0.02      0.65                    0.02               0.65            1       True         11\n",
      "6      RandomForestGini       0.86           0.11      1.44                    0.11               1.44            1       True          5\n",
      "7      RandomForestEntr       0.86           0.11      1.73                    0.11               1.73            1       True          6\n",
      "8        ExtraTreesGini       0.85           0.11      1.23                    0.11               1.23            1       True          8\n",
      "9        ExtraTreesEntr       0.85           0.11      1.33                    0.11               1.33            1       True          9\n",
      "10       KNeighborsUnif       0.78           0.11      0.20                    0.11               0.20            1       True          1\n",
      "11       KNeighborsDist       0.77           0.11      0.20                    0.11               0.20            1       True          2\n",
      "Number of models trained: 12\n",
      "Types of models trained:\n",
      "{'LGBModel', 'CatBoostModel', 'KNNModel', 'XGBoostModel', 'XTModel', 'RFModel', 'WeightedEnsembleModel'}\n",
      "Bagging used: False \n",
      "Multi-layer stack-ensembling used: False \n",
      "Feature Metadata (Processed):\n",
      "(raw dtype, special dtypes):\n",
      "('category', [])  : 7 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "('int', [])       : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "('int', ['bool']) : 1 | ['sex']\n",
      "Plot summary of models saved to file: agModels-predictClass/SummaryOfModels.html\n",
      "*** End of fit() summary ***\n"
     ]
    }
   ],
   "source": [
    "results = predictor.fit_summary(show_plot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a7454038",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoGluon infers problem type is:  binary\n",
      "AutoGluon identified the following types of features:\n",
      "('category', [])  : 7 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "('int', [])       : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "('int', ['bool']) : 1 | ['sex']\n"
     ]
    }
   ],
   "source": [
    "print(\"AutoGluon infers problem type is: \", predictor.problem_type)\n",
    "print(\"AutoGluon identified the following types of features:\")\n",
    "print(predictor.feature_metadata)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c0ceaeb",
   "metadata": {},
   "source": [
    "## Maximizing predictive performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fe2d7373",
   "metadata": {},
   "outputs": [],
   "source": [
    "time_limit = 60  # for quick demonstration only, you should set this to longest time you are willing to wait (in seconds)\n",
    "\n",
    "metric = 'roc_auc'  # specify your evaluation metric here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5e17ac29",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No path specified. Models will be saved in: \"AutogluonModels/ag-20220428_150632/\"\n",
      "Presets specified: ['best_quality']\n",
      "Beginning AutoGluon training ... Time limit = 60s\n",
      "AutoGluon will save models to \"AutogluonModels/ag-20220428_150632/\"\n",
      "AutoGluon Version:  0.3.1\n",
      "Train Data Rows:    39073\n",
      "Train Data Columns: 14\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [' <=50K', ' >50K']\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type argument in fit() (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 =  >50K, class 0 =  <=50K\n",
      "\tNote: For your binary classification, AutoGluon arbitrarily selected which label-value represents positive ( >50K) vs negative ( <=50K) class.\n",
      "\tTo explicitly set the positive_class, either rename classes to 1 and 0, or specify positive_class in Predictor init.\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    63207.28 MB\n",
      "\tTrain Data (Original)  Memory Usage: 22.92 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('int', [])    : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "\t\t('object', []) : 8 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 7 | ['workclass', 'education', 'marital-status', 'occupation', 'relationship', ...]\n",
      "\t\t('int', [])       : 6 | ['age', 'fnlwgt', 'education-num', 'capital-gain', 'capital-loss', ...]\n",
      "\t\t('int', ['bool']) : 1 | ['sex']\n",
      "\t0.2s = Fit runtime\n",
      "\t14 features in original data used to generate 14 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 2.19 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.28s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'roc_auc'\n",
      "\tThis metric expects predicted probabilities rather than predicted class labels, so you'll need to use predict_proba() instead of predict()\n",
      "\tTo change this, specify the eval_metric argument of fit()\n",
      "AutoGluon will fit 2 stack levels (L1 to L2) ...\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif_BAG_L1 ... Training model for up to 39.8s of the 59.72s of remaining time.\n",
      "\t0.6865\t = Validation score   (roc_auc)\n",
      "\t0.33s\t = Training   runtime\n",
      "\t0.31s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist_BAG_L1 ... Training model for up to 39.03s of the 58.94s of remaining time.\n",
      "\t0.7025\t = Validation score   (roc_auc)\n",
      "\t0.33s\t = Training   runtime\n",
      "\t0.21s\t = Validation runtime\n",
      "Fitting model: LightGBMXT_BAG_L1 ... Training model for up to 38.35s of the 58.27s of remaining time.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1000]\ttrain_set's binary_logloss: 0.22337\tvalid_set's binary_logloss: 0.279745\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\t0.9244\t = Validation score   (roc_auc)\n",
      "\t15.39s\t = Training   runtime\n",
      "\t0.32s\t = Validation runtime\n",
      "Fitting model: LightGBM_BAG_L1 ... Training model for up to 22.4s of the 42.31s of remaining time.\n",
      "\t0.9289\t = Validation score   (roc_auc)\n",
      "\t7.15s\t = Training   runtime\n",
      "\t0.19s\t = Validation runtime\n",
      "Fitting model: RandomForestGini_BAG_L1 ... Training model for up to 14.95s of the 34.86s of remaining time.\n",
      "\t0.9061\t = Validation score   (roc_auc)\n",
      "\t1.53s\t = Training   runtime\n",
      "\t1.35s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr_BAG_L1 ... Training model for up to 11.51s of the 31.42s of remaining time.\n",
      "\t0.9067\t = Validation score   (roc_auc)\n",
      "\t1.93s\t = Training   runtime\n",
      "\t1.34s\t = Validation runtime\n",
      "Fitting model: CatBoost_BAG_L1 ... Training model for up to 7.69s of the 27.6s of remaining time.\n",
      "\tTime limit exceeded... Skipping CatBoost_BAG_L1.\n",
      "Completed 1/20 k-fold bagging repeats ...\n",
      "Fitting model: WeightedEnsemble_L2 ... Training model for up to 59.72s of the 18.33s of remaining time.\n",
      "\t0.929\t = Validation score   (roc_auc)\n",
      "\t6.66s\t = Training   runtime\n",
      "\t0.01s\t = Validation runtime\n",
      "Fitting 11 L2 models ...\n",
      "Fitting model: LightGBMXT_BAG_L2 ... Training model for up to 11.66s of the 11.65s of remaining time.\n",
      "\tRan out of time, early stopping on iteration 22. Best iteration is:\n",
      "\t[22]\ttrain_set's binary_logloss: 0.332317\tvalid_set's binary_logloss: 0.33301\n",
      "\tRan out of time, early stopping on iteration 99. Best iteration is:\n",
      "\t[99]\ttrain_set's binary_logloss: 0.264852\tvalid_set's binary_logloss: 0.280098\n",
      "\t0.9265\t = Validation score   (roc_auc)\n",
      "\t7.07s\t = Training   runtime\n",
      "\t0.19s\t = Validation runtime\n",
      "Fitting model: LightGBM_BAG_L2 ... Training model for up to 4.29s of the 4.28s of remaining time.\n",
      "\tRan out of time, early stopping on iteration 87. Best iteration is:\n",
      "\t[87]\ttrain_set's binary_logloss: 0.254884\tvalid_set's binary_logloss: 0.277604\n",
      "\tRan out of time, early stopping on iteration 85. Best iteration is:\n",
      "\t[85]\ttrain_set's binary_logloss: 0.255426\tvalid_set's binary_logloss: 0.274211\n",
      "\tRan out of time, early stopping on iteration 94. Best iteration is:\n",
      "\t[94]\ttrain_set's binary_logloss: 0.25254\tvalid_set's binary_logloss: 0.279827\n",
      "\tRan out of time, early stopping on iteration 92. Best iteration is:\n",
      "\t[85]\ttrain_set's binary_logloss: 0.256332\tvalid_set's binary_logloss: 0.273322\n",
      "\tRan out of time, early stopping on iteration 67. Best iteration is:\n",
      "\t[67]\ttrain_set's binary_logloss: 0.26288\tvalid_set's binary_logloss: 0.274267\n",
      "\tRan out of time, early stopping on iteration 101. Best iteration is:\n",
      "\t[98]\ttrain_set's binary_logloss: 0.251703\tvalid_set's binary_logloss: 0.273798\n",
      "\tRan out of time, early stopping on iteration 104. Best iteration is:\n",
      "\t[101]\ttrain_set's binary_logloss: 0.250457\tvalid_set's binary_logloss: 0.28027\n",
      "\tRan out of time, early stopping on iteration 114. Best iteration is:\n",
      "\t[107]\ttrain_set's binary_logloss: 0.247913\tvalid_set's binary_logloss: 0.287344\n",
      "\tRan out of time, early stopping on iteration 122. Best iteration is:\n",
      "\t[122]\ttrain_set's binary_logloss: 0.246258\tvalid_set's binary_logloss: 0.274056\n",
      "\tRan out of time, early stopping on iteration 145. Best iteration is:\n",
      "\t[111]\ttrain_set's binary_logloss: 0.249128\tvalid_set's binary_logloss: 0.273688\n",
      "\t0.9284\t = Validation score   (roc_auc)\n",
      "\t4.02s\t = Training   runtime\n",
      "\t0.14s\t = Validation runtime\n",
      "Fitting model: RandomForestGini_BAG_L2 ... Training model for up to 0.04s of the 0.03s of remaining time.\n",
      "\t0.9261\t = Validation score   (roc_auc)\n",
      "\t2.03s\t = Training   runtime\n",
      "\t1.25s\t = Validation runtime\n",
      "Completed 1/20 k-fold bagging repeats ...\n",
      "Fitting model: WeightedEnsemble_L3 ... Training model for up to 59.72s of the -3.82s of remaining time.\n",
      "\t0.929\t = Validation score   (roc_auc)\n",
      "\t3.4s\t = Training   runtime\n",
      "\t0.01s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 67.25s ...\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"AutogluonModels/ag-20220428_150632/\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11min 2s, sys: 10.6 s, total: 11min 12s\n",
      "Wall time: 1min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "predictor_max = (\n",
    "    TabularPredictor(label, eval_metric=metric)\n",
    "    .fit(train_data, time_limit=time_limit, presets='best_quality')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "fddc9c4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LightGBMXT_BAG_L2</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>2.20</td>\n",
       "      <td>3.91</td>\n",
       "      <td>33.73</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.19</td>\n",
       "      <td>7.07</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LightGBM_BAG_L1</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.19</td>\n",
       "      <td>7.15</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.19</td>\n",
       "      <td>7.15</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.41</td>\n",
       "      <td>14.14</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>6.66</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LightGBM_BAG_L2</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>2.16</td>\n",
       "      <td>3.86</td>\n",
       "      <td>30.68</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.14</td>\n",
       "      <td>4.02</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WeightedEnsemble_L3</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>2.68</td>\n",
       "      <td>5.30</td>\n",
       "      <td>43.19</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>3.40</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>LightGBMXT_BAG_L1</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.32</td>\n",
       "      <td>15.39</td>\n",
       "      <td>0.74</td>\n",
       "      <td>0.32</td>\n",
       "      <td>15.39</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RandomForestGini_BAG_L2</td>\n",
       "      <td>0.93</td>\n",
       "      <td>0.93</td>\n",
       "      <td>2.29</td>\n",
       "      <td>4.96</td>\n",
       "      <td>28.69</td>\n",
       "      <td>0.31</td>\n",
       "      <td>1.25</td>\n",
       "      <td>2.03</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestEntr_BAG_L1</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.34</td>\n",
       "      <td>1.93</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.34</td>\n",
       "      <td>1.93</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>RandomForestGini_BAG_L1</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.35</td>\n",
       "      <td>1.53</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.35</td>\n",
       "      <td>1.53</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>KNeighborsDist_BAG_L1</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.33</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>KNeighborsUnif_BAG_L1</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.33</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      model  score_test  score_val  pred_time_test  \\\n",
       "0         LightGBMXT_BAG_L2        0.93       0.93            2.20   \n",
       "1           LightGBM_BAG_L1        0.93       0.93            0.32   \n",
       "2       WeightedEnsemble_L2        0.93       0.93            0.43   \n",
       "3           LightGBM_BAG_L2        0.93       0.93            2.16   \n",
       "4       WeightedEnsemble_L3        0.93       0.93            2.68   \n",
       "5         LightGBMXT_BAG_L1        0.93       0.92            0.74   \n",
       "6   RandomForestGini_BAG_L2        0.93       0.93            2.29   \n",
       "7   RandomForestEntr_BAG_L1        0.91       0.91            0.36   \n",
       "8   RandomForestGini_BAG_L1        0.91       0.91            0.36   \n",
       "9     KNeighborsDist_BAG_L1        0.69       0.70            0.11   \n",
       "10    KNeighborsUnif_BAG_L1        0.67       0.69            0.11   \n",
       "\n",
       "    pred_time_val  fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0            3.91     33.73                     0.21                    0.19   \n",
       "1            0.19      7.15                     0.32                    0.19   \n",
       "2            0.41     14.14                     0.00                    0.01   \n",
       "3            3.86     30.68                     0.17                    0.14   \n",
       "4            5.30     43.19                     0.00                    0.01   \n",
       "5            0.32     15.39                     0.74                    0.32   \n",
       "6            4.96     28.69                     0.31                    1.25   \n",
       "7            1.34      1.93                     0.36                    1.34   \n",
       "8            1.35      1.53                     0.36                    1.35   \n",
       "9            0.21      0.33                     0.11                    0.21   \n",
       "10           0.31      0.33                     0.11                    0.31   \n",
       "\n",
       "    fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0                7.07            2       True          8  \n",
       "1                7.15            1       True          4  \n",
       "2                6.66            2       True          7  \n",
       "3                4.02            2       True          9  \n",
       "4                3.40            3       True         11  \n",
       "5               15.39            1       True          3  \n",
       "6                2.03            2       True         10  \n",
       "7                1.93            1       True          6  \n",
       "8                1.53            1       True          5  \n",
       "9                0.33            1       True          2  \n",
       "10               0.33            1       True          1  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor_max.leaderboard(test_data, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "623edf15",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the fitted model to make predictions on the test dataset\n",
    "test_predictions = predictor.predict(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "30d79db5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7081  848]\n",
      " [ 370 1470]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       <=50K       0.95      0.89      0.92      7929\n",
      "        >50K       0.63      0.80      0.71      1840\n",
      "\n",
      "    accuracy                           0.88      9769\n",
      "   macro avg       0.79      0.85      0.81      9769\n",
      "weighted avg       0.89      0.88      0.88      9769\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(test_predictions, y_test))\n",
    "print(classification_report(test_predictions, y_test))\n",
    "# print(\"Accuracy (validation):\", accuracy_score(test_predictions, y_test))\n",
    "# print(\"ROC AUC (validation):\", roc_auc_score(test_predictions, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1d5c61d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}