lab8.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 8\n",
    "\n",
    "## TODO:\n",
    "\n",
    "- [x] Split into training and test set (stratified)\n",
    "- [x] Decision tree on dataset\n",
    "- [ ] Add option to run on the smile dataset (optional for more consistent results)\n",
    "- [x] Make table (plot accuracy vs some hyperparams)\n",
    "  - [x] 10-fold CV\n",
    "  - [x] Accuracy\n",
    "  - [ ] TP rate\n",
    "  - [ ] FP rate\n",
    "  - [ ] precision\n",
    "  - [ ] recall\n",
    "  - [ ] F measure\n",
    "  - [ ] ROC area\n",
    "  - [ ] Graph table\n",
    "- [x] Random search hyperparameters\n",
    "- [x] 3 trees with different train/test sets (optional, exact details unclear)\n",
    "\n",
    "## Notes:\n",
    "- When doing k-fold validation make sure that the folds are stratified"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sklearn\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>failures</th>\n",
       "      <th>higher</th>\n",
       "      <th>studytime</th>\n",
       "      <th>Medu</th>\n",
       "      <th>Fedu</th>\n",
       "      <th>Dalc</th>\n",
       "      <th>age</th>\n",
       "      <th>reason_reputation</th>\n",
       "      <th>school</th>\n",
       "      <th>address</th>\n",
       "      <th>internet</th>\n",
       "      <th>G3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1043</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1044 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      failures  higher  studytime  Medu  Fedu  Dalc  age  reason_reputation  \\\n",
       "0            0       1          2     4     4     1   18                  0   \n",
       "1            0       1          2     1     1     1   17                  0   \n",
       "2            3       1          2     1     1     2   15                  0   \n",
       "3            0       1          3     4     2     1   15                  0   \n",
       "4            0       1          2     3     3     1   16                  0   \n",
       "...        ...     ...        ...   ...   ...   ...  ...                ...   \n",
       "1039         1       1          3     2     3     1   19                  0   \n",
       "1040         0       1          2     3     1     1   18                  0   \n",
       "1041         0       1          2     1     1     1   18                  0   \n",
       "1042         0       1          1     3     1     3   17                  0   \n",
       "1043         0       1          1     3     2     3   18                  0   \n",
       "\n",
       "      school  address  internet  G3  \n",
       "0          0        0         0   6  \n",
       "1          0        0         1   6  \n",
       "2          0        0         1  10  \n",
       "3          0        0         1  15  \n",
       "4          0        0         0  10  \n",
       "...      ...      ...       ...  ..  \n",
       "1039       1        1         1  10  \n",
       "1040       1        0         1  16  \n",
       "1041       1        0         0   9  \n",
       "1042       1        0         1  10  \n",
       "1043       1        1         1  11  \n",
       "\n",
       "[1044 rows x 12 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"5\"\n",
    "\n",
    "df3 = pd.read_csv(\"Data/data3.csv\")\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3_copy = df3.copy()\n",
    "y = df3_copy.pop(\"G3\")\n",
    "y_binary = [0 if y_i > np.mean(y) else 1 for y_i in y] #making the target class into binary (over/under average score)\n",
    "X = df3_copy\n",
    "# display(X)\n",
    "# print(y_binary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(835, 11)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.2, stratify=y_binary, random_state=42,)\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: No need to scale, trees are not affected by different scales."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tree 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV, cross_val_score\n",
    "from scipy.stats import randint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=10, estimator=DecisionTreeClassifier(),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [3, None],\n",
       "                                        &#x27;max_features&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1CAA00&gt;,\n",
       "                                        &#x27;min_samples_leaf&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808DB41AF0&gt;},\n",
       "                   random_state=10)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=10, estimator=DecisionTreeClassifier(),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [3, None],\n",
       "                                        &#x27;max_features&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1CAA00&gt;,\n",
       "                                        &#x27;min_samples_leaf&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808DB41AF0&gt;},\n",
       "                   random_state=10)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=10, estimator=DecisionTreeClassifier(),\n",
       "                   param_distributions={'criterion': ['gini', 'entropy'],\n",
       "                                        'max_depth': [3, None],\n",
       "                                        'max_features': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1CAA00>,\n",
       "                                        'min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808DB41AF0>},\n",
       "                   random_state=10)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {\"max_depth\": [3, None],\n",
    "              \"max_features\": randint(1, X.shape[1]),\n",
    "              \"min_samples_leaf\": randint(1, 9),\n",
    "              \"criterion\": [\"gini\", \"entropy\"]}\n",
    "\n",
    "clf = DecisionTreeClassifier()\n",
    "tree_cv = RandomizedSearchCV(clf, parameters, cv=10, random_state = 10)\n",
    "tree_cv.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#takes a few seconds and hard to see but just for sanity check\n",
    "# tree.plot_tree(tree_cv.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Decision Tree Parameters: {'criterion': 'entropy', 'max_depth': None, 'max_features': 7, 'min_samples_leaf': 7}\n",
      "Best score is 0.6633820998278829\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Decision Tree Parameters: {}\".format(tree_cv.best_params_))\n",
    "print(\"Best score is {}\".format(tree_cv.best_score_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tree_cv.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = pd.DataFrame(tree_cv.cv_results_[\"params\"])\n",
    "table[\"classifier\"] = \"DecisionTreeClassifier\"\n",
    "table[\"test_method\"] = \"10CV\"\n",
    "table[\"accuracy\"] = tree_cv.cv_results_[\"mean_test_score\"]\n",
    "table = table.sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test set results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.52380952 0.47619048 0.57142857 0.57142857 0.57142857 0.61904762\n",
      " 0.66666667 0.66666667 0.61904762 0.65      ]\n",
      "Mean test accuracy:  0.5935714285714285\n"
     ]
    }
   ],
   "source": [
    "#cross validation\n",
    "\n",
    "CV_test = cross_val_score(tree_cv, X_test, y_test, cv=10)\n",
    "print(CV_test)\n",
    "print(\"Mean test accuracy: \",np.mean(CV_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy:  0.7365269461077845\n",
      "Test accuracy:  0.6220095693779905\n"
     ]
    }
   ],
   "source": [
    "#overall test\n",
    "print(\"Training accuracy: \",tree_cv.score(X_train, y_train))\n",
    "print(\"Test accuracy: \", tree_cv.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tree 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_tree = tree_cv.best_estimator_\n",
    "\n",
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.2, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"DecisionTreeClassifier\", \"test_method\": \"20% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tree 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.5, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"DecisionTreeClassifier\", \"test_method\": \"50% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tree 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.8, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"DecisionTreeClassifier\", \"test_method\": \"80% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forrest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=10, estimator=RandomForestClassifier(),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [3, None],\n",
       "                                        &#x27;max_features&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E2769D0&gt;,\n",
       "                                        &#x27;min_samples_leaf&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1F6130&gt;},\n",
       "                   random_state=10)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=10, estimator=RandomForestClassifier(),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [3, None],\n",
       "                                        &#x27;max_features&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E2769D0&gt;,\n",
       "                                        &#x27;min_samples_leaf&#x27;: &lt;scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1F6130&gt;},\n",
       "                   random_state=10)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=10, estimator=RandomForestClassifier(),\n",
       "                   param_distributions={'criterion': ['gini', 'entropy'],\n",
       "                                        'max_depth': [3, None],\n",
       "                                        'max_features': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E2769D0>,\n",
       "                                        'min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x000001808E1F6130>},\n",
       "                   random_state=10)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.2, stratify=y_binary, random_state=42,)\n",
    "\n",
    "parameters = {\"max_depth\": [3, None],\n",
    "              \"max_features\": randint(1, X.shape[1]),\n",
    "              \"min_samples_leaf\": randint(1, 9),\n",
    "              \"criterion\": [\"gini\", \"entropy\"]}\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "forrest_cv = RandomizedSearchCV(clf, parameters, cv=10, random_state = 10)\n",
    "\n",
    "forrest_cv.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabel_temp = pd.DataFrame(forrest_cv.cv_results_[\"params\"])\n",
    "tabel_temp[\"classifier\"] = \"RandomForrestClassifier\"\n",
    "tabel_temp[\"test_method\"] = \"10CV\"\n",
    "tabel_temp[\"accuracy\"] = forrest_cv.cv_results_[\"mean_test_score\"]\n",
    "table = pd.concat([table, tabel_temp], ignore_index=True)\n",
    "table = table.sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_tree = tree_cv.best_estimator_\n",
    "\n",
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.2, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"RandomForrestClassifier\", \"test_method\": \"20% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.5, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"RandomForrestClassifier\", \"test_method\": \"50% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_accs = []\n",
    "train_accs = []\n",
    "for i in range(10):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y_binary, test_size=0.8, stratify=y_binary, random_state=42,)\n",
    "    best_tree.fit(X_train,y_train)\n",
    "    test_accs.append(best_tree.score(X_test, y_test))\n",
    "    train_accs.append(best_tree.score(X_train, y_train))\n",
    "\n",
    "test_acc = sum(test_accs) / len(test_accs)\n",
    "train_acc = sum(train_accs) / len(train_accs)\n",
    "\n",
    "row = pd.Series({k: best_tree.get_params()[k] for k in [\"criterion\",\"max_depth\",\"max_features\",\"min_samples_leaf\"]} | {\"classifier\": \"RandomForrestClassifier\", \"test_method\": \"80% Test Split\", \"accuracy\": test_acc, \"train_accuracy\": train_acc})\n",
    "table = pd.concat([table, row.to_frame().T], ignore_index=True).sort_values(by=[\"accuracy\"], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Binary Class split:  0.5114942528735632\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>criterion</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>max_features</th>\n",
       "      <th>min_samples_leaf</th>\n",
       "      <th>classifier</th>\n",
       "      <th>test_method</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>train_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.671801</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.669478</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entropy</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.66714</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.663382</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.659854</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.658649</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.657487</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.65743</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.656239</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>20% Test Split</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>0.741317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.650158</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.647849</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.646629</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>50% Test Split</td>\n",
       "      <td>0.64387</td>\n",
       "      <td>0.73659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>20% Test Split</td>\n",
       "      <td>0.643541</td>\n",
       "      <td>0.737844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>50% Test Split</td>\n",
       "      <td>0.641762</td>\n",
       "      <td>0.735632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.639443</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.631096</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>80% Test Split</td>\n",
       "      <td>0.628589</td>\n",
       "      <td>0.726442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.626291</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.620324</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>entropy</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.620209</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.617814</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>entropy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>RandomForrestClassifier</td>\n",
       "      <td>80% Test Split</td>\n",
       "      <td>0.616029</td>\n",
       "      <td>0.734615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>gini</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.608405</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>gini</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>DecisionTreeClassifier</td>\n",
       "      <td>10CV</td>\n",
       "      <td>0.605938</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   criterion  max_depth max_features min_samples_leaf  \\\n",
       "0       gini        3.0            2                4   \n",
       "1       gini        3.0            5                4   \n",
       "2    entropy        3.0            2                1   \n",
       "3    entropy        NaN            7                7   \n",
       "4       gini        NaN            2                3   \n",
       "5       gini        NaN            1                3   \n",
       "6       gini        3.0            7                1   \n",
       "7    entropy        NaN            5                8   \n",
       "8    entropy        NaN            7                7   \n",
       "9    entropy        NaN            7                7   \n",
       "10   entropy        NaN            5                8   \n",
       "11      gini        NaN            6                6   \n",
       "12      gini        NaN           10                6   \n",
       "13   entropy        NaN            7                7   \n",
       "14   entropy        NaN            7                7   \n",
       "15   entropy        NaN            7                7   \n",
       "16      gini        NaN           10                6   \n",
       "17      gini        3.0            7                1   \n",
       "18   entropy        NaN            7                7   \n",
       "19      gini        NaN            1                3   \n",
       "20      gini        3.0            5                4   \n",
       "21   entropy        3.0            2                1   \n",
       "22      gini        NaN            6                6   \n",
       "25   entropy        NaN            7                7   \n",
       "23      gini        3.0            2                4   \n",
       "24      gini        NaN            2                3   \n",
       "\n",
       "                 classifier     test_method  accuracy train_accuracy  \n",
       "0   RandomForrestClassifier            10CV  0.671801            NaN  \n",
       "1   RandomForrestClassifier            10CV  0.669478            NaN  \n",
       "2   RandomForrestClassifier            10CV   0.66714            NaN  \n",
       "3    DecisionTreeClassifier            10CV  0.663382            NaN  \n",
       "4   RandomForrestClassifier            10CV  0.659854            NaN  \n",
       "5   RandomForrestClassifier            10CV  0.658649            NaN  \n",
       "6   RandomForrestClassifier            10CV  0.657487            NaN  \n",
       "7   RandomForrestClassifier            10CV   0.65743            NaN  \n",
       "8   RandomForrestClassifier            10CV  0.656239            NaN  \n",
       "9   RandomForrestClassifier  20% Test Split  0.654545       0.741317  \n",
       "10   DecisionTreeClassifier            10CV  0.650158            NaN  \n",
       "11  RandomForrestClassifier            10CV  0.647849            NaN  \n",
       "12  RandomForrestClassifier            10CV  0.646629            NaN  \n",
       "13   DecisionTreeClassifier  50% Test Split   0.64387        0.73659  \n",
       "14   DecisionTreeClassifier  20% Test Split  0.643541       0.737844  \n",
       "15  RandomForrestClassifier  50% Test Split  0.641762       0.735632  \n",
       "16   DecisionTreeClassifier            10CV  0.639443            NaN  \n",
       "17   DecisionTreeClassifier            10CV  0.631096            NaN  \n",
       "18   DecisionTreeClassifier  80% Test Split  0.628589       0.726442  \n",
       "19   DecisionTreeClassifier            10CV  0.626291            NaN  \n",
       "20   DecisionTreeClassifier            10CV  0.620324            NaN  \n",
       "21   DecisionTreeClassifier            10CV  0.620209            NaN  \n",
       "22   DecisionTreeClassifier            10CV  0.617814            NaN  \n",
       "25  RandomForrestClassifier  80% Test Split  0.616029       0.734615  \n",
       "23   DecisionTreeClassifier            10CV  0.608405            NaN  \n",
       "24   DecisionTreeClassifier            10CV  0.605938            NaN  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Binary Class split: \", sum(y_binary)/len(y_binary))\n",
    "table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "96cc367a3e274e4efb54435191b44d84ced746980f71f260d488ba56050b94ee"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}