11_Hyper_Parameter_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Which model to use for classification?\n",
    "- suppose you want to perform classification on iris dataset.\n",
    "- there are many models to choose from (Random forest, SVM, Decisin Tree, ogistic Regression, Naive Bayes, etc.)\n",
    "- supose you choose SVM\n",
    "- model = svm .SVC(kernel=\"rbf\", C=30, gamma=\"auto\")\n",
    "- how to choose the hyperparameters(kernal, C, gemma)\n",
    "- **Hyperparameter tuning** - the process of choosing the optimal parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm, datasets\n",
    "iris = datasets.load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>flower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>5.3</td>\n",
       "      <td>3.7</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.3</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>7.0</td>\n",
       "      <td>3.2</td>\n",
       "      <td>4.7</td>\n",
       "      <td>1.4</td>\n",
       "      <td>versicolor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>6.4</td>\n",
       "      <td>3.2</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>versicolor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "47                4.6               3.2                1.4               0.2   \n",
       "48                5.3               3.7                1.5               0.2   \n",
       "49                5.0               3.3                1.4               0.2   \n",
       "50                7.0               3.2                4.7               1.4   \n",
       "51                6.4               3.2                4.5               1.5   \n",
       "\n",
       "        flower  \n",
       "47      setosa  \n",
       "48      setosa  \n",
       "49      setosa  \n",
       "50  versicolor  \n",
       "51  versicolor  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "iris_df = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
    "iris_df[\"flower\"] = iris.target\n",
    "iris_df[\"flower\"] = iris_df[\"flower\"].apply(lambda x: iris.target_names[x])\n",
    "iris_df[47:52]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(iris.data, iris.target, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9111111111111111"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = svm.SVC(kernel=\"rbf\", C=30, gamma=\"auto\")\n",
    "model.fit(X_train, Y_train)\n",
    "model.score(X_test, Y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The score changes if the train and test data is changed\n",
    "- We use K Fold Cross Validation to overcome this issue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 1.        , 0.9       , 0.96666667, 1.        ])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cross_val_score(svm.SVC(kernel=\"linear\", C=10, gamma=\"auto\"), iris.data, iris.target, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96666667, 1.        , 0.96666667, 0.96666667, 1.        ])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(svm.SVC(kernel=\"rbf\", C=10, gamma=\"auto\"), iris.data, iris.target, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 1.        , 0.9       , 0.93333333, 1.        ])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(svm.SVC(kernel=\"linear\", C=20, gamma=\"auto\"), iris.data, iris.target, cv=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- This method is very manual and repeatetive. Becoz there are so many values you can supply as combination.\n",
    "- Use for loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'rbf_1': 0.9800000000000001,\n",
       " 'rbf_10': 0.9800000000000001,\n",
       " 'rbf_20': 0.9666666666666668,\n",
       " 'linear_1': 0.9800000000000001,\n",
       " 'linear_10': 0.9733333333333334,\n",
       " 'linear_20': 0.9666666666666666}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kernels = [\"rbf\", \"linear\"]\n",
    "C = [1,10,20]\n",
    "avg_scores = {}\n",
    "for kval in kernels:\n",
    "    for cval in C:\n",
    "        cv_scores = cross_val_score(svm.SVC(kernel=kval, C=cval, gamma=\"auto\"), iris.data, iris.target, cv=5)\n",
    "        avg_scores[kval + \"_\" + str(cval)] = np.average(cv_scores)\n",
    "\n",
    "avg_scores\n",
    "\n",
    "# this is also not good since as the c vals increase the iterations will increase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sklearn provides an api which do the exact same thing as above with a single line of code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00260277, 0.0020062 , 0.00140162, 0.00159731, 0.00200624,\n",
       "        0.00199518]),\n",
       " 'std_fit_time': array([0.00049608, 0.00061782, 0.00048624, 0.00048751, 0.00089324,\n",
       "        0.00062955]),\n",
       " 'mean_score_time': array([0.00260425, 0.00159841, 0.0013968 , 0.00060096, 0.00139632,\n",
       "        0.00140414]),\n",
       " 'std_score_time': array([0.00049165, 0.00079466, 0.00049545, 0.0004907 , 0.00048719,\n",
       "        0.00049306]),\n",
       " 'param_C': masked_array(data=[1, 1, 10, 10, 20, 20],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value=999999),\n",
       " 'param_kernel': masked_array(data=['rbf', 'linear', 'rbf', 'linear', 'rbf', 'linear'],\n",
       "              mask=[False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 1, 'kernel': 'rbf'},\n",
       "  {'C': 1, 'kernel': 'linear'},\n",
       "  {'C': 10, 'kernel': 'rbf'},\n",
       "  {'C': 10, 'kernel': 'linear'},\n",
       "  {'C': 20, 'kernel': 'rbf'},\n",
       "  {'C': 20, 'kernel': 'linear'}],\n",
       " 'split0_test_score': array([0.96666667, 0.96666667, 0.96666667, 1.        , 0.96666667,\n",
       "        1.        ]),\n",
       " 'split1_test_score': array([1., 1., 1., 1., 1., 1.]),\n",
       " 'split2_test_score': array([0.96666667, 0.96666667, 0.96666667, 0.9       , 0.9       ,\n",
       "        0.9       ]),\n",
       " 'split3_test_score': array([0.96666667, 0.96666667, 0.96666667, 0.96666667, 0.96666667,\n",
       "        0.93333333]),\n",
       " 'split4_test_score': array([1., 1., 1., 1., 1., 1.]),\n",
       " 'mean_test_score': array([0.98      , 0.98      , 0.98      , 0.97333333, 0.96666667,\n",
       "        0.96666667]),\n",
       " 'std_test_score': array([0.01632993, 0.01632993, 0.01632993, 0.03887301, 0.03651484,\n",
       "        0.0421637 ]),\n",
       " 'rank_test_score': array([1, 1, 1, 4, 5, 6])}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "classifier = GridSearchCV(svm.SVC(gamma=\"auto\"), {\n",
    "    \"C\":[1,10,20],\n",
    "    \"kernel\":[\"rbf\", \"linear\"]\n",
    "}, cv=5, return_train_score=False)\n",
    "\n",
    "classifier.fit(iris.data, iris.target)\n",
    "classifier.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_kernel</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002603</td>\n",
       "      <td>0.000496</td>\n",
       "      <td>0.002604</td>\n",
       "      <td>0.000492</td>\n",
       "      <td>1</td>\n",
       "      <td>rbf</td>\n",
       "      <td>{'C': 1, 'kernel': 'rbf'}</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.016330</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.002006</td>\n",
       "      <td>0.000618</td>\n",
       "      <td>0.001598</td>\n",
       "      <td>0.000795</td>\n",
       "      <td>1</td>\n",
       "      <td>linear</td>\n",
       "      <td>{'C': 1, 'kernel': 'linear'}</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.016330</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.001402</td>\n",
       "      <td>0.000486</td>\n",
       "      <td>0.001397</td>\n",
       "      <td>0.000495</td>\n",
       "      <td>10</td>\n",
       "      <td>rbf</td>\n",
       "      <td>{'C': 10, 'kernel': 'rbf'}</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.016330</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.001597</td>\n",
       "      <td>0.000488</td>\n",
       "      <td>0.000601</td>\n",
       "      <td>0.000491</td>\n",
       "      <td>10</td>\n",
       "      <td>linear</td>\n",
       "      <td>{'C': 10, 'kernel': 'linear'}</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.973333</td>\n",
       "      <td>0.038873</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.002006</td>\n",
       "      <td>0.000893</td>\n",
       "      <td>0.001396</td>\n",
       "      <td>0.000487</td>\n",
       "      <td>20</td>\n",
       "      <td>rbf</td>\n",
       "      <td>{'C': 20, 'kernel': 'rbf'}</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.036515</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.001995</td>\n",
       "      <td>0.000630</td>\n",
       "      <td>0.001404</td>\n",
       "      <td>0.000493</td>\n",
       "      <td>20</td>\n",
       "      <td>linear</td>\n",
       "      <td>{'C': 20, 'kernel': 'linear'}</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.042164</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time  param_C  \\\n",
       "0       0.002603      0.000496         0.002604        0.000492        1   \n",
       "1       0.002006      0.000618         0.001598        0.000795        1   \n",
       "2       0.001402      0.000486         0.001397        0.000495       10   \n",
       "3       0.001597      0.000488         0.000601        0.000491       10   \n",
       "4       0.002006      0.000893         0.001396        0.000487       20   \n",
       "5       0.001995      0.000630         0.001404        0.000493       20   \n",
       "\n",
       "  param_kernel                         params  split0_test_score  \\\n",
       "0          rbf      {'C': 1, 'kernel': 'rbf'}           0.966667   \n",
       "1       linear   {'C': 1, 'kernel': 'linear'}           0.966667   \n",
       "2          rbf     {'C': 10, 'kernel': 'rbf'}           0.966667   \n",
       "3       linear  {'C': 10, 'kernel': 'linear'}           1.000000   \n",
       "4          rbf     {'C': 20, 'kernel': 'rbf'}           0.966667   \n",
       "5       linear  {'C': 20, 'kernel': 'linear'}           1.000000   \n",
       "\n",
       "   split1_test_score  split2_test_score  split3_test_score  split4_test_score  \\\n",
       "0                1.0           0.966667           0.966667                1.0   \n",
       "1                1.0           0.966667           0.966667                1.0   \n",
       "2                1.0           0.966667           0.966667                1.0   \n",
       "3                1.0           0.900000           0.966667                1.0   \n",
       "4                1.0           0.900000           0.966667                1.0   \n",
       "5                1.0           0.900000           0.933333                1.0   \n",
       "\n",
       "   mean_test_score  std_test_score  rank_test_score  \n",
       "0         0.980000        0.016330                1  \n",
       "1         0.980000        0.016330                1  \n",
       "2         0.980000        0.016330                1  \n",
       "3         0.973333        0.038873                4  \n",
       "4         0.966667        0.036515                5  \n",
       "5         0.966667        0.042164                6  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(classifier.cv_results_)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_kernel</th>\n",
       "      <th>mean_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>rbf</td>\n",
       "      <td>0.980000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>linear</td>\n",
       "      <td>0.980000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>rbf</td>\n",
       "      <td>0.980000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>linear</td>\n",
       "      <td>0.973333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20</td>\n",
       "      <td>rbf</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20</td>\n",
       "      <td>linear</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   param_C param_kernel  mean_test_score\n",
       "0        1          rbf         0.980000\n",
       "1        1       linear         0.980000\n",
       "2       10          rbf         0.980000\n",
       "3       10       linear         0.973333\n",
       "4       20          rbf         0.966667\n",
       "5       20       linear         0.966667"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\"param_C\", \"param_kernel\", \"mean_test_score\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9800000000000001\n",
      "{'C': 1, 'kernel': 'rbf'}\n"
     ]
    }
   ],
   "source": [
    "print(classifier.best_score_)\n",
    "print(classifier.best_params_) # know you kow the best paramrters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- One isuue that can happen with GridSearchCV is that the computation cost, our dataset now is very limited but imagine having millions of data point in the dataset and for parameters you have so many values then the no of possible permutation will become very high.\n",
    "- to tackle this issue sklearn with another class called RandomizedSearchCV\n",
    "- RandomizedSearchCV will not try every single permutation and combination of parameter but it will try random combibnations of these parameter value and you can choose what those iteration culd be."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_kernel</th>\n",
       "      <th>mean_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>rbf</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>rbf</td>\n",
       "      <td>0.98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   param_C param_kernel  mean_test_score\n",
       "0       10          rbf             0.98\n",
       "1        1          rbf             0.98"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "rs_classifier = RandomizedSearchCV(svm.SVC(gamma=\"auto\"), {\n",
    "    \"C\":[1,10,20],\n",
    "    \"kernel\":[\"rbf\", \"linear\"]\n",
    "}, cv=5, return_train_score=False, n_iter=2) # trying only 2 combinations here above we tried total 6\n",
    "\n",
    "rs_classifier.fit(iris.data, iris.target)\n",
    "pd.DataFrame(rs_classifier.cv_results_)[[\"param_C\", \"param_kernel\", \"mean_test_score\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now, Which Model to Choose?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_params = {\n",
    "    \"svm\" : {\n",
    "        \"model\" : svm.SVC(gamma=\"auto\"),\n",
    "        \"params\" : {\n",
    "            \"C\" : [1,10,20],\n",
    "            \"kernel\" : [\"rbf\", \"linear\"]\n",
    "        }\n",
    "    },\n",
    "    \"random_forest\" : {\n",
    "        \"model\" : RandomForestClassifier(),\n",
    "        \"params\" : {\n",
    "            \"n_estimators\" : [1,5,10]\n",
    "        }\n",
    "    },\n",
    "    \"logictic_regression\" : {\n",
    "        \"model\" : LogisticRegression(solver=\"liblinear\", multi_class=\"auto\"),\n",
    "        \"params\" : {\n",
    "            \"C\" : [1,5,10]\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# suppress warning for clean notebook\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = []\n",
    "\n",
    "for model_name, mp in model_params.items():\n",
    "    clf = GridSearchCV(mp[\"model\"], mp[\"params\"], cv=5, return_train_score=False)\n",
    "    clf.fit(iris.data, iris.target)\n",
    "    scores.append({\n",
    "        \"model\" : model_name,\n",
    "        \"best_score\" : clf.best_score_,\n",
    "        \"best_params\" : clf.best_params_\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>best_score</th>\n",
       "      <th>best_params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>svm</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>{'C': 1, 'kernel': 'rbf'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>random_forest</td>\n",
       "      <td>0.953333</td>\n",
       "      <td>{'n_estimators': 5}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>logictic_regression</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>{'C': 5}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  best_score                best_params\n",
       "0                  svm    0.980000  {'C': 1, 'kernel': 'rbf'}\n",
       "1        random_forest    0.953333        {'n_estimators': 5}\n",
       "2  logictic_regression    0.966667                   {'C': 5}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(scores, columns=[\"model\", \"best_score\", \"best_params\"])\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".ml_algo_py_venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}