14.2.6.주요 교차 검증 방법.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc3a1c7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 필요한 패키지 설치\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold, LeaveOneOut, StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fc8c31e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RI</th>\n",
       "      <th>Na</th>\n",
       "      <th>Mg</th>\n",
       "      <th>Al</th>\n",
       "      <th>Si</th>\n",
       "      <th>K</th>\n",
       "      <th>Ca</th>\n",
       "      <th>Ba</th>\n",
       "      <th>Fe</th>\n",
       "      <th>Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.52101</td>\n",
       "      <td>13.64</td>\n",
       "      <td>4.49</td>\n",
       "      <td>1.10</td>\n",
       "      <td>71.78</td>\n",
       "      <td>0.06</td>\n",
       "      <td>8.75</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.51761</td>\n",
       "      <td>13.89</td>\n",
       "      <td>3.60</td>\n",
       "      <td>1.36</td>\n",
       "      <td>72.73</td>\n",
       "      <td>0.48</td>\n",
       "      <td>7.83</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.51618</td>\n",
       "      <td>13.53</td>\n",
       "      <td>3.55</td>\n",
       "      <td>1.54</td>\n",
       "      <td>72.99</td>\n",
       "      <td>0.39</td>\n",
       "      <td>7.78</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.51766</td>\n",
       "      <td>13.21</td>\n",
       "      <td>3.69</td>\n",
       "      <td>1.29</td>\n",
       "      <td>72.61</td>\n",
       "      <td>0.57</td>\n",
       "      <td>8.22</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.51742</td>\n",
       "      <td>13.27</td>\n",
       "      <td>3.62</td>\n",
       "      <td>1.24</td>\n",
       "      <td>73.08</td>\n",
       "      <td>0.55</td>\n",
       "      <td>8.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        RI     Na    Mg    Al     Si     K    Ca   Ba   Fe  Type\n",
       "0  1.52101  13.64  4.49  1.10  71.78  0.06  8.75  0.0  0.0     1\n",
       "1  1.51761  13.89  3.60  1.36  72.73  0.48  7.83  0.0  0.0     1\n",
       "2  1.51618  13.53  3.55  1.54  72.99  0.39  7.78  0.0  0.0     1\n",
       "3  1.51766  13.21  3.69  1.29  72.61  0.57  8.22  0.0  0.0     1\n",
       "4  1.51742  13.27  3.62  1.24  73.08  0.55  8.07  0.0  0.0     1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 데이터 불러오기\n",
    "# https://www.kaggle.com/datasets/uciml/glass\n",
    "df = pd.read_csv(\"datasets/glass.csv\")\n",
    "\n",
    "# 데이터 샘플 확인\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b6f13a8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:676: UserWarning: The least populated class in y has only 5 members, which is less than n_splits=7.\n",
      "  warnings.warn(\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k-fold 교차 검증 SCORE :  [0.89473684 0.68421053 0.61111111 0.55555556 0.94444444 0.72222222\n",
      " 0.88888889]\n",
      "k-fold 교차 검증 SCORE 평균 : 0.76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n",
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:680: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  estimator.fit(X_train, y_train, **fit_params)\n"
     ]
    }
   ],
   "source": [
    "# k-fold 방법 1\n",
    "\n",
    "# 종속변수 문자형 변환\n",
    "df[\"Type_str\"]=df[\"Type\"].apply(str)\n",
    "\n",
    "# 독립변수, 종속변수 분리\n",
    "df_x1 = df[['RI','Na','Mg','Al','Si','K','Ca','Ba','Fe']]\n",
    "df_y1 = df[['Type_str']]\n",
    "\n",
    "# 기본 모델 설정\n",
    "rnf_model = RandomForestClassifier(n_estimators=100, max_depth=5,random_state=0)\n",
    "\n",
    "# 학습셋과 테스트셋 분리하여 생성(6:4)\n",
    "x_train, x_test, y_train, y_test = train_test_split(\n",
    "    df_x1,df_y1,test_size=0.4,random_state=10)\n",
    "\n",
    "# cross_val_score 함수로 k-fold 성능 측정\n",
    "kfold_scores_1 = cross_val_score(rnf_model, x_train, y_train, cv = 7)\n",
    "\n",
    "print('k-fold 교차 검증 SCORE : ', kfold_scores_1)\n",
    "print(\"k-fold 교차 검증 SCORE 평균 : {:.2f}\".format(kfold_scores_1.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9f111ee6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k-fold 교차 검증 SCORE 평균 : 0.7526881720430108\n"
     ]
    }
   ],
   "source": [
    "# k-fold 방법 2\n",
    "\n",
    "# 종속변수 문자형 변환 및 분리\n",
    "df[\"Type_str\"]=df[\"Type\"].apply(str)\n",
    "df_x2 = np.array(df[['RI','Na','Mg','Al','Si','K','Ca','Ba','Fe']])\n",
    "df_y2 = df['Type_str']\n",
    "\n",
    "# k-fold 설정\n",
    "kfold = KFold(n_splits=7, shuffle = True, random_state = 37)\n",
    "kfold_scores_2_mean = []\n",
    "fold_idx = 0\n",
    "\n",
    "# k-fold 수행\n",
    "for train_idx, test_idx in kfold.split(df_x2):\n",
    "    \n",
    "    train_x, train_y = df_x2[train_idx], df_y2[train_idx]\n",
    "    test_x, test_y = df_x2[test_idx], df_y2[test_idx]\n",
    "    \n",
    "    # 기본 모델 설정\n",
    "    rnf_model = RandomForestClassifier(n_estimators=100, max_depth=5,random_state=0)    \n",
    "    rnf_model.fit(train_x, train_y)\n",
    "    pred_y = rnf_model.predict(test_x)\n",
    "    kfold_scores_2 = accuracy_score(test_y, pred_y)    \n",
    "    fold_idx += 1    \n",
    "    kfold_scores_2_mean.append(kfold_scores_2)\n",
    "\n",
    "print(f\"k-fold 교차 검증 SCORE 평균 : {np.mean(kfold_scores_2_mean)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "06b3edc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "테스트 셋 전체 관측치 수 :  30\n",
      "LOOCV 검증 분할 횟수 :  30\n",
      "LOOCV 교차 검증 SCORE 평균 : 0.70\n"
     ]
    }
   ],
   "source": [
    "# LOOCV (Leave-one-out Cross-validation)\n",
    "\n",
    "# 기본 모델 설정\n",
    "rnf_model = RandomForestClassifier(n_estimators=100, max_depth=5,random_state=0)\n",
    "rnf_model.fit(train_x, train_y)\n",
    "pred_y = rnf_model.predict(test_x)\n",
    "\n",
    "loocv = LeaveOneOut()\n",
    "loocv_scores = cross_val_score(rnf_model,test_x,test_y, cv = loocv)\n",
    "\n",
    "print(\"테스트 셋 전체 관측치 수 : \", len(test_y))\n",
    "print(\"LOOCV 검증 분할 횟수 : \", len(loocv_scores))\n",
    "print(\"LOOCV 교차 검증 SCORE 평균 : {:.2f}\".format(loocv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "56ae8000",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "전체 데이터셋 범주 별 관측치 수 :\n",
      " 2    76\n",
      "1    70\n",
      "7    29\n",
      "3    17\n",
      "5    13\n",
      "6     9\n",
      "Name: Type_str, dtype: int64\n",
      "분할 1\n",
      "학습 셋 범주 별 관측치 수:\n",
      " 2    51\n",
      "1    46\n",
      "7    19\n",
      "3    11\n",
      "5     9\n",
      "6     6\n",
      "Name: Type_str, dtype: int64\n",
      "검증 셋 범주 별 관측치 수:\n",
      " 2    25\n",
      "1    24\n",
      "7    10\n",
      "3     6\n",
      "5     4\n",
      "6     3\n",
      "Name: Type_str, dtype: int64\n",
      "분할 2\n",
      "학습 셋 범주 별 관측치 수:\n",
      " 2    50\n",
      "1    47\n",
      "7    20\n",
      "3    12\n",
      "5     8\n",
      "6     6\n",
      "Name: Type_str, dtype: int64\n",
      "검증 셋 범주 별 관측치 수:\n",
      " 2    26\n",
      "1    23\n",
      "7     9\n",
      "3     5\n",
      "5     5\n",
      "6     3\n",
      "Name: Type_str, dtype: int64\n",
      "분할 3\n",
      "학습 셋 범주 별 관측치 수:\n",
      " 2    51\n",
      "1    47\n",
      "7    19\n",
      "3    11\n",
      "5     9\n",
      "6     6\n",
      "Name: Type_str, dtype: int64\n",
      "검증 셋 범주 별 관측치 수:\n",
      " 2    25\n",
      "1    23\n",
      "7    10\n",
      "3     6\n",
      "5     4\n",
      "6     3\n",
      "Name: Type_str, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\angry\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:676: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=3.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stratified K-fold 교차 검증 SCORE 평균 : 0.67\n"
     ]
    }
   ],
   "source": [
    "# Stratified K-fold Cross Validation\n",
    "\n",
    "# Stratified K-fold 교차검증 설정\n",
    "strati = StratifiedKFold(n_splits=3)\n",
    "\n",
    "n_iter=0\n",
    "print(\"전체 데이터셋 범주 별 관측치 수 :\\n\",df_y2.value_counts())\n",
    "for train_index, test_index in strati.split(df_x2,df_y2):\n",
    "    n_iter +=1\n",
    "    strati_train_y = df_y2.iloc[train_index]\n",
    "    strati_test_y = df_y2.iloc[test_index]\n",
    "    print('분할 {0}'.format(n_iter))\n",
    "    print('학습 셋 범주 별 관측치 수:\\n', strati_train_y.value_counts())\n",
    "    print('검증 셋 범주 별 관측치 수:\\n', strati_test_y.value_counts())\n",
    "\n",
    "strati_scores = cross_val_score(rnf_model,test_x,test_y, cv = strati)\n",
    "print(\"Stratified K-fold 교차 검증 SCORE 평균 : {:.2f}\".format(strati_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8cb11f58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "각 TRIAL 별 SCORE 평균 : \n",
      " [0.71059432 0.71077889 0.67884828 0.72646733 0.71834625 0.69545958\n",
      " 0.72683647 0.68752307 0.6720192  0.68014027 0.72757475 0.7113326\n",
      " 0.68014027 0.7113326  0.66426726 0.68807678 0.73366556 0.73421927\n",
      " 0.67109635 0.70302695]\n",
      "전체 TRIAL SCORE 평균 : 0.70\n",
      "최대 TRIAL SCORE : 0.73\n",
      "최적 하이퍼 파라미터 : \n",
      " {'max_depth': 4, 'min_samples_split': 2}\n"
     ]
    }
   ],
   "source": [
    "# Nested Cross Validation 및 Grid Search Cross Validation\n",
    "\n",
    "# 학습셋과 테스트셋 분리하여 생성(6:4)\n",
    "x_train, x_test, y_train, y_test = train_test_split(\n",
    "    df_x2,df_y2,test_size=0.4,random_state=10)\n",
    "\n",
    "# 모델 시도 횟수 설정\n",
    "NUM_TRIALS = 20\n",
    "\n",
    "# 그리드 하이퍼 파라미터 설정\n",
    "hp_para = {'max_depth':[2,3,4], 'min_samples_split':[2,3]}\n",
    "\n",
    "# 스코어를 array로 저장\n",
    "nested_scores = np.zeros(NUM_TRIALS)\n",
    "\n",
    "# 교차 검증 반복 수행\n",
    "for i in range(NUM_TRIALS):\n",
    "\n",
    "    inner_loop = KFold(n_splits=3, shuffle=True, random_state=i)\n",
    "    outer_loop = KFold(n_splits=3, shuffle=True, random_state=i)\n",
    "\n",
    "    # Nested Cross Validation 하이퍼 파라미터 최적화\n",
    "    gs_cv = GridSearchCV(rnf_model, param_grid=hp_para, cv=inner_loop)\n",
    "    nested_score = cross_val_score(gs_cv, X=x_train, y=y_train, cv=outer_loop)\n",
    "    nested_scores[i] = nested_score.mean()\n",
    "\n",
    "# 테스트셋에 모델 적용\n",
    "gs_cv.fit(x_test, y_test)\n",
    "\n",
    "print(\"각 TRIAL 별 SCORE 평균 : \\n\", nested_scores)\n",
    "print(\"전체 TRIAL SCORE 평균 : {:.2f}\".format(nested_scores.mean()))\n",
    "print(\"최대 TRIAL SCORE : {:.2f}\".format(nested_scores.max()))\n",
    "print(\"최적 하이퍼 파라미터 : \\n\", gs_cv.best_params_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}