Data Cleaning Techniques/Data Cleaning_Missing value imputation sklearn II.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.5 64-bit"
  },
  "interpreter": {
   "hash": "21e1b569344c375d63bdfc9656685154aa52feb3c0e941d62c4e6de15ab1dc2d"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Data Cleaning\n",
    "## Missing value imputation using Scikit-Learn\n",
    "### Different strategy for Different variables"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "shape of train df =  (1460, 81)\nshape of test df =  (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(r\"K:\\KaustavR25\\SKILLZ\\Data Science & Analysis, ML\\Data-Science_ML----Projects-Resources-main\\Study & Resource\\Data Cleaning\\train.csv\")\n",
    "test = pd.read_csv(r\"K:\\KaustavR25\\SKILLZ\\Data Science & Analysis, ML\\Data-Science_ML----Projects-Resources-main\\Study & Resource\\Data Cleaning\\test.csv\")\n",
    "print(\"shape of train df = \", train.shape)\n",
    "print(\"shape of test df = \", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "shape of X_train df =  (1460, 80)\nshape of y_train df =  (1460,)\nshape of X_test df =  (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "X_train = train.drop(columns='SalePrice', axis=1)\n",
    "y_train = train['SalePrice']\n",
    "X_test = test.copy()\n",
    "print(\"shape of X_train df = \", X_train.shape)\n",
    "print(\"shape of y_train df = \", y_train.shape)\n",
    "print(\"shape of X_test df = \", X_test.shape)"
   ]
  },
  {
   "source": [
    "## Missing value Imputation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Id                 0\n",
       "MSSubClass         0\n",
       "MSZoning           0\n",
       "LotFrontage      259\n",
       "LotArea            0\n",
       "                ... \n",
       "MiscVal            0\n",
       "MoSold             0\n",
       "YrSold             0\n",
       "SaleType           0\n",
       "SaleCondition      0\n",
       "Length: 80, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "isnull_sum = X_train.isnull().sum()\n",
    "isnull_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_vars = X_train.select_dtypes(include=['int64', 'float64']).columns\n",
    "num_vars_miss = [var for var in num_vars if isnull_sum[var]>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['LotFrontage', 'MasVnrArea', 'GarageYrBlt']"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "num_vars_miss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Alley',\n",
       " 'MasVnrType',\n",
       " 'BsmtQual',\n",
       " 'BsmtCond',\n",
       " 'BsmtExposure',\n",
       " 'BsmtFinType1',\n",
       " 'BsmtFinType2',\n",
       " 'Electrical',\n",
       " 'FireplaceQu',\n",
       " 'GarageType',\n",
       " 'GarageFinish',\n",
       " 'GarageQual',\n",
       " 'GarageCond',\n",
       " 'PoolQC',\n",
       " 'Fence',\n",
       " 'MiscFeature']"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "cat_vars = X_train.select_dtypes(include=['O']).columns\n",
    "cat_vars_miss = [var for var in cat_vars if isnull_sum[var]>0]\n",
    "cat_vars_miss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_var_mean = [\"LotFrontage\"]\n",
    "num_var_median = ['MasVnrArea', 'GarageYrBlt']\n",
    "cat_vars_mode = ['Alley',\n",
    " 'MasVnrType',\n",
    " 'BsmtQual',\n",
    " 'BsmtCond',\n",
    " 'BsmtExposure',\n",
    " 'BsmtFinType1',\n",
    " 'BsmtFinType2',\n",
    " 'Electrical',\n",
    " 'FireplaceQu',]\n",
    "cat_vars_missing = ['GarageType',\n",
    " 'GarageFinish',\n",
    " 'GarageQual',\n",
    " 'GarageCond',\n",
    " 'PoolQC',\n",
    " 'Fence',\n",
    " 'MiscFeature']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_var_mean_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"mean\"))])\n",
    "num_var_median_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"median\"))])\n",
    "cat_vars_mode_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"most_frequent\"))])\n",
    "cat_vars_missing_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\"))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = ColumnTransformer(transformers=[(\"mean_imputer\", num_var_mean_imputer, num_var_mean),\n",
    "                                (\"median_imputer\", num_var_median_imputer, num_var_median),\n",
    "                               (\"mode_imputer\", cat_vars_mode_imputer, cat_vars_mode),\n",
    "                               (\"missing_imputer\", cat_vars_missing_imputer,cat_vars_missing)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "ColumnTransformer(transformers=[('mean_imputer',\n",
       "                                 Pipeline(steps=[('imputer', SimpleImputer())]),\n",
       "                                 ['LotFrontage']),\n",
       "                                ('median_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(strategy='median'))]),\n",
       "                                 ['MasVnrArea', 'GarageYrBlt']),\n",
       "                                ('mode_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(strategy='most_frequent'))]),\n",
       "                                 ['Alley', 'MasVnrType', 'BsmtQual', 'BsmtCond',\n",
       "                                  'BsmtExposure', 'BsmtFinType1',\n",
       "                                  'BsmtFinType2', 'Electrical',\n",
       "                                  'FireplaceQu']),\n",
       "                                ('missing_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(fill_value='missing',\n",
       "                                                                strategy='constant'))]),\n",
       "                                 ['GarageType', 'GarageFinish', 'GarageQual',\n",
       "                                  'GarageCond', 'PoolQC', 'Fence',\n",
       "                                  'MiscFeature'])])"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "preprocessor.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<bound method ColumnTransformer.transform of ColumnTransformer(transformers=[('mean_imputer',\n",
       "                                 Pipeline(steps=[('imputer', SimpleImputer())]),\n",
       "                                 ['LotFrontage']),\n",
       "                                ('median_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(strategy='median'))]),\n",
       "                                 ['MasVnrArea', 'GarageYrBlt']),\n",
       "                                ('mode_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(strategy='most_frequent'))]),\n",
       "                                 ['Alley', 'MasVnrType', 'BsmtQual', 'BsmtCond',\n",
       "                                  'BsmtExposure', 'BsmtFinType1',\n",
       "                                  'BsmtFinType2', 'Electrical',\n",
       "                                  'FireplaceQu']),\n",
       "                                ('missing_imputer',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(fill_value='missing',\n",
       "                                                                strategy='constant'))]),\n",
       "                                 ['GarageType', 'GarageFinish', 'GarageQual',\n",
       "                                  'GarageCond', 'PoolQC', 'Fence',\n",
       "                                  'MiscFeature'])])>"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "preprocessor.transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([70.04995837])"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "preprocessor.named_transformers_[\"mean_imputer\"].named_steps[\"imputer\"].statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "70.04995836802665"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "train[\"LotFrontage\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array(['Grvl', 'None', 'TA', 'TA', 'No', 'Unf', 'Unf', 'SBrkr', 'Gd'],\n",
       "      dtype=object)"
      ]
     },
     "metadata": {},
     "execution_count": 15
    }
   ],
   "source": [
    "preprocessor.named_transformers_[\"mode_imputer\"].named_steps[\"imputer\"].statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array(['missing', 'missing', 'missing', 'missing', 'missing', 'missing',\n",
       "       'missing'], dtype=object)"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ],
   "source": [
    "preprocessor.named_transformers_[\"missing_imputer\"].named_steps[\"imputer\"].statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_clean = preprocessor.transform(X_train)\n",
    "X_test_clean = preprocessor.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[65.0, 196.0, 2003.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [80.0, 0.0, 1976.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [68.0, 162.0, 2001.0, ..., 'missing', 'missing', 'missing'],\n",
       "       ...,\n",
       "       [66.0, 0.0, 1941.0, ..., 'missing', 'GdPrv', 'Shed'],\n",
       "       [68.0, 0.0, 1950.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [75.0, 0.0, 1965.0, ..., 'missing', 'missing', 'missing']],\n",
       "      dtype=object)"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "X_train_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('mean_imputer',\n",
       "  Pipeline(steps=[('imputer', SimpleImputer())]),\n",
       "  ['LotFrontage']),\n",
       " ('median_imputer',\n",
       "  Pipeline(steps=[('imputer', SimpleImputer(strategy='median'))]),\n",
       "  ['MasVnrArea', 'GarageYrBlt']),\n",
       " ('mode_imputer',\n",
       "  Pipeline(steps=[('imputer', SimpleImputer(strategy='most_frequent'))]),\n",
       "  ['Alley',\n",
       "   'MasVnrType',\n",
       "   'BsmtQual',\n",
       "   'BsmtCond',\n",
       "   'BsmtExposure',\n",
       "   'BsmtFinType1',\n",
       "   'BsmtFinType2',\n",
       "   'Electrical',\n",
       "   'FireplaceQu']),\n",
       " ('missing_imputer',\n",
       "  Pipeline(steps=[('imputer',\n",
       "                   SimpleImputer(fill_value='missing', strategy='constant'))]),\n",
       "  ['GarageType',\n",
       "   'GarageFinish',\n",
       "   'GarageQual',\n",
       "   'GarageCond',\n",
       "   'PoolQC',\n",
       "   'Fence',\n",
       "   'MiscFeature']),\n",
       " ('remainder',\n",
       "  'drop',\n",
       "  [0,\n",
       "   1,\n",
       "   2,\n",
       "   4,\n",
       "   5,\n",
       "   7,\n",
       "   8,\n",
       "   9,\n",
       "   10,\n",
       "   11,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   20,\n",
       "   21,\n",
       "   22,\n",
       "   23,\n",
       "   24,\n",
       "   27,\n",
       "   28,\n",
       "   29,\n",
       "   34,\n",
       "   36,\n",
       "   37,\n",
       "   38,\n",
       "   39,\n",
       "   40,\n",
       "   41,\n",
       "   43,\n",
       "   44,\n",
       "   45,\n",
       "   46,\n",
       "   47,\n",
       "   48,\n",
       "   49,\n",
       "   50,\n",
       "   51,\n",
       "   52,\n",
       "   53,\n",
       "   54,\n",
       "   55,\n",
       "   56,\n",
       "   61,\n",
       "   62,\n",
       "   65,\n",
       "   66,\n",
       "   67,\n",
       "   68,\n",
       "   69,\n",
       "   70,\n",
       "   71,\n",
       "   75,\n",
       "   76,\n",
       "   77,\n",
       "   78,\n",
       "   79])]"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "preprocessor.transformers_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_clean_miss_var = pd.DataFrame(X_train_clean, columns=num_var_mean+num_var_median+cat_vars_mode+cat_vars_missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "  LotFrontage MasVnrArea GarageYrBlt Alley MasVnrType BsmtQual BsmtCond  \\\n",
       "0        65.0      196.0      2003.0  Grvl    BrkFace       Gd       TA   \n",
       "1        80.0        0.0      1976.0  Grvl       None       Gd       TA   \n",
       "2        68.0      162.0      2001.0  Grvl    BrkFace       Gd       TA   \n",
       "3        60.0        0.0      1998.0  Grvl       None       TA       Gd   \n",
       "4        84.0      350.0      2000.0  Grvl    BrkFace       Gd       TA   \n",
       "\n",
       "  BsmtExposure BsmtFinType1 BsmtFinType2 Electrical FireplaceQu GarageType  \\\n",
       "0           No          GLQ          Unf      SBrkr          Gd     Attchd   \n",
       "1           Gd          ALQ          Unf      SBrkr          TA     Attchd   \n",
       "2           Mn          GLQ          Unf      SBrkr          TA     Attchd   \n",
       "3           No          ALQ          Unf      SBrkr          Gd     Detchd   \n",
       "4           Av          GLQ          Unf      SBrkr          TA     Attchd   \n",
       "\n",
       "  GarageFinish GarageQual GarageCond   PoolQC    Fence MiscFeature  \n",
       "0          RFn         TA         TA  missing  missing     missing  \n",
       "1          RFn         TA         TA  missing  missing     missing  \n",
       "2          RFn         TA         TA  missing  missing     missing  \n",
       "3          Unf         TA         TA  missing  missing     missing  \n",
       "4          RFn         TA         TA  missing  missing     missing  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>LotFrontage</th>\n      <th>MasVnrArea</th>\n      <th>GarageYrBlt</th>\n      <th>Alley</th>\n      <th>MasVnrType</th>\n      <th>BsmtQual</th>\n      <th>BsmtCond</th>\n      <th>BsmtExposure</th>\n      <th>BsmtFinType1</th>\n      <th>BsmtFinType2</th>\n      <th>Electrical</th>\n      <th>FireplaceQu</th>\n      <th>GarageType</th>\n      <th>GarageFinish</th>\n      <th>GarageQual</th>\n      <th>GarageCond</th>\n      <th>PoolQC</th>\n      <th>Fence</th>\n      <th>MiscFeature</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>65.0</td>\n      <td>196.0</td>\n      <td>2003.0</td>\n      <td>Grvl</td>\n      <td>BrkFace</td>\n      <td>Gd</td>\n      <td>TA</td>\n      <td>No</td>\n      <td>GLQ</td>\n      <td>Unf</td>\n      <td>SBrkr</td>\n      <td>Gd</td>\n      <td>Attchd</td>\n      <td>RFn</td>\n      <td>TA</td>\n      <td>TA</td>\n      <td>missing</td>\n      <td>missing</td>\n      <td>missing</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>80.0</td>\n      <td>0.0</td>\n      <td>1976.0</td>\n      <td>Grvl</td>\n      <td>None</td>\n      <td>Gd</td>\n      <td>TA</td>\n      <td>Gd</td>\n      <td>ALQ</td>\n      <td>Unf</td>\n      <td>SBrkr</td>\n      <td>TA</td>\n      <td>Attchd</td>\n      <td>RFn</td>\n      <td>TA</td>\n      <td>TA</td>\n      <td>missing</td>\n      <td>missing</td>\n      <td>missing</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>68.0</td>\n      <td>162.0</td>\n      <td>2001.0</td>\n      <td>Grvl</td>\n      <td>BrkFace</td>\n      <td>Gd</td>\n      <td>TA</td>\n      <td>Mn</td>\n      <td>GLQ</td>\n      <td>Unf</td>\n      <td>SBrkr</td>\n      <td>TA</td>\n      <td>Attchd</td>\n      <td>RFn</td>\n      <td>TA</td>\n      <td>TA</td>\n      <td>missing</td>\n      <td>missing</td>\n      <td>missing</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>60.0</td>\n      <td>0.0</td>\n      <td>1998.0</td>\n      <td>Grvl</td>\n      <td>None</td>\n      <td>TA</td>\n      <td>Gd</td>\n      <td>No</td>\n      <td>ALQ</td>\n      <td>Unf</td>\n      <td>SBrkr</td>\n      <td>Gd</td>\n      <td>Detchd</td>\n      <td>Unf</td>\n      <td>TA</td>\n      <td>TA</td>\n      <td>missing</td>\n      <td>missing</td>\n      <td>missing</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>84.0</td>\n      <td>350.0</td>\n      <td>2000.0</td>\n      <td>Grvl</td>\n      <td>BrkFace</td>\n      <td>Gd</td>\n      <td>TA</td>\n      <td>Av</td>\n      <td>GLQ</td>\n      <td>Unf</td>\n      <td>SBrkr</td>\n      <td>TA</td>\n      <td>Attchd</td>\n      <td>RFn</td>\n      <td>TA</td>\n      <td>TA</td>\n      <td>missing</td>\n      <td>missing</td>\n      <td>missing</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "X_train_clean_miss_var.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "source": [
    "X_train_clean_miss_var.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Grvl    50\n",
       "Pave    41\n",
       "Name: Alley, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "train[\"Alley\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Grvl    1419\n",
       "Pave      41\n",
       "Name: Alley, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "X_train_clean_miss_var[\"Alley\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "missing    1406\n",
       "Shed         49\n",
       "Gar2          2\n",
       "Othr          2\n",
       "TenC          1\n",
       "Name: MiscFeature, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "X_train_clean_miss_var[\"MiscFeature\"].value_counts()"
   ]
  },
  {
   "source": [
    "## Create Clean X_train DataFrame with call variables"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "remainder_vars_index = [0,\n",
    "   1,\n",
    "   2,\n",
    "   4,\n",
    "   5,\n",
    "   7,\n",
    "   8,\n",
    "   9,\n",
    "   10,\n",
    "   11,\n",
    "   12,\n",
    "   13,\n",
    "   14,\n",
    "   15,\n",
    "   16,\n",
    "   17,\n",
    "   18,\n",
    "   19,\n",
    "   20,\n",
    "   21,\n",
    "   22,\n",
    "   23,\n",
    "   24,\n",
    "   27,\n",
    "   28,\n",
    "   29,\n",
    "   34,\n",
    "   36,\n",
    "   37,\n",
    "   38,\n",
    "   39,\n",
    "   40,\n",
    "   41,\n",
    "   43,\n",
    "   44,\n",
    "   45,\n",
    "   46,\n",
    "   47,\n",
    "   48,\n",
    "   49,\n",
    "   50,\n",
    "   51,\n",
    "   52,\n",
    "   53,\n",
    "   54,\n",
    "   55,\n",
    "   56,\n",
    "   61,\n",
    "   62,\n",
    "   65,\n",
    "   66,\n",
    "   67,\n",
    "   68,\n",
    "   69,\n",
    "   70,\n",
    "   71,\n",
    "   75,\n",
    "   76,\n",
    "   77,\n",
    "   78,\n",
    "   79]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['Id',\n",
       " 'MSSubClass',\n",
       " 'MSZoning',\n",
       " 'LotArea',\n",
       " 'Street',\n",
       " 'LotShape',\n",
       " 'LandContour',\n",
       " 'Utilities',\n",
       " 'LotConfig',\n",
       " 'LandSlope',\n",
       " 'Neighborhood',\n",
       " 'Condition1',\n",
       " 'Condition2',\n",
       " 'BldgType',\n",
       " 'HouseStyle',\n",
       " 'OverallQual',\n",
       " 'OverallCond',\n",
       " 'YearBuilt',\n",
       " 'YearRemodAdd',\n",
       " 'RoofStyle',\n",
       " 'RoofMatl',\n",
       " 'Exterior1st',\n",
       " 'Exterior2nd',\n",
       " 'ExterQual',\n",
       " 'ExterCond',\n",
       " 'Foundation',\n",
       " 'BsmtFinSF1',\n",
       " 'BsmtFinSF2',\n",
       " 'BsmtUnfSF',\n",
       " 'TotalBsmtSF',\n",
       " 'Heating',\n",
       " 'HeatingQC',\n",
       " 'CentralAir',\n",
       " '1stFlrSF',\n",
       " '2ndFlrSF',\n",
       " 'LowQualFinSF',\n",
       " 'GrLivArea',\n",
       " 'BsmtFullBath',\n",
       " 'BsmtHalfBath',\n",
       " 'FullBath',\n",
       " 'HalfBath',\n",
       " 'BedroomAbvGr',\n",
       " 'KitchenAbvGr',\n",
       " 'KitchenQual',\n",
       " 'TotRmsAbvGrd',\n",
       " 'Functional',\n",
       " 'Fireplaces',\n",
       " 'GarageCars',\n",
       " 'GarageArea',\n",
       " 'PavedDrive',\n",
       " 'WoodDeckSF',\n",
       " 'OpenPorchSF',\n",
       " 'EnclosedPorch',\n",
       " '3SsnPorch',\n",
       " 'ScreenPorch',\n",
       " 'PoolArea',\n",
       " 'MiscVal',\n",
       " 'MoSold',\n",
       " 'YrSold',\n",
       " 'SaleType',\n",
       " 'SaleCondition']"
      ]
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "# get no missing values variables name using there index\n",
    "remainder_vars = [isnull_sum.keys()[var_index] for var_index in remainder_vars_index]\n",
    "remainder_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "metadata": {},
     "execution_count": 28
    }
   ],
   "source": [
    "len(remainder_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatinate X_train_clean_miss_var df and remainder_vars\n",
    "X_train =  pd.concat([X_train_clean_miss_var,train[remainder_vars]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(1460, 80)"
      ]
     },
     "metadata": {},
     "execution_count": 30
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "X_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(1459, 19)"
      ]
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "# Create test DataFrame with missing value imputed variables\n",
    "X_test_clean_miss_var = pd.DataFrame(X_test_clean, columns=num_var_mean+num_var_median+cat_vars_mode+cat_vars_missing)\n",
    "X_test_clean_miss_var.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "execution_count": 33
    }
   ],
   "source": [
    "X_test_clean_miss_var.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(1459, 80)"
      ]
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "# concatinate X_test_clean_miss_var df and remainder_vars\n",
    "X_test =  pd.concat([X_test_clean_miss_var,test[remainder_vars]], axis=1)\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "X_test.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "LotFrontage      0\n",
       "MasVnrArea       0\n",
       "GarageYrBlt      0\n",
       "Alley            0\n",
       "MasVnrType       0\n",
       "                ..\n",
       "MiscVal          0\n",
       "MoSold           0\n",
       "YrSold           0\n",
       "SaleType         1\n",
       "SaleCondition    0\n",
       "Length: 80, dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "isnull_sum_test = X_test.isnull().sum()\n",
    "isnull_sum_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['BsmtFinSF1',\n",
       " 'BsmtFinSF2',\n",
       " 'BsmtUnfSF',\n",
       " 'TotalBsmtSF',\n",
       " 'BsmtFullBath',\n",
       " 'BsmtHalfBath',\n",
       " 'GarageCars',\n",
       " 'GarageArea']"
      ]
     },
     "metadata": {},
     "execution_count": 37
    }
   ],
   "source": [
    "# finding the numerical variable which have mising value\n",
    "num_vars_test = X_test.select_dtypes(include=[\"int64\", \"float64\"]).columns\n",
    "num_vars_miss_test = [var for var in num_vars_test if isnull_sum_test[var]>0]\n",
    "num_vars_miss_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['MSZoning',\n",
       " 'Utilities',\n",
       " 'Exterior1st',\n",
       " 'Exterior2nd',\n",
       " 'KitchenQual',\n",
       " 'Functional',\n",
       " 'SaleType']"
      ]
     },
     "metadata": {},
     "execution_count": 38
    }
   ],
   "source": [
    "# finding the categorical variable which have mising value\n",
    "cat_vars_test = X_test.select_dtypes(include=[\"O\"]).columns\n",
    "cat_vars_miss_test = [var for var in cat_vars_test if isnull_sum_test[var]>0]\n",
    "cat_vars_miss_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}