Data Cleaning Techniques/Missing value imputation using Scikit-Learn III .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning\n",
    "## Missing value imputation using Scikit-Learn\n",
    "### Different strategy for different variables(Numerical & Categorical) with Scikit-Learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of train df =  (1460, 81)\n",
      "Shape of test df = (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "train = pd.read_csv(r\"G:\\DataSet\\House Price Prediction\\train.csv\")\n",
    "test = pd.read_csv(r\"G:\\DataSet\\House Price Prediction\\test.csv\")\n",
    "print(\"Shape of train df = \", train.shape)\n",
    "print(\"Shape of test df =\", test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X_train =  (1460, 80)\n",
      "Shape of y_train =  (1460,)\n",
      "Shape of X_test = (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "X_train = train.drop(columns=\"SalePrice\", axis=1)\n",
    "y_train = train[\"SalePrice\"]\n",
    "X_test = test.copy()\n",
    "print(\"Shape of X_train = \", X_train.shape)\n",
    "print(\"Shape of y_train = \", y_train.shape)\n",
    "print(\"Shape of X_test =\", X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing value imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id                 0\n",
       "MSSubClass         0\n",
       "MSZoning           0\n",
       "LotFrontage      259\n",
       "LotArea            0\n",
       "                ... \n",
       "MiscVal            0\n",
       "MoSold             0\n",
       "YrSold             0\n",
       "SaleType           0\n",
       "SaleCondition      0\n",
       "Length: 80, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isnull_sum = X_train.isnull().sum()\n",
    "isnull_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finding the numerical variable which have mising value\n",
    "num_vars = X_train.select_dtypes(include=[\"int64\", \"float64\"]).columns\n",
    "num_vars_miss = [var for var in num_vars if isnull_sum[var]>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LotFrontage', 'MasVnrArea', 'GarageYrBlt']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_vars_miss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alley',\n",
       " 'MasVnrType',\n",
       " 'BsmtQual',\n",
       " 'BsmtCond',\n",
       " 'BsmtExposure',\n",
       " 'BsmtFinType1',\n",
       " 'BsmtFinType2',\n",
       " 'Electrical',\n",
       " 'FireplaceQu',\n",
       " 'GarageType',\n",
       " 'GarageFinish',\n",
       " 'GarageQual',\n",
       " 'GarageCond',\n",
       " 'PoolQC',\n",
       " 'Fence',\n",
       " 'MiscFeature']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding the categorical variable which have mising value\n",
    "cat_vars = X_train.select_dtypes(include=[\"O\"]).columns\n",
    "cat_vars_miss = [var for var in cat_vars if isnull_sum[var]>0]\n",
    "cat_vars_miss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_var_mean = [\"LotFrontage\"]\n",
    "num_var_median = ['MasVnrArea', 'GarageYrBlt']\n",
    "cat_vars_mode = ['Alley',\n",
    " 'MasVnrType',\n",
    " 'BsmtQual',\n",
    " 'BsmtCond',\n",
    " 'BsmtExposure',\n",
    " 'BsmtFinType1',\n",
    " 'BsmtFinType2',\n",
    " 'Electrical',\n",
    " 'FireplaceQu',]\n",
    "cat_vars_missing = ['GarageType',\n",
    " 'GarageFinish',\n",
    " 'GarageQual',\n",
    " 'GarageCond',\n",
    " 'PoolQC',\n",
    " 'Fence',\n",
    " 'MiscFeature']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_var_mean_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"mean\"))])\n",
    "num_var_median_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"median\"))])\n",
    "cat_vars_mode_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"most_frequent\"))])\n",
    "cat_vars_missing_imputer = Pipeline(steps=[(\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\"))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = ColumnTransformer(transformers=[(\"mean_imputer\", num_var_mean_imputer, num_var_mean),\n",
    "                                (\"median_imputer\", num_var_median_imputer, num_var_median),\n",
    "                               (\"mode_imputer\", cat_vars_mode_imputer, cat_vars_mode),\n",
    "                               (\"missing_imputer\", cat_vars_missing_imputer,cat_vars_missing)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,\n",
       "                  transformer_weights=None,\n",
       "                  transformers=[('mean_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=False,\n",
       "                                                                copy=True,\n",
       "                                                                fill_value=None,\n",
       "                                                                missing_values=nan,\n",
       "                                                                strategy='mean',\n",
       "                                                                verbose=0))],\n",
       "                                          verbose=False),\n",
       "                                 ['LotFrontage']),\n",
       "                                ('median_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('...\n",
       "                                  'BsmtExposure', 'BsmtFinType1',\n",
       "                                  'BsmtFinType2', 'Electrical',\n",
       "                                  'FireplaceQu']),\n",
       "                                ('missing_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=False,\n",
       "                                                                copy=True,\n",
       "                                                                fill_value='missing',\n",
       "                                                                missing_values=nan,\n",
       "                                                                strategy='constant',\n",
       "                                                                verbose=0))],\n",
       "                                          verbose=False),\n",
       "                                 ['GarageType', 'GarageFinish', 'GarageQual',\n",
       "                                  'GarageCond', 'PoolQC', 'Fence',\n",
       "                                  'MiscFeature'])],\n",
       "                  verbose=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method ColumnTransformer.transform of ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,\n",
       "                  transformer_weights=None,\n",
       "                  transformers=[('mean_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=False,\n",
       "                                                                copy=True,\n",
       "                                                                fill_value=None,\n",
       "                                                                missing_values=nan,\n",
       "                                                                strategy='mean',\n",
       "                                                                verbose=0))],\n",
       "                                          verbose=False),\n",
       "                                 ['LotFrontage']),\n",
       "                                ('median_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('...\n",
       "                                  'BsmtExposure', 'BsmtFinType1',\n",
       "                                  'BsmtFinType2', 'Electrical',\n",
       "                                  'FireplaceQu']),\n",
       "                                ('missing_imputer',\n",
       "                                 Pipeline(memory=None,\n",
       "                                          steps=[('imputer',\n",
       "                                                  SimpleImputer(add_indicator=False,\n",
       "                                                                copy=True,\n",
       "                                                                fill_value='missing',\n",
       "                                                                missing_values=nan,\n",
       "                                                                strategy='constant',\n",
       "                                                                verbose=0))],\n",
       "                                          verbose=False),\n",
       "                                 ['GarageType', 'GarageFinish', 'GarageQual',\n",
       "                                  'GarageCond', 'PoolQC', 'Fence',\n",
       "                                  'MiscFeature'])],\n",
       "                  verbose=False)>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([70.04995837])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.named_transformers_[\"mean_imputer\"].named_steps[\"imputer\"].statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70.04995836802665"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[\"LotFrontage\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Grvl', 'None', 'TA', 'TA', 'No', 'Unf', 'Unf', 'SBrkr', 'Gd'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.named_transformers_[\"mode_imputer\"].named_steps[\"imputer\"].statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_clean = preprocessor.transform(X_train)\n",
    "X_test_clean = preprocessor.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[65.0, 196.0, 2003.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [80.0, 0.0, 1976.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [68.0, 162.0, 2001.0, ..., 'missing', 'missing', 'missing'],\n",
       "       ...,\n",
       "       [66.0, 0.0, 1941.0, ..., 'missing', 'GdPrv', 'Shed'],\n",
       "       [68.0, 0.0, 1950.0, ..., 'missing', 'missing', 'missing'],\n",
       "       [75.0, 0.0, 1965.0, ..., 'missing', 'missing', 'missing']],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('mean_imputer',\n",
       "  Pipeline(memory=None,\n",
       "           steps=[('imputer',\n",
       "                   SimpleImputer(add_indicator=False, copy=True, fill_value=None,\n",
       "                                 missing_values=nan, strategy='mean',\n",
       "                                 verbose=0))],\n",
       "           verbose=False),\n",
       "  ['LotFrontage']),\n",
       " ('median_imputer',\n",
       "  Pipeline(memory=None,\n",
       "           steps=[('imputer',\n",
       "                   SimpleImputer(add_indicator=False, copy=True, fill_value=None,\n",
       "                                 missing_values=nan, strategy='median',\n",
       "                                 verbose=0))],\n",
       "           verbose=False),\n",
       "  ['MasVnrArea', 'GarageYrBlt']),\n",
       " ('mode_imputer',\n",
       "  Pipeline(memory=None,\n",
       "           steps=[('imputer',\n",
       "                   SimpleImputer(add_indicator=False, copy=True, fill_value=None,\n",
       "                                 missing_values=nan, strategy='most_frequent',\n",
       "                                 verbose=0))],\n",
       "           verbose=False),\n",
       "  ['Alley',\n",
       "   'MasVnrType',\n",
       "   'BsmtQual',\n",
       "   'BsmtCond',\n",
       "   'BsmtExposure',\n",
       "   'BsmtFinType1',\n",
       "   'BsmtFinType2',\n",
       "   'Electrical',\n",
       "   'FireplaceQu']),\n",
       " ('missing_imputer',\n",
       "  Pipeline(memory=None,\n",
       "           steps=[('imputer',\n",
       "                   SimpleImputer(add_indicator=False, copy=True,\n",
       "                                 fill_value='missing', missing_values=nan,\n",
       "                                 strategy='constant', verbose=0))],\n",
       "           verbose=False),\n",
       "  ['GarageType',\n",
       "   'GarageFinish',\n",
       "   'GarageQual',\n",
       "   'GarageCond',\n",
       "   'PoolQC',\n",
       "   'Fence',\n",
       "   'MiscFeature']),\n",
       " ('remainder',\n",
       "  'drop',\n",
       "  [0,\n",
       "   1,\n",
       "   2,\n",
       "   4,\n",
       "   5,\n",
       "   7,\n",
       "   8,\n",
       "   9,\n",
       "   10,\n",
       "   11,\n",
       "   12,\n",
       "   13,\n",
       "   14,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   18,\n",
       "   19,\n",
       "   20,\n",
       "   21,\n",
       "   22,\n",
       "   23,\n",
       "   24,\n",
       "   27,\n",
       "   28,\n",
       "   29,\n",
       "   34,\n",
       "   36,\n",
       "   37,\n",
       "   38,\n",
       "   39,\n",
       "   40,\n",
       "   41,\n",
       "   43,\n",
       "   44,\n",
       "   45,\n",
       "   46,\n",
       "   47,\n",
       "   48,\n",
       "   49,\n",
       "   50,\n",
       "   51,\n",
       "   52,\n",
       "   53,\n",
       "   54,\n",
       "   55,\n",
       "   56,\n",
       "   61,\n",
       "   62,\n",
       "   65,\n",
       "   66,\n",
       "   67,\n",
       "   68,\n",
       "   69,\n",
       "   70,\n",
       "   71,\n",
       "   75,\n",
       "   76,\n",
       "   77,\n",
       "   78,\n",
       "   79])]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.transformers_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_clean_miss_var = pd.DataFrame(X_train_clean, columns=num_var_mean+num_var_median+cat_vars_mode+cat_vars_missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "      <th>Alley</th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>Electrical</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65</td>\n",
       "      <td>196</td>\n",
       "      <td>2003</td>\n",
       "      <td>Grvl</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>Unf</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>1976</td>\n",
       "      <td>Grvl</td>\n",
       "      <td>None</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>Unf</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>68</td>\n",
       "      <td>162</td>\n",
       "      <td>2001</td>\n",
       "      <td>Grvl</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Mn</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>Unf</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "      <td>1998</td>\n",
       "      <td>Grvl</td>\n",
       "      <td>None</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>No</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>Unf</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>Unf</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84</td>\n",
       "      <td>350</td>\n",
       "      <td>2000</td>\n",
       "      <td>Grvl</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Av</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>Unf</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>RFn</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "      <td>missing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  LotFrontage MasVnrArea GarageYrBlt Alley MasVnrType BsmtQual BsmtCond  \\\n",
       "0          65        196        2003  Grvl    BrkFace       Gd       TA   \n",
       "1          80          0        1976  Grvl       None       Gd       TA   \n",
       "2          68        162        2001  Grvl    BrkFace       Gd       TA   \n",
       "3          60          0        1998  Grvl       None       TA       Gd   \n",
       "4          84        350        2000  Grvl    BrkFace       Gd       TA   \n",
       "\n",
       "  BsmtExposure BsmtFinType1 BsmtFinType2 Electrical FireplaceQu GarageType  \\\n",
       "0           No          GLQ          Unf      SBrkr          Gd     Attchd   \n",
       "1           Gd          ALQ          Unf      SBrkr          TA     Attchd   \n",
       "2           Mn          GLQ          Unf      SBrkr          TA     Attchd   \n",
       "3           No          ALQ          Unf      SBrkr          Gd     Detchd   \n",
       "4           Av          GLQ          Unf      SBrkr          TA     Attchd   \n",
       "\n",
       "  GarageFinish GarageQual GarageCond   PoolQC    Fence MiscFeature  \n",
       "0          RFn         TA         TA  missing  missing     missing  \n",
       "1          RFn         TA         TA  missing  missing     missing  \n",
       "2          RFn         TA         TA  missing  missing     missing  \n",
       "3          Unf         TA         TA  missing  missing     missing  \n",
       "4          RFn         TA         TA  missing  missing     missing  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_clean_miss_var.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_clean_miss_var.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Grvl    50\n",
       "Pave    41\n",
       "Name: Alley, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[\"Alley\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Grvl    1419\n",
       "Pave      41\n",
       "Name: Alley, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_clean_miss_var[\"Alley\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "missing    1406\n",
       "Shed         49\n",
       "Gar2          2\n",
       "Othr          2\n",
       "TenC          1\n",
       "Name: MiscFeature, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_clean_miss_var[\"MiscFeature\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Clean X_train DataFrame with call variables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# no missing values variables index\n",
    "remainder_vars_index = [0,\n",
    "   1,\n",
    "   2,\n",
    "   4,\n",
    "   5,\n",
    "   7,\n",
    "   8,\n",
    "   9,\n",
    "   10,\n",
    "   11,\n",
    "   12,\n",
    "   13,\n",
    "   14,\n",
    "   15,\n",
    "   16,\n",
    "   17,\n",
    "   18,\n",
    "   19,\n",
    "   20,\n",
    "   21,\n",
    "   22,\n",
    "   23,\n",
    "   24,\n",
    "   27,\n",
    "   28,\n",
    "   29,\n",
    "   34,\n",
    "   36,\n",
    "   37,\n",
    "   38,\n",
    "   39,\n",
    "   40,\n",
    "   41,\n",
    "   43,\n",
    "   44,\n",
    "   45,\n",
    "   46,\n",
    "   47,\n",
    "   48,\n",
    "   49,\n",
    "   50,\n",
    "   51,\n",
    "   52,\n",
    "   53,\n",
    "   54,\n",
    "   55,\n",
    "   56,\n",
    "   61,\n",
    "   62,\n",
    "   65,\n",
    "   66,\n",
    "   67,\n",
    "   68,\n",
    "   69,\n",
    "   70,\n",
    "   71,\n",
    "   75,\n",
    "   76,\n",
    "   77,\n",
    "   78,\n",
    "   79]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Id',\n",
       " 'MSSubClass',\n",
       " 'MSZoning',\n",
       " 'LotArea',\n",
       " 'Street',\n",
       " 'LotShape',\n",
       " 'LandContour',\n",
       " 'Utilities',\n",
       " 'LotConfig',\n",
       " 'LandSlope',\n",
       " 'Neighborhood',\n",
       " 'Condition1',\n",
       " 'Condition2',\n",
       " 'BldgType',\n",
       " 'HouseStyle',\n",
       " 'OverallQual',\n",
       " 'OverallCond',\n",
       " 'YearBuilt',\n",
       " 'YearRemodAdd',\n",
       " 'RoofStyle',\n",
       " 'RoofMatl',\n",
       " 'Exterior1st',\n",
       " 'Exterior2nd',\n",
       " 'ExterQual',\n",
       " 'ExterCond',\n",
       " 'Foundation',\n",
       " 'BsmtFinSF1',\n",
       " 'BsmtFinSF2',\n",
       " 'BsmtUnfSF',\n",
       " 'TotalBsmtSF',\n",
       " 'Heating',\n",
       " 'HeatingQC',\n",
       " 'CentralAir',\n",
       " '1stFlrSF',\n",
       " '2ndFlrSF',\n",
       " 'LowQualFinSF',\n",
       " 'GrLivArea',\n",
       " 'BsmtFullBath',\n",
       " 'BsmtHalfBath',\n",
       " 'FullBath',\n",
       " 'HalfBath',\n",
       " 'BedroomAbvGr',\n",
       " 'KitchenAbvGr',\n",
       " 'KitchenQual',\n",
       " 'TotRmsAbvGrd',\n",
       " 'Functional',\n",
       " 'Fireplaces',\n",
       " 'GarageCars',\n",
       " 'GarageArea',\n",
       " 'PavedDrive',\n",
       " 'WoodDeckSF',\n",
       " 'OpenPorchSF',\n",
       " 'EnclosedPorch',\n",
       " '3SsnPorch',\n",
       " 'ScreenPorch',\n",
       " 'PoolArea',\n",
       " 'MiscVal',\n",
       " 'MoSold',\n",
       " 'YrSold',\n",
       " 'SaleType',\n",
       " 'SaleCondition']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get no missing values variables name using there index\n",
    "remainder_vars = [isnull_sum.keys()[var_index] for var_index in remainder_vars_index]\n",
    "remainder_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(remainder_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatinate X_train_clean_miss_var df and remainder_vars\n",
    "X_train =  pd.concat([X_train_clean_miss_var,train[remainder_vars]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 80)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1459, 19)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create test DataFrame with missing value imputed variables\n",
    "X_test_clean_miss_var = pd.DataFrame(X_test_clean, columns=num_var_mean+num_var_median+cat_vars_mode+cat_vars_missing)\n",
    "X_test_clean_miss_var.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_clean_miss_var.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1459, 80)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatinate X_test_clean_miss_var df and remainder_vars\n",
    "X_test =  pd.concat([X_test_clean_miss_var,test[remainder_vars]], axis=1)\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 22 <= What is this, in X_test df still missing values as available but why \n",
    "#because we fill missing values in those columns which have missing value present in only X_train df\n",
    "# Basicaly we get df then  find missing values variables then split df into X_train, X_test, y_train, y_test\n",
    "# after that we fill missing value\n",
    "\n",
    "# so if we have train and test df seperatly then first thing we should concatinate then find the missing values variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LotFrontage      0\n",
       "MasVnrArea       0\n",
       "GarageYrBlt      0\n",
       "Alley            0\n",
       "MasVnrType       0\n",
       "                ..\n",
       "MiscVal          0\n",
       "MoSold           0\n",
       "YrSold           0\n",
       "SaleType         1\n",
       "SaleCondition    0\n",
       "Length: 80, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isnull_sum_test = X_test.isnull().sum()\n",
    "isnull_sum_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BsmtFinSF1',\n",
       " 'BsmtFinSF2',\n",
       " 'BsmtUnfSF',\n",
       " 'TotalBsmtSF',\n",
       " 'BsmtFullBath',\n",
       " 'BsmtHalfBath',\n",
       " 'GarageCars',\n",
       " 'GarageArea']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding the numerical variable which have mising value\n",
    "num_vars_test = X_test.select_dtypes(include=[\"int64\", \"float64\"]).columns\n",
    "num_vars_miss_test = [var for var in num_vars_test if isnull_sum_test[var]>0]\n",
    "num_vars_miss_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MSZoning',\n",
       " 'Utilities',\n",
       " 'Exterior1st',\n",
       " 'Exterior2nd',\n",
       " 'KitchenQual',\n",
       " 'Functional',\n",
       " 'SaleType']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding the categorical variable which have mising value\n",
    "cat_vars_test = X_test.select_dtypes(include=[\"O\"]).columns\n",
    "cat_vars_miss_test = [var for var in cat_vars_test if isnull_sum_test[var]>0]\n",
    "cat_vars_miss_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}