Session 1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"application_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "\n",
       "   ...  FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21  \\\n",
       "0  ...                 0                0                0                0   \n",
       "1  ...                 0                0                0                0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "0                        0.0                       0.0   \n",
       "1                        0.0                       0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                        0.0                         1.0  \n",
       "1                        0.0                         0.0  \n",
       "\n",
       "[2 rows x 122 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 122)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 307511 entries, 0 to 307510\n",
      "Columns: 122 entries, SK_ID_CURR to AMT_REQ_CREDIT_BUREAU_YEAR\n",
      "dtypes: float64(65), int64(41), object(16)\n",
      "memory usage: 286.2+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick out diff datatype columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>DAYS_BIRTH</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>3.075110e+05</td>\n",
       "      <td>3.075110e+05</td>\n",
       "      <td>307499.000000</td>\n",
       "      <td>3.072330e+05</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>307511.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "      <td>265992.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>278180.518577</td>\n",
       "      <td>0.080729</td>\n",
       "      <td>0.417052</td>\n",
       "      <td>1.687979e+05</td>\n",
       "      <td>5.990260e+05</td>\n",
       "      <td>27108.573909</td>\n",
       "      <td>5.383962e+05</td>\n",
       "      <td>0.020868</td>\n",
       "      <td>-16036.995067</td>\n",
       "      <td>63815.045904</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008130</td>\n",
       "      <td>0.000595</td>\n",
       "      <td>0.000507</td>\n",
       "      <td>0.000335</td>\n",
       "      <td>0.006402</td>\n",
       "      <td>0.007000</td>\n",
       "      <td>0.034362</td>\n",
       "      <td>0.267395</td>\n",
       "      <td>0.265474</td>\n",
       "      <td>1.899974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>102790.175348</td>\n",
       "      <td>0.272419</td>\n",
       "      <td>0.722121</td>\n",
       "      <td>2.371231e+05</td>\n",
       "      <td>4.024908e+05</td>\n",
       "      <td>14493.737315</td>\n",
       "      <td>3.694465e+05</td>\n",
       "      <td>0.013831</td>\n",
       "      <td>4363.988632</td>\n",
       "      <td>141275.766519</td>\n",
       "      <td>...</td>\n",
       "      <td>0.089798</td>\n",
       "      <td>0.024387</td>\n",
       "      <td>0.022518</td>\n",
       "      <td>0.018299</td>\n",
       "      <td>0.083849</td>\n",
       "      <td>0.110757</td>\n",
       "      <td>0.204685</td>\n",
       "      <td>0.916002</td>\n",
       "      <td>0.794056</td>\n",
       "      <td>1.869295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>100002.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.565000e+04</td>\n",
       "      <td>4.500000e+04</td>\n",
       "      <td>1615.500000</td>\n",
       "      <td>4.050000e+04</td>\n",
       "      <td>0.000290</td>\n",
       "      <td>-25229.000000</td>\n",
       "      <td>-17912.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>189145.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.125000e+05</td>\n",
       "      <td>2.700000e+05</td>\n",
       "      <td>16524.000000</td>\n",
       "      <td>2.385000e+05</td>\n",
       "      <td>0.010006</td>\n",
       "      <td>-19682.000000</td>\n",
       "      <td>-2760.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>278202.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.471500e+05</td>\n",
       "      <td>5.135310e+05</td>\n",
       "      <td>24903.000000</td>\n",
       "      <td>4.500000e+05</td>\n",
       "      <td>0.018850</td>\n",
       "      <td>-15750.000000</td>\n",
       "      <td>-1213.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>367142.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.025000e+05</td>\n",
       "      <td>8.086500e+05</td>\n",
       "      <td>34596.000000</td>\n",
       "      <td>6.795000e+05</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>-12413.000000</td>\n",
       "      <td>-289.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>456255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>1.170000e+08</td>\n",
       "      <td>4.050000e+06</td>\n",
       "      <td>258025.500000</td>\n",
       "      <td>4.050000e+06</td>\n",
       "      <td>0.072508</td>\n",
       "      <td>-7489.000000</td>\n",
       "      <td>365243.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>27.000000</td>\n",
       "      <td>261.000000</td>\n",
       "      <td>25.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          SK_ID_CURR         TARGET   CNT_CHILDREN  AMT_INCOME_TOTAL  \\\n",
       "count  307511.000000  307511.000000  307511.000000      3.075110e+05   \n",
       "mean   278180.518577       0.080729       0.417052      1.687979e+05   \n",
       "std    102790.175348       0.272419       0.722121      2.371231e+05   \n",
       "min    100002.000000       0.000000       0.000000      2.565000e+04   \n",
       "25%    189145.500000       0.000000       0.000000      1.125000e+05   \n",
       "50%    278202.000000       0.000000       0.000000      1.471500e+05   \n",
       "75%    367142.500000       0.000000       1.000000      2.025000e+05   \n",
       "max    456255.000000       1.000000      19.000000      1.170000e+08   \n",
       "\n",
       "         AMT_CREDIT    AMT_ANNUITY  AMT_GOODS_PRICE  \\\n",
       "count  3.075110e+05  307499.000000     3.072330e+05   \n",
       "mean   5.990260e+05   27108.573909     5.383962e+05   \n",
       "std    4.024908e+05   14493.737315     3.694465e+05   \n",
       "min    4.500000e+04    1615.500000     4.050000e+04   \n",
       "25%    2.700000e+05   16524.000000     2.385000e+05   \n",
       "50%    5.135310e+05   24903.000000     4.500000e+05   \n",
       "75%    8.086500e+05   34596.000000     6.795000e+05   \n",
       "max    4.050000e+06  258025.500000     4.050000e+06   \n",
       "\n",
       "       REGION_POPULATION_RELATIVE     DAYS_BIRTH  DAYS_EMPLOYED  ...  \\\n",
       "count               307511.000000  307511.000000  307511.000000  ...   \n",
       "mean                     0.020868  -16036.995067   63815.045904  ...   \n",
       "std                      0.013831    4363.988632  141275.766519  ...   \n",
       "min                      0.000290  -25229.000000  -17912.000000  ...   \n",
       "25%                      0.010006  -19682.000000   -2760.000000  ...   \n",
       "50%                      0.018850  -15750.000000   -1213.000000  ...   \n",
       "75%                      0.028663  -12413.000000    -289.000000  ...   \n",
       "max                      0.072508   -7489.000000  365243.000000  ...   \n",
       "\n",
       "       FLAG_DOCUMENT_18  FLAG_DOCUMENT_19  FLAG_DOCUMENT_20  FLAG_DOCUMENT_21  \\\n",
       "count     307511.000000     307511.000000     307511.000000     307511.000000   \n",
       "mean           0.008130          0.000595          0.000507          0.000335   \n",
       "std            0.089798          0.024387          0.022518          0.018299   \n",
       "min            0.000000          0.000000          0.000000          0.000000   \n",
       "25%            0.000000          0.000000          0.000000          0.000000   \n",
       "50%            0.000000          0.000000          0.000000          0.000000   \n",
       "75%            0.000000          0.000000          0.000000          0.000000   \n",
       "max            1.000000          1.000000          1.000000          1.000000   \n",
       "\n",
       "       AMT_REQ_CREDIT_BUREAU_HOUR  AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "count               265992.000000              265992.000000   \n",
       "mean                     0.006402                   0.007000   \n",
       "std                      0.083849                   0.110757   \n",
       "min                      0.000000                   0.000000   \n",
       "25%                      0.000000                   0.000000   \n",
       "50%                      0.000000                   0.000000   \n",
       "75%                      0.000000                   0.000000   \n",
       "max                      4.000000                   9.000000   \n",
       "\n",
       "       AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "count               265992.000000              265992.000000   \n",
       "mean                     0.034362                   0.267395   \n",
       "std                      0.204685                   0.916002   \n",
       "min                      0.000000                   0.000000   \n",
       "25%                      0.000000                   0.000000   \n",
       "50%                      0.000000                   0.000000   \n",
       "75%                      0.000000                   0.000000   \n",
       "max                      8.000000                  27.000000   \n",
       "\n",
       "       AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "count              265992.000000               265992.000000  \n",
       "mean                    0.265474                    1.899974  \n",
       "std                     0.794056                    1.869295  \n",
       "min                     0.000000                    0.000000  \n",
       "25%                     0.000000                    0.000000  \n",
       "50%                     0.000000                    1.000000  \n",
       "75%                     0.000000                    3.000000  \n",
       "max                   261.000000                   25.000000  \n",
       "\n",
       "[8 rows x 106 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['SK_ID_CURR', 'TARGET', 'CNT_CHILDREN', 'AMT_INCOME_TOTAL',\n",
       "       'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE',\n",
       "       'REGION_POPULATION_RELATIVE', 'DAYS_BIRTH', 'DAYS_EMPLOYED',\n",
       "       ...\n",
       "       'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20',\n",
       "       'FLAG_DOCUMENT_21', 'AMT_REQ_CREDIT_BUREAU_HOUR',\n",
       "       'AMT_REQ_CREDIT_BUREAU_DAY', 'AMT_REQ_CREDIT_BUREAU_WEEK',\n",
       "       'AMT_REQ_CREDIT_BUREAU_MON', 'AMT_REQ_CREDIT_BUREAU_QRT',\n",
       "       'AMT_REQ_CREDIT_BUREAU_YEAR'],\n",
       "      dtype='object', length=106)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe().columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.columns) - len(data.describe().columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove null/ NA values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307506</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307507</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307508</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307509</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307510</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>307511 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        SK_ID_CURR  TARGET  NAME_CONTRACT_TYPE  CODE_GENDER  FLAG_OWN_CAR  \\\n",
       "0            False   False               False        False         False   \n",
       "1            False   False               False        False         False   \n",
       "2            False   False               False        False         False   \n",
       "3            False   False               False        False         False   \n",
       "4            False   False               False        False         False   \n",
       "...            ...     ...                 ...          ...           ...   \n",
       "307506       False   False               False        False         False   \n",
       "307507       False   False               False        False         False   \n",
       "307508       False   False               False        False         False   \n",
       "307509       False   False               False        False         False   \n",
       "307510       False   False               False        False         False   \n",
       "\n",
       "        FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  \\\n",
       "0                 False         False             False       False   \n",
       "1                 False         False             False       False   \n",
       "2                 False         False             False       False   \n",
       "3                 False         False             False       False   \n",
       "4                 False         False             False       False   \n",
       "...                 ...           ...               ...         ...   \n",
       "307506            False         False             False       False   \n",
       "307507            False         False             False       False   \n",
       "307508            False         False             False       False   \n",
       "307509            False         False             False       False   \n",
       "307510            False         False             False       False   \n",
       "\n",
       "        AMT_ANNUITY  ...  FLAG_DOCUMENT_18  FLAG_DOCUMENT_19  \\\n",
       "0             False  ...             False             False   \n",
       "1             False  ...             False             False   \n",
       "2             False  ...             False             False   \n",
       "3             False  ...             False             False   \n",
       "4             False  ...             False             False   \n",
       "...             ...  ...               ...               ...   \n",
       "307506        False  ...             False             False   \n",
       "307507        False  ...             False             False   \n",
       "307508        False  ...             False             False   \n",
       "307509        False  ...             False             False   \n",
       "307510        False  ...             False             False   \n",
       "\n",
       "        FLAG_DOCUMENT_20  FLAG_DOCUMENT_21  AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                  False             False                       False   \n",
       "1                  False             False                       False   \n",
       "2                  False             False                       False   \n",
       "3                  False             False                        True   \n",
       "4                  False             False                       False   \n",
       "...                  ...               ...                         ...   \n",
       "307506             False             False                        True   \n",
       "307507             False             False                        True   \n",
       "307508             False             False                       False   \n",
       "307509             False             False                       False   \n",
       "307510             False             False                       False   \n",
       "\n",
       "        AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "0                           False                       False   \n",
       "1                           False                       False   \n",
       "2                           False                       False   \n",
       "3                            True                        True   \n",
       "4                           False                       False   \n",
       "...                           ...                         ...   \n",
       "307506                       True                        True   \n",
       "307507                       True                        True   \n",
       "307508                      False                       False   \n",
       "307509                      False                       False   \n",
       "307510                      False                       False   \n",
       "\n",
       "        AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                           False                      False   \n",
       "1                           False                      False   \n",
       "2                           False                      False   \n",
       "3                            True                       True   \n",
       "4                           False                      False   \n",
       "...                           ...                        ...   \n",
       "307506                       True                       True   \n",
       "307507                       True                       True   \n",
       "307508                      False                      False   \n",
       "307509                      False                      False   \n",
       "307510                      False                      False   \n",
       "\n",
       "        AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                            False  \n",
       "1                            False  \n",
       "2                            False  \n",
       "3                             True  \n",
       "4                            False  \n",
       "...                            ...  \n",
       "307506                        True  \n",
       "307507                        True  \n",
       "307508                       False  \n",
       "307509                       False  \n",
       "307510                       False  \n",
       "\n",
       "[307511 rows x 122 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         0.0\n",
       "1         0.0\n",
       "2         0.0\n",
       "3         NaN\n",
       "4         0.0\n",
       "         ... \n",
       "307506    NaN\n",
       "307507    NaN\n",
       "307508    0.0\n",
       "307509    0.0\n",
       "307510    0.0\n",
       "Name: AMT_REQ_CREDIT_BUREAU_DAY, Length: 307511, dtype: float64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['AMT_REQ_CREDIT_BUREAU_DAY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,     12,    278,   1292,      0,      0,      0,      0,\n",
       "            0,      0,      0,      0,      0, 202929,      0,      0,\n",
       "            0,      0,      0,      0,  96391,      2,      0,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0, 173378,    660,  60965, 156061, 179943, 150007, 204488,\n",
       "       214865, 163891, 154828, 153020, 208642, 182590, 210199, 154350,\n",
       "       213514, 169682, 156061, 179943, 150007, 204488, 214865, 163891,\n",
       "       154828, 153020, 208642, 182590, 210199, 154350, 213514, 169682,\n",
       "       156061, 179943, 150007, 204488, 214865, 163891, 154828, 153020,\n",
       "       208642, 182590, 210199, 154350, 213514, 169682, 210295, 154297,\n",
       "       148431, 156341, 145755,   1021,   1021,   1021,   1021,      1,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,      0,      0,      0,  41519,  41519,  41519,  41519,\n",
       "        41519,  41519])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "153755.5"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_elig_cnt = len(data) * 0.5\n",
    "max_elig_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     81\n",
       "False    41\n",
       "dtype: int64"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data.isna().sum() < max_elig_cnt).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['OWN_CAR_AGE', 'EXT_SOURCE_1', 'APARTMENTS_AVG', 'BASEMENTAREA_AVG',\n",
       "       'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'ELEVATORS_AVG', 'ENTRANCES_AVG',\n",
       "       'FLOORSMIN_AVG', 'LANDAREA_AVG', 'LIVINGAPARTMENTS_AVG',\n",
       "       'LIVINGAREA_AVG', 'NONLIVINGAPARTMENTS_AVG', 'NONLIVINGAREA_AVG',\n",
       "       'APARTMENTS_MODE', 'BASEMENTAREA_MODE', 'YEARS_BUILD_MODE',\n",
       "       'COMMONAREA_MODE', 'ELEVATORS_MODE', 'ENTRANCES_MODE', 'FLOORSMIN_MODE',\n",
       "       'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE',\n",
       "       'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE', 'APARTMENTS_MEDI',\n",
       "       'BASEMENTAREA_MEDI', 'YEARS_BUILD_MEDI', 'COMMONAREA_MEDI',\n",
       "       'ELEVATORS_MEDI', 'ENTRANCES_MEDI', 'FLOORSMIN_MEDI', 'LANDAREA_MEDI',\n",
       "       'LIVINGAPARTMENTS_MEDI', 'LIVINGAREA_MEDI', 'NONLIVINGAPARTMENTS_MEDI',\n",
       "       'NONLIVINGAREA_MEDI', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE',\n",
       "       'WALLSMATERIAL_MODE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns[data.isna().sum() > max_elig_cnt]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get in Numpy Array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['OWN_CAR_AGE', 'EXT_SOURCE_1', 'APARTMENTS_AVG',\n",
       "       'BASEMENTAREA_AVG', 'YEARS_BUILD_AVG', 'COMMONAREA_AVG',\n",
       "       'ELEVATORS_AVG', 'ENTRANCES_AVG', 'FLOORSMIN_AVG', 'LANDAREA_AVG',\n",
       "       'LIVINGAPARTMENTS_AVG', 'LIVINGAREA_AVG',\n",
       "       'NONLIVINGAPARTMENTS_AVG', 'NONLIVINGAREA_AVG', 'APARTMENTS_MODE',\n",
       "       'BASEMENTAREA_MODE', 'YEARS_BUILD_MODE', 'COMMONAREA_MODE',\n",
       "       'ELEVATORS_MODE', 'ENTRANCES_MODE', 'FLOORSMIN_MODE',\n",
       "       'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE',\n",
       "       'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE',\n",
       "       'APARTMENTS_MEDI', 'BASEMENTAREA_MEDI', 'YEARS_BUILD_MEDI',\n",
       "       'COMMONAREA_MEDI', 'ELEVATORS_MEDI', 'ENTRANCES_MEDI',\n",
       "       'FLOORSMIN_MEDI', 'LANDAREA_MEDI', 'LIVINGAPARTMENTS_MEDI',\n",
       "       'LIVINGAREA_MEDI', 'NONLIVINGAPARTMENTS_MEDI',\n",
       "       'NONLIVINGAREA_MEDI', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE',\n",
       "       'WALLSMATERIAL_MODE'], dtype=object)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns[data.isna().sum() > max_elig_cnt].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick Good Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_good = data[data.columns[data.isna().sum() < max_elig_cnt]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 81)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,     12,    278,   1292,      0,      0,      0,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,      0,      0,  96391,      2,      0,      0,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "          660,  60965, 150007, 153020, 150007, 153020, 150007, 153020,\n",
       "       148431, 145755,   1021,   1021,   1021,   1021,      1,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,      0,      0,      0,      0,      0,      0,      0,\n",
       "            0,      0,      0,  41519,  41519,  41519,  41519,  41519,\n",
       "        41519])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good.isna().sum().values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numerical column Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "168797.9192969845"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "147150.0"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL'].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    135000.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL'].mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/avishek/opt/anaconda3/lib/python3.8/site-packages/pandas/core/generic.py:6245: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._update_inplace(new_data)\n"
     ]
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL'].fillna(df_good['AMT_INCOME_TOTAL'].median(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         202500.0\n",
       "1         270000.0\n",
       "2          67500.0\n",
       "3         135000.0\n",
       "4         121500.0\n",
       "            ...   \n",
       "307506    157500.0\n",
       "307507     72000.0\n",
       "307508    153000.0\n",
       "307509    171000.0\n",
       "307510    157500.0\n",
       "Name: AMT_INCOME_TOTAL, Length: 307511, dtype: float64"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_good['AMT_INCOME_TOTAL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}